Skip to content
  • Rasmus Dahlberg's avatar
    Automate handling of notice file · ad9fb496
    Rasmus Dahlberg authored
    Here's a hacky tool to migrate our ongoing v0.0.1 measurement once it's
    done.  I.e., just split-up the NOTICE prints we have in collect.stdout,
    putting them in per-log notice files that happens automatically now.
    
    ```
    // Package main provides a hacky tool that extracts NOTICE: <log desc> prints
    // from a file collect.stdout, putting them in the logs data directories as
    // notice.txt.  Only meant to migrate away from v0.0.1 that did not store
    // per-log notice files automatically, which makes things less error-prone.
    package main
    
    import (
    	"bytes"
    	"encoding/json"
    	"fmt"
    	logger "log"
    	"os"
    	"strings"
    
    	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
    )
    
    func main() {
    	directory := "../data"
    	logDirectory := fmt.Sprintf("%s/logs", directory)
    	noticeFile := "../collect.stdout"
    
    	b, err := os.ReadFile(fmt.Sprintf("%s/metadata.json", directory))
    	if err != nil {
    		logger.Fatal(err)
    	}
    	var md metadata.Metadata
    	if err := json.Unmarshal(b, &md); err != nil {
    		logger.Fatal(err)
    	}
    	if b, err = os.ReadFile(noticeFile); err != nil {
    		logger.Fatal(err)
    	}
    
    	lines := bytes.Split(b, []byte("\n"))
    	for _, log := range logs(md) {
    		id, _ := log.Key.ID()
    		desc := *log.Description
    
    		var notes []byte
    		var numNotes int
    		for _, line := range lines[:len(lines)-1] {
    			if strings.Contains(string(line), fmt.Sprintf("NOTICE: %s", desc)) {
    				notes = append(notes, line...)
    				notes = append(notes, []byte("\n")...)
    				numNotes += 1
    			}
    		}
    
    		if len(notes) == 0 {
    			logger.Printf("%s: no notices", desc)
    			continue
    		}
    
    		logger.Printf("%s: %d notices", desc, numNotes)
    		if err := os.WriteFile(fmt.Sprintf("%s/%x/notice.txt", logDirectory, id[:]), notes, 0644); err != nil {
    			logger.Fatal(err)
    		}
    	}
    }
    
    func logs(md metadata.Metadata) (logs []metadata.Log) {
    	for _, operators := range md.Operators {
    		for _, log := range operators.Logs {
    			if log.Description == nil {
    				logger.Printf("WARNING: skipping log without description")
    				continue
    			}
    			if log.State == nil {
    				continue // skip logs with unknown states
    			}
    			if log.State.Name == metadata.LogStatePending {
    				continue // pending logs do not count towards CT-compliance
    			}
    			if log.State.Name == metadata.LogStateRetired {
    				continue // retired logs are not necessarily reachable
    			}
    			if log.State.Name == metadata.LogStateRejected {
    				continue // rejected logs do not count towards CT-compliance
    			}
    
    			logs = append(logs, log)
    		}
    	}
    	return
    }
    ```
    ad9fb496