certstream

README

CertStream

Small library wrapping github.com/google/certificate-transparency-go.

Adds no new dependencies except github.com/linkdata/bwlimit, which has no dependencies outside the standard library.

Documentation

Index

• Constants
• Variables
• func GetLogList(ctx context.Context, httpClient *http.Client, listUrl string) (logList *loglist3.LogList, err error)
• func OperatorDomain(urlString string) string
• func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
• func ScanDnsname(row Scanner, p *PgDnsname) error
• func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
• func ScanIdent(row Scanner, ident *PgIdent) error
• func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
• type CertStream
  • func Start(ctx context.Context, wg *sync.WaitGroup, cfg *Config) (cs *CertStream, err error)
  • func (cs *CertStream) Close()
  • func (cs *CertStream) CountStreams() (n int)
  • func (cs *CertStream) LogError(err error, msg string, args ...any) error
  • func (cs *CertStream) LogInfo(msg string, args ...any)
  • func (cs *CertStream) Operators() (operators []*LogOperator)
• type Certificate
• type Config
  • func NewConfig() *Config
• type JsonCertificate
  • func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)
• type JsonIdentity
  • func (id *JsonIdentity) Fill(name *pkix.Name)
• type LogEntry
  • func (le *LogEntry) Cert() (crt *Certificate)
  • func (le *LogEntry) Index() (index int64)
  • func (le *LogEntry) String() (s string)
• type LogOperator
  • func (lo *LogOperator) StreamCount() (n int)
  • func (lo *LogOperator) Streams() (sl []*LogStream)
• type LogStream
  • func (ls *LogStream) GetRawEntries(ctx context.Context, start, end int64, historical bool, ...) (youngest time.Duration)
  • func (ls *LogStream) NewLastIndex(ctx context.Context) (lastIndex int64, err error)
  • func (ls *LogStream) String() string
• type Logger
• type PgCertificate
• type PgDB
  • func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)
  • func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)
  • func (cdb *PgDB) Close()
  • func (cdb *PgDB) Estimate(table string) (f float64)
  • func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)
  • func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)
  • func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
  • func (cdb *PgDB) QueueUsage() (pct int)
• type PgDnsname
• type PgDnsnamesView
• type PgIdent
• type PgLogEntry
• type Scanner

Constants

const SelectDnsnameLike = `SELECT * FROM CERTDB_dnsnames WHERE dnsname LIKE $1;`

const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`

const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`

const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`

Variables

var BatchSize = 1024

var BulkRange = int64(4096)

var CreateSchema string

var DefaultTransport = &http.Transport{
	TLSHandshakeTimeout:   30 * time.Second,
	ResponseHeaderTimeout: 30 * time.Second,
	MaxIdleConnsPerHost:   2,
	DisableKeepAlives:     false,
	ExpectContinueTimeout: 1 * time.Second,
	ForceAttemptHTTP2:     true,
}

var FunctionOperatorID string

var FunctionStreamID string

var ProcAddNewEntry string

var SelectGaps string

Functions

func GetLogList

func GetLogList(ctx context.Context, httpClient *http.Client, listUrl string) (logList *loglist3.LogList, err error)

GetLogList fetches a CT log list from the given listUrl. Usually you would pass loglist3.AllLogListURL for the listUrl.

func OperatorDomain

func OperatorDomain(urlString string) string

OperatorDomain returns the TLD+1 given an URL.

func ScanCertificate

func ScanCertificate(row Scanner, cert *PgCertificate) (err error)

func ScanDnsname

func ScanDnsname(row Scanner, p *PgDnsname) error

func ScanDnsnamesView

func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)

func ScanIdent

func ScanIdent(row Scanner, ident *PgIdent) error

func ScanLogEntry

func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)

Types

type CertStream

type CertStream struct {
	Config                      // copy of config
	C          <-chan *LogEntry // log entry channel
	HeadClient *http.Client     // main HTTP client, uses Config.HeadDialer
	TailClient *http.Client     // may be nil if not backfilling
	DB         *PgDB
	// contains filtered or unexported fields
}

func Start

func Start(ctx context.Context, wg *sync.WaitGroup, cfg *Config) (cs *CertStream, err error)

func (*CertStream) Close

func (cs *CertStream) Close()

func (*CertStream) CountStreams

func (cs *CertStream) CountStreams() (n int)

func (*CertStream) LogError

func (cs *CertStream) LogError(err error, msg string, args ...any) error

func (*CertStream) LogInfo

func (cs *CertStream) LogInfo(msg string, args ...any)

func (*CertStream) Operators

func (cs *CertStream) Operators() (operators []*LogOperator)

type Certificate

type Certificate struct {
	PreCert   bool
	Seen      time.Time
	Signature []byte
	*x509.Certificate
}

type Config

type Config struct {
	Logger     Logger              // if not nil Logger to use, no default
	HeadDialer proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
	TailDialer proxy.ContextDialer // if not nil, backfill db using this dialer, no default
	PgUser     string              // PostgreSQL user, default "certstream"
	PgPass     string              // PostgreSQL password, default "certstream"
	PgName     string              // PostgreSQL db name, default "certstream"
	PgAddr     string              // PostgreSQL address, no default
	PgPrefix   string              // PostgreSQL naming prefix, default "certdb_"
	PgConns    int                 // max number of database connections, default 100
	PgMaxAge   int                 // maximum age in days to backfill
}

func NewConfig

func NewConfig() *Config

NewConfig returns a new default Config

type JsonCertificate

type JsonCertificate struct {
	PreCert        bool         `json:",omitempty"`
	Seen           time.Time    `json:",omitempty"`
	Signature      []byte       `json:",omitempty"` // SHA256 signature, searchable on crt.sh
	Issuer         JsonIdentity `json:",omitempty"`
	Subject        JsonIdentity `json:",omitempty"`
	DNSNames       []string     `json:",omitempty"`
	EmailAddresses []string     `json:",omitempty"`
	IPAddresses    []string     `json:",omitempty"`
	URIs           []string     `json:",omitempty"`
	NotBefore      time.Time    `json:",omitempty"`
	NotAfter       time.Time    `json:",omitempty"`
}

func NewJSONCertificate

func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)

type JsonIdentity

type JsonIdentity struct {
	Country      string `json:",omitempty"`
	Organization string `json:",omitempty"`
	Province     string `json:",omitempty"`
	CommonName   string `json:",omitempty"`
}

func (*JsonIdentity) Fill

func (id *JsonIdentity) Fill(name *pkix.Name)

type LogEntry

type LogEntry struct {
	*LogStream
	Err          error           // error from RawLogEntryFromLeaf or ToLogEntry, or nil
	RawLogEntry  *ct.RawLogEntry // may be nil in case of error
	*ct.LogEntry                 // may be nil in case of error
	Id           int64           // database id, if available
	Historical   bool            // true if the entry is from gap or backfilling
}

func (*LogEntry) Cert

func (le *LogEntry) Cert() (crt *Certificate)

Cert returns the Certificate given a LogEntry or nil.

func (*LogEntry) Index

func (le *LogEntry) Index() (index int64)

Index returns the log index or -1 if none is available.

func (*LogEntry) String

func (le *LogEntry) String() (s string)

type LogOperator

type LogOperator struct {
	*CertStream
	*loglist3.Operator
	Domain string       // e.g. "letsencrypt.org" or "googleapis.com"
	Count  atomic.Int64 // atomic; sum of the stream's Count
	Id     int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogOperator) StreamCount

func (lo *LogOperator) StreamCount() (n int)

func (*LogOperator) Streams

func (lo *LogOperator) Streams() (sl []*LogStream)

type LogStream

type LogStream struct {
	*LogOperator
	*loglist3.Log
	HeadClient *client.LogClient
	TailClient *client.LogClient
	Count      atomic.Int64 // number of certificates sent to the channel
	MinIndex   atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
	MaxIndex   atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
	LastIndex  atomic.Int64 // atomic: highest index that is available from stream source
	InsideGaps atomic.Int64 // atomic: number of remaining entries inside gaps
	Id         int32        // database ID, if available
}

func (*LogStream) GetRawEntries

func (ls *LogStream) GetRawEntries(ctx context.Context, start, end int64, historical bool, gapcounter *atomic.Int64) (youngest time.Duration)

func (*LogStream) NewLastIndex

func (ls *LogStream) NewLastIndex(ctx context.Context) (lastIndex int64, err error)

func (*LogStream) String

func (ls *LogStream) String() string

type Logger

type Logger interface {
	Info(msg string, args ...any)
	Error(msg string, args ...any)
}

type PgCertificate

type PgCertificate struct {
	Id         int64
	NotBefore  time.Time
	NotAfter   time.Time
	CommonName string
	SubjectID  int
	IssuerID   int
	Sha256     []byte
	PreCert    bool
}

type PgDB

type PgDB struct {
	*CertStream
	*pgxpool.Pool
	Pfx     func(string) string // prefix replacer
	Workers atomic.Int32
	// contains filtered or unexported fields
}

PgDB integrates with sql.DB to manage certificate stream data for a PostgreSQL database

func NewPgDB

func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)

NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.

func (*PgDB) AverageNewEntryTime

func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)

func (*PgDB) Close

func (cdb *PgDB) Close()

func (*PgDB) Estimate

func (cdb *PgDB) Estimate(table string) (f float64)

func (*PgDB) GetCertificateByHash

func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByID

func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByLogEntry

func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)

func (*PgDB) QueueUsage

func (cdb *PgDB) QueueUsage() (pct int)

type PgDnsname

type PgDnsname struct {
	Dnsname string
	CertID  int64
}

type PgDnsnamesView

type PgDnsnamesView struct {
	CertID    int64
	DNSName   string
	NotBefore time.Time
	Idna      bool
	Valid     bool
	PreCert   bool
	Issuer    string
	Subject   string
	Crtsh     string
}

type PgIdent

type PgIdent struct {
	Id           int
	Organization string
	Province     string
	Country      string
}

type PgLogEntry

type PgLogEntry struct {
	Seen     time.Time // CT log entry timestamp
	LogIndex int64     // CT log index for the stream
	CertID   int64     // database ID of cert
	StreamID int       // database ID of stream
}

type Scanner

type Scanner interface {
	Scan(dest ...any) error
}