importer

Documentation

Index

• Constants
• Variables
• type ConnEntry
• type DNSEntry
• type DoneChans
• type EntryChans
• type HTTPEntry
• type Importer
  • func NewImporter(db *database.DB, cfg *config.Config, importStartedAt time.Time, ...) (*Importer, error)
  • func (importer *Importer) Import(afs afero.Fs, files map[string][]string) error
• type MetaDBFile
• type ResultCounts
• type SSLEntry
• type UniqueConn
• type UniqueFQDN
• type WaitGroups
• type ZeekHeader
• type ZeekUIDRecord

Constants

const ConnPrefix = "conn"

const ConnSummaryPrefixHyphen = "conn-summary"

const ConnSummaryPrefixUnderscore = "conn_summary"

const DNSPrefix = "dns"

const HTTPPrefix = "http"

const OpenConnPrefix = "open_conn"

const OpenHTTPPrefix = "open_http"

const OpenSSLPrefix = "open_ssl"

const SSLPrefix = "ssl"

const ZeekDateTimeFmt = "2006-01-02-15-04-05"

ZeekDateTimeFmt is the common format for zeek header datetimes

Variables

var ErrAllFilesPreviouslyImported = errors.New("all files were previously imported")

Types

type ConnEntry

type ConnEntry struct {
	ImportTime           time.Time        `ch:"import_time"`
	ZeekUID              util.FixedString `ch:"zeek_uid"`
	ImportID             util.FixedString `ch:"import_id"`
	Filtered             bool             `ch:"filtered"`
	Hash                 util.FixedString `ch:"hash"`
	Timestamp            time.Time        `ch:"ts"`
	Src                  net.IP           `ch:"src"`
	Dst                  net.IP           `ch:"dst"`
	SrcNUID              uuid.UUID        `ch:"src_nuid"`
	DstNUID              uuid.UUID        `ch:"dst_nuid"`
	SrcPort              uint16           `ch:"src_port"`
	DstPort              uint16           `ch:"dst_port"`
	MissingHostHeader    bool             `ch:"missing_host_header"`    // used to mark HTTP entries that have a missing host header
	MissingHostUseragent string           `ch:"missing_host_useragent"` // useragent for connections that have a missing host header
	Proto                string           `ch:"proto"`
	Service              string           `ch:"service"`
	Duration             float64          `ch:"duration"`
	SrcLocal             bool             `ch:"src_local"`
	DstLocal             bool             `ch:"dst_local"`
	ICMPType             int              `ch:"icmp_type"`
	ICMPCode             int              `ch:"icmp_code"`
	SrcBytes             int64            `ch:"src_bytes"`
	DstBytes             int64            `ch:"dst_bytes"`
	SrcIPBytes           int64            `ch:"src_ip_bytes"`
	DstIPBytes           int64            `ch:"dst_ip_bytes"`
	SrcPackets           int64            `ch:"src_packets"`
	DstPackets           int64            `ch:"dst_packets"`
	ConnState            string           `ch:"conn_state"`
	MissedBytes          int64            `ch:"missed_bytes"`
	ZeekHistory          string           `ch:"zeek_history"`
}

type DNSEntry

type DNSEntry struct {
	ImportTime          time.Time        `ch:"import_time"`
	ZeekUID             util.FixedString `ch:"zeek_uid"`
	Hash                util.FixedString `ch:"hash"`
	Timestamp           time.Time        `ch:"ts"`
	Src                 net.IP           `ch:"src"`
	Dst                 net.IP           `ch:"dst"`
	SrcNUID             uuid.UUID        `ch:"src_nuid"`
	DstNUID             uuid.UUID        `ch:"dst_nuid"`
	SrcPort             uint16           `ch:"src_port"`
	DstPort             uint16           `ch:"dst_port"`
	SrcLocal            bool             `ch:"src_local"`
	DstLocal            bool             `ch:"dst_local"`
	TransactionID       uint16           `ch:"transaction_id"`
	RoundTripTime       float64          `ch:"round_trip_time"`
	Query               string           `ch:"query"`
	QueryClassCode      uint16           `ch:"query_class_code"`
	QueryClassName      string           `ch:"query_class_name"`
	QueryTypeCode       uint16           `ch:"query_type_code"`
	QueryTypeName       string           `ch:"query_type_name"`
	ResponseCode        uint16           `ch:"response_code"`
	ResponseCodeName    string           `ch:"response_code_name"`
	AuthoritativeAnswer bool             `ch:"authoritative_answer"`
	RecursionDesired    bool             `ch:"recursion_desired"`
	RecursionAvailable  bool             `ch:"recursion_available"`
	Z                   uint16           `ch:"z"`
	Answers             []string         `ch:"answers"`
	TTLs                []float64        `ch:"ttls"`
	Rejected            bool             `ch:"rejected"`
	// PDNS field
	ResolvedIP net.IP `ch:"resolved_ip"`
}

type DoneChans

type DoneChans struct {
	// contains filtered or unexported fields
}

type EntryChans

type EntryChans struct {
	Conn     chan zeektypes.Conn
	OpenConn chan zeektypes.Conn
	DNS      chan zeektypes.DNS
	HTTP     chan zeektypes.HTTP
	OpenHTTP chan zeektypes.HTTP
	SSL      chan zeektypes.SSL
	OpenSSL  chan zeektypes.SSL
}

type HTTPEntry

type HTTPEntry struct {
	ImportTime   time.Time        `ch:"import_time"`
	ZeekUID      util.FixedString `ch:"zeek_uid"`
	Hash         util.FixedString `ch:"hash"`
	Timestamp    time.Time        `ch:"ts"`
	Src          net.IP           `ch:"src"`
	Dst          net.IP           `ch:"dst"`
	SrcNUID      uuid.UUID        `ch:"src_nuid"`
	DstNUID      uuid.UUID        `ch:"dst_nuid"`
	MultiRequest bool             `ch:"multi_request"`
	SrcPort      uint16           `ch:"src_port"`
	DstPort      uint16           `ch:"dst_port"`
	Duration     float64          `ch:"duration"`
	SrcLocal     bool             `ch:"src_local"`
	DstLocal     bool             `ch:"dst_local"`
	SrcBytes     int64            `ch:"src_bytes"`
	DstBytes     int64            `ch:"dst_bytes"`
	SrcIPBytes   int64            `ch:"src_ip_bytes"`
	DstIPBytes   int64            `ch:"dst_ip_bytes"`
	SrcPackets   int64            `ch:"src_packets"`
	DstPackets   int64            `ch:"dst_packets"`
	Proto        string           `ch:"proto"`
	Service      string           `ch:"service"`
	ConnState    string           `ch:"conn_state"`
	TransDepth   uint16           `ch:"trans_depth"`
	Method       string           `ch:"method"`
	Host         string           `ch:"host"`
	URI          string           `ch:"uri"`
	Referrer     string           `ch:"referrer"`
	HTTPVersion  string           `ch:"http_version"`
	UserAgent    string           `ch:"useragent"`
	Origin       string           `ch:"origin"`
	StatusCode   int64            `ch:"status_code"`
	StatusMsg    string           `ch:"status_msg"`
	InfoCode     int64            `ch:"info_code"`
	InfoMsg      string           `ch:"info_msg"`
	Username     string           `ch:"username"`
	Password     string           `ch:"password"`
	SrcFUIDs     []string         `ch:"src_fuids"`
	SrcFileNames []string         `ch:"src_file_names"`
	SrcMIMETypes []string         `ch:"src_mime_types"`
	DstFUIDs     []string         `ch:"dst_fuids"`
	DstFileNames []string         `ch:"dst_file_names"`
	DstMIMETypes []string         `ch:"dst_mime_types"`
}

type Importer

type Importer struct {
	Cfg               *config.Config
	Database          *database.DB
	ImportID          util.FixedString
	LogDirectory      string
	FileMap           map[string][]string
	EntryChannels     EntryChans
	MetaDBChannel     chan MetaDBFile
	Paths             chan string
	ErrChannel        chan error
	TotalFileCount    int
	DoneChannels      DoneChans
	Writers           writers
	WriteLimiter      *rate.Limiter
	ProgressBar       *mpb.Progress
	FileProgressBar   *mpb.Bar
	ProgressLogger    *log.Logger
	HTTPLinkMutex     sync.Mutex
	OpenHTTPLinkMutex sync.Mutex
	NumParsers        int
	NumDigesters      int
	NumWriters        int
	ResultCounts      ResultCounts
	// contains filtered or unexported fields
}

func NewImporter

func NewImporter(db *database.DB, cfg *config.Config, importStartedAt time.Time, numDigesters int, numParsers int, numWriters int) (*Importer, error)

NewImporter creates and returns a new Importer object

func (*Importer) Import

func (importer *Importer) Import(afs afero.Fs, files map[string][]string) error

type MetaDBFile

type MetaDBFile struct {
	// contains filtered or unexported fields
}

type ResultCounts

type ResultCounts struct {
	ZeekUIDs       uint64
	OpenZeekUIDs   uint64
	UnfilteredConn uint64
	Conn           uint64
	OpenConn       uint64
	HTTP           uint64
	OpenHTTP       uint64
	DNS            uint64
	UDNS           int64
	PDNSRaw        uint64
	SSL            uint64
	OpenSSL        uint64
}

type SSLEntry

type SSLEntry struct {
	ImportTime       time.Time        `ch:"import_time"`
	ZeekUID          util.FixedString `ch:"zeek_uid"`
	Hash             util.FixedString `ch:"hash"`
	Timestamp        time.Time        `ch:"ts"`
	Src              net.IP           `ch:"src"`
	Dst              net.IP           `ch:"dst"`
	SrcNUID          uuid.UUID        `ch:"src_nuid"`
	DstNUID          uuid.UUID        `ch:"dst_nuid"`
	SrcPort          uint16           `ch:"src_port"`
	DstPort          uint16           `ch:"dst_port"`
	Duration         float64          `ch:"duration"`
	SrcLocal         bool             `ch:"src_local"`
	DstLocal         bool             `ch:"dst_local"`
	SrcBytes         int64            `ch:"src_bytes"`
	DstBytes         int64            `ch:"dst_bytes"`
	SrcIPBytes       int64            `ch:"src_ip_bytes"`
	DstIPBytes       int64            `ch:"dst_ip_bytes"`
	SrcPackets       int64            `ch:"src_packets"`
	DstPackets       int64            `ch:"dst_packets"`
	Proto            string           `ch:"proto"`
	Service          string           `ch:"service"`
	ConnState        string           `ch:"conn_state"`
	Version          string           `ch:"version"`
	Cipher           string           `ch:"cipher"`
	Curve            string           `ch:"curve"`
	ServerName       string           `ch:"server_name"`
	Resumed          bool             `ch:"resumed"`
	NextProtocol     string           `ch:"next_protocol"`
	Established      bool             `ch:"established"`
	ServerCertFUIDs  []string         `ch:"server_cert_fuids"`
	ClientCertFUIDs  []string         `ch:"client_cert_fuids"`
	ServerSubject    string           `ch:"server_subject"`
	ServerIssuer     string           `ch:"server_issuer"`
	ClientSubject    string           `ch:"client_subject"`
	ClientIssuer     string           `ch:"client_issuer"`
	ValidationStatus string           `ch:"validation_status"`
	JA3              string           `ch:"ja3"`
	JA3S             string           `ch:"ja3s"`
}

type UniqueConn

type UniqueConn struct {
	Hash      util.FixedString `ch:"hash"`
	Src       net.IP           `ch:"src"`
	Dst       net.IP           `ch:"dst"`
	SrcNUID   uuid.UUID        `ch:"src_nuid"`
	DstNUID   uuid.UUID        `ch:"dst_nuid"`
	ConnCount uint64
	ConnType  string
}

type UniqueFQDN

type UniqueFQDN struct {
	Hash    util.FixedString `ch:"hash"`
	Src     net.IP           `ch:"src"`
	Dst     net.IP           `ch:"dst"`
	FQDN    string           `ch:"fqdn"`
	SrcNUID uuid.UUID        `ch:"src_nuid"`
	DstNUID uuid.UUID        `ch:"dst_nuid"`
}

type WaitGroups

type WaitGroups struct {
	Digester sync.WaitGroup
	MetaDB   sync.WaitGroup
	OpenConn sync.WaitGroup
	Conn     sync.WaitGroup
	DNS      sync.WaitGroup
	HTTP     sync.WaitGroup
	OpenHTTP sync.WaitGroup
	SSL      sync.WaitGroup
	OpenSSL  sync.WaitGroup
}

type ZeekHeader

type ZeekHeader[Z zeekRecord] struct {
	// contains filtered or unexported fields
}

ZeekHeader stores vars in the header of the zeek log

type ZeekUIDRecord

type ZeekUIDRecord struct {
	UID               util.FixedString
	Timestamp         time.Time
	UsedByFQDNBeacon  bool
	UsedByDNS         bool
	LinkedToHTTPEntry bool
	NumUsedByHTTP     int
	Duration          float64
	SrcBytes          int64
	DstBytes          int64
	SrcIPBytes        int64
	DstIPBytes        int64
	SrcPackets        int64
	DstPackets        int64
	ConnState         string
	Proto             string
	Service           string
}