storage

Documentation

Index

• func OrderPerception(enabled bool) func(db *gorm.DB) *gorm.DB
• type ConsoleLog
• type Db
  • func NewDb() *Db
  • func (db *Db) Get() (*gorm.DB, error)
• type Header
• type NetworkLog
• type RequestType
• type TLS
• type TLSCertificate
  • func (tlsCert *TLSCertificate) AddDNSName(name string)
• type TLSCertificateDNSName
• type Technologie
• type URL
  • func (url *URL) AddHeader(key string, value string)
  • func (url *URL) AddTechnologie(value string)
  • func (url *URL) MarshalJSON() ([]byte, error)
  • func (url *URL) MarshallCSV() (res []string)

Functions

func OrderPerception

func OrderPerception(enabled bool) func(db *gorm.DB) *gorm.DB

OrderPerception orders by perception hash if enabled

Types

type ConsoleLog

type ConsoleLog struct {
	gorm.Model

	URLID uint

	Time  time.Time
	Type  string
	Value string
}

ConsoleLog contains the console log, and exceptions emitted

type Db

type Db struct {
	Location      string
	SkipMigration bool

	// cli flags
	Disabled bool
	Debug    bool
}

Db is the SQLite3 db handler ype

func NewDb

func NewDb() *Db

NewDb sets up a new DB

func (*Db) Get

func (db *Db) Get() (*gorm.DB, error)

Get gets a db handle

type Header

type Header struct {
	gorm.Model

	URLID uint

	Key   string
	Value string
}

Header contains an HTTP header

type NetworkLog

type NetworkLog struct {
	gorm.Model

	URLID uint

	RequestID   string
	RequestType RequestType
	StatusCode  int64
	URL         string
	FinalURL    string
	IP          string
	Time        time.Time
	Error       string
}

NetworkLog contains Chrome networks events that were emitted

type RequestType

type RequestType int

RequestType are network log types

const (
	HTTP RequestType = 0
	WS
)

type TLS

type TLS struct {
	gorm.Model

	URLID uint

	Version         uint16
	ServerName      string
	TLSCertificates []TLSCertificate
}

TLS contains TLS information for a URL

type TLSCertificate

type TLSCertificate struct {
	gorm.Model

	TLSID uint

	Raw                []byte
	DNSNames           []TLSCertificateDNSName
	SubjectCommonName  string
	IssuerCommonName   string
	SignatureAlgorithm string
	PubkeyAlgorithm    string
}

TLSCertificate contain TLS Certificate information

func (*TLSCertificate) AddDNSName

func (tlsCert *TLSCertificate) AddDNSName(name string)

AddDNSName adds a new DNS Name to a Certificate

type TLSCertificateDNSName

type TLSCertificateDNSName struct {
	gorm.Model

	TLSCertificateID uint
	Name             string
}

TLSCertificateDNSName has DNS names for a TLS certificate

type Technologie

type Technologie struct {
	gorm.Model

	URLID uint

	Value string
}

Technologie contains a technologie

type URL

type URL struct {
	gorm.Model

	URL            string
	FinalURL       string
	ResponseCode   int
	ResponseReason string
	Proto          string
	ContentLength  int64
	Title          string
	Filename       string
	IsPDF          bool
	PerceptionHash string
	DOM            string
	Screenshot     string

	TLS TLS

	Headers      []Header
	Technologies []Technologie
	Console      []ConsoleLog
	Network      []NetworkLog
}

URL contains information about a URL

func (*URL) AddHeader

func (url *URL) AddHeader(key string, value string)

AddHeader adds a new header to a URL

func (*URL) AddTechnologie

func (url *URL) AddTechnologie(value string)

AddTechnlogies adds a new technologies to a URL

func (*URL) MarshalJSON

func (url *URL) MarshalJSON() ([]byte, error)

MarshalJSON returns JSON encoding of url. Implements json.Marshaler.

func (*URL) MarshallCSV

func (url *URL) MarshallCSV() (res []string)

MarshallCSV returns values as a slice