core

package
v0.0.0-...-aeffa90 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 29, 2024 License: BSD-3-Clause Imports: 24 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ErrNotImplemented = errors.New("not implemented")

ErrNotImplemented means this feature is not implemented yet

View Source 
var GLOBAL_JS_OBJECTS = []string{

	`Console`,
	`Save`,
	`hasOwnProperty`,
	`setImmediate`,
	`setTimeout`,
	`clearImmediate`,

	`globalThis`,
	`Infinity`,
	`NaN`,
	`undefined`,
	`eval`,
	`isFinite`,
	`isNaN`,
	`parseFloat`,
	`parseInt`,
	`decodeURI`,
	`decodeURIComponent`,
	`encodeURI`,
	`encodeURIComponent`,
	`escape`,
	`unescape`,
	`Object`,
	`Function`,
	`Boolean`,
	`Symbol`,
	`Error`,
	`AggregateError`,
	`EvalError`,
	`RangeError`,
	`ReferenceError`,
	`SyntaxError`,
	`TypeError`,
	`URIError`,
	`InternalError`,
	`Number`,
	`BigInt`,
	`Math`,
	`Date`,
	`Array`,
	`Int8Array`,
	`Uint8Array`,
	`Uint8ClampedArray`,
	`Int16Array`,
	`Uint16Array`,
	`Int32Array`,
	`Uint32Array`,
	`BigInt64Array`,
	`BigUint64Array`,
	`Float32Array`,
	`Float64Array`,
	`String`,
	`RegExp`,
	`Map`,
	`Set`,
	`WeakMap`,
	`WeakSet`,
	`ArrayBuffer`,
	`SharedArrayBuffer`,
	`DataView`,
	`Atomics`,
	`JSON`,
	`Iterator`,
	`AsyncIterator`,
	`Promise`,
	`GeneratorFunction`,
	`AsyncGeneratorFunction`,
	`Generator`,
	`AsyncGenerator`,
	`AsyncFunction`,
	`WeakRef`,
	`FinalizationRegistry`,
	`Intl`,
	`Reflect`,
	`Proxy`,
}

Functions

func AnnotateStream 

func AnnotateStream(stream io.Reader, aggCtx *AggregationContext) error

AnnotateStream is a simplified version of the core parser that just annotates raw lines

func BulkInsertRows 

func BulkInsertRows(sqlDb *sql.DB, functionName, tableName string, fieldNames []string, generator BulkFieldGenerator) (int64, error)

BulkInsertRows performs a bulk-insert transaction, streaming callback-provided data into a temp import table

func CreateImportTable 

func CreateImportTable(sqlDb *sql.DB, likeTable, importTableName string) error

CreateImportTable creates a temp table copying the schema of a given prototype table

func FilterName 

func FilterName(name string) bool

FilterName identifies V8 object member names that should be filtered out of analysis

func GetEnvDefault 

func GetEnvDefault(key, def string) string

GetEnvDefault gets the named ENV var _or_ a default value if it's not there

func GetRootDomain 

func GetRootDomain(db *sql.DB, ln *LogInfo) (string, error)

func IDLTreeTest 

func IDLTreeTest()

IDLTreeTest is a horribly broken test since I lost wat.json...

func NullableBytes 

func NullableBytes(val []byte) interface{}

NullableBytes returns either <val> (if not length 0) or nil

func NullableInt 

func NullableInt(val int) interface{}

NullableInt returns either <val> (if not 0) or nil

func NullableMongoOID 

func NullableMongoOID(val string) interface{}

NullableMongoOID returns either <val> (if .Valid() == true) or nil

func NullableRune 

func NullableRune(val rune) interface{}

NullableRune returns either string(<val>) (if not 0) or nil

func NullableString 

func NullableString(val string) interface{}

NullableString returns either <val> (if not "") or nil

func NullableTimestamp 

func NullableTimestamp(val time.Time) interface{}

NullableTimestamp returns either <val> (if not .IsZero()) or nil

func StripCurlies 

func StripCurlies(s string) (string, bool)

StripCurlies removes a bracketing '{' '}' pair of characters if present

func StripQuotes 

func StripQuotes(s string) (string, bool)

StripQuotes removes a bracketing '"' '"' pair of characters if present

Types

type AggregationContext 

type AggregationContext struct {
	LogOid       primitive.ObjectID // if present, used to tag db.vv8log[_id: LogId] complete
	LogID        uuid.UUID          // a unique ID assigned to each and every log
	SubmissionID uuid.UUID          // if present, used to provide the URL of the specific submission being processed
	RootName     string             // if present, used to provide root-filename of possibly-fragmented log file
	Formats      FormatSet          // what formats are we outputting?
	Ln           *LogInfo           // actual context structure
	MongoDb      *mongo.Database    // shared MongoDB connection (may be nil)
	SQLDb        *sql.DB            // shared PG connection (may be nil)
	RootDomain   string             // if present, used to provide the root domain of the submission (only used by causality right now)
}

AggregationContext provides configuration/options

type Aggregator 

type Aggregator interface {
	IngestRecord(ctx *ExecutionContext, lineNumber int, op byte, fields []string) error
}

Aggregator is the abstract interface of "payload" implementations (i.e., what we are actually trying to aggregate)

type BulkFieldGenerator 

type BulkFieldGenerator func() ([]interface{}, error)

BulkFieldGenerator is a callback-iterator pattern for streaming row values into a bulk-import table/transaction

type ClosingReader 

type ClosingReader struct {
	// contains filtered or unexported fields
}

ClosingReader attempts to close the underlying reader on EOF

func NewClosingReader 

func NewClosingReader(rawReader io.Reader) *ClosingReader

NewClosingReader wraps an existing io.Reader to auto-close on EOF

func (*ClosingReader) Read 

func (cr *ClosingReader) Read(p []byte) (int, error)

type DumpDriver 

type DumpDriver func(agg Aggregator, ctx *AggregationContext) error

A DumpDriver is a function that invokes a given kind of output logic on the given aggregator (if possible)

func NewPostgresqlDumpDriver 

func NewPostgresqlDumpDriver(sqlDb *sql.DB) DumpDriver

NewPostgresqlDumpDriver creates a driver function to invoke PostgresqlDumper logic on the given aggregator if possible, falling back to MongoDumper logic if necessary (and possible)

func NewStreamDumpDriver 

func NewStreamDumpDriver(stream io.Writer) DumpDriver

NewStreamDumpDriver creates a driver function to invoke StreamDumper logic on the given aggregator (if possible)

type ExecutionContext 

type ExecutionContext struct {
	Script *ScriptInfo
	Origin *Origin
}

ExecutionContext provides context to a trace record: the active script and the enforced SOP domain (if any)

type FormatSet 

type FormatSet map[string]bool

FormatSet tracks which output formats are enabled (by name, without duplicates)

type IDLInfo 

type IDLInfo struct {
	BaseInterface string // The base-interface name for this feature
	MemberRole    rune   // The member role (p = property, m = method, a = alias)
}

IDLInfo is information about a feature name obtained from an IDL database

type IDLInterface 

type IDLInterface struct {
	ParentName string   `json:"parent"`
	AliasFor   string   `json:"aliasFor"`
	Aliases    []string `json:"aliases"`
	Members    []string `json:"members"`
	Methods    []string `json:"methods"`
	Properties []string `json:"properties"`
}

IDLInterface is a JSON-unmarshalling structure for reading records from idldata.json

type IDLTree 

type IDLTree map[string]*IDLInterface

IDLTree is a convenience type alias for a map-of-string-to-IDLInterface

func LoadDefaultIDLData 

func LoadDefaultIDLData() (IDLTree, error)

LoadDefaultIDLData is a sanity-retainer, so that all aggregators use the same ENV variable

func LoadIDLData 

func LoadIDLData(path string) (IDLTree, error)

LoadIDLData loads a idldata.json-like file into an IDLTree

func (IDLTree) IsAPIInIDLFile 

func (tree IDLTree) IsAPIInIDLFile(op byte, class, member string) bool

func (IDLTree) LookupInfo 

func (tree IDLTree) LookupInfo(class, member string) (IDLInfo, error)

LookupInfo finds the base-interface and member-role information (if any) for a class/member name pair

func (IDLTree) NormalizeMember 

func (tree IDLTree) NormalizeMember(class, member string) (string, error)

NormalizeMember attempts to look up the base class/interface defining "member" and replace "class" with that name (for backwards compatability, basically)

type IsolateInfo 

type IsolateInfo struct {
	// The original identifying "pointer" string
	ID string

	// What are all the script IDs in this isolate mapped to?
	Scripts map[int]*ScriptInfo

	// What is our current context (active script and security origin) in this isolate?
	Context ExecutionContext
}

IsolateInfo tracks a V8 isolate (i.e., script namespace, for our purposes) during processing

func NewIsolateInfo 

func NewIsolateInfo(id string) *IsolateInfo

NewIsolateInfo constructs a fresh, empty IsolateInfo for a given hex-string pointer tag

type JSONArray 

type JSONArray []interface{}

JSONArray is a friendly type alias for "array of any" (useful for constructing JSON arrays)

type JSONObject 

type JSONObject map[string]interface{}

JSONObject is a friendly type alias for "map of string to any" (useful for constructing JSON objects)

type LogInfo 

type LogInfo struct {
	// A unique ID assigned to the log
	ID uuid.UUID

	// Database id of the vv8log record being processed
	MongoID primitive.ObjectID

	// Root filename of this log stream
	RootName string

	// What is this log's discovered submission ID?
	SubmissionID uuid.UUID

	// What is the current isolate for this log?
	World *IsolateInfo

	// Any other isolates we know about
	Isolates map[string]*IsolateInfo

	// Has a entry for this log been added to the database?
	Tabled bool

	// Statistics on log size
	Stats struct {
		Lines int
		Bytes int64
	}
}

A LogInfo tracks all essential context information for a VV8 log under processing

func NewLogInfo 

func NewLogInfo(oid primitive.ObjectID, rootName string, submissionID uuid.UUID) *LogInfo

NewLogInfo constructs a fresh LogInfo for the given vv8log Mongo oid (if available) and root log filename (if available)

func (*LogInfo) IngestStream 

func (ln *LogInfo) IngestStream(stream io.Reader, aggs ...Aggregator) error

IngestStream is the entry point for parsing a given log and feeding the records into zero or more aggregators

func (*LogInfo) InsertLogfile 

func (ln *LogInfo) InsertLogfile(sqldb *sql.DB) (int, error)

InsertLogfile inserts (if not present) a record about this log file into PG

type MongoConnection 

type MongoConnection struct {
	URL    string        // connection URL dialed ("mongodb://$host:$port/$db")
	User   string        // username authenticated as (if applicable; "" otherwise)
	Client *mongo.Client // Active (and possibly authenticated) session to Mongo
}

MongoConnection bundles together essential state and stats for a live MongoDB connection

func DialMongo 

func DialMongo() (MongoConnection, error)

DialMongo creates a possibly authenticated Mongo session based on ENV configs Returns a MongoConnection on success; error otherwise (in all cases, the `url` field of MongoConnection will be set)

func (MongoConnection) String 

func (mc MongoConnection) String() string

type Origin 

type Origin struct {
	Origin              string
	OriginSecurityToken string
}

type PostgresqlDumper 

type PostgresqlDumper interface {
	DumpToPostgresql(ctx *AggregationContext, sqlDb *sql.DB) error
}

PostgresqlDumper implements dumping of output data to Mongo and/or Postgres backends together

type SHA2Block 

type SHA2Block [sha256.Size]byte

SHA2Block is storage for a SHA256 digest

type ScriptHash 

type ScriptHash struct {
	Length int
	SHA2   [sha256.Size]byte
	SHA3   [32]byte
}

ScriptHash was originally just an alias for SHA2-256 digest, but then we discovered collisions; now it's a tuple (length, SHA2-256, SHA3-256)

func NewScriptHash 

func NewScriptHash(code string) ScriptHash

NewScriptHash produces a new (length, SHA2-256, SHA3-256) triple from a JS code string (i.e., a script)

type ScriptInfo 

type ScriptInfo struct {
	// Parent pointer
	Isolate *IsolateInfo

	// Self ID (per parent isolate)
	ID int

	// Is this a "visible-v8://" script (which shouldn't be included in usage stats)?
	VisibleV8 bool

	// Script code
	Code string

	// CodeHash tries very hard to uniquely identify the script by its (length, SHA2-256, SHA3-256) tuple
	CodeHash ScriptHash

	// URL from which this script was loaded (if any--eval'd scripts will have nil for this)
	URL string

	// What script eval'd us? (if any--we might not be an eval'd script)
	EvaledBy *ScriptInfo

	// Active origin at moment of creation in the logs?
	FirstOrigin *Origin
}

ScriptInfo bundles all metadata/data available about a logged script

func NewScriptInfo 

func NewScriptInfo(iso *IsolateInfo, id int, code string, activeOrigin *Origin) *ScriptInfo

NewScriptInfo constructs a new script in a given Isolate with the given runtime ID and code body

type StreamDumper 

type StreamDumper interface {
	DumpToStream(ctx *AggregationContext, stream io.Writer) error
}

StreamDumper implements dumping of aggregate output data as some kind of plain text (typically for development/debugging)

type URLBakery 

type URLBakery struct {
	// contains filtered or unexported fields
}

URLBakery keeps a stash of cooked URLs pending insertion

func NewURLBakery 

func NewURLBakery() *URLBakery

NewURLBakery instantiates a fresh, empty URLBakery

func (*URLBakery) InsertBakedURLs 

func (ub *URLBakery) InsertBakedURLs(sqlDb *sql.DB) error

InsertBakedURLs performs a de-duping bulk insert of cooked URL records into PG's `urls` table

func (*URLBakery) URLToHash 

func (ub *URLBakery) URLToHash(rawurl string) SHA2Block

URLToHash takes a raw URL string and returns its SHA256 hash (after stashing it if it is new/unseen)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.