engine

Documentation

Index

• func DefaultDetectorTypesImplementing[T any]() map[detectorspb.DetectorType]struct{}
• func DefaultDetectors() []detectors.Detector
• func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64)
• func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool)
• func SetResultLineNumber(chunk *sources.Chunk, result *detectors.Result, fragStart int64, mdLine *int64) bool
• func SupportsLineNumbers(sourceType sourcespb.SourceType) bool
• type Engine
  • func Start(ctx context.Context, options ...EngineOption) (*Engine, error)
  • func (e *Engine) ChunksChan() <-chan *sources.Chunk
  • func (e *Engine) DetectorAvgTime() map[string][]time.Duration
  • func (e *Engine) Finish(ctx context.Context) error
  • func (e *Engine) GetDetectorsMetrics() map[string]time.Duration
  • func (e *Engine) GetMetrics() Metrics
  • func (e *Engine) HasFoundResults() bool
  • func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata
  • func (e *Engine) ScanChunk(chunk *sources.Chunk)
  • func (e *Engine) ScanCircleCI(ctx context.Context, token string) error
  • func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error
  • func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error
  • func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error
  • func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error
  • func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error
  • func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error
  • func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error
  • func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error
• type EngineOption
  • func WithConcurrency(concurrency uint8) EngineOption
  • func WithDecoders(decoders ...decoders.Decoder) EngineOption
  • func WithDetectors(verify bool, d ...detectors.Detector) EngineOption
  • func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption
  • func WithFilterUnverified(filter bool) EngineOption
  • func WithOnlyVerified(onlyVerified bool) EngineOption
  • func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption
  • func WithPrinter(printer Printer) EngineOption
• type Metrics
• type Printer

Functions

func DefaultDetectorTypesImplementing

func DefaultDetectorTypesImplementing[T any]() map[detectorspb.DetectorType]struct{}

func DefaultDetectors

func DefaultDetectors() []detectors.Detector

func FragmentFirstLine

func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64)

FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the chunk metadata.

func FragmentLineOffset

func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool)

FragmentLineOffset sets the line number for a provided source chunk with a given detector result.

func SetResultLineNumber

func SetResultLineNumber(chunk *sources.Chunk, result *detectors.Result, fragStart int64, mdLine *int64) bool

SetResultLineNumber sets the line number in the provided result.

func SupportsLineNumbers

func SupportsLineNumbers(sourceType sourcespb.SourceType) bool

SupportsLineNumbers determines if a line number can be found for a source type.

Types

type Engine

type Engine struct {
	WgNotifier sync.WaitGroup
	// contains filtered or unexported fields
}

func Start

func Start(ctx context.Context, options ...EngineOption) (*Engine, error)

Start the engine with options.

func (*Engine) ChunksChan

func (e *Engine) ChunksChan() <-chan *sources.Chunk

func (*Engine) DetectorAvgTime

func (e *Engine) DetectorAvgTime() map[string][]time.Duration

DetectorAvgTime returns the average time taken by each detector.

func (*Engine) Finish

func (e *Engine) Finish(ctx context.Context) error

Finish waits for running sources to complete and workers to finish scanning chunks before closing their respective channels. Once Finish is called, no more sources may be scanned by the engine.

func (*Engine) GetDetectorsMetrics

func (e *Engine) GetDetectorsMetrics() map[string]time.Duration

GetDetectorsMetrics returns a copy of the average time taken by each detector.

func (*Engine) GetMetrics

func (e *Engine) GetMetrics() Metrics

GetMetrics returns a copy of Metrics. It's safe for concurrent use, and the caller can't modify the original data.

func (*Engine) HasFoundResults

func (e *Engine) HasFoundResults() bool

HasFoundResults returns true if any results are found.

func (*Engine) ResultsChan

func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata

func (*Engine) ScanChunk

func (e *Engine) ScanChunk(chunk *sources.Chunk)

ScanChunk injects a chunk into the output stream of chunks to be scanned. This method should rarely be used. TODO: Remove when dependencies no longer rely on this functionality.

func (*Engine) ScanCircleCI

func (e *Engine) ScanCircleCI(ctx context.Context, token string) error

ScanCircleCI scans CircleCI logs.

func (*Engine) ScanDocker

func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error

ScanDocker scans a given docker connection.

func (*Engine) ScanFileSystem

func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error

ScanFileSystem scans a given file system.

func (*Engine) ScanGCS

func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error

ScanGCS with the provided options.

func (*Engine) ScanGit

func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error

ScanGit scans any git source.

func (*Engine) ScanGitHub

func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error

ScanGitHub scans Github with the provided options.

func (*Engine) ScanGitLab

func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error

ScanGitLab scans GitLab with the provided configuration.

func (*Engine) ScanS3

func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error

ScanS3 scans S3 buckets.

func (*Engine) ScanSyslog

func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error

ScanSyslog is a source that scans syslog files.

type EngineOption

type EngineOption func(*Engine)

func WithConcurrency

func WithConcurrency(concurrency uint8) EngineOption

func WithDecoders

func WithDecoders(decoders ...decoders.Decoder) EngineOption

func WithDetectors

func WithDetectors(verify bool, d ...detectors.Detector) EngineOption

func WithFilterDetectors

func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption

WithFilterDetectors applies a filter to the configured list of detectors. If the filterFunc returns true, the detector will be included for scanning. This option applies to the existing list of detectors configured, so the order this option appears matters. All filtering happens before scanning.

func WithFilterUnverified

func WithFilterUnverified(filter bool) EngineOption

WithFilterUnverified sets the filterUnverified flag on the engine. If set to true, the engine will only return the first unverified result for a chunk for a detector.

func WithOnlyVerified

func WithOnlyVerified(onlyVerified bool) EngineOption

WithOnlyVerified sets the onlyVerified flag on the engine. If set to true, the engine will only print verified results.

func WithPrintAvgDetectorTime

func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption

WithPrintAvgDetectorTime sets the printAvgDetectorTime flag on the engine. If set to true, the engine will print the average time taken by each detector. This option allows us to measure the time taken for each detector ONLY if the engine is configured to print the results. Calculating the average time taken by each detector is an expensive operation and should be avoided unless specified by the user.

func WithPrinter

func WithPrinter(printer Printer) EngineOption

WithPrinter sets the Printer on the engine.

type Metrics

type Metrics struct {
	BytesScanned           uint64
	ChunksScanned          uint64
	VerifiedSecretsFound   uint64
	UnverifiedSecretsFound uint64
	AvgDetectorTime        map[string]time.Duration

	ScanDuration time.Duration
	// contains filtered or unexported fields
}

Metrics for the scan engine for external consumption.

type Printer

type Printer interface {
	Print(ctx context.Context, r *detectors.ResultWithMetadata) error
}

Printer is used to format found results and output them to the user. Ex JSON, plain text, etc. Please note printer implementations SHOULD BE thread safe.