scanner

Documentation

Overview

Package scanner provides functionality for concurrent directory scanning with support for filtering, content extraction, and progress reporting.

The scanner uses a worker pool for concurrent processing and provides detailed logging of all operations. It supports various configuration options including depth limiting, pattern-based ignoring, and rate limiting.

Basic usage:

config := scanner.Config{
	Workers: 4,
	MaxDepth: -1,
	IgnorePatterns: []string{".git", "node_modules"},
	IncludeContent: true,
}

scanner := scanner.NewScanner(config, fs, log)
result, err := scanner.Scan(ctx, "/path/to/scan")

Index

• type BasicSymlinkFs
  • func (fs *BasicSymlinkFs) ReadlinkIfPossible(name string) (string, error)
• type BatchConfig
• type Config
• type MaxDepthError
  • func (e *MaxDepthError) Error() string
• type Node
• type NodeType
• type PermissionError
  • func (e *PermissionError) Error() string
• type Progress
• type Result
• type ScanStats
• type Scanner
  • func NewScanner(config Config, fs afero.Fs, log logger.Logger) Scanner
• type ScannerStats
  • func NewScannerStats() *ScannerStats
  • func (s *ScannerStats) AddBytesRead(delta int64) int64
  • func (s *ScannerStats) AddDirectoriesScanned(delta int64) int64
  • func (s *ScannerStats) AddFilesFound(delta int64) int64
  • func (s *ScannerStats) AddFilesScanned(delta int64) int64
  • func (s *ScannerStats) GetBytesRead() int64
  • func (s *ScannerStats) GetCurrentDepth() int32
  • func (s *ScannerStats) GetFilesFound() int64
  • func (s *ScannerStats) GetFilesScanned() int64
  • func (s *ScannerStats) SetCurrentDepth(depth int32) int32
• type SymlinkFs
• type TestSymlinkFs
  • func NewTestSymlinkFs(fs afero.Fs) *TestSymlinkFs
  • func (fs *TestSymlinkFs) CreateSymlink(source, target string) error
  • func (fs *TestSymlinkFs) ReadlinkIfPossible(name string) (string, error)

Types

type BasicSymlinkFs

type BasicSymlinkFs struct {
	afero.Fs
}

BasicSymlinkFs implements SymlinkFs with no symlink support

func (*BasicSymlinkFs) ReadlinkIfPossible

func (fs *BasicSymlinkFs) ReadlinkIfPossible(name string) (string, error)

ReadlinkIfPossible implements SymlinkFs for BasicSymlinkFs

type BatchConfig

type BatchConfig struct {
	BatchSize      int
	NodesChan      chan<- *Node
	ErrorsChan     chan<- error
	Logger         logger.Logger
	IncludeContent bool
	Pool           worker.Pool
}

BatchConfig holds configuration for batch processor

type Config

type Config struct {
	Workers    int
	MaxDepth   int
	BufferSize int
	RateLimit  int
}

Config contains scanner configuration options

type MaxDepthError

type MaxDepthError struct {
	Path     string
	MaxDepth int
}

MaxDepthError represents an error when maximum depth is reached

func (*MaxDepthError) Error

func (e *MaxDepthError) Error() string

type Node

type Node struct {
	Name     string
	Path     string
	Type     NodeType
	Size     int64
	Mode     uint32
	ModTime  time.Time
	Children []*Node
}

Node represents a filesystem node in the scanned tree

type NodeType

type NodeType int

NodeType represents the type of filesystem node

const (
	// FileNode represents a regular file
	FileNode NodeType = iota
	// DirNode represents a directory
	DirNode
	// SymlinkNode represents a symbolic link
	SymlinkNode
)

type PermissionError

type PermissionError struct {
	Path string
	Err  error
}

PermissionError represents a permission-related error during scanning

func (*PermissionError) Error

func (e *PermissionError) Error() string

type Progress

type Progress struct {
	CurrentPath    string
	TotalFiles     int64
	ProcessedFiles int64
	CurrentDepth   int
	StartTime      time.Time
	BytesRead      int64
}

Progress represents the current progress of the scanning operation

type Result

type Result struct {
	Root     *Node
	Contents map[string][]byte
	Errors   map[string]error
	Stats    ScanStats
}

Result contains the complete scan results

type ScanStats

type ScanStats struct {
	StartTime    time.Time
	EndTime      time.Time
	Duration     time.Duration
	TotalFiles   int64
	TotalDirs    int64
	TotalSize    int64
	ErrorCount   int
	SkippedFiles int64
}

ScanStats contains statistics about the scanning operation

type Scanner

type Scanner interface {
	// Scan performs a directory scan starting from the given root path
	Scan(ctx context.Context, root string, ignorePatterns []string, includeContent bool, followSymlinks bool) (Result, error)

	// Progress returns the current scanning progress
	Progress() Progress
}

Scanner defines the interface for directory scanning operations

func NewScanner

func NewScanner(config Config, fs afero.Fs, log logger.Logger) Scanner

type ScannerStats

type ScannerStats struct {
	// contains filtered or unexported fields
}

ScannerStats holds the atomic counters for scanner statistics

func NewScannerStats

func NewScannerStats() *ScannerStats

NewScannerStats creates and initializes a new ScannerStats instance

func (*ScannerStats) AddBytesRead

func (s *ScannerStats) AddBytesRead(delta int64) int64

func (*ScannerStats) AddDirectoriesScanned

func (s *ScannerStats) AddDirectoriesScanned(delta int64) int64

func (*ScannerStats) AddFilesFound

func (s *ScannerStats) AddFilesFound(delta int64) int64

func (*ScannerStats) AddFilesScanned

func (s *ScannerStats) AddFilesScanned(delta int64) int64

func (*ScannerStats) GetBytesRead

func (s *ScannerStats) GetBytesRead() int64

func (*ScannerStats) GetCurrentDepth

func (s *ScannerStats) GetCurrentDepth() int32

func (*ScannerStats) GetFilesFound

func (s *ScannerStats) GetFilesFound() int64

func (*ScannerStats) GetFilesScanned

func (s *ScannerStats) GetFilesScanned() int64

func (*ScannerStats) SetCurrentDepth

func (s *ScannerStats) SetCurrentDepth(depth int32) int32

type SymlinkFs

type SymlinkFs interface {
	afero.Fs
	ReadlinkIfPossible(name string) (string, error)
}

SymlinkFs extends afero.Fs with symlink support

type TestSymlinkFs

type TestSymlinkFs struct {
	afero.Fs
	// contains filtered or unexported fields
}

TestSymlinkFs implements SymlinkFs for testing

func NewTestSymlinkFs

func NewTestSymlinkFs(fs afero.Fs) *TestSymlinkFs

NewTestSymlinkFs creates a new TestSymlinkFs

func (*TestSymlinkFs) CreateSymlink

func (fs *TestSymlinkFs) CreateSymlink(source, target string) error

CreateSymlink creates a new symlink for testing

func (*TestSymlinkFs) ReadlinkIfPossible

func (fs *TestSymlinkFs) ReadlinkIfPossible(name string) (string, error)

ReadlinkIfPossible implements SymlinkFs for TestSymlinkFs