filter

Documentation

Index

• Constants
• type ProcessedFile
• type Processor
  • func New(ctx *c.Context, deps *c.Dependencies) (*Processor, error)
  • func (p *Processor) Run(ctx context.Context) error
• type State

Constants

const (
	MaxLineSize     = 10 << 20 // 10MB max line size
	DefaultMapSize  = 16       // Default size for JSON object maps
	FlushInterval   = time.Second
	LogInterval     = 5 * time.Second
	OutputBatchSize = 10000 // Number of lines to buffer before writing
)

Types

type ProcessedFile

type ProcessedFile struct {
	Filename     string    `json:"filename"`
	ProcessedAt  time.Time `json:"processed_at"`
	MatchCount   int64     `json:"match_count"`
	TotalLines   int64     `json:"total_lines"`
	HashedConfig string    `json:"hashed_config"` // To detect config changes
}

type Processor

type Processor struct {
	// contains filtered or unexported fields
}

Processor handles the processing of Reddit data files

func New

func New(ctx *c.Context, deps *c.Dependencies) (*Processor, error)

New creates a new filter instance with optimized pools

func (*Processor) Run

func (p *Processor) Run(ctx context.Context) error

Run executes the processing pipeline

type State

type State struct {
	ProcessedFiles map[string]ProcessedFile `json:"processed_files"`
	// contains filtered or unexported fields
}