index

Documentation

Index

• Constants
• func NewIndexer(w RecordWriter, options ...Option) func(string)
• type AutoIndexOption
  • func WithExcludeDirs(res ...*regexp.Regexp) AutoIndexOption
  • func WithMaxDepth(depth int) AutoIndexOption
  • func WithPaths(paths []string) AutoIndexOption
• type AutoIndexOptions
• type AutoIndexer
  • func NewAutoIndexer(s Queue, options ...AutoIndexOption) AutoIndexer
  • func (a AutoIndexer) Run(ctx context.Context) error
• type Cdx
  • func (c Cdx) Write(rec Record) error
• type CdxAPI
• type CdxJ
  • func (c CdxJ) Write(rec Record) error
• type CdxPb
  • func (c CdxPb) Write(rec Record) error
• type CdxResponse
• type ClosestRequest
• type DateRange
• type FileAPI
• type FileResponse
• type Filter
• type IdAPI
• type IdResponse
• type Indexer
• type KafkaIndexOption
  • func WithBrokers(brokers []string) KafkaIndexOption
  • func WithGroupID(groupID string) KafkaIndexOption
  • func WithMaxBytes(maxBytes int) KafkaIndexOption
  • func WithMaxWait(maxWait time.Duration) KafkaIndexOption
  • func WithMinBytes(minBytes int) KafkaIndexOption
  • func WithTopic(topic string) KafkaIndexOption
• type KafkaIndexer
  • func NewKafkaIndexer(q Queue, options ...KafkaIndexOption) KafkaIndexer
  • func (k KafkaIndexer) Run(ctx context.Context) (err error)
• type Option
  • func WithExcludes(res ...*regexp.Regexp) Option
  • func WithIncludes(res ...*regexp.Regexp) Option
• type Options
• type Queue
• type Record
  • func (r Record) Marshal() ([]byte, error)
  • func (r Record) String() string
• type RecordWriter
• type Runner
• type SearchRequest
• type Sort
• type Toc
  • func (t *Toc) Write(rec Record) error
• type WorkQueue
  • func NewWorkQueue(execute Worker, nrOfWorkers int) *WorkQueue
  • func (iw *WorkQueue) Add(job string)
  • func (iw *WorkQueue) Close()
• type Worker

Constants

const AlreadyIndexedError indexError = "already indexed"

Functions

func NewIndexer

func NewIndexer(w RecordWriter, options ...Option) func(string)

Types

type AutoIndexOption

type AutoIndexOption func(*AutoIndexOptions)

func WithExcludeDirs

func WithExcludeDirs(res ...*regexp.Regexp) AutoIndexOption

func WithMaxDepth

func WithMaxDepth(depth int) AutoIndexOption

func WithPaths

func WithPaths(paths []string) AutoIndexOption

type AutoIndexOptions

type AutoIndexOptions struct {
	MaxDepth int
	Paths    []string
	Options
}

type AutoIndexer

type AutoIndexer struct {
	Queue
	// contains filtered or unexported fields
}

func NewAutoIndexer

func NewAutoIndexer(s Queue, options ...AutoIndexOption) AutoIndexer

func (AutoIndexer) Run

func (a AutoIndexer) Run(ctx context.Context) error

type Cdx

type Cdx struct{}

func (Cdx) Write

func (c Cdx) Write(rec Record) error

type CdxAPI

type CdxAPI interface {
	List(context.Context, int, chan<- CdxResponse) error
	Search(context.Context, SearchRequest, chan<- CdxResponse) error
	Closest(context.Context, ClosestRequest, chan<- CdxResponse) error
}

type CdxJ

type CdxJ struct{}

func (CdxJ) Write

func (c CdxJ) Write(rec Record) error

type CdxPb

type CdxPb struct{}

func (CdxPb) Write

func (c CdxPb) Write(rec Record) error

type CdxResponse

type CdxResponse struct {
	*schema.Cdx
	Error error
}

type ClosestRequest

type ClosestRequest interface {
	Key() string
	Closest() string
	Limit() int
}

type DateRange

type DateRange interface {
	ContainsTime(time.Time) (bool, error)
	ContainsStr(string) (bool, error)
	Contains(int64) bool
}

type FileAPI

type FileAPI interface {
	GetFileInfo(ctx context.Context, filename string) (*schema.Fileinfo, error)
	ListFileInfo(context.Context, int, chan<- FileResponse) error
}

type FileResponse

type FileResponse struct {
	*schema.Fileinfo
	Error error
}

type Filter

type Filter interface {
	Eval(*schema.Cdx) bool
}

type IdAPI

type IdAPI interface {
	GetStorageRef(ctx context.Context, warcId string) (string, error)
	ListStorageRef(context.Context, int, chan<- IdResponse) error
}

type IdResponse

type IdResponse struct {
	Key   string
	Value string
	Error error
}

type Indexer

type Indexer interface {
	Index(string) error
}

type KafkaIndexOption

type KafkaIndexOption func(options *kafka.ReaderConfig)

func WithBrokers

func WithBrokers(brokers []string) KafkaIndexOption

func WithGroupID

func WithGroupID(groupID string) KafkaIndexOption

func WithMaxBytes

func WithMaxBytes(maxBytes int) KafkaIndexOption

func WithMaxWait

func WithMaxWait(maxWait time.Duration) KafkaIndexOption

func WithMinBytes

func WithMinBytes(minBytes int) KafkaIndexOption

func WithTopic

func WithTopic(topic string) KafkaIndexOption

type KafkaIndexer

type KafkaIndexer struct {
	kafka.ReaderConfig
	Queue
}

func NewKafkaIndexer

func NewKafkaIndexer(q Queue, options ...KafkaIndexOption) KafkaIndexer

func (KafkaIndexer) Run

func (k KafkaIndexer) Run(ctx context.Context) (err error)

type Option

type Option func(*Options)

func WithExcludes

func WithExcludes(res ...*regexp.Regexp) Option

func WithIncludes

func WithIncludes(res ...*regexp.Regexp) Option

type Options

type Options struct {
	Includes []*regexp.Regexp
	Excludes []*regexp.Regexp
	// contains filtered or unexported fields
}

type Queue

type Queue interface {
	Add(path string)
}

type Record

type Record struct {
	*schema.Cdx
}

func (Record) Marshal

func (r Record) Marshal() ([]byte, error)

func (Record) String

func (r Record) String() string

type RecordWriter

type RecordWriter interface {
	Write(Record) error
}

type Runner

type Runner interface {
	Run(context.Context) error
}

type SearchRequest

type SearchRequest interface {
	Keys() []string
	Sort() Sort
	DateRange() DateRange
	Filter() Filter
	Limit() int
	Closest() string
	MatchType() string
}

type Sort

type Sort int

const (
	SortNone Sort = iota
	SortDesc
	SortAsc
	SortClosest
)

type Toc

type Toc struct {
	*bloom.BloomFilter
	// contains filtered or unexported fields
}

func (*Toc) Write

func (t *Toc) Write(rec Record) error

type WorkQueue

type WorkQueue struct {
	// contains filtered or unexported fields
}

func NewWorkQueue

func NewWorkQueue(execute Worker, nrOfWorkers int) *WorkQueue

func (*WorkQueue) Add

func (iw *WorkQueue) Add(job string)

Add job to work queue

func (*WorkQueue) Close

func (iw *WorkQueue) Close()

type Worker

type Worker func(string)