duplicate

Documentation

Index

• type DetectOptions
• type Detector
  • func NewDetector(sorter extsort.ExternalSorter, logger log.Logger) *Detector
  • func (d *Detector) Detect(ctx context.Context, opts *DetectOptions) (numDups int64, _ error)
  • func (d *Detector) KeyAdder(ctx context.Context) (*KeyAdder, error)
• type Handler
• type HandlerConstructor
• type KeyAdder
  • func (k *KeyAdder) Add(key, keyID []byte) error
  • func (k *KeyAdder) Close() error
  • func (k *KeyAdder) Flush() error

Types

type DetectOptions

type DetectOptions struct {
	// Concurrency is the maximum number of concurrent workers.
	//
	// The default value is runtime.GOMAXPROCS(0).
	Concurrency int
	// HandlerConstructor specifies how to handle duplicate keys.
	// If it is nil, a nop handler constructor will be used.
	HandlerConstructor HandlerConstructor
}

DetectOptions holds optional arguments for the Detect method.

type Detector

type Detector struct {
	// contains filtered or unexported fields
}

Detector is used to detect duplicate keys.

func NewDetector

func NewDetector(sorter extsort.ExternalSorter, logger log.Logger) *Detector

NewDetector creates a new Detector.

func (*Detector) Detect

func (d *Detector) Detect(ctx context.Context, opts *DetectOptions) (numDups int64, _ error)

Detect detects duplicate keys that have been added to the Detector. It returns the number of duplicate keys detected and any error encountered.

func (*Detector) KeyAdder

func (d *Detector) KeyAdder(ctx context.Context) (*KeyAdder, error)

KeyAdder returns a KeyAdder that can be used to add keys to the Detector.

type Handler

type Handler interface {
	// Begin is called when a new duplicate key is detected.
	Begin(key []byte) error
	// Append appends a keyID to the current duplicate key.
	// Multiple keyIDs are appended in lexicographical order.
	// NOTE: keyID may be changed after the call.
	Append(keyID []byte) error
	// End is called when all keyIDs of the current duplicate key have been appended.
	End() error
	// Close closes the Handler. It is called when the Detector finishes detecting.
	// It is guaranteed to be called after all Begin/Append/End calls.
	Close() error
}

Handler is used to handle duplicate keys.

Handler behaves like a state machine. It is called in the following order: Begin -> Append -> Append -> ... -> Append -> End -> Begin -> ... -> End -> Close

type HandlerConstructor

type HandlerConstructor func(ctx context.Context) (Handler, error)

HandlerConstructor is used to construct a Handler to handle duplicate keys.

type KeyAdder

type KeyAdder struct {
	// contains filtered or unexported fields
}

KeyAdder is used to add keys to the Detector.

func (*KeyAdder) Add

func (k *KeyAdder) Add(key, keyID []byte) error

Add adds a key and its keyID to the KeyAdder.

If key is duplicated, the keyID can be used to locate the duplicate key in the original data source.

func (*KeyAdder) Close

func (k *KeyAdder) Close() error

Close closes the KeyAdder with buffered keys flushed.

func (*KeyAdder) Flush

func (k *KeyAdder) Flush() error

Flush flushes buffered keys to the detector.