dedup

Documentation

Index

• func DefaultRudderPath() string
• func Init()
• type DedupHandleT
  • func New(path string, fns ...OptFn) *DedupHandleT
  • func (d *DedupHandleT) Close()
  • func (d *DedupHandleT) FindDuplicates(messageIDs []string, allMessageIDsSet map[string]struct{}) (duplicateIndexes []int)
  • func (d *DedupHandleT) MarkProcessed(messageIDs []string)
  • func (d *DedupHandleT) PrintHistogram()
• type DedupI
  • func GetInstance(clearDB *bool) DedupI
• type OptFn
  • func FromConfig() OptFn
  • func WithClearDB() OptFn
  • func WithWindow(d time.Duration) OptFn

Functions

func DefaultRudderPath

func DefaultRudderPath() string

func Init

func Init()

Types

type DedupHandleT

type DedupHandleT struct {
	// contains filtered or unexported fields
}

func New

func New(path string, fns ...OptFn) *DedupHandleT

func (*DedupHandleT) Close

func (d *DedupHandleT) Close()

func (*DedupHandleT) FindDuplicates

func (d *DedupHandleT) FindDuplicates(messageIDs []string, allMessageIDsSet map[string]struct{}) (duplicateIndexes []int)

func (*DedupHandleT) MarkProcessed

func (d *DedupHandleT) MarkProcessed(messageIDs []string)

MarkProcessed persist messageIDs in Disk, with expiry time of dedupWindow Any message mark here will appear in FindDuplicates() if queried inside the dedupWindow

func (*DedupHandleT) PrintHistogram

func (d *DedupHandleT) PrintHistogram()

type DedupI

type DedupI interface {
	FindDuplicates(messageIDs []string, allMessageIDsSet map[string]struct{}) (duplicateIndexes []int)
	MarkProcessed(messageIDs []string)
	PrintHistogram()
	Close()
}

func GetInstance

func GetInstance(clearDB *bool) DedupI

GetInstance returns an instance of DedupI

type OptFn

type OptFn func(*DedupHandleT)

func FromConfig

func FromConfig() OptFn

func WithClearDB

func WithClearDB() OptFn

func WithWindow

func WithWindow(d time.Duration) OptFn