gobloom

README

gobloom

Small example of detecting duplicate keys among ~10M protobuf messages using a Bloom Filter.

How to run

# Generate 10M protobuf messages, put them in out.ldproto
go run cmd/main.go gen-file

# Read 10M messages, use a HashMap to detect duplicates
go run cmd/main.go find-dup-with-map

# Read 10M messages, use a Bloom Filter to detect duplicates
go run cmd/main.go find-dup-with-filter

Documentation

Index

• Variables
• type Filter
  • func NewFilter(nitems int, p float64) *Filter
  • func (f *Filter) TestAndAdd(key []byte) bool
• type Prediction
  • func (*Prediction) Descriptor() ([]byte, []int)
  • func (x *Prediction) GetCapabilityId() string
  • func (x *Prediction) GetCategory() int32
  • func (x *Prediction) GetLocationId() string
  • func (x *Prediction) GetTs() *timestamppb.Timestamp
  • func (x *Prediction) GetValue() float32
  • func (*Prediction) ProtoMessage()
  • func (x *Prediction) ProtoReflect() protoreflect.Message
  • func (x *Prediction) Reset()
  • func (x *Prediction) String() string

Variables

var File_msg_proto protoreflect.FileDescriptor

Types

type Filter

type Filter struct {
	// contains filtered or unexported fields
}

func NewFilter

func NewFilter(nitems int, p float64) *Filter

NewFilter returns a new bloom filter. nitems is the number of items that will pass through the filter. p is the probability of a false positive.

func (*Filter) TestAndAdd

func (f *Filter) TestAndAdd(key []byte) bool

TestAndAdd returns false if the key has not been seen before, and true if it is likely that it has seen the key before. This operation will add the key to the bloom filter.

type Prediction

type Prediction struct {
	Ts           *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=ts,proto3" json:"ts,omitempty"`
	LocationId   string                 `protobuf:"bytes,2,opt,name=location_id,json=locationId,proto3" json:"location_id,omitempty"`
	CapabilityId string                 `protobuf:"bytes,3,opt,name=capability_id,json=capabilityId,proto3" json:"capability_id,omitempty"`
	Value        float32                `protobuf:"fixed32,4,opt,name=value,proto3" json:"value,omitempty"`
	Category     int32                  `protobuf:"varint,5,opt,name=category,proto3" json:"category,omitempty"`
	// contains filtered or unexported fields
}

func (*Prediction) Descriptor

func (*Prediction) Descriptor() ([]byte, []int)

Deprecated: Use Prediction.ProtoReflect.Descriptor instead.

func (*Prediction) GetCapabilityId

func (x *Prediction) GetCapabilityId() string

func (*Prediction) GetCategory

func (x *Prediction) GetCategory() int32

func (*Prediction) GetLocationId

func (x *Prediction) GetLocationId() string

func (*Prediction) GetTs

func (x *Prediction) GetTs() *timestamppb.Timestamp

func (*Prediction) GetValue

func (x *Prediction) GetValue() float32

func (*Prediction) ProtoMessage

func (*Prediction) ProtoMessage()

func (*Prediction) ProtoReflect

func (x *Prediction) ProtoReflect() protoreflect.Message

func (*Prediction) Reset

func (x *Prediction) Reset()

func (*Prediction) String

func (x *Prediction) String() string