Documentation ¶
Index ¶
- Variables
- func MsgpGetVersion(in []byte) (b []byte, v uint8)
- func NewSegmentElements(parent *Segments, e Elements) *lockedSegment
- func RandString(numRnd ...int) string
- type Bucket
- type Cache
- type Element
- func (e Element) Above(b Element) bool
- func (e Element) AboveThis() (score uint64, tie uint32)
- func (e Element) AsIndex(s SegmentID) IndexElement
- func (z *Element) DecodeMsg(dc *msgp.Reader) (err error)
- func (z *Element) EncodeMsg(en *msgp.Writer) (err error)
- func (z *Element) MarshalMsg(b []byte) (o []byte, err error)
- func (z *Element) Msgsize() (s int)
- func (e Element) PrevMax() (score uint64, tie uint32)
- func (e Element) String() string
- func (z *Element) UnmarshalMsg(bts []byte) (o []byte, err error)
- type ElementID
- type ElementIDs
- func (e ElementIDs) AsScore() []uint64
- func (z *ElementIDs) DecodeMsg(dc *msgp.Reader) (err error)
- func (e *ElementIDs) Deduplicate()
- func (z ElementIDs) EncodeMsg(en *msgp.Writer) (err error)
- func (e ElementIDs) Map() map[ElementID]struct{}
- func (z ElementIDs) MarshalMsg(b []byte) (o []byte, err error)
- func (z ElementIDs) Msgsize() (s int)
- func (e ElementIDs) NotIn(b ElementIDs) ElementIDs
- func (e ElementIDs) Overlap(b ElementIDs) ElementIDs
- func (e *ElementIDs) Sort()
- func (z *ElementIDs) UnmarshalMsg(bts []byte) (o []byte, err error)
- type Elements
- func (l *Elements) Add(e Element) (*Rank, error)
- func (l Elements) Clone(payloads bool) Elements
- func (z *Elements) DecodeMsg(dc *msgp.Reader) (err error)
- func (l *Elements) Deduplicate() (changed bool)
- func (l *Elements) Delete(id ElementID) error
- func (l Elements) ElementIDs(id SegmentID) IndexElements
- func (z Elements) EncodeMsg(en *msgp.Writer) (err error)
- func (l Elements) Find(id ElementID) (*Element, error)
- func (l Elements) FindIdx(id ElementID) (int, error)
- func (l Elements) FindScoreIdx(score uint64) (int, error)
- func (e Elements) FirstElementsWithScore(scores []uint64) Elements
- func (l *Elements) HasDuplicates() error
- func (l Elements) IDSorter() func(i, j int) bool
- func (l Elements) IDs() ElementIDs
- func (l *Elements) Insert(e Element) int
- func (z Elements) MarshalMsg(b []byte) (o []byte, err error)
- func (l *Elements) Merge(ins Elements, sliced bool)
- func (l *Elements) MergeDeduplicate(ins Elements)
- func (l Elements) MinMax() (min, max uint64, minTie, maxTie uint32)
- func (z Elements) Msgsize() (s int)
- func (l Elements) Ranked(topOffset, total int) RankedElements
- func (l *Elements) Sort() (changed bool)
- func (l Elements) Sorter() func(i, j int) bool
- func (l Elements) SplitSize(inEach int) []Elements
- func (l Elements) String() string
- func (z *Elements) UnmarshalMsg(bts []byte) (o []byte, err error)
- func (l *Elements) Update(e Element) (*Rank, error)
- func (l *Elements) UpdateTime(t time.Time)
- type IndexElement
- func (z *IndexElement) DecodeMsg(dc *msgp.Reader) (err error)
- func (z *IndexElement) EncodeMsg(en *msgp.Writer) (err error)
- func (z *IndexElement) MarshalMsg(b []byte) (o []byte, err error)
- func (z *IndexElement) Msgsize() (s int)
- func (z *IndexElement) UnmarshalMsg(bts []byte) (o []byte, err error)
- type IndexElements
- func (z *IndexElements) DecodeMsg(dc *msgp.Reader) (err error)
- func (z *IndexElements) EncodeMsg(en *msgp.Writer) (err error)
- func (z *IndexElements) MarshalMsg(b []byte) (o []byte, err error)
- func (z *IndexElements) Msgsize() (s int)
- func (e IndexElements) SegmentSorter() func(i, j int) bool
- func (z *IndexElements) UnmarshalMsg(bts []byte) (o []byte, err error)
- type IndexError
- type List
- func (l *List) Backup(ctx context.Context, bs blobstore.Store, w *WriterMsgp) error
- func (z *List) DecodeMsg(dc *msgp.Reader) (err error)
- func (l *List) DeleteAll(ctx context.Context, bs blobstore.Store) error
- func (l *List) DeleteElements(ctx context.Context, bs blobstore.Store, ids []ElementID) error
- func (z *List) EncodeMsg(en *msgp.Writer) (err error)
- func (l *List) ForceSplit(ctx context.Context, bs blobstore.Store) error
- func (l *List) GetElements(ctx context.Context, bs blobstore.Store, ids []ElementID, radius int) (RankedElements, error)
- func (l *List) GetPercentile(ctx context.Context, bs blobstore.Store, percentile float64, radius int) (*RankedElement, error)
- func (l *List) GetRankBottom(ctx context.Context, bs blobstore.Store, offset, elements int) (Elements, error)
- func (l *List) GetRankScoreAsc(ctx context.Context, bs blobstore.Store, score uint64, n int) (Elements, error)
- func (l *List) GetRankScoreDesc(ctx context.Context, bs blobstore.Store, score uint64, n int) (RankedElements, error)
- func (l *List) GetRankTop(ctx context.Context, bs blobstore.Store, offset, elements int) (Elements, error)
- func (l *List) Insert(ctx context.Context, bs blobstore.Store, e Elements) error
- func (l *List) Len(ctx context.Context, bs blobstore.Store) (int, error)
- func (z *List) MarshalMsg(b []byte) (o []byte, err error)
- func (z *List) Msgsize() (s int)
- func (l *List) Populate(ctx context.Context, bs blobstore.Store, e Elements) error
- func (l *List) Reindex(ctx context.Context, bs blobstore.Store) error
- func (l *List) ReleaseSegments(ctx context.Context)
- func (l *List) Repair(ctx context.Context, bs blobstore.Store, clearIfErr bool) error
- func (l *List) Stats(ctx context.Context, bs blobstore.Store, elements bool) (*ListStats, error)
- func (l *List) String() string
- func (z *List) UnmarshalMsg(bts []byte) (o []byte, err error)
- func (l *List) UpdateElements(ctx context.Context, bs blobstore.Store, elems Elements, radius int, ...) (RankedElements, error)
- func (l *List) Verify(ctx context.Context, bs blobstore.Store) error
- func (l *List) VerifyElements(ctx context.Context, bs blobstore.Store) error
- func (l *List) VerifyUnlocked(ctx context.Context, timeout time.Duration) error
- type ListID
- type ListIDs
- func (z *ListIDs) DecodeMsg(dc *msgp.Reader) (err error)
- func (ids *ListIDs) Deduplicate() (changed bool)
- func (z ListIDs) EncodeMsg(en *msgp.Writer) (err error)
- func (z ListIDs) MarshalMsg(b []byte) (o []byte, err error)
- func (z ListIDs) Msgsize() (s int)
- func (ids ListIDs) Sort()
- func (z *ListIDs) UnmarshalMsg(bts []byte) (o []byte, err error)
- type ListOption
- func (l ListOption) Cache(cache Cache) ListOption
- func (l ListOption) Clone(lst *List) ListOption
- func (l ListOption) LoadIndex(b bool) ListOption
- func (l ListOption) MergeSplitSize(merge, split int) ListOption
- func (l ListOption) Metadata(m map[string]string) ListOption
- func (l ListOption) Populate(e []Element) ListOption
- type ListStats
- type Lists
- func (l *Lists) Add(lists ...*List)
- func (l *Lists) All() ListIDs
- func (l Lists) ByID(id ListID) (*List, bool)
- func (l Lists) ByIDs(ids ...ListID) []*List
- func (l *Lists) Delete(id ListID)
- func (l *Lists) Exists(ids ...ListID) []ListID
- func (l *Lists) Load(ctx context.Context, bs blobstore.Store, b []byte) error
- func (l *Lists) MatchAll(q map[string]string, sets []string) []ListID
- func (l *Lists) Prune(ctx context.Context) error
- func (l Lists) Save(ctx context.Context, w io.Writer) error
- func (l *Lists) SortedIDsAfter(from ListID, n int) ([]*List, PageInfo)
- func (l *Lists) SortedIDsBefore(to ListID, n int) ([]*List, PageInfo)
- type Manager
- func (m *Manager) DeleteList(ctx context.Context, id ListID) error
- func (m *Manager) LoadLists(ctx context.Context, cache Cache) error
- func (m *Manager) NewLists(cache Cache) error
- func (m *Manager) SaveLists(ctx context.Context, backup bool) error
- func (m *Manager) StartIntervalSaver(ctx context.Context, every time.Duration, shutdown chan chan struct{})
- func (m *Manager) StartListPruner(ctx context.Context, every time.Duration, shutdown chan chan struct{})
- func (m *Manager) StartListSplitter(ctx context.Context, store blobstore.Store, shutdown chan chan struct{})
- type PageInfo
- type Rank
- type RankedElement
- type RankedElements
- type ReaderMsgp
- type ScoreError
- type Segment
- func (z *Segment) DecodeMsg(dc *msgp.Reader) (err error)
- func (z *Segment) EncodeMsg(en *msgp.Writer) (err error)
- func (s *Segment) Filter(e Elements) Elements
- func (s *Segment) FilterIdx(e Elements) (start, end int)
- func (s *Segment) FilterScoresIdx(scores []uint64) (start, end int)
- func (z *Segment) MarshalMsg(b []byte) (o []byte, err error)
- func (z *Segment) Msgsize() (s int)
- func (s *Segment) StorageKey() string
- func (s *Segment) String() string
- func (z *Segment) UnmarshalMsg(bts []byte) (o []byte, err error)
- type SegmentID
- type Segments
- func (s *Segments) AlignIndex()
- func (s *Segments) Append(ls *lockedSegment)
- func (z *Segments) DecodeMsg(dc *msgp.Reader) (err error)
- func (s *Segments) Delete(ctx context.Context, store blobstore.WithSet) error
- func (s *Segments) DeleteElements(ctx context.Context, bs blobstore.WithSet, e Elements, withIdx bool) (IndexElements, error)
- func (s *Segments) DeleteElementsIdx(ctx context.Context, bs blobstore.WithSet, ids IndexElements, withIdx bool) (IndexElements, error)
- func (s *Segments) ElementIndexAll(ctx context.Context, bs blobstore.WithSet) (IndexElements, error)
- func (s *Segments) Elements() int
- func (z *Segments) EncodeMsg(en *msgp.Writer) (err error)
- func (s *Segments) FindElements(ctx context.Context, bs blobstore.WithSet, ids IndexElements, radius int) ([]RankedElement, error)
- func (s *Segments) Insert(ctx context.Context, bs blobstore.WithSet, in Elements, withIdx bool) (IndexElements, error)
- func (z *Segments) MarshalMsg(b []byte) (o []byte, err error)
- func (z *Segments) Msgsize() (s int)
- func (s *Segments) Save(ctx context.Context, store blobstore.WithSet) error
- func (s *Segments) String() string
- func (z *Segments) UnmarshalMsg(bts []byte) (o []byte, err error)
- func (s *Segments) Verify(ctx context.Context, bs blobstore.WithSet) error
- func (s *Segments) VerifyElements(ctx context.Context, bs blobstore.WithSet, IDs *map[ElementID]SegmentID) error
- type SegmentsID
- func (z *SegmentsID) DecodeMsg(dc *msgp.Reader) (err error)
- func (z SegmentsID) EncodeMsg(en *msgp.Writer) (err error)
- func (s SegmentsID) Load(ctx context.Context, store blobstore.WithSet, cache Cache) (*Segments, error)
- func (z SegmentsID) MarshalMsg(b []byte) (o []byte, err error)
- func (z SegmentsID) Msgsize() (s int)
- func (z *SegmentsID) UnmarshalMsg(bts []byte) (o []byte, err error)
- func (s SegmentsID) Unset() bool
- type Token
- type WriterMsgp
Constants ¶
This section is empty.
Variables ¶
var ( // ErrNotImplemented is returned if the functionality is not yet implemented. ErrNotImplemented = errors.New("not implemented") // ErrNotFound is returned if the requested item could not be found. ErrNotFound = errors.New("not found") // ErrVersionMismatch is returned when unable to decode a content because of version mismatch. ErrVersionMismatch = errors.New("version mismatch") // ErrOffsetOutOfBounds is returned if requested offset is outside list bounds. ErrOffsetOutOfBounds = errors.New("offset out of list bounds") // ErrEmptySet is returned if a set was empty. ErrEmptySet = errors.New("set cannot be empty") )
var WithListOption = ListOption(nil)
WithListOption provides an element to create list parameters.
Functions ¶
func MsgpGetVersion ¶
MsgpGetVersion reads the version and returns it from a byte slice.
func NewSegmentElements ¶
NewSegmentElements creates a segment with a number of elements. If elements are provided min/max will be populated with values from slice. Elements are not saved to segment.
func RandString ¶
RandString returns a random string. x Random bytes + 8 bytes derived from current time. The length is x + 8 characters (same in bytes). The returned string has values 0-9,a-z,A-Z. The number of random bytes is prefixed to help avoid collisions and make strings sort with random distribution. Default number of random bytes is 8, giving 62^8 = 2e14 combinations for the given OS time resolution. Note that creation time is leaked.
Types ¶
type Bucket ¶
type Bucket chan struct{}
A Bucket contains a number of tokens. A bucket can safely be concurrently accessed. A user is free to access the channel directly.
type Cache ¶
type Cache interface { Add(key, value interface{}) Get(key interface{}) (interface{}, bool) Contains(key interface{}) bool Remove(key interface{}) }
Cache provides a caching interface.
type Element ¶
Element contains information about a single element in a list.
func (Element) Above ¶
Above returns true if e should be ranked above b. If score is equal, the tiebreaker is used, descending. If tiebreaker is equal, last update time wins. Final tiebreaker is element ID, where lowest ID gets first.
func (Element) AsIndex ¶
func (e Element) AsIndex(s SegmentID) IndexElement
AsIndex returns the index element of this element in the given collection.
func (*Element) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
func (*Element) Msgsize ¶
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
type ElementID ¶
type ElementID uint64
ElementID is the ID of an element in a list.
func (ElementID) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
type ElementIDs ¶
type ElementIDs []ElementID
ElementIds is a slice of collection ids.
func (ElementIDs) AsScore ¶
func (e ElementIDs) AsScore() []uint64
AsScore returns the element ids as slice of uint64.
func (*ElementIDs) DecodeMsg ¶
func (z *ElementIDs) DecodeMsg(dc *msgp.Reader) (err error)
DecodeMsg implements msgp.Decodable
func (*ElementIDs) Deduplicate ¶
func (e *ElementIDs) Deduplicate()
Deduplicate and sort the element ids.
func (ElementIDs) EncodeMsg ¶
func (z ElementIDs) EncodeMsg(en *msgp.Writer) (err error)
EncodeMsg implements msgp.Encodable
func (ElementIDs) Map ¶
func (e ElementIDs) Map() map[ElementID]struct{}
Map returns the element ids as a map.
func (ElementIDs) MarshalMsg ¶
func (z ElementIDs) MarshalMsg(b []byte) (o []byte, err error)
MarshalMsg implements msgp.Marshaler
func (ElementIDs) Msgsize ¶
func (z ElementIDs) Msgsize() (s int)
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (ElementIDs) NotIn ¶
func (e ElementIDs) NotIn(b ElementIDs) ElementIDs
NotIn returns the elements not in b.
func (ElementIDs) Overlap ¶
func (e ElementIDs) Overlap(b ElementIDs) ElementIDs
Overlap returns the overlap between the Element IDs.
func (*ElementIDs) UnmarshalMsg ¶
func (z *ElementIDs) UnmarshalMsg(bts []byte) (o []byte, err error)
UnmarshalMsg implements msgp.Unmarshaler
type Elements ¶
type Elements []Element
Elements is a collection of elements. When this type is used elements can be assumed to be sorted.
func NewElements ¶
NewElements converts an (unsorted) slice of elements into a sorted slice of elements. Duplicate elements are removed.
func (*Elements) Deduplicate ¶
Deduplicate will remove entries with duplicate Element IDs. If duplicates are found, the element with latest update time is kept. If update time is equal, the one with the highest score is kept. The element list is always re-sorted.
func (*Elements) Delete ¶
Delete element from list. Returns ErrNotFound if element could not be found.
func (Elements) ElementIDs ¶
func (l Elements) ElementIDs(id SegmentID) IndexElements
ElementIDs returns element ids as ranked elements, where score is the element id and payload is the segment to which they belong.
func (Elements) FindScoreIdx ¶
FindScoreIdx returns index of first element that matches score.
func (Elements) FirstElementsWithScore ¶
firstElementsWithScore returns the first element that has each of the supplied scores. If no element exist with the supplied score, the element below is returned. Scores must be sorted descending.
func (*Elements) HasDuplicates ¶
HasDuplicates returns true if elements contains duplicates.
func (Elements) IDs ¶
func (l Elements) IDs() ElementIDs
IDSorter returns a sorting function that sorts by ID.
func (Elements) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
func (*Elements) Merge ¶
Merge other elements into this list. Provided elements must be sorted. Provide information on whether l is shared with other slices. Use true if in doubt. Does not deduplicate on IDs, use MergeDeduplicate for this (approximately 1 order of magnitude slower).
func (*Elements) MergeDeduplicate ¶
MergeDeduplicate will merge other elements into this list. IDs are checked for duplicates and inserted elements overwrite existing. ins is used, so content is overwritten. Each list must be de-duplicated. It is not a strict requirement that lists are sorted.
func (Elements) MinMax ¶
MinMax returns the minimum and maximum values of the elements. If no elements are provided the entire range is returned.
func (Elements) Msgsize ¶
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (Elements) Ranked ¶
func (l Elements) Ranked(topOffset, total int) RankedElements
Ranked converts elements to ranked elements. The offset from the top of the list of the first element and total number of elements in the list must be provided.
func (Elements) Sorter ¶
Sorter returns a sorter that will sort the elements by score, descending. If score is equal, the tiebreaker is used, descending. If tiebreaker is equal, earliest update time wins. Final tiebreaker is element ID, where lowest ID gets first.
func (*Elements) UnmarshalMsg ¶
UnmarshalMsg implements msgp.Unmarshaler
func (*Elements) Update ¶
Update will delete the previous element in the list with the same ID and insert the new element. Returns ErrNotFound if element could not be found.
func (*Elements) UpdateTime ¶
UpdateTime will update time of all elements to the provided time.
type IndexElement ¶
type IndexElement struct {
Element
}
IndexElement is an element that is used as an index.
func (*IndexElement) DecodeMsg ¶
func (z *IndexElement) DecodeMsg(dc *msgp.Reader) (err error)
DecodeMsg implements msgp.Decodable
func (*IndexElement) EncodeMsg ¶
func (z *IndexElement) EncodeMsg(en *msgp.Writer) (err error)
EncodeMsg implements msgp.Encodable
func (*IndexElement) MarshalMsg ¶
func (z *IndexElement) MarshalMsg(b []byte) (o []byte, err error)
MarshalMsg implements msgp.Marshaler
func (*IndexElement) Msgsize ¶
func (z *IndexElement) Msgsize() (s int)
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (*IndexElement) UnmarshalMsg ¶
func (z *IndexElement) UnmarshalMsg(bts []byte) (o []byte, err error)
UnmarshalMsg implements msgp.Unmarshaler
type IndexElements ¶
type IndexElements struct {
Elements
}
IndexElements contains elements that are used to index other elements.
func (*IndexElements) DecodeMsg ¶
func (z *IndexElements) DecodeMsg(dc *msgp.Reader) (err error)
DecodeMsg implements msgp.Decodable
func (*IndexElements) EncodeMsg ¶
func (z *IndexElements) EncodeMsg(en *msgp.Writer) (err error)
EncodeMsg implements msgp.Encodable
func (*IndexElements) MarshalMsg ¶
func (z *IndexElements) MarshalMsg(b []byte) (o []byte, err error)
MarshalMsg implements msgp.Marshaler
func (*IndexElements) Msgsize ¶
func (z *IndexElements) Msgsize() (s int)
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (IndexElements) SegmentSorter ¶
func (e IndexElements) SegmentSorter() func(i, j int) bool
SegmentSorter returns a sorted that orders by segment first and score secondly.
func (*IndexElements) UnmarshalMsg ¶
func (z *IndexElements) UnmarshalMsg(bts []byte) (o []byte, err error)
UnmarshalMsg implements msgp.Unmarshaler
type IndexError ¶
type IndexError error
IndexError is returned when error is related to index and can be fixed by a re-index.
type List ¶
type List struct { sync.RWMutex `msg:"-" json:"-"` ID ListID Set string Metadata map[string]string SplitSize int MergeSize int LoadIndex bool Scores SegmentsID Index SegmentsID // contains filtered or unexported fields }
List contains the segments representing a list as well as the index to look up segments by object ID.
func NewList ¶
func NewList(ctx context.Context, id ListID, set string, bs blobstore.Store, opts ...ListOption) (*List, error)
NewList creates a new list. A list ID and storage set must be provided. Use WithListOption to access additional options.
func RestoreList ¶
func RestoreList(ctx context.Context, bs blobstore.Store, r *ReaderMsgp, c Cache, newID *ListID) (*List, error)
RestoreList will restore a list. If "newID" is provided, the list is saved with its new new and segments are given new ids. Otherwise the exact same list is restored.
func (*List) Backup ¶
Backup serializes all content of a lists and writes it to the provided writer.
func (*List) DeleteElements ¶
deleteElements will delete elements with the supplied ids. If no elements are not found, no error is returned.
func (*List) ForceSplit ¶
ForceSplit will force list splitting/Merging.
func (*List) GetElements ¶
func (l *List) GetElements(ctx context.Context, bs blobstore.Store, ids []ElementID, radius int) (RankedElements, error)
GetElements will look up the provided elements and return them as ranked. The returned elements are sorted by rank. Elements that are not found are not returned.
func (*List) GetPercentile ¶
func (l *List) GetPercentile(ctx context.Context, bs blobstore.Store, percentile float64, radius int) (*RankedElement, error)
GetPercentile returns the element at a given percentile. The percentile must be normalized to 0->1.
func (*List) GetRankBottom ¶
func (l *List) GetRankBottom(ctx context.Context, bs blobstore.Store, offset, elements int) (Elements, error)
GetRankBottom returns a number of elements at a specific offset from the top of the list. Elements are returned in descending order. First element returned is (list length) - offset - 1. Requesting offset 0 will return the bottom element. If offset is outside list range ErrOffsetOutOfBounds is returned.
func (*List) GetRankScoreAsc ¶
func (l *List) GetRankScoreAsc(ctx context.Context, bs blobstore.Store, score uint64, n int) (Elements, error)
GetRankScoreAsc is not implemented.
func (*List) GetRankScoreDesc ¶
func (l *List) GetRankScoreDesc(ctx context.Context, bs blobstore.Store, score uint64, n int) (RankedElements, error)
GetRankScoreDesc returns ranked elements starting with first elements at a specific score. If the exact score isn't found, the list will start with the following next entry below the score. If no scores are at or below the supplied score, an empty array is returned. Results are returned in descending order.
func (*List) GetRankTop ¶
func (l *List) GetRankTop(ctx context.Context, bs blobstore.Store, offset, elements int) (Elements, error)
GetRankTop returns a number of elements at a specific offset from the top of the list. Elements are returned in descending order. First element returned is rank offset+1. Requesting offset 0 will start with top ranked element. Requesting offset equal to or grater than list length will return ErrOffsetOutOfBounds.
func (*List) Insert ¶
Insert elements into list. Elements are not checked if they exist and will result in duplicates if they do. Provided elements are not required to be sorted.
func (*List) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
func (*List) Msgsize ¶
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (*List) Populate ¶
Populate will replace content of list with the supplied elements. Elements are assumed to be de-duplicated.
func (*List) ReleaseSegments ¶
ReleaseSegments will release all segments from list including all elements.
func (*List) Repair ¶
Repair will repair the list. So far this is fairly brutally done by loading all segments and recreating scores and indexes.
func (*List) Stats ¶
Stats returns general stats about the list. If elements is true, top and bottom elements will be loaded.
func (*List) UnmarshalMsg ¶
UnmarshalMsg implements msgp.Unmarshaler
func (*List) UpdateElements ¶
func (l *List) UpdateElements(ctx context.Context, bs blobstore.Store, elems Elements, radius int, results bool) (RankedElements, error)
UpdateElements will update or add elements to the list. Elements should be deduplicated when provided. Concurrent calls to UpdateElements with the same elements can result in duplicate entries in the list. If results are not requested, the returned slice will always be empty.
func (*List) VerifyElements ¶
VerifyElements verifies elements in list.
type ListID ¶
type ListID string
ListID is the ID of a list.
func ListToListID ¶
ListToListID returns the IDs of the supplied lists. Elements that are nil are ignored.
func (ListID) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
type ListIDs ¶
type ListIDs []ListID
ListIDs is a slice of list IDs.
func (*ListIDs) Deduplicate ¶
Deduplicate list ids. The id list will be sorted as a side effect.
func (ListIDs) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
type ListOption ¶
type ListOption func(*listOptions) error
ListOption can be used to specify options when creating a list. Use either directly or use WithListOption.
func (ListOption) Cache ¶
func (l ListOption) Cache(cache Cache) ListOption
Cache will set cache of the list.
func (ListOption) Clone ¶
func (l ListOption) Clone(lst *List) ListOption
Clone will populate the list with the content of another list.
func (ListOption) LoadIndex ¶
func (l ListOption) LoadIndex(b bool) ListOption
LoadIndex will signify that lists indexes should be loaded on startup. If not, they are loaded on demand, and reclaimed by the server based on global policy. By default indexes are loaded.
func (ListOption) MergeSplitSize ¶
func (l ListOption) MergeSplitSize(merge, split int) ListOption
Provide custom split/merge sizes for list. Merge must be < split.
func (ListOption) Metadata ¶
func (l ListOption) Metadata(m map[string]string) ListOption
Metadata will set metadata.
func (ListOption) Populate ¶
func (l ListOption) Populate(e []Element) ListOption
Populate will populate the list with supplied elements.
type ListStats ¶
type ListStats struct { Elements int Segments int Top *RankedElement Bottom *RankedElement CacheHits uint64 CacheMisses uint64 CachePct float64 }
ListStats provides overall stats for a list
func (*ListStats) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
type Lists ¶
type Lists struct {
// contains filtered or unexported fields
}
Lists contains an id indexed
func (Lists) ByIDs ¶
ByIDs returns lists by ID. This is an O(n) operation. Lists that are not found are returned as nil.
func (*Lists) MatchAll ¶
MatchAll will return all lists where all metadata fields match the provided. A numbers of sets can also be provided. All lists from sets are included. If q and sets both have 0 entries nothing is returned. Results are returned in random order.
func (*Lists) Prune ¶
Prune will release all in-memory elements and for lists that has LoadIndex set to false.
func (*Lists) SortedIDsAfter ¶
SortedIDsAfter returns a number of lists sorted by ID. The first entry returned will be the one following the provided ID. Provide an empty ID to get from the first list. If n <= 0 an empty list and page info will be returned.
func (*Lists) SortedIDsBefore ¶
SortedIDsBefore returns a number of lists sorted by ID. The last entry in the list will be the one preceding the provided ID. Provide an empty ID to get from the first list. If n <= 0 an empty list and page info will be returned.
type Manager ¶
type Manager struct { Storage blobstore.Store Backup blobstore.WithSet Set string Lists Lists BackupEvery int // contains filtered or unexported fields }
func NewManager ¶
NewManager will create a new manager that manages the lists of the server.
func (*Manager) DeleteList ¶
DeleteList will delete a list and all data associated.
func (*Manager) StartIntervalSaver ¶
func (m *Manager) StartIntervalSaver(ctx context.Context, every time.Duration, shutdown chan chan struct{})
StartIntervalSaver will start a saver that will save the lists at fixed intervals. If the interval is 0 the lists are never saved. The lists are saved on shutdown. The provided channel is closed when save has finished.
func (*Manager) StartListPruner ¶
func (m *Manager) StartListPruner(ctx context.Context, every time.Duration, shutdown chan chan struct{})
StartListPruner will start an async list pruner that will unload elements and segments at regular intervals. If the interval provided is 0 the lists are never pruned.
type PageInfo ¶
type PageInfo struct {
Before, After int
}
PageInfo contains information about the number of lists/elements before and after the ones in the list.
type RankedElement ¶
type RankedElement struct { Element // FromTop indicates how many items are above this in the list. FromTop int // FromBottom indicates how many items are below this in the list. FromBottom int // Above contains elements above the current element. // Ends with element just above current. Above []Element `json:",omitempty"` // Below contains elements below the current element. // Starts with element just below current. Below []Element `json:",omitempty"` // Before contains the element placement before update. Before *RankedElement `json:"before,omitempty"` }
RankedElement is a ranked element.
func (*RankedElement) CalculateFromBottom ¶
func (r *RankedElement) CalculateFromBottom(total int)
CalculateFromBottom recalculates the FromBottom with the supplied list total.
func (RankedElement) String ¶
func (r RankedElement) String() string
String returns a human readble representation of the ranked element.
type RankedElements ¶
type RankedElements []RankedElement
RankedElements is a collection od ranked elements.
func (RankedElements) CalculateFromBottom ¶
func (r RankedElements) CalculateFromBottom(total int)
CalculateFromBottom recalculates the FromBottom with the supplied list total.
func (RankedElements) Elements ¶
func (r RankedElements) Elements() Elements
Elements returns the underlying elements in the same order as the ranked elements.
func (RankedElements) IDs ¶
func (r RankedElements) IDs() ElementIDs
ElementIDs returns the element ids.
func (RankedElements) Offset ¶
func (r RankedElements) Offset(offset int)
Offset will add a specific offset to FromTop and subtract it from FromBottom.
type ReaderMsgp ¶
type ReaderMsgp struct {
// contains filtered or unexported fields
}
ReaderMsgp
func NewReaderMsgp ¶
func NewReaderMsgp(b []byte) *ReaderMsgp
func NewReaderMsgpReader ¶
func NewReaderMsgpReader(r io.Reader) *ReaderMsgp
func (*ReaderMsgp) Close ¶
func (w *ReaderMsgp) Close()
Call Close to signify you are done with serialization. This will recycle the Reader. This may only be called once, otherwise races will occur.
func (ReaderMsgp) GetVersion ¶
func (w ReaderMsgp) GetVersion() (v uint8)
func (ReaderMsgp) Reader ¶
func (w ReaderMsgp) Reader() *msgp.Reader
type Segment ¶
type Segment struct { ID SegmentID // ID of this segment Min, Max uint64 // Min/Max score of the elements in the segment. MinTie, MaxTie uint32 // Min/Max Tiebreaker of the elements in the segment. N int Updated int64 Parent SegmentsID // contains filtered or unexported fields }
A Segment describes a part of the list. All elements in the segment are guaranteed to be within Min/Max values (inclusive). A segment way describe a larger range than the elements themselves represent, it is up to splitting functions to determine this.
func NewSegment ¶
NewSegment creates a new, empty segment.
func (*Segment) Filter ¶
Filter returns the slice of elements that falls within the current segment.
func (*Segment) FilterIdx ¶
FilterIdx returns the indexes of the start and end of the slice of elements that fall within the range of the segment.
func (*Segment) FilterScoresIdx ¶
FilterScoresIdx returns the indexes of the start and end of the slice of elements that fall within the range of the segment. The supplied scores must be sorted descending.
func (*Segment) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
func (*Segment) Msgsize ¶
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (*Segment) StorageKey ¶
StorageKey returns the storage key for a segment.
type SegmentID ¶
type SegmentID uint32
SegmentID is the ID of a segment. It does not correspond to the placement in the list, but is used to reference it.
func (SegmentID) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
type Segments ¶
type Segments struct { ID SegmentsID Segments []Segment // Individual segments SegmentsLock sync.RWMutex `msg:"-" json:"-"` // Global lock for all stats on all segments (above) NextID SegmentID // Next unused ID IsIndex bool // These segments represents an index. // contains filtered or unexported fields }
Segments contains all segments of a list and contains and index to quickly look up the index of a segment.
func NewSegments ¶
NewSegments creates new segments and preallocates space for segments. Optionally it will preallocate internal index as well.
func NewSegmentsElements ¶
func NewSegmentsElements(ctx context.Context, bs blobstore.WithSet, e []Elements, idx *IndexElements) (*Segments, error)
NewSegmentsElements creates segments from slices of elements. Each member of the slice will result in one segment. The Segment range will cover the entire possible space, so first segment will always be from MaxElement, etc. If idx is supplied it will be populated with index elements. If idx is not supplied it is assumed that this is index segments.
func (*Segments) AlignIndex ¶
func (s *Segments) AlignIndex()
AlignIndex will align segments to be used for indexes. This ensure that segment splits are on score boundaries so changes in tiebreaker does not affect segment assignment. This should only be used on segment creation and not on segments with content.
func (*Segments) Append ¶
func (s *Segments) Append(ls *lockedSegment)
Append a segment to the end of segments. The locked segment is updated with new index.
func (*Segments) DeleteElements ¶
func (s *Segments) DeleteElements(ctx context.Context, bs blobstore.WithSet, e Elements, withIdx bool) (IndexElements, error)
deleteElements deletes specific elements in segments. Optionally returns index elements of deleted items. Provided elements must be sorted.
func (*Segments) DeleteElementsIdx ¶
func (s *Segments) DeleteElementsIdx(ctx context.Context, bs blobstore.WithSet, ids IndexElements, withIdx bool) (IndexElements, error)
deleteElements deletes specific elements in segments places at supplied index. Optionally returns index elements of deleted items. Read lock must be held for segments.
func (*Segments) ElementIndexAll ¶
func (s *Segments) ElementIndexAll(ctx context.Context, bs blobstore.WithSet) (IndexElements, error)
ElementIndexAll will return index for all elements in the list.
func (*Segments) Elements ¶
Elements returns the number of elements in all segments. Caller must hold at least read lock for segment.
func (*Segments) FindElements ¶
func (s *Segments) FindElements(ctx context.Context, bs blobstore.WithSet, ids IndexElements, radius int) ([]RankedElement, error)
FindElements returns elements with the supplied indices. Not found errors are logged, but not fatal. Caller must hold s.scores read lock.
func (*Segments) Insert ¶
func (s *Segments) Insert(ctx context.Context, bs blobstore.WithSet, in Elements, withIdx bool) (IndexElements, error)
Insert sorted elements into segments. Returns indexes of elements if withIdx is true. Read lock must be held
func (*Segments) MarshalMsg ¶
MarshalMsg implements msgp.Marshaler
func (*Segments) Msgsize ¶
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (*Segments) UnmarshalMsg ¶
UnmarshalMsg implements msgp.Unmarshaler
func (*Segments) VerifyElements ¶
func (s *Segments) VerifyElements(ctx context.Context, bs blobstore.WithSet, IDs *map[ElementID]SegmentID) error
VerifyElements verifies elements in segments. Supply a non-nil "IDs". It will be populated with the Element IDs found. The supplied map is overwritten. Segments must be read locked by callers.
type SegmentsID ¶
type SegmentsID string
SegmentsID is the ID of a collection of segments.
func (*SegmentsID) DecodeMsg ¶
func (z *SegmentsID) DecodeMsg(dc *msgp.Reader) (err error)
DecodeMsg implements msgp.Decodable
func (SegmentsID) EncodeMsg ¶
func (z SegmentsID) EncodeMsg(en *msgp.Writer) (err error)
EncodeMsg implements msgp.Encodable
func (SegmentsID) Load ¶
func (s SegmentsID) Load(ctx context.Context, store blobstore.WithSet, cache Cache) (*Segments, error)
Load will load the segments (but no elements) of the specified ID.
func (SegmentsID) MarshalMsg ¶
func (z SegmentsID) MarshalMsg(b []byte) (o []byte, err error)
MarshalMsg implements msgp.Marshaler
func (SegmentsID) Msgsize ¶
func (z SegmentsID) Msgsize() (s int)
Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (*SegmentsID) UnmarshalMsg ¶
func (z *SegmentsID) UnmarshalMsg(bts []byte) (o []byte, err error)
UnmarshalMsg implements msgp.Unmarshaler
func (SegmentsID) Unset ¶
func (s SegmentsID) Unset() bool
SegmentsID returns whether the segment ID is non-zero.
type Token ¶
type Token struct {
// contains filtered or unexported fields
}
A Token from a bucket. Can be put buck using the Release function.
type WriterMsgp ¶
type WriterMsgp struct {
// contains filtered or unexported fields
}
func NewWriterMsg ¶
func NewWriterMsg() *WriterMsgp
func (WriterMsgp) Buffer ¶
func (w WriterMsgp) Buffer() *bytes.Buffer
Buffer returns the buffer containing the encoded content. The encoder is flushed to buffer.
func (*WriterMsgp) Close ¶
func (w *WriterMsgp) Close()
Call Close to signify you are done with serialization and you no longer need the data kept in the buffer. This will recycle the Writer. This may only be called once, otherwise races will occur.
func (WriterMsgp) ReplaceWriter ¶
func (w WriterMsgp) ReplaceWriter(writer io.Writer)
ReplaceWriter replaces the writer
func (WriterMsgp) SetVersion ¶
func (w WriterMsgp) SetVersion(v uint8) error
SetVersion will write a version number.
func (WriterMsgp) Writer ¶
func (w WriterMsgp) Writer() *msgp.Writer
Writer returns the msgpack writer.
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
design
Package design contains the API design used for generating goa models and controllers.
|
Package design contains the API design used for generating goa models and controllers. |
bstest
Package bstest supplies helpers to test blobstores.
|
Package bstest supplies helpers to test blobstores. |
cmd
|
|
testlogger
Package testlogger provides a logging adapter for tests and benchmarks.
|
Package testlogger provides a logging adapter for tests and benchmarks. |