part2

Documentation

Index

• Variables
• type BinaryStore
  • func (store BinaryStore) Insert(dataParts []Elements) ([]data.Block, error)
  • func (store BinaryStore) Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)
• type Element
  • func GenerateData(len int) []Element
  • func (*Element) Descriptor() ([]byte, []int)
  • func (m *Element) GetTimestamp() int64
  • func (m *Element) GetValue() float64
  • func (m *Element) Marshal() (dAtA []byte, err error)
  • func (m *Element) MarshalTo(dAtA []byte) (int, error)
  • func (m *Element) MarshalToSizedBuffer(dAtA []byte) (int, error)
  • func (*Element) ProtoMessage()
  • func (m *Element) Reset()
  • func (m *Element) Size() (n int)
  • func (m *Element) String() string
  • func (m *Element) Unmarshal(dAtA []byte) error
  • func (m *Element) XXX_DiscardUnknown()
  • func (m *Element) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
  • func (m *Element) XXX_Merge(src proto.Message)
  • func (m *Element) XXX_Size() int
  • func (m *Element) XXX_Unmarshal(b []byte) error
• type Elements
  • func (*Elements) Descriptor() ([]byte, []int)
  • func (m *Elements) GetData() []Element
  • func (m *Elements) Marshal() (dAtA []byte, err error)
  • func (m *Elements) MarshalTo(dAtA []byte) (int, error)
  • func (m *Elements) MarshalToSizedBuffer(dAtA []byte) (int, error)
  • func (*Elements) ProtoMessage()
  • func (m *Elements) Reset()
  • func (m *Elements) Size() (n int)
  • func (m *Elements) String() string
  • func (m *Elements) Unmarshal(dAtA []byte) error
  • func (m *Elements) XXX_DiscardUnknown()
  • func (m *Elements) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
  • func (m *Elements) XXX_Merge(src proto.Message)
  • func (m *Elements) XXX_Size() int
  • func (m *Elements) XXX_Unmarshal(b []byte) error
• type GobStore
  • func (store GobStore) Insert(dataParts []Elements) ([]data.Block, error)
  • func (store GobStore) Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)
• type Index
  • func NewIndex(id string, blockInterval int64, storeType string, storePath string, ...) (Index, error)
  • func (idx Index) Extract(start, finish int64) ([]Element, error)
• type ParquetStore
  • func (store ParquetStore) Insert(dataParts []Elements) ([]data.Block, error)
  • func (store ParquetStore) Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)
• type ProtoStore
  • func (store ProtoStore) Insert(dataParts []Elements) ([]data.Block, error)
  • func (store ProtoStore) Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)
• type Store
  • func NewStore(sType string, path string) Store

Variables

var (
	ErrInvalidLengthElements        = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowElements          = fmt.Errorf("proto: integer overflow")
	ErrUnexpectedEndOfGroupElements = fmt.Errorf("proto: unexpected end of group")
)

Types

type BinaryStore

type BinaryStore struct {
	// contains filtered or unexported fields
}

BinaryStore is an implementation of Store that serializes input time series into binary representation by calculating deltas of timestamps and values and placing them one after another.

func (BinaryStore) Insert

func (store BinaryStore) Insert(dataParts []Elements) ([]data.Block, error)

Insert preallocates binary slice for serialized data, calculates deltas, converts them to bytes, fills binary slice with them and writes binary slice to the file.

func (BinaryStore) Read

func (store BinaryStore) Read(blockIds []int, blockSizes []int, blockNums []int,
	offset int64) (Elements, error)

Read uses input meta-information to extract only required binary representations of blocks from the file, convert them to Elements, and return the result.

type Element

type Element struct {
	Timestamp            int64    `protobuf:"varint,1,req,name=timestamp" json:"timestamp"`
	Value                float64  `protobuf:"fixed64,2,req,name=value" json:"value"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func GenerateData

func GenerateData(len int) []Element

GenerateData is a simple function for creating datasets that are used in stores benchmarking.

func (*Element) Descriptor

func (*Element) Descriptor() ([]byte, []int)

func (*Element) GetTimestamp

func (m *Element) GetTimestamp() int64

func (*Element) GetValue

func (m *Element) GetValue() float64

func (*Element) Marshal

func (m *Element) Marshal() (dAtA []byte, err error)

func (*Element) MarshalTo

func (m *Element) MarshalTo(dAtA []byte) (int, error)

func (*Element) MarshalToSizedBuffer

func (m *Element) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Element) ProtoMessage

func (*Element) ProtoMessage()

func (*Element) Reset

func (m *Element) Reset()

func (*Element) Size

func (m *Element) Size() (n int)

func (*Element) String

func (m *Element) String() string

func (*Element) Unmarshal

func (m *Element) Unmarshal(dAtA []byte) error

func (*Element) XXX_DiscardUnknown

func (m *Element) XXX_DiscardUnknown()

func (*Element) XXX_Marshal

func (m *Element) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Element) XXX_Merge

func (m *Element) XXX_Merge(src proto.Message)

func (*Element) XXX_Size

func (m *Element) XXX_Size() int

func (*Element) XXX_Unmarshal

func (m *Element) XXX_Unmarshal(b []byte) error

type Elements

type Elements struct {
	Data                 []Element `protobuf:"bytes,1,rep,name=data" json:"data"`
	XXX_NoUnkeyedLiteral struct{}  `json:"-"`
	XXX_unrecognized     []byte    `json:"-"`
	XXX_sizecache        int32     `json:"-"`
}

func (*Elements) Descriptor

func (*Elements) Descriptor() ([]byte, []int)

func (*Elements) GetData

func (m *Elements) GetData() []Element

func (*Elements) Marshal

func (m *Elements) Marshal() (dAtA []byte, err error)

func (*Elements) MarshalTo

func (m *Elements) MarshalTo(dAtA []byte) (int, error)

func (*Elements) MarshalToSizedBuffer

func (m *Elements) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Elements) ProtoMessage

func (*Elements) ProtoMessage()

func (*Elements) Reset

func (m *Elements) Reset()

func (*Elements) Size

func (m *Elements) Size() (n int)

func (*Elements) String

func (m *Elements) String() string

func (*Elements) Unmarshal

func (m *Elements) Unmarshal(dAtA []byte) error

func (*Elements) XXX_DiscardUnknown

func (m *Elements) XXX_DiscardUnknown()

func (*Elements) XXX_Marshal

func (m *Elements) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Elements) XXX_Merge

func (m *Elements) XXX_Merge(src proto.Message)

func (*Elements) XXX_Size

func (m *Elements) XXX_Size() int

func (*Elements) XXX_Unmarshal

func (m *Elements) XXX_Unmarshal(b []byte) error

type GobStore

type GobStore struct {
	// contains filtered or unexported fields
}

GobStore is an implementation of Store that serializes data parts into gob binary form.

func (GobStore) Insert

func (store GobStore) Insert(dataParts []Elements) ([]data.Block, error)

Insert converts every data part into gob binary representation, combines all representations into the file, and creates block for every data part.

func (GobStore) Read

func (store GobStore) Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)

Read uses input meta-information to extract only required binary representations of blocks from the file, convert them to Elements, and return the result.

type Index

type Index struct {
	ID            string
	StartTime     int64
	BlockInterval int64
	Store         Store
	Blocks        []data.Block
}

Index is data structure that implements indexing logic for a time series database.

ID - name of the sensor.

StartTime - Unix timestamp of the first entry in the index.

BlockInterval - time interval for blocks generation.

Store - backend storage that is responsible for data
serialization and deserialization.

Blocks - is a slice of index blocks. Slices were chosen
instead of trees in part1.

func NewIndex

func NewIndex(id string, blockInterval int64, storeType string, storePath string, d []Element) (Index, error)

NewIndex creates new index, initializes Store based on the storeType parameter, loads input data d to the Store and loads output blocks into the index.

func (Index) Extract

func (idx Index) Extract(start, finish int64) ([]Element, error)

Extract based on start and finish parameters calculates blocks that need to be extracted, and reads them from the Store implementation.

type ParquetStore

type ParquetStore struct {
	// contains filtered or unexported fields
}

ParquetStore is an implementation of Store that serializes data parts into Parquet binary form.

func (ParquetStore) Insert

func (store ParquetStore) Insert(dataParts []Elements) ([]data.Block, error)

Insert converts every data part into Parquet binary representation, combines all representations into the file, and creates block for every data part.

func (ParquetStore) Read

func (store ParquetStore) Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)

Read uses input meta-information to extract only required binary representations of blocks from the file, convert them to Elements, and return the result.

type ProtoStore

type ProtoStore struct {
	// contains filtered or unexported fields
}

ProtoStore is an implementation of Store that serializes data parts into Protobuf binary form.

func (ProtoStore) Insert

func (store ProtoStore) Insert(dataParts []Elements) ([]data.Block, error)

Insert converts every data part into gob binary representation, combines all representations into the file, and creates block for every data part.

func (ProtoStore) Read

func (store ProtoStore) Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)

Read uses input meta-information to extract only required binary representations of blocks from the file, convert them to Elements, and return the result.

type Store

type Store interface {
	Insert(dataParts []Elements) ([]data.Block, error)
	Read(blockIds []int, blockSizes []int, blockNums []int, offset int64) (Elements, error)
}

Store is the inferface that describes a block storage for time series data.

Insert loads slice of data parts, each of which is represented as Elements, to the storage and if successful returns a slice of index blocks.

Read uses meta-information about data blocks to extract data from the store.

func NewStore

func NewStore(sType string, path string) Store

NewStore initializes new storage based on the provided type and sets the path for storing the serialized binary data.