repository

Documentation

Index

• Variables
• type ChunkDynamoDB
  • func NewChunkDynamoDB(region, tableName string) *ChunkDynamoDB
  • func (x *ChunkDynamoDB) DeleteChunk(chunk *models.Chunk) (*models.Chunk, error)
  • func (x *ChunkDynamoDB) FreezeChunk(chunk *models.Chunk) (*models.Chunk, error)
  • func (x *ChunkDynamoDB) GetMergableChunks(schema string, createdBefore time.Time, minChunkSize int64) ([]*models.Chunk, error)
  • func (x *ChunkDynamoDB) GetWritableChunks(schema, partition string, writableTotalSize int64) ([]*models.Chunk, error)
  • func (x *ChunkDynamoDB) PutChunk(recordID string, size int64, schema, partition string, created time.Time) error
  • func (x *ChunkDynamoDB) UpdateChunk(chunk *models.Chunk, recordID string, objSize, writableSize int64) error
• type ChunkRepository
• type MetaDynamoDB
  • func (x *MetaDynamoDB) GetObjecID(s3path string) (int64, error)
  • func (x *MetaDynamoDB) GetRecordObjects(recordIDs []string, schema models.ParquetSchemaName) ([]*MetaRecordObject, error)
  • func (x *MetaDynamoDB) HeadPartition(partitionKey string) (bool, error)
  • func (x *MetaDynamoDB) PutPartition(partitionKey string) error
  • func (x *MetaDynamoDB) PutRecordObjects(records []*MetaRecordObject) error
• type MetaRecordObject
  • func (x *MetaRecordObject) HashKey() interface{}
  • func (x *MetaRecordObject) RangeKey() interface{}
• type MetaRepository
  • func NewMetaDynamoDB(region, tableName string) MetaRepository

Variables

var (
	// ErrChunkNotWritable means updating after FreezedAt or over TotalSize
	ErrChunkNotWritable = fmt.Errorf("Update condition is not matched")
)

Types

type ChunkDynamoDB

type ChunkDynamoDB struct {
	KeyPrefix string
	// contains filtered or unexported fields
}

ChunkDynamoDB is implementation of ChunkRepository for DynamoDB

func NewChunkDynamoDB

func NewChunkDynamoDB(region, tableName string) *ChunkDynamoDB

NewChunkDynamoDB is constructor of ChunkDynamoDB

func (*ChunkDynamoDB) DeleteChunk

func (x *ChunkDynamoDB) DeleteChunk(chunk *models.Chunk) (*models.Chunk, error)

func (*ChunkDynamoDB) FreezeChunk

func (x *ChunkDynamoDB) FreezeChunk(chunk *models.Chunk) (*models.Chunk, error)

func (*ChunkDynamoDB) GetMergableChunks

func (x *ChunkDynamoDB) GetMergableChunks(schema string, createdBefore time.Time, minChunkSize int64) ([]*models.Chunk, error)

GetMergableChunks returns mergable chunks exceeding freezedAt or minChunkSize

func (*ChunkDynamoDB) GetWritableChunks

func (x *ChunkDynamoDB) GetWritableChunks(schema, partition string, writableTotalSize int64) ([]*models.Chunk, error)

GetWritableChunks returns writable chunks for now (because chunks are not locked)

func (*ChunkDynamoDB) PutChunk

func (x *ChunkDynamoDB) PutChunk(recordID string, size int64, schema, partition string, created time.Time) error

func (*ChunkDynamoDB) UpdateChunk

func (x *ChunkDynamoDB) UpdateChunk(chunk *models.Chunk, recordID string, objSize, writableSize int64) error

type ChunkRepository

type ChunkRepository interface {
	GetWritableChunks(schema, partition string, writableTotalSize int64) ([]*models.Chunk, error)
	GetMergableChunks(schema string, createdBefore time.Time, minChunkSize int64) ([]*models.Chunk, error)
	PutChunk(recordID string, objSize int64, schema, partition string, created time.Time) error
	UpdateChunk(chunk *models.Chunk, recordID string, objSize, writableSize int64) error
	FreezeChunk(chunk *models.Chunk) (*models.Chunk, error)
	DeleteChunk(chunk *models.Chunk) (*models.Chunk, error)
}

type MetaDynamoDB

type MetaDynamoDB struct {
	// contains filtered or unexported fields
}

MetaDynamoDB is implementation of MetaRepository

func (*MetaDynamoDB) GetObjecID

func (x *MetaDynamoDB) GetObjecID(s3path string) (int64, error)

func (*MetaDynamoDB) GetRecordObjects

func (x *MetaDynamoDB) GetRecordObjects(recordIDs []string, schema models.ParquetSchemaName) ([]*MetaRecordObject, error)

GetRecordObjects retrieves S3 path of record file from DynamoDB

func (*MetaDynamoDB) HeadPartition

func (x *MetaDynamoDB) HeadPartition(partitionKey string) (bool, error)

func (*MetaDynamoDB) PutPartition

func (x *MetaDynamoDB) PutPartition(partitionKey string) error

func (*MetaDynamoDB) PutRecordObjects

func (x *MetaDynamoDB) PutRecordObjects(records []*MetaRecordObject) error

PutRecordObjects puts set of S3 path of record file to DynamoDB

type MetaRecordObject

type MetaRecordObject struct {
	models.S3Object

	RecordID string                   `dynamo:"record_id"`
	Schema   models.ParquetSchemaName `dynamo:"schema"`
	// contains filtered or unexported fields
}

func (*MetaRecordObject) HashKey

func (x *MetaRecordObject) HashKey() interface{}

func (*MetaRecordObject) RangeKey

func (x *MetaRecordObject) RangeKey() interface{}

type MetaRepository

type MetaRepository interface {
	GetObjecID(s3path string) (int64, error)
	PutRecordObjects(objects []*MetaRecordObject) error
	GetRecordObjects(recordIDs []string, schema models.ParquetSchemaName) ([]*MetaRecordObject, error)
	HeadPartition(partitionKey string) (bool, error)
	PutPartition(partitionKey string) error
}

MetaRepository is interface of object repository

func NewMetaDynamoDB

func NewMetaDynamoDB(region, tableName string) MetaRepository

NewMetaDynamoDB is a constructor of MetaDynamoDB as MetaAccessor