datastore

package
v0.0.0-...-089553b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 20, 2024 License: Apache-2.0, Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type DataStore 

type DataStore interface {
	GetFileMetadata(ctx context.Context, path string) (map[string]string, error)
	GetFile(ctx context.Context, path string) (io.ReadCloser, error)
	PutFile(ctx context.Context, path string, in io.WriterTo, metaData map[string]string) error
	PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo, metaData map[string]string) (bool, error)
	Exists(ctx context.Context, path string) (bool, error)
	Size(ctx context.Context, path string) (int64, error)
	GetSchema() DataStoreSchema
	Close() error
}

DataStore defines an interface for interacting with data storage

func FromGCSClient 

func FromGCSClient(ctx context.Context, client *storage.Client, bucketPath string, schema DataStoreSchema) (DataStore, error)

func NewDataStore 

func NewDataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error)

NewDataStore factory, it creates a new DataStore based on the config type

func NewGCSDataStore 

func NewGCSDataStore(ctx context.Context, bucketPath string, schema DataStoreSchema) (DataStore, error)

type DataStoreConfig 

type DataStoreConfig struct {
	Type   string            `toml:"type"`
	Params map[string]string `toml:"params"`
	Schema DataStoreSchema   `toml:"schema"`
}

type DataStoreSchema 

type DataStoreSchema struct {
	LedgersPerFile    uint32 `toml:"ledgers_per_file"`
	FilesPerPartition uint32 `toml:"files_per_partition"`
}

func (DataStoreSchema) GetObjectKeyFromSequenceNumber 

func (ec DataStoreSchema) GetObjectKeyFromSequenceNumber(ledgerSeq uint32) string

GetObjectKeyFromSequenceNumber generates the object key name from the ledger sequence number based on configuration.

func (DataStoreSchema) GetSequenceNumberEndBoundary 

func (ec DataStoreSchema) GetSequenceNumberEndBoundary(ledgerSeq uint32) uint32

func (DataStoreSchema) GetSequenceNumberStartBoundary 

func (ec DataStoreSchema) GetSequenceNumberStartBoundary(ledgerSeq uint32) uint32

type GCSDataStore 

type GCSDataStore struct {
	// contains filtered or unexported fields
}

GCSDataStore implements DataStore for GCS

func (GCSDataStore) Close 

func (b GCSDataStore) Close() error

Close closes the GCS client connection.

func (GCSDataStore) Exists 

func (b GCSDataStore) Exists(ctx context.Context, pth string) (bool, error)

Exists checks if a file exists in the GCS bucket.

func (GCSDataStore) GetFile 

func (b GCSDataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)

GetFile retrieves a file from the GCS bucket.

func (GCSDataStore) GetFileMetadata 

func (b GCSDataStore) GetFileMetadata(ctx context.Context, filePath string) (map[string]string, error)

GetFileMetadata retrieves the metadata for the specified file in the GCS bucket.

func (GCSDataStore) GetSchema 

func (b GCSDataStore) GetSchema() DataStoreSchema

GetSchema returns the schema information which defines the structure and organization of data in the datastore.

func (GCSDataStore) PutFile 

func (b GCSDataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) error

PutFile uploads a file to GCS

func (GCSDataStore) PutFileIfNotExists 

func (b GCSDataStore) PutFileIfNotExists(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) (bool, error)

PutFileIfNotExists uploads a file to GCS only if it doesn't already exist.

func (GCSDataStore) Size 

func (b GCSDataStore) Size(ctx context.Context, pth string) (int64, error)

Size retrieves the size of a file in the GCS bucket.

type MetaData 

type MetaData struct {
	StartLedger          uint32
	EndLedger            uint32
	StartLedgerCloseTime int64
	EndLedgerCloseTime   int64
	ProtocolVersion      uint32
	CoreVersion          string
	NetworkPassPhrase    string
	CompressionType      string
	Version              string
}

func NewMetaDataFromMap 

func NewMetaDataFromMap(data map[string]string) (MetaData, error)

func (MetaData) ToMap 

func (m MetaData) ToMap() map[string]string

type MockDataStore 

type MockDataStore struct {
	mock.Mock
}

MockDataStore is a mock implementation for the Storage interface.

func (*MockDataStore) Close 

func (m *MockDataStore) Close() error

func (*MockDataStore) Exists 

func (m *MockDataStore) Exists(ctx context.Context, path string) (bool, error)

func (*MockDataStore) GetFile 

func (m *MockDataStore) GetFile(ctx context.Context, path string) (io.ReadCloser, error)

func (*MockDataStore) GetFileMetadata 

func (m *MockDataStore) GetFileMetadata(ctx context.Context, path string) (map[string]string, error)

func (*MockDataStore) GetSchema 

func (m *MockDataStore) GetSchema() DataStoreSchema

func (*MockDataStore) PutFile 

func (m *MockDataStore) PutFile(ctx context.Context, path string, in io.WriterTo, metadata map[string]string) error

func (*MockDataStore) PutFileIfNotExists 

func (m *MockDataStore) PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo, metadata map[string]string) (bool, error)

func (*MockDataStore) Size 

func (m *MockDataStore) Size(ctx context.Context, path string) (int64, error)

type MockResumableManager 

type MockResumableManager struct {
	mock.Mock
}

func (*MockResumableManager) FindStart 

func (m *MockResumableManager) FindStart(ctx context.Context, start, end uint32) (absentLedger uint32, ok bool, err error)

type ResumableManager 

type ResumableManager interface {
	// Given a requested ledger range, return the first absent ledger within the
	// requested range of [start, end].
	//
	// start - begin search inclusive from this ledger, must be greater than 0.
	// end   - stop search inclusive to this ledger.
	//
	// If start=0, invalid, error will be returned.
	//
	// If end=0, is provided as a convenience, to allow requesting an effectively
	// dynamic end value for the range, which will be an approximation of the network's
	// most recent checkpointed ledger + (2 * checkpoint_frequency).
	//
	// return:
	// absentLedger      - will be non-zero, the oldest ledger sequence between range of [start, end]
	//                     which is not populated on data store.
	// ok                - if true, 'absentLedger' has a usable non-zero value, if false, there is no absent ledger in the requested range and 'absentLedger' is set to zero.
	// err               - the search was cancelled due to this unexpected error, 'absentLedger' and 'ok' return values should be ignored.
	//
	// When no error, the two return values will compose the following truth table:
	//    1. datastore had no data in the requested range: absentLedger={start}, ok=true
	//    2. datastore had partial data in the requested range: absentLedger={a value > start and <= end}, ok=true
	//    3. datastore had all data in the requested range: absentLedger=0, ok=false
	FindStart(ctx context.Context, start, end uint32) (absentLedger uint32, ok bool, err error)
}

func NewResumableManager 

func NewResumableManager(dataStore DataStore,
	ledgerBatchConfig DataStoreSchema,
	archive historyarchive.ArchiveInterface) ResumableManager

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.