storage

Documentation

Index

• Variables
• func CheckVariablesForStorage() error
• func ReadFiles(keys []string, storageInstance Storer) ([]bytes.Buffer, error)
• func ReadFilesAsync(keys []string, threads int, storageInstance Storer) ([]bytes.Buffer, error)
• func SetStorageBucketFromEnv() error
• type FileStorage
  • func NewFileStorage(basePath string) *FileStorage
  • func (fs *FileStorage) Delete(key string) error
  • func (fs *FileStorage) List(ctx context.Context, delim, blockBatch string, timeout int, ...) ([]string, error)
  • func (fs *FileStorage) Read(key string) (bytes.Buffer, error)
  • func (fs *FileStorage) ReadBatch(readItems []ReadItem) (map[string][]string, error)
  • func (fs *FileStorage) Save(batchDir, filename string, bf bytes.Buffer) error
• type GCS
  • func NewGCSStorage(client *storage.Client, basePath string) *GCS
  • func (g *GCS) Delete(key string) error
  • func (g *GCS) List(ctx context.Context, delim, blockBatch string, timeout int, ...) ([]string, error)
  • func (g *GCS) Read(key string) (bytes.Buffer, error)
  • func (g *GCS) ReadBatch(readItems []ReadItem) (map[string][]string, error)
  • func (g *GCS) Save(batchDir, filename string, bf bytes.Buffer) error
• type ListReturnFunc
• type ReadItem
• type S3
  • func NewS3Storage(basePath string) *S3
  • func (s *S3) Delete(key string) error
  • func (s *S3) List(ctx context.Context, delim, blockBatch string, timeout int, ...) ([]string, error)
  • func (s *S3) Read(key string) (bytes.Buffer, error)
  • func (s *S3) ReadBatch(readItems []ReadItem) (map[string][]string, error)
  • func (s *S3) Save(batchDir, filename string, bf bytes.Buffer) error
• type Storer
  • func NewStorage(storageType, basePath string) (Storer, error)

Variables

var (
	GCSListReturnNameFunc = func(item any) string {
		if attr, ok := item.(*storage.ObjectAttrs); ok {
			return attr.Name
		}
		return ""
	}

	GCSListReturnPrefixFunc = func(item any) string {
		if attr, ok := item.(*storage.ObjectAttrs); ok {
			return attr.Prefix
		}
		return ""
	}
)

var (
	SeerCrawlerStorageType            string
	SeerCrawlerStorageBucket          string
	GCPStorageServiceAccountCredsPath string
	SeerCrawlerStoragePath            string = "data"
)

var Blockchains = map[string]string{
	"ethereum":                     "ethereum_smartcontract",
	"polygon":                      "polygon_smartcontract",
	"arbitrum_one":                 "arbitrum_one_smartcontract",
	"arbitrum_sepolia":             "arbitrum_sepolia_smartcontract",
	"game7_orbit_arbitrum_sepolia": "game7_orbit_arbitrum_sepolia_smartcontract",
	"xai":                          "xai_smartcontract",
	"xai_sepolia":                  "xai_sepolia_smartcontract",
	"mantle":                       "mantle_smartcontract",
	"mantle_sepolia":               "mantle_sepolia_smartcontract",
}

Blockchains map for storage or database models

Functions

func CheckVariablesForStorage

func CheckVariablesForStorage() error

func ReadFiles

func ReadFiles(keys []string, storageInstance Storer) ([]bytes.Buffer, error)

func ReadFilesAsync

func ReadFilesAsync(keys []string, threads int, storageInstance Storer) ([]bytes.Buffer, error)

func SetStorageBucketFromEnv

func SetStorageBucketFromEnv() error

Types

type FileStorage

type FileStorage struct {
	BasePath string
}

func NewFileStorage

func NewFileStorage(basePath string) *FileStorage

func (*FileStorage) Delete

func (fs *FileStorage) Delete(key string) error

func (*FileStorage) List

func (fs *FileStorage) List(ctx context.Context, delim, blockBatch string, timeout int, returnFunc ListReturnFunc) ([]string, error)

func (*FileStorage) Read

func (fs *FileStorage) Read(key string) (bytes.Buffer, error)

func (*FileStorage) ReadBatch

func (fs *FileStorage) ReadBatch(readItems []ReadItem) (map[string][]string, error)

func (*FileStorage) Save

func (fs *FileStorage) Save(batchDir, filename string, bf bytes.Buffer) error

type GCS

type GCS struct {
	Client   *storage.Client
	BasePath string
}

GCS implements the Storer interface for Google Cloud Storage

func NewGCSStorage

func NewGCSStorage(client *storage.Client, basePath string) *GCS

NewGCSStorage initializes a GCS storage with the provided client

func (*GCS) Delete

func (g *GCS) Delete(key string) error

func (*GCS) List

func (g *GCS) List(ctx context.Context, delim, blockBatch string, timeout int, returnFunc ListReturnFunc) ([]string, error)

func (*GCS) Read

func (g *GCS) Read(key string) (bytes.Buffer, error)

func (*GCS) ReadBatch

func (g *GCS) ReadBatch(readItems []ReadItem) (map[string][]string, error)

func (*GCS) Save

func (g *GCS) Save(batchDir, filename string, bf bytes.Buffer) error

type ListReturnFunc

type ListReturnFunc func(any) string

type ReadItem

type ReadItem struct {
	Key    string
	RowIds []uint64
}

type S3

type S3 struct {
	BasePath string
}

S3 implements the Storer interface for Amazon S3 Bucket

func NewS3Storage

func NewS3Storage(basePath string) *S3

func (*S3) Delete

func (s *S3) Delete(key string) error

func (*S3) List

func (s *S3) List(ctx context.Context, delim, blockBatch string, timeout int, returnFunc ListReturnFunc) ([]string, error)

func (*S3) Read

func (s *S3) Read(key string) (bytes.Buffer, error)

func (*S3) ReadBatch

func (s *S3) ReadBatch(readItems []ReadItem) (map[string][]string, error)

func (*S3) Save

func (s *S3) Save(batchDir, filename string, bf bytes.Buffer) error

type Storer

type Storer interface {
	Save(batchDir, filename string, bf bytes.Buffer) error
	Read(key string) (bytes.Buffer, error)
	ReadBatch(readItems []ReadItem) (map[string][]string, error)
	Delete(key string) error
	List(ctx context.Context, delim, blockBatch string, timeout int, returnFunc ListReturnFunc) ([]string, error)
}

func NewStorage

func NewStorage(storageType, basePath string) (Storer, error)

NewStorage initialize storage placement for protobuf batch data.