historyarchive

package
v0.0.0-...-446988d Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 28, 2023 License: Apache-2.0, Apache-2.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source
const DefaultCheckpointFrequency = uint32(64)
View Source
const NumLevels = 11

Variables

This section is empty.

Functions

func BucketPath

func BucketPath(bucket Hash) string

func Categories

func Categories() []string

func CategoryCheckpointPath

func CategoryCheckpointPath(cat string, chk uint32) string

func DumpXdrAsJson

func DumpXdrAsJson(args []string) error

func Mirror

func Mirror(src *Archive, dst *Archive, opts *CommandOptions) error

Mirror mirrors an archive, it assumes that the source and destination have the same checkpoint ledger frequency

func RangePaths

func RangePaths(r Range) []string

Returns an array of path prefixes to walk to enumerate all the objects in the provided range.

func Repair

func Repair(src *Archive, dst *Archive, opts *CommandOptions) error

Repair repairs a destination archive based on a source archive, it assumes that the source and destination have the same checkpoint ledger frequency

func SortTxsForHash

func SortTxsForHash(txset *xdr.TransactionSet) error

Types

type Archive

type Archive struct {
	// contains filtered or unexported fields
}

func Connect

func Connect(u string, opts ConnectOptions) (*Archive, error)

func MustConnect

func MustConnect(u string, opts ConnectOptions) *Archive

func (*Archive) BucketExists

func (a *Archive) BucketExists(bucket Hash) (bool, error)

func (*Archive) BucketSize

func (a *Archive) BucketSize(bucket Hash) (int64, error)

func (*Archive) CategoryCheckpointExists

func (a *Archive) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (*Archive) CheckBucketsMissing

func (arch *Archive) CheckBucketsMissing() map[Hash]bool

func (*Archive) CheckCheckpointFilesMissing

func (arch *Archive) CheckCheckpointFilesMissing(opts *CommandOptions) map[string][]uint32

func (*Archive) ClearCachedInfo

func (arch *Archive) ClearCachedInfo()

func (*Archive) GetBucketPathForHash

func (a *Archive) GetBucketPathForHash(hash Hash) string

func (*Archive) GetCheckpointHAS

func (a *Archive) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (*Archive) GetCheckpointManager

func (arch *Archive) GetCheckpointManager() CheckpointManager

func (*Archive) GetLedgerHeader

func (a *Archive) GetLedgerHeader(ledger uint32) (xdr.LedgerHeaderHistoryEntry, error)

func (*Archive) GetLedgers

func (a *Archive) GetLedgers(start, end uint32) (map[uint32]*Ledger, error)

func (*Archive) GetPathHAS

func (a *Archive) GetPathHAS(path string) (HistoryArchiveState, error)

func (*Archive) GetRootHAS

func (a *Archive) GetRootHAS() (HistoryArchiveState, error)

func (*Archive) GetXdrStream

func (a *Archive) GetXdrStream(pth string) (*XdrStream, error)

func (*Archive) GetXdrStreamForHash

func (a *Archive) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (*Archive) ListAllBucketHashes

func (a *Archive) ListAllBucketHashes() (chan Hash, chan error)

func (*Archive) ListAllBuckets

func (a *Archive) ListAllBuckets() (chan string, chan error)

func (*Archive) ListBucket

func (a *Archive) ListBucket(dp DirPrefix) (chan string, chan error)

func (*Archive) ListCategoryCheckpoints

func (a *Archive) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (*Archive) Log

func (arch *Archive) Log(opts *CommandOptions) error

func (*Archive) MustGetBucketSize

func (arch *Archive) MustGetBucketSize(hash Hash) int64

func (*Archive) MustGetLedgerHeaderHistoryEntries

func (arch *Archive) MustGetLedgerHeaderHistoryEntries(chk uint32) []xdr.LedgerHeaderHistoryEntry

func (*Archive) MustGetTransactionHistoryEntries

func (arch *Archive) MustGetTransactionHistoryEntries(chk uint32) []xdr.TransactionHistoryEntry

func (*Archive) NoteCheckpointFile

func (arch *Archive) NoteCheckpointFile(cat string, chk uint32, present bool)

func (*Archive) NoteExistingBucket

func (arch *Archive) NoteExistingBucket(bucket Hash)

func (*Archive) NoteReferencedBucket

func (arch *Archive) NoteReferencedBucket(bucket Hash) bool

func (*Archive) PutCheckpointHAS

func (a *Archive) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutPathHAS

func (a *Archive) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutRootHAS

func (a *Archive) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) ReportBucketStats

func (arch *Archive) ReportBucketStats()

func (*Archive) ReportCheckpointStats

func (arch *Archive) ReportCheckpointStats()

func (*Archive) ReportInvalid

func (arch *Archive) ReportInvalid(opts *CommandOptions) (bool, error)

func (*Archive) ReportMissing

func (arch *Archive) ReportMissing(opts *CommandOptions) (bool, error)

func (*Archive) Scan

func (arch *Archive) Scan(opts *CommandOptions) error

func (*Archive) ScanAllBuckets

func (arch *Archive) ScanAllBuckets() error

func (*Archive) ScanBuckets

func (arch *Archive) ScanBuckets(opts *CommandOptions) error

func (*Archive) ScanCheckpoints

func (arch *Archive) ScanCheckpoints(opts *CommandOptions) error

func (*Archive) ScanCheckpointsFast

func (arch *Archive) ScanCheckpointsFast(opts *CommandOptions) error

func (*Archive) ScanCheckpointsSlow

func (arch *Archive) ScanCheckpointsSlow(opts *CommandOptions) error

func (*Archive) VerifyBucketEntries

func (arch *Archive) VerifyBucketEntries(h Hash) error

func (*Archive) VerifyBucketHash

func (arch *Archive) VerifyBucketHash(h Hash) error

func (*Archive) VerifyCategoryCheckpoint

func (arch *Archive) VerifyCategoryCheckpoint(cat string, chk uint32) error

func (*Archive) VerifyLedgerHeaderHistoryEntry

func (arch *Archive) VerifyLedgerHeaderHistoryEntry(entry *xdr.LedgerHeaderHistoryEntry) error

func (*Archive) VerifyTransactionHistoryEntry

func (arch *Archive) VerifyTransactionHistoryEntry(entry *xdr.TransactionHistoryEntry) error

func (*Archive) VerifyTransactionHistoryResultEntry

func (arch *Archive) VerifyTransactionHistoryResultEntry(entry *xdr.TransactionHistoryResultEntry) error

type ArchiveBackend

type ArchiveBackend interface {
	Exists(path string) (bool, error)
	Size(path string) (int64, error)
	GetFile(path string) (io.ReadCloser, error)
	PutFile(path string, in io.ReadCloser) error
	ListFiles(path string) (chan string, chan error)
	CanListFiles() bool
}

type ArchiveInterface

type ArchiveInterface interface {
	GetPathHAS(path string) (HistoryArchiveState, error)
	PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error
	BucketExists(bucket Hash) (bool, error)
	BucketSize(bucket Hash) (int64, error)
	CategoryCheckpointExists(cat string, chk uint32) (bool, error)
	GetLedgerHeader(chk uint32) (xdr.LedgerHeaderHistoryEntry, error)
	GetRootHAS() (HistoryArchiveState, error)
	GetLedgers(start, end uint32) (map[uint32]*Ledger, error)
	GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)
	PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error
	PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error
	ListBucket(dp DirPrefix) (chan string, chan error)
	ListAllBuckets() (chan string, chan error)
	ListAllBucketHashes() (chan Hash, chan error)
	ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)
	GetXdrStreamForHash(hash Hash) (*XdrStream, error)
	GetXdrStream(pth string) (*XdrStream, error)
	GetCheckpointManager() CheckpointManager
}

type ArchivePool

type ArchivePool []ArchiveInterface

A ArchivePool is just a collection of `ArchiveInterface`s so that we can distribute requests fairly throughout the pool.

func NewArchivePool

func NewArchivePool(archiveURLs []string, config ConnectOptions) (ArchivePool, error)

NewArchivePool tries connecting to each of the provided history archive URLs, returning a pool of valid archives.

If none of the archives work, this returns the error message of the last failed archive. Note that the errors for each individual archive are hard to track if there's success overall.

func (ArchivePool) BucketExists

func (pa ArchivePool) BucketExists(bucket Hash) (bool, error)

func (ArchivePool) BucketSize

func (pa ArchivePool) BucketSize(bucket Hash) (int64, error)

func (ArchivePool) CategoryCheckpointExists

func (pa ArchivePool) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (ArchivePool) GetAnyArchive

func (pa ArchivePool) GetAnyArchive() ArchiveInterface

func (ArchivePool) GetCheckpointHAS

func (pa ArchivePool) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (ArchivePool) GetCheckpointManager

func (pa ArchivePool) GetCheckpointManager() CheckpointManager

func (ArchivePool) GetLedgerHeader

func (pa ArchivePool) GetLedgerHeader(chk uint32) (xdr.LedgerHeaderHistoryEntry, error)

func (ArchivePool) GetLedgers

func (pa ArchivePool) GetLedgers(start, end uint32) (map[uint32]*Ledger, error)

func (ArchivePool) GetPathHAS

func (pa ArchivePool) GetPathHAS(path string) (HistoryArchiveState, error)

func (ArchivePool) GetRootHAS

func (pa ArchivePool) GetRootHAS() (HistoryArchiveState, error)

func (ArchivePool) GetXdrStream

func (pa ArchivePool) GetXdrStream(pth string) (*XdrStream, error)

func (ArchivePool) GetXdrStreamForHash

func (pa ArchivePool) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (ArchivePool) ListAllBucketHashes

func (pa ArchivePool) ListAllBucketHashes() (chan Hash, chan error)

func (ArchivePool) ListAllBuckets

func (pa ArchivePool) ListAllBuckets() (chan string, chan error)

func (ArchivePool) ListBucket

func (pa ArchivePool) ListBucket(dp DirPrefix) (chan string, chan error)

func (ArchivePool) ListCategoryCheckpoints

func (pa ArchivePool) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (ArchivePool) PutCheckpointHAS

func (pa ArchivePool) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (ArchivePool) PutPathHAS

func (pa ArchivePool) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (ArchivePool) PutRootHAS

func (pa ArchivePool) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

type CheckpointManager

type CheckpointManager struct {
	// contains filtered or unexported fields
}

func NewCheckpointManager

func NewCheckpointManager(checkpointFrequency uint32) CheckpointManager

NewCheckpointManager creates a CheckpointManager based on a checkpoint frequency (the number of ledgers between ledger checkpoints). If checkpointFrequency is 0 DefaultCheckpointFrequency will be used.

func (CheckpointManager) GetCheckpoint

func (c CheckpointManager) GetCheckpoint(i uint32) uint32

GetCheckPoint gets the checkpoint containing information about the given ledger sequence

func (CheckpointManager) GetCheckpointFrequency

func (c CheckpointManager) GetCheckpointFrequency() uint32

func (CheckpointManager) GetCheckpointRange

func (c CheckpointManager) GetCheckpointRange(i uint32) Range

GetCheckpointRange gets the range of the checkpoint containing information for the given ledger sequence

func (CheckpointManager) IsCheckpoint

func (c CheckpointManager) IsCheckpoint(i uint32) bool

func (CheckpointManager) MakeRange

func (c CheckpointManager) MakeRange(low uint32, high uint32) Range

func (CheckpointManager) NextCheckpoint

func (c CheckpointManager) NextCheckpoint(i uint32) uint32

func (CheckpointManager) PrevCheckpoint

func (c CheckpointManager) PrevCheckpoint(i uint32) uint32

type CommandOptions

type CommandOptions struct {
	Concurrency  int
	Range        Range
	DryRun       bool
	Force        bool
	Verify       bool
	Thorough     bool
	SkipOptional bool
}

type ConnectOptions

type ConnectOptions struct {
	Context context.Context
	// NetworkPassphrase defines the expected network of history archive. It is
	// checked when getting HAS. If network passphrase does not match, error is
	// returned.
	NetworkPassphrase string
	S3Region          string
	S3Endpoint        string
	UnsignedRequests  bool
	// CheckpointFrequency is the number of ledgers between checkpoints
	// if unset, DefaultCheckpointFrequency will be used
	CheckpointFrequency uint32
	// UserAgent is the value of `User-Agent` header. Applicable only for HTTP client.
	UserAgent string
}

type DirPrefix

type DirPrefix [3]uint8

func CheckpointPrefix

func CheckpointPrefix(seq uint32) DirPrefix

func HashPrefix

func HashPrefix(h Hash) DirPrefix

func (DirPrefix) Path

func (d DirPrefix) Path() string

type FsArchiveBackend

type FsArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*FsArchiveBackend) CanListFiles

func (b *FsArchiveBackend) CanListFiles() bool

func (*FsArchiveBackend) Exists

func (b *FsArchiveBackend) Exists(pth string) (bool, error)

func (*FsArchiveBackend) GetFile

func (b *FsArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*FsArchiveBackend) ListFiles

func (b *FsArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*FsArchiveBackend) PutFile

func (b *FsArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*FsArchiveBackend) Size

func (b *FsArchiveBackend) Size(pth string) (int64, error)

type Hash

type Hash [sha256.Size]byte

func DecodeHash

func DecodeHash(s string) (Hash, error)

func EmptyXdrArrayHash

func EmptyXdrArrayHash() Hash

func HashEmptyTxSet

func HashEmptyTxSet(previousLedgerHash Hash) Hash

func HashTxSet

func HashTxSet(txset *xdr.TransactionSet) (Hash, error)

func HashXdr

func HashXdr(x interface{}) (Hash, error)

func MustDecodeHash

func MustDecodeHash(s string) Hash

func (Hash) IsZero

func (h Hash) IsZero() bool

func (Hash) String

func (h Hash) String() string

type HistoryArchiveState

type HistoryArchiveState struct {
	Version       int    `json:"version"`
	Server        string `json:"server"`
	CurrentLedger uint32 `json:"currentLedger"`
	// NetworkPassphrase was added in Stellar-Core v14.1.0. Can be missing
	// in HAS created by previous versions.
	NetworkPassphrase string `json:"networkPassphrase"`
	CurrentBuckets    [NumLevels]struct {
		Curr string `json:"curr"`
		Snap string `json:"snap"`
		Next struct {
			State  uint32 `json:"state"`
			Output string `json:"output,omitempty"`
		} `json:"next"`
	} `json:"currentBuckets"`
}

func (*HistoryArchiveState) BucketListHash

func (h *HistoryArchiveState) BucketListHash() (xdr.Hash, error)

BucketListHash calculates the hash of bucket list in the HistoryArchiveState. This can be later compared with LedgerHeader.BucketListHash of the checkpoint ledger to ensure data in history archive has not been changed by a malicious actor. Warning: Ledger header should be fetched from a trusted (!) gravity instead of ex. history archives!

func (*HistoryArchiveState) Buckets

func (h *HistoryArchiveState) Buckets() ([]Hash, error)

func (*HistoryArchiveState) GetChangedBuckets

func (has *HistoryArchiveState) GetChangedBuckets(arch *Archive, prevHas *HistoryArchiveState) (string, int, int64)

func (*HistoryArchiveState) LevelSummary

func (h *HistoryArchiveState) LevelSummary() (string, int, error)

func (*HistoryArchiveState) Range

func (h *HistoryArchiveState) Range() Range

type HttpArchiveBackend

type HttpArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*HttpArchiveBackend) CanListFiles

func (b *HttpArchiveBackend) CanListFiles() bool

func (*HttpArchiveBackend) Exists

func (b *HttpArchiveBackend) Exists(pth string) (bool, error)

func (*HttpArchiveBackend) GetFile

func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*HttpArchiveBackend) Head

func (b *HttpArchiveBackend) Head(pth string) (*http.Response, error)

func (*HttpArchiveBackend) ListFiles

func (b *HttpArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*HttpArchiveBackend) PutFile

func (b *HttpArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*HttpArchiveBackend) Size

func (b *HttpArchiveBackend) Size(pth string) (int64, error)

type Ledger

type Ledger struct {
	Header            xdr.LedgerHeaderHistoryEntry
	Transaction       xdr.TransactionHistoryEntry
	TransactionResult xdr.TransactionHistoryResultEntry
}

type MockArchive

type MockArchive struct {
	mock.Mock
}

MockArchive is a mockable archive.

func (*MockArchive) BucketExists

func (m *MockArchive) BucketExists(bucket Hash) (bool, error)

func (*MockArchive) BucketSize

func (m *MockArchive) BucketSize(bucket Hash) (int64, error)

func (*MockArchive) CategoryCheckpointExists

func (m *MockArchive) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (*MockArchive) GetCheckpointHAS

func (m *MockArchive) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (*MockArchive) GetCheckpointManager

func (m *MockArchive) GetCheckpointManager() CheckpointManager

func (*MockArchive) GetLedgerHeader

func (m *MockArchive) GetLedgerHeader(chk uint32) (xdr.LedgerHeaderHistoryEntry, error)

func (*MockArchive) GetLedgers

func (m *MockArchive) GetLedgers(start, end uint32) (map[uint32]*Ledger, error)

func (*MockArchive) GetPathHAS

func (m *MockArchive) GetPathHAS(path string) (HistoryArchiveState, error)

func (*MockArchive) GetRootHAS

func (m *MockArchive) GetRootHAS() (HistoryArchiveState, error)

func (*MockArchive) GetXdrStream

func (m *MockArchive) GetXdrStream(pth string) (*XdrStream, error)

func (*MockArchive) GetXdrStreamForHash

func (m *MockArchive) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (*MockArchive) ListAllBucketHashes

func (m *MockArchive) ListAllBucketHashes() (chan Hash, chan error)

func (*MockArchive) ListAllBuckets

func (m *MockArchive) ListAllBuckets() (chan string, chan error)

func (*MockArchive) ListBucket

func (m *MockArchive) ListBucket(dp DirPrefix) (chan string, chan error)

func (*MockArchive) ListCategoryCheckpoints

func (m *MockArchive) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (*MockArchive) PutCheckpointHAS

func (m *MockArchive) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (*MockArchive) PutPathHAS

func (m *MockArchive) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (*MockArchive) PutRootHAS

func (m *MockArchive) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

type MockArchiveBackend

type MockArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*MockArchiveBackend) CanListFiles

func (b *MockArchiveBackend) CanListFiles() bool

func (*MockArchiveBackend) Exists

func (b *MockArchiveBackend) Exists(pth string) (bool, error)

func (*MockArchiveBackend) GetFile

func (b *MockArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*MockArchiveBackend) ListFiles

func (b *MockArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*MockArchiveBackend) PutFile

func (b *MockArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*MockArchiveBackend) Size

func (b *MockArchiveBackend) Size(pth string) (int64, error)

type Range

type Range struct {
	Low  uint32
	High uint32
}

func (Range) GenerateCheckpoints

func (r Range) GenerateCheckpoints(cManager CheckpointManager) chan uint32

func (Range) InRange

func (r Range) InRange(sequence uint32) bool

func (Range) SizeInCheckPoints

func (r Range) SizeInCheckPoints(cManager CheckpointManager) int

func (Range) String

func (r Range) String() string

type S3ArchiveBackend

type S3ArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*S3ArchiveBackend) CanListFiles

func (b *S3ArchiveBackend) CanListFiles() bool

func (*S3ArchiveBackend) Exists

func (b *S3ArchiveBackend) Exists(pth string) (bool, error)

func (*S3ArchiveBackend) GetFile

func (b *S3ArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*S3ArchiveBackend) Head

func (b *S3ArchiveBackend) Head(pth string) (*http.Response, error)

func (*S3ArchiveBackend) ListFiles

func (b *S3ArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*S3ArchiveBackend) PutFile

func (b *S3ArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*S3ArchiveBackend) Size

func (b *S3ArchiveBackend) Size(pth string) (int64, error)

type XdrStream

type XdrStream struct {
	// contains filtered or unexported fields
}

func CreateXdrStream

func CreateXdrStream(entries ...xdr.BucketEntry) *XdrStream

func NewXdrGzStream

func NewXdrGzStream(in io.ReadCloser) (*XdrStream, error)

func NewXdrStream

func NewXdrStream(in io.ReadCloser) *XdrStream

func (*XdrStream) BytesRead

func (x *XdrStream) BytesRead() int64

BytesRead returns the number of bytes read in the stream

func (*XdrStream) Close

func (x *XdrStream) Close() error

Close closes all internal readers and checks if the expected hash (if set by SetExpectedHash) matches the actual hash of the stream.

func (*XdrStream) Discard

func (x *XdrStream) Discard(n int64) (int64, error)

Discard removes n bytes from the stream

func (*XdrStream) ExpectedHash

func (x *XdrStream) ExpectedHash() ([sha256.Size]byte, bool)

ExpectedHash returns the expected hash and a boolean indicating if the expected hash was set

func (*XdrStream) GzipBytesRead

func (x *XdrStream) GzipBytesRead() int64

GzipBytesRead returns the number of gzip bytes read in the stream. Returns -1 if underlying reader is not gzipped.

func (*XdrStream) ReadOne

func (x *XdrStream) ReadOne(in xdr.DecoderFrom) error

func (*XdrStream) SetExpectedHash

func (x *XdrStream) SetExpectedHash(hash [sha256.Size]byte)

SetExpectedHash sets expected hash that will be checked in Close(). This (obviously) needs to be set before Close() is called.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL