historyarchive

package
v0.1.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 13, 2021 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const DefaultCheckpointFrequency = uint32(64)
View Source 
const NumLevels = 11

Variables

This section is empty.

Functions

func BucketPath 

func BucketPath(bucket Hash) string

func Categories 

func Categories() []string

func CategoryCheckpointPath 

func CategoryCheckpointPath(cat string, chk uint32) string

func DumpXdrAsJson 

func DumpXdrAsJson(args []string) error

func Mirror 

func Mirror(src *Archive, dst *Archive, opts *CommandOptions) error

Mirror mirrors an archive, it assumes that the source and destination have the same checkpoint ledger frequency

func RangePaths 

func RangePaths(r Range) []string

Returns an array of path prefixes to walk to enumerate all the objects in the provided range.

func Repair 

func Repair(src *Archive, dst *Archive, opts *CommandOptions) error

Repair repairs a destination archive based on a source archive, it assumes that the source and destination have the same checkpoint ledger frequency

func SortTxsForHash 

func SortTxsForHash(txset *xdr.TransactionSet) error

Types

type Archive 

type Archive struct {
	// contains filtered or unexported fields
}

func Connect 

func Connect(u string, opts ConnectOptions) (*Archive, error)

func MustConnect 

func MustConnect(u string, opts ConnectOptions) *Archive

func (*Archive) BucketExists 

func (a *Archive) BucketExists(bucket Hash) (bool, error)

func (*Archive) CategoryCheckpointExists 

func (a *Archive) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (*Archive) CheckBucketsMissing 

func (arch *Archive) CheckBucketsMissing() map[Hash]bool

func (*Archive) CheckCheckpointFilesMissing 

func (arch *Archive) CheckCheckpointFilesMissing(opts *CommandOptions) map[string][]uint32

func (*Archive) ClearCachedInfo 

func (arch *Archive) ClearCachedInfo()

func (*Archive) GetBucketPathForHash 

func (a *Archive) GetBucketPathForHash(hash Hash) string

func (*Archive) GetCheckpointHAS 

func (a *Archive) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (*Archive) GetCheckpointManager 

func (arch *Archive) GetCheckpointManager() CheckpointManager

func (*Archive) GetLedgerHeader 

func (a *Archive) GetLedgerHeader(ledger uint32) (xdr.LedgerHeaderHistoryEntry, error)

func (*Archive) GetLedgers 

func (a *Archive) GetLedgers(start, end uint32) (map[uint32]*Ledger, error)

func (*Archive) GetPathHAS 

func (a *Archive) GetPathHAS(path string) (HistoryArchiveState, error)

func (*Archive) GetRootHAS 

func (a *Archive) GetRootHAS() (HistoryArchiveState, error)

func (*Archive) GetXdrStream 

func (a *Archive) GetXdrStream(pth string) (*XdrStream, error)

func (*Archive) GetXdrStreamForHash 

func (a *Archive) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (*Archive) ListAllBucketHashes 

func (a *Archive) ListAllBucketHashes() (chan Hash, chan error)

func (*Archive) ListAllBuckets 

func (a *Archive) ListAllBuckets() (chan string, chan error)

func (*Archive) ListBucket 

func (a *Archive) ListBucket(dp DirPrefix) (chan string, chan error)

func (*Archive) ListCategoryCheckpoints 

func (a *Archive) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (*Archive) Log 

func (arch *Archive) Log(opts *CommandOptions) error

func (*Archive) MustGetBucketSize 

func (arch *Archive) MustGetBucketSize(hash Hash) int64

func (*Archive) MustGetLedgerHeaderHistoryEntries 

func (arch *Archive) MustGetLedgerHeaderHistoryEntries(chk uint32) []xdr.LedgerHeaderHistoryEntry

func (*Archive) MustGetTransactionHistoryEntries 

func (arch *Archive) MustGetTransactionHistoryEntries(chk uint32) []xdr.TransactionHistoryEntry

func (*Archive) NoteCheckpointFile 

func (arch *Archive) NoteCheckpointFile(cat string, chk uint32, present bool)

func (*Archive) NoteExistingBucket 

func (arch *Archive) NoteExistingBucket(bucket Hash)

func (*Archive) NoteReferencedBucket 

func (arch *Archive) NoteReferencedBucket(bucket Hash) bool

func (*Archive) PutCheckpointHAS 

func (a *Archive) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutPathHAS 

func (a *Archive) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutRootHAS 

func (a *Archive) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) ReportBucketStats 

func (arch *Archive) ReportBucketStats()

func (*Archive) ReportCheckpointStats 

func (arch *Archive) ReportCheckpointStats()

func (*Archive) ReportInvalid 

func (arch *Archive) ReportInvalid(opts *CommandOptions) (bool, error)

func (*Archive) ReportMissing 

func (arch *Archive) ReportMissing(opts *CommandOptions) (bool, error)

func (*Archive) Scan 

func (arch *Archive) Scan(opts *CommandOptions) error

func (*Archive) ScanAllBuckets 

func (arch *Archive) ScanAllBuckets() error

func (*Archive) ScanBuckets 

func (arch *Archive) ScanBuckets(opts *CommandOptions) error

func (*Archive) ScanCheckpoints 

func (arch *Archive) ScanCheckpoints(opts *CommandOptions) error

func (*Archive) ScanCheckpointsFast 

func (arch *Archive) ScanCheckpointsFast(opts *CommandOptions) error

func (*Archive) ScanCheckpointsSlow 

func (arch *Archive) ScanCheckpointsSlow(opts *CommandOptions) error

func (*Archive) VerifyBucketEntries 

func (arch *Archive) VerifyBucketEntries(h Hash) error

func (*Archive) VerifyBucketHash 

func (arch *Archive) VerifyBucketHash(h Hash) error

func (*Archive) VerifyCategoryCheckpoint 

func (arch *Archive) VerifyCategoryCheckpoint(cat string, chk uint32) error

func (*Archive) VerifyLedgerHeaderHistoryEntry 

func (arch *Archive) VerifyLedgerHeaderHistoryEntry(entry *xdr.LedgerHeaderHistoryEntry) error

func (*Archive) VerifyTransactionHistoryEntry 

func (arch *Archive) VerifyTransactionHistoryEntry(entry *xdr.TransactionHistoryEntry) error

func (*Archive) VerifyTransactionHistoryResultEntry 

func (arch *Archive) VerifyTransactionHistoryResultEntry(entry *xdr.TransactionHistoryResultEntry) error

type ArchiveBackend 

type ArchiveBackend interface {
	Exists(path string) (bool, error)
	Size(path string) (int64, error)
	GetFile(path string) (io.ReadCloser, error)
	PutFile(path string, in io.ReadCloser) error
	ListFiles(path string) (chan string, chan error)
	CanListFiles() bool
}

type ArchiveInterface 

type ArchiveInterface interface {
	GetPathHAS(path string) (HistoryArchiveState, error)
	PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error
	BucketExists(bucket Hash) (bool, error)
	CategoryCheckpointExists(cat string, chk uint32) (bool, error)
	GetLedgerHeader(chk uint32) (xdr.LedgerHeaderHistoryEntry, error)
	GetRootHAS() (HistoryArchiveState, error)
	GetLedgers(start, end uint32) (map[uint32]*Ledger, error)
	GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)
	PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error
	PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error
	ListBucket(dp DirPrefix) (chan string, chan error)
	ListAllBuckets() (chan string, chan error)
	ListAllBucketHashes() (chan Hash, chan error)
	ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)
	GetXdrStreamForHash(hash Hash) (*XdrStream, error)
	GetXdrStream(pth string) (*XdrStream, error)
	GetCheckpointManager() CheckpointManager
}

type ArchivePool 

type ArchivePool []ArchiveInterface

A ArchivePool is just a collection of `ArchiveInterface`s so that we can distribute requests fairly throughout the pool.

func NewArchivePool 

func NewArchivePool(archiveURLs []string, config ConnectOptions) (ArchivePool, error)

NewArchivePool tries connecting to each of the provided history archive URLs, returning a pool of valid archives.

If none of the archives work, this returns the error message of the last failed archive. Note that the errors for each individual archive are hard to track if there's success overall.

func (ArchivePool) BucketExists 

func (pa ArchivePool) BucketExists(bucket Hash) (bool, error)

func (ArchivePool) CategoryCheckpointExists 

func (pa ArchivePool) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (ArchivePool) GetAnyArchive 

func (pa ArchivePool) GetAnyArchive() ArchiveInterface

func (ArchivePool) GetCheckpointHAS 

func (pa ArchivePool) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (ArchivePool) GetCheckpointManager 

func (pa ArchivePool) GetCheckpointManager() CheckpointManager

func (ArchivePool) GetLedgerHeader 

func (pa ArchivePool) GetLedgerHeader(chk uint32) (xdr.LedgerHeaderHistoryEntry, error)

func (ArchivePool) GetLedgers 

func (pa ArchivePool) GetLedgers(start, end uint32) (map[uint32]*Ledger, error)

func (ArchivePool) GetPathHAS 

func (pa ArchivePool) GetPathHAS(path string) (HistoryArchiveState, error)

func (ArchivePool) GetRootHAS 

func (pa ArchivePool) GetRootHAS() (HistoryArchiveState, error)

func (ArchivePool) GetXdrStream 

func (pa ArchivePool) GetXdrStream(pth string) (*XdrStream, error)

func (ArchivePool) GetXdrStreamForHash 

func (pa ArchivePool) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (ArchivePool) ListAllBucketHashes 

func (pa ArchivePool) ListAllBucketHashes() (chan Hash, chan error)

func (ArchivePool) ListAllBuckets 

func (pa ArchivePool) ListAllBuckets() (chan string, chan error)

func (ArchivePool) ListBucket 

func (pa ArchivePool) ListBucket(dp DirPrefix) (chan string, chan error)

func (ArchivePool) ListCategoryCheckpoints 

func (pa ArchivePool) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (ArchivePool) PutCheckpointHAS 

func (pa ArchivePool) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (ArchivePool) PutPathHAS 

func (pa ArchivePool) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (ArchivePool) PutRootHAS 

func (pa ArchivePool) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

type CheckpointManager 

type CheckpointManager struct {
	// contains filtered or unexported fields
}

func NewCheckpointManager 

func NewCheckpointManager(checkpointFrequency uint32) CheckpointManager

NewCheckpointManager creates a CheckpointManager based on a checkpoint frequency (the number of ledgers between ledger checkpoints). If checkpointFrequency is 0 DefaultCheckpointFrequency will be used.

func (CheckpointManager) GetCheckpoint 

func (c CheckpointManager) GetCheckpoint(i uint32) uint32

GetCheckPoint gets the checkpoint containing information about the given ledger sequence

func (CheckpointManager) GetCheckpointFrequency 

func (c CheckpointManager) GetCheckpointFrequency() uint32

func (CheckpointManager) GetCheckpointRange 

func (c CheckpointManager) GetCheckpointRange(i uint32) Range

GetCheckpointRange gets the range of the checkpoint containing information for the given ledger sequence

func (CheckpointManager) IsCheckpoint 

func (c CheckpointManager) IsCheckpoint(i uint32) bool

func (CheckpointManager) MakeRange 

func (c CheckpointManager) MakeRange(low uint32, high uint32) Range

func (CheckpointManager) NextCheckpoint 

func (c CheckpointManager) NextCheckpoint(i uint32) uint32

func (CheckpointManager) PrevCheckpoint 

func (c CheckpointManager) PrevCheckpoint(i uint32) uint32

type CommandOptions 

type CommandOptions struct {
	Concurrency int
	Range       Range
	DryRun      bool
	Force       bool
	Verify      bool
	Thorough    bool
}

type ConnectOptions 

type ConnectOptions struct {
	Context context.Context
	// NetworkPassphrase defines the expected network of history archive. It is
	// checked when getting HAS. If network passphrase does not match, error is
	// returned.
	NetworkPassphrase string
	S3Region          string
	S3Endpoint        string
	UnsignedRequests  bool
	// CheckpointFrequency is the number of ledgers between checkpoints
	// if unset, DefaultCheckpointFrequency will be used
	CheckpointFrequency uint32
}

type DirPrefix 

type DirPrefix [3]uint8

func CheckpointPrefix 

func CheckpointPrefix(seq uint32) DirPrefix

func HashPrefix 

func HashPrefix(h Hash) DirPrefix

func (DirPrefix) Path 

func (d DirPrefix) Path() string

type FsArchiveBackend 

type FsArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*FsArchiveBackend) CanListFiles 

func (b *FsArchiveBackend) CanListFiles() bool

func (*FsArchiveBackend) Exists 

func (b *FsArchiveBackend) Exists(pth string) (bool, error)

func (*FsArchiveBackend) GetFile 

func (b *FsArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*FsArchiveBackend) ListFiles 

func (b *FsArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*FsArchiveBackend) PutFile 

func (b *FsArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*FsArchiveBackend) Size 

func (b *FsArchiveBackend) Size(pth string) (int64, error)

type Hash 

type Hash [sha256.Size]byte

func DecodeHash 

func DecodeHash(s string) (Hash, error)

func EmptyXdrArrayHash 

func EmptyXdrArrayHash() Hash

func HashEmptyTxSet 

func HashEmptyTxSet(previousLedgerHash Hash) Hash

func HashTxSet 

func HashTxSet(txset *xdr.TransactionSet) (Hash, error)

func HashXdr 

func HashXdr(x interface{}) (Hash, error)

func MustDecodeHash 

func MustDecodeHash(s string) Hash

func (Hash) IsZero 

func (h Hash) IsZero() bool

func (Hash) String 

func (h Hash) String() string

type HistoryArchiveState 

type HistoryArchiveState struct {
	Version       int    `json:"version"`
	Server        string `json:"server"`
	CurrentLedger uint32 `json:"currentLedger"`
	// NetworkPassphrase was added in Stellar-Core v14.1.0. Can be missing
	// in HAS created by previous versions.
	NetworkPassphrase string `json:"networkPassphrase"`
	CurrentBuckets    [NumLevels]struct {
		Curr string `json:"curr"`
		Snap string `json:"snap"`
		Next struct {
			State  uint32 `json:"state"`
			Output string `json:"output,omitempty"`
		} `json:"next"`
	} `json:"currentBuckets"`
}

func (*HistoryArchiveState) BucketListHash 

func (h *HistoryArchiveState) BucketListHash() (xdr.Hash, error)

BucketListHash calculates the hash of bucket list in the HistoryArchiveState. This can be later compared with LedgerHeader.BucketListHash of the checkpoint ledger to ensure data in history archive has not been changed by a malicious actor. Warning: Ledger header should be fetched from a trusted (!) stellar-core instead of ex. history archives!

func (*HistoryArchiveState) Buckets 

func (h *HistoryArchiveState) Buckets() ([]Hash, error)

func (*HistoryArchiveState) GetChangedBuckets 

func (has *HistoryArchiveState) GetChangedBuckets(arch *Archive, prevHas *HistoryArchiveState) (string, int, int64)

func (*HistoryArchiveState) LevelSummary 

func (h *HistoryArchiveState) LevelSummary() (string, int, error)

func (*HistoryArchiveState) Range 

func (h *HistoryArchiveState) Range() Range

type HttpArchiveBackend 

type HttpArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*HttpArchiveBackend) CanListFiles 

func (b *HttpArchiveBackend) CanListFiles() bool

func (*HttpArchiveBackend) Exists 

func (b *HttpArchiveBackend) Exists(pth string) (bool, error)

func (*HttpArchiveBackend) GetFile 

func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*HttpArchiveBackend) Head 

func (b *HttpArchiveBackend) Head(pth string) (*http.Response, error)

func (*HttpArchiveBackend) ListFiles 

func (b *HttpArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*HttpArchiveBackend) PutFile 

func (b *HttpArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*HttpArchiveBackend) Size 

func (b *HttpArchiveBackend) Size(pth string) (int64, error)

type Ledger 

type Ledger struct {
	Header            xdr.LedgerHeaderHistoryEntry
	Transaction       xdr.TransactionHistoryEntry
	TransactionResult xdr.TransactionHistoryResultEntry
}

type MockArchive 

type MockArchive struct {
	mock.Mock
}

MockArchive is a mockable archive.

func (*MockArchive) BucketExists 

func (m *MockArchive) BucketExists(bucket Hash) (bool, error)

func (*MockArchive) CategoryCheckpointExists 

func (m *MockArchive) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (*MockArchive) GetCheckpointHAS 

func (m *MockArchive) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (*MockArchive) GetCheckpointManager 

func (m *MockArchive) GetCheckpointManager() CheckpointManager

func (*MockArchive) GetLedgerHeader 

func (m *MockArchive) GetLedgerHeader(chk uint32) (xdr.LedgerHeaderHistoryEntry, error)

func (*MockArchive) GetLedgers 

func (m *MockArchive) GetLedgers(start, end uint32) (map[uint32]*Ledger, error)

func (*MockArchive) GetPathHAS 

func (m *MockArchive) GetPathHAS(path string) (HistoryArchiveState, error)

func (*MockArchive) GetRootHAS 

func (m *MockArchive) GetRootHAS() (HistoryArchiveState, error)

func (*MockArchive) GetXdrStream 

func (m *MockArchive) GetXdrStream(pth string) (*XdrStream, error)

func (*MockArchive) GetXdrStreamForHash 

func (m *MockArchive) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (*MockArchive) ListAllBucketHashes 

func (m *MockArchive) ListAllBucketHashes() (chan Hash, chan error)

func (*MockArchive) ListAllBuckets 

func (m *MockArchive) ListAllBuckets() (chan string, chan error)

func (*MockArchive) ListBucket 

func (m *MockArchive) ListBucket(dp DirPrefix) (chan string, chan error)

func (*MockArchive) ListCategoryCheckpoints 

func (m *MockArchive) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (*MockArchive) PutCheckpointHAS 

func (m *MockArchive) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (*MockArchive) PutPathHAS 

func (m *MockArchive) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (*MockArchive) PutRootHAS 

func (m *MockArchive) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

type MockArchiveBackend 

type MockArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*MockArchiveBackend) CanListFiles 

func (b *MockArchiveBackend) CanListFiles() bool

func (*MockArchiveBackend) Exists 

func (b *MockArchiveBackend) Exists(pth string) (bool, error)

func (*MockArchiveBackend) GetFile 

func (b *MockArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*MockArchiveBackend) ListFiles 

func (b *MockArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*MockArchiveBackend) PutFile 

func (b *MockArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*MockArchiveBackend) Size 

func (b *MockArchiveBackend) Size(pth string) (int64, error)

type Range 

type Range struct {
	Low  uint32
	High uint32
}

func (Range) GenerateCheckpoints 

func (r Range) GenerateCheckpoints(cManager CheckpointManager) chan uint32

func (Range) InRange 

func (r Range) InRange(sequence uint32) bool

func (Range) SizeInCheckPoints 

func (r Range) SizeInCheckPoints(cManager CheckpointManager) int

func (Range) String 

func (r Range) String() string

type S3ArchiveBackend 

type S3ArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*S3ArchiveBackend) CanListFiles 

func (b *S3ArchiveBackend) CanListFiles() bool

func (*S3ArchiveBackend) Exists 

func (b *S3ArchiveBackend) Exists(pth string) (bool, error)

func (*S3ArchiveBackend) GetFile 

func (b *S3ArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*S3ArchiveBackend) Head 

func (b *S3ArchiveBackend) Head(pth string) (*http.Response, error)

func (*S3ArchiveBackend) ListFiles 

func (b *S3ArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*S3ArchiveBackend) PutFile 

func (b *S3ArchiveBackend) PutFile(pth string, in io.ReadCloser) error

func (*S3ArchiveBackend) Size 

func (b *S3ArchiveBackend) Size(pth string) (int64, error)

type XdrStream 

type XdrStream struct {
	// contains filtered or unexported fields
}

func NewXdrGzStream 

func NewXdrGzStream(in io.ReadCloser) (*XdrStream, error)

func NewXdrStream 

func NewXdrStream(in io.ReadCloser) *XdrStream

func (*XdrStream) BytesRead 

func (x *XdrStream) BytesRead() int64

BytesRead returns the number of bytes read in the stream

func (*XdrStream) Close 

func (x *XdrStream) Close() error

Close closes all internal readers and checks if the expected hash (if set by SetExpectedHash) matches the actual hash of the stream.

func (*XdrStream) Discard 

func (x *XdrStream) Discard(n int64) (int64, error)

Discard removes n bytes from the stream

func (*XdrStream) ExpectedHash 

func (x *XdrStream) ExpectedHash() ([sha256.Size]byte, bool)

ExpectedHash returns the expected hash and a boolean indicating if the expected hash was set

func (*XdrStream) ReadOne 

func (x *XdrStream) ReadOne(in interface{}) error

func (*XdrStream) SetExpectedHash 

func (x *XdrStream) SetExpectedHash(hash [sha256.Size]byte)

SetExpectedHash sets expected hash that will be checked in Close(). This (obviously) needs to be set before Close() is called.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.