historyarchive

package
v0.0.0-...-c9303cd Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 24, 2020 License: Apache-2.0 Imports: 30 Imported by: 0

Documentation

Index

Constants

View Source 
const CheckpointFreq = uint32(64)
View Source 
const NumLevels = 11

Variables

This section is empty.

Functions

func BucketPath 

func BucketPath(bucket Hash) string

func Categories 

func Categories() []string

func CategoryCheckpointPath 

func CategoryCheckpointPath(cat string, chk uint32) string

func DumpXdrAsJson 

func DumpXdrAsJson(args []string) error

func Mirror 

func Mirror(src *Archive, dst *Archive, opts *CommandOptions) error

func NextCheckpoint 

func NextCheckpoint(i uint32) uint32

func PrevCheckpoint 

func PrevCheckpoint(i uint32) uint32

func RangePaths 

func RangePaths(r Range) []string

Returns an array of path prefixes to walk to enumerate all the objects in the provided range.

func Repair 

func Repair(src *Archive, dst *Archive, opts *CommandOptions) error

func SortTxsForHash 

func SortTxsForHash(txset *xdr.TransactionSet) error

func WriteFramedXdr 

func WriteFramedXdr(out io.Writer, in interface{}) error

Types

type Archive 

type Archive struct {
	// contains filtered or unexported fields
}

func Connect 

func Connect(u string, opts ConnectOptions) (*Archive, error)

func MustConnect 

func MustConnect(u string, opts ConnectOptions) *Archive

func (*Archive) BucketExists 

func (a *Archive) BucketExists(bucket Hash) (bool, error)

func (*Archive) CategoryCheckpointExists 

func (a *Archive) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (*Archive) CheckBucketsMissing 

func (arch *Archive) CheckBucketsMissing() map[Hash]bool

func (*Archive) CheckCheckpointFilesMissing 

func (arch *Archive) CheckCheckpointFilesMissing(opts *CommandOptions) map[string][]uint32

func (*Archive) ClearCachedInfo 

func (arch *Archive) ClearCachedInfo()

func (*Archive) GetCheckpointHAS 

func (a *Archive) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (*Archive) GetPathHAS 

func (a *Archive) GetPathHAS(path string) (HistoryArchiveState, error)

func (*Archive) GetRootHAS 

func (a *Archive) GetRootHAS() (HistoryArchiveState, error)

func (*Archive) GetXdrStream 

func (a *Archive) GetXdrStream(pth string) (*XdrStream, error)

func (*Archive) GetXdrStreamForHash 

func (a *Archive) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (*Archive) ListAllBucketHashes 

func (a *Archive) ListAllBucketHashes() (chan Hash, chan error)

func (*Archive) ListAllBuckets 

func (a *Archive) ListAllBuckets() (chan string, chan error)

func (*Archive) ListBucket 

func (a *Archive) ListBucket(dp DirPrefix) (chan string, chan error)

func (*Archive) ListCategoryCheckpoints 

func (a *Archive) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (*Archive) NoteCheckpointFile 

func (arch *Archive) NoteCheckpointFile(cat string, chk uint32, present bool)

func (*Archive) NoteExistingBucket 

func (arch *Archive) NoteExistingBucket(bucket Hash)

func (*Archive) NoteReferencedBucket 

func (arch *Archive) NoteReferencedBucket(bucket Hash) bool

func (*Archive) PutCheckpointHAS 

func (a *Archive) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutPathHAS 

func (a *Archive) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutRootHAS 

func (a *Archive) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) ReportBucketStats 

func (arch *Archive) ReportBucketStats()

func (*Archive) ReportCheckpointStats 

func (arch *Archive) ReportCheckpointStats()

func (*Archive) ReportInvalid 

func (arch *Archive) ReportInvalid(opts *CommandOptions) error

func (*Archive) ReportMissing 

func (arch *Archive) ReportMissing(opts *CommandOptions) error

func (*Archive) Scan 

func (arch *Archive) Scan(opts *CommandOptions) error

func (*Archive) ScanAllBuckets 

func (arch *Archive) ScanAllBuckets() error

func (*Archive) ScanBuckets 

func (arch *Archive) ScanBuckets(opts *CommandOptions) error

func (*Archive) ScanCheckpoints 

func (arch *Archive) ScanCheckpoints(opts *CommandOptions) error

func (*Archive) ScanCheckpointsFast 

func (arch *Archive) ScanCheckpointsFast(opts *CommandOptions) error

func (*Archive) ScanCheckpointsSlow 

func (arch *Archive) ScanCheckpointsSlow(opts *CommandOptions) error

func (*Archive) VerifyBucketEntries 

func (arch *Archive) VerifyBucketEntries(h Hash) error

func (*Archive) VerifyBucketHash 

func (arch *Archive) VerifyBucketHash(h Hash) error

func (*Archive) VerifyCategoryCheckpoint 

func (arch *Archive) VerifyCategoryCheckpoint(cat string, chk uint32) error

func (*Archive) VerifyLedgerHeaderHistoryEntry 

func (arch *Archive) VerifyLedgerHeaderHistoryEntry(entry *xdr.LedgerHeaderHistoryEntry) error

func (*Archive) VerifyTransactionHistoryEntry 

func (arch *Archive) VerifyTransactionHistoryEntry(entry *xdr.TransactionHistoryEntry) error

func (*Archive) VerifyTransactionHistoryResultEntry 

func (arch *Archive) VerifyTransactionHistoryResultEntry(entry *xdr.TransactionHistoryResultEntry) error

type ArchiveBackend 

type ArchiveBackend interface {
	Exists(path string) (bool, error)
	GetFile(path string) (io.ReadCloser, error)
	PutFile(path string, in io.ReadCloser) error
	ListFiles(path string) (chan string, chan error)
	CanListFiles() bool
}

type ArchiveInterface 

type ArchiveInterface interface {
	GetPathHAS(path string) (HistoryArchiveState, error)
	PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error
	BucketExists(bucket Hash) (bool, error)
	CategoryCheckpointExists(cat string, chk uint32) (bool, error)
	GetRootHAS() (HistoryArchiveState, error)
	GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)
	PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error
	PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error
	ListBucket(dp DirPrefix) (chan string, chan error)
	ListAllBuckets() (chan string, chan error)
	ListAllBucketHashes() (chan Hash, chan error)
	ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)
	GetXdrStreamForHash(hash Hash) (*XdrStream, error)
	GetXdrStream(pth string) (*XdrStream, error)
}

type CommandOptions 

type CommandOptions struct {
	Concurrency int
	Range       Range
	DryRun      bool
	Force       bool
	Verify      bool
	Thorough    bool
}

type ConnectOptions 

type ConnectOptions struct {
	S3Region         string
	S3Endpoint       string
	UnsignedRequests bool
}

type DirPrefix 

type DirPrefix [3]uint8

func CheckpointPrefix 

func CheckpointPrefix(seq uint32) DirPrefix

func HashPrefix 

func HashPrefix(h Hash) DirPrefix

func (DirPrefix) Path 

func (d DirPrefix) Path() string

type FsArchiveBackend 

type FsArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*FsArchiveBackend) CanListFiles 

func (b *FsArchiveBackend) CanListFiles() bool

func (*FsArchiveBackend) Exists 

func (b *FsArchiveBackend) Exists(pth string) (bool, error)

func (*FsArchiveBackend) GetFile 

func (b *FsArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*FsArchiveBackend) ListFiles 

func (b *FsArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*FsArchiveBackend) PutFile 

func (b *FsArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type Hash 

type Hash [sha256.Size]byte

func DecodeHash 

func DecodeHash(s string) (Hash, error)

func EmptyXdrArrayHash 

func EmptyXdrArrayHash() Hash

func HashEmptyTxSet 

func HashEmptyTxSet(previousLedgerHash Hash) Hash

func HashTxSet 

func HashTxSet(txset *xdr.TransactionSet) (Hash, error)

func HashXdr 

func HashXdr(x interface{}) (Hash, error)

func MustDecodeHash 

func MustDecodeHash(s string) Hash

func (Hash) IsZero 

func (h Hash) IsZero() bool

func (Hash) String 

func (h Hash) String() string

type HistoryArchiveState 

type HistoryArchiveState struct {
	Version        int    `json:"version"`
	Server         string `json:"server"`
	CurrentLedger  uint32 `json:"currentLedger"`
	CurrentBuckets [NumLevels]struct {
		Curr string `json:"curr"`
		Snap string `json:"snap"`
		Next struct {
			State  uint32 `json:"state"`
			Output string `json:"output,omitempty"`
		} `json:"next"`
	} `json:"currentBuckets"`
}

func (*HistoryArchiveState) BucketListHash 

func (h *HistoryArchiveState) BucketListHash() (xdr.Hash, error)

BucketListHash calculates the hash of bucket list in the HistoryArchiveState. This can be later compared with LedgerHeader.BucketListHash of the checkpoint ledger to ensure data in history archive has not been changed by a malicious actor. Warning: Ledger header should be fetched from a trusted (!) diamnet-core instead of ex. history archives!

func (*HistoryArchiveState) Buckets 

func (h *HistoryArchiveState) Buckets() ([]Hash, error)

func (*HistoryArchiveState) LevelSummary 

func (h *HistoryArchiveState) LevelSummary() (string, int, error)

func (*HistoryArchiveState) Range 

func (h *HistoryArchiveState) Range() Range

type HttpArchiveBackend 

type HttpArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*HttpArchiveBackend) CanListFiles 

func (b *HttpArchiveBackend) CanListFiles() bool

func (*HttpArchiveBackend) Exists 

func (b *HttpArchiveBackend) Exists(pth string) (bool, error)

func (*HttpArchiveBackend) GetFile 

func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*HttpArchiveBackend) ListFiles 

func (b *HttpArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*HttpArchiveBackend) PutFile 

func (b *HttpArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type MockArchive 

type MockArchive struct {
	mock.Mock
}

MockArchive is a mockable archive.

func (*MockArchive) BucketExists 

func (m *MockArchive) BucketExists(bucket Hash) (bool, error)

func (*MockArchive) CategoryCheckpointExists 

func (m *MockArchive) CategoryCheckpointExists(cat string, chk uint32) (bool, error)

func (*MockArchive) GetCheckpointHAS 

func (m *MockArchive) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (*MockArchive) GetPathHAS 

func (m *MockArchive) GetPathHAS(path string) (HistoryArchiveState, error)

func (*MockArchive) GetRootHAS 

func (m *MockArchive) GetRootHAS() (HistoryArchiveState, error)

func (*MockArchive) GetXdrStream 

func (m *MockArchive) GetXdrStream(pth string) (*XdrStream, error)

func (*MockArchive) GetXdrStreamForHash 

func (m *MockArchive) GetXdrStreamForHash(hash Hash) (*XdrStream, error)

func (*MockArchive) ListAllBucketHashes 

func (m *MockArchive) ListAllBucketHashes() (chan Hash, chan error)

func (*MockArchive) ListAllBuckets 

func (m *MockArchive) ListAllBuckets() (chan string, chan error)

func (*MockArchive) ListBucket 

func (m *MockArchive) ListBucket(dp DirPrefix) (chan string, chan error)

func (*MockArchive) ListCategoryCheckpoints 

func (m *MockArchive) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (*MockArchive) PutCheckpointHAS 

func (m *MockArchive) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (*MockArchive) PutPathHAS 

func (m *MockArchive) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (*MockArchive) PutRootHAS 

func (m *MockArchive) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

type MockArchiveBackend 

type MockArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*MockArchiveBackend) CanListFiles 

func (b *MockArchiveBackend) CanListFiles() bool

func (*MockArchiveBackend) Exists 

func (b *MockArchiveBackend) Exists(pth string) (bool, error)

func (*MockArchiveBackend) GetFile 

func (b *MockArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*MockArchiveBackend) ListFiles 

func (b *MockArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*MockArchiveBackend) PutFile 

func (b *MockArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type Range 

type Range struct {
	Low  uint32
	High uint32
}

func MakeRange 

func MakeRange(low uint32, high uint32) Range

func (Range) Checkpoints 

func (r Range) Checkpoints() chan uint32

func (Range) Size 

func (r Range) Size() int

func (Range) String 

func (r Range) String() string

type S3ArchiveBackend 

type S3ArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*S3ArchiveBackend) CanListFiles 

func (b *S3ArchiveBackend) CanListFiles() bool

func (*S3ArchiveBackend) Exists 

func (b *S3ArchiveBackend) Exists(pth string) (bool, error)

func (*S3ArchiveBackend) GetFile 

func (b *S3ArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*S3ArchiveBackend) ListFiles 

func (b *S3ArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*S3ArchiveBackend) PutFile 

func (b *S3ArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type XdrStream 

type XdrStream struct {
	// contains filtered or unexported fields
}

func NewXdrGzStream 

func NewXdrGzStream(in io.ReadCloser) (*XdrStream, error)

func NewXdrStream 

func NewXdrStream(in io.ReadCloser) *XdrStream

func (*XdrStream) Close 

func (x *XdrStream) Close() error

Close closes all internal readers and checks if the expected hash (if set by SetExpectedHash) matches the actual hash of the stream.

func (*XdrStream) ReadOne 

func (x *XdrStream) ReadOne(in interface{}) error

func (*XdrStream) SetExpectedHash 

func (x *XdrStream) SetExpectedHash(hash [sha256.Size]byte)

SetExpectedHash sets expected hash that will be checked in Close(). This (obviously) needs to be set before Close() is called.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.