Documentation
¶
Index ¶
- func PumpIt(enumerator Enumerator, collector Collector, drain Drain, ...)
- type APICollector
- type APIDrain
- type APIPinEnumerator
- type Block
- type BlockInfo
- type Collector
- type CountedDrain
- type CounterDrain
- type DatastoreCollector
- type DatastoreDrain
- type DatastoreEnumerator
- type Drain
- type Enumerator
- type FailedBlocksWriter
- type FileEnumerator
- type FileEnumeratorWriter
- type MockCollector
- type MockEnumerator
- type NullProgressWriter
- type NullableFileEnumeratorWriter
- type ProgressBarWriter
- type ProgressWriter
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func PumpIt ¶
func PumpIt(enumerator Enumerator, collector Collector, drain Drain, failedBlocksWriter FailedBlocksWriter, progressWriter ProgressWriter, worker uint)
Types ¶
type APICollector ¶
type APICollector struct {
URL string
}
func NewAPICollector ¶
func NewAPICollector(URL string) *APICollector
type APIDrain ¶
type APIDrain struct {
// contains filtered or unexported fields
}
func NewAPIDrain ¶
func NewAPIDrainWithShell ¶
type APIPinEnumerator ¶
type APIPinEnumerator struct {
URL string
// contains filtered or unexported fields
}
func NewAPIPinEnumerator ¶
func NewAPIPinEnumerator(URL string, stream bool) *APIPinEnumerator
func (*APIPinEnumerator) CIDs ¶
func (a *APIPinEnumerator) CIDs(out chan<- BlockInfo) error
func (*APIPinEnumerator) TotalCount ¶
func (a *APIPinEnumerator) TotalCount() int
type Collector ¶
type Collector interface {
// Blocks read each CID from the input, retrieve the corresponding
// block and emit it to the output
Blocks(in <-chan BlockInfo, out chan<- Block) error
}
A Collector is able to read a block from a source
type CountedDrain ¶
type CounterDrain ¶
type CounterDrain struct {
// contains filtered or unexported fields
}
func NewCountedDrain ¶
func NewCountedDrain(drain Drain) *CounterDrain
func (*CounterDrain) Drain ¶
func (c *CounterDrain) Drain(block Block) error
func (*CounterDrain) SuccessfulBlocksCount ¶
func (c *CounterDrain) SuccessfulBlocksCount() uint64
type DatastoreCollector ¶
type DatastoreCollector struct {
// contains filtered or unexported fields
}
func NewBadgerCollector ¶
func NewBadgerCollector(path string) (*DatastoreCollector, error)
func NewDatastoreCollector ¶
func NewDatastoreCollector(dstore ds.Datastore) *DatastoreCollector
func NewFlatFSCollector ¶
func NewFlatFSCollector(path string) (*DatastoreCollector, error)
func NewS3Collector ¶
func NewS3Collector(config s3ds.Config) (*DatastoreCollector, error)
type DatastoreDrain ¶
type DatastoreDrain struct {
// contains filtered or unexported fields
}
func NewBadgerDrain ¶
func NewBadgerDrain(path string) (*DatastoreDrain, error)
func NewDatastoreDrain ¶
func NewDatastoreDrain(dstore ds.Datastore) *DatastoreDrain
func NewFlatFSDrain ¶
func NewFlatFSDrain(path string) (*DatastoreDrain, error)
func NewS3Drain ¶
func NewS3Drain(config s3ds.Config) (*DatastoreDrain, error)
func (*DatastoreDrain) Drain ¶
func (d *DatastoreDrain) Drain(block Block) error
type DatastoreEnumerator ¶
type DatastoreEnumerator struct {
// contains filtered or unexported fields
}
func NewBadgerEnumerator ¶
func NewBadgerEnumerator(path string) (*DatastoreEnumerator, error)
func NewDatastoreEnumerator ¶
func NewDatastoreEnumerator(dstore ds.Datastore) *DatastoreEnumerator
func NewFlatFSEnumerator ¶
func NewFlatFSEnumerator(path string) (*DatastoreEnumerator, error)
func NewS3Enumerator ¶
func NewS3Enumerator(config s3ds.Config) (*DatastoreEnumerator, error)
func (*DatastoreEnumerator) CIDs ¶
func (d *DatastoreEnumerator) CIDs(out chan<- BlockInfo) error
func (*DatastoreEnumerator) TotalCount ¶
func (*DatastoreEnumerator) TotalCount() int
type Enumerator ¶
type Enumerator interface {
// TotalCount return the total number of existing blocks,
// or -1 if unknown/unsupported.
TotalCount() int
// CIDs emit in the given channel each CID existing in the source
CIDs(out chan<- BlockInfo) error
}
An Enumerator is able to enumerate the blocks from a source
type FailedBlocksWriter ¶
func NewFileEnumeratorWriter ¶
func NewFileEnumeratorWriter(path string) (enumWriter FailedBlocksWriter, close func() error, err error)
func NewNullableFileEnumeratorWriter ¶
func NewNullableFileEnumeratorWriter() FailedBlocksWriter
type FileEnumerator ¶
type FileEnumerator struct {
// contains filtered or unexported fields
}
func NewFileEnumerator ¶
func NewFileEnumerator(file io.ReadSeeker) (*FileEnumerator, error)
func (*FileEnumerator) CIDs ¶
func (f *FileEnumerator) CIDs(out chan<- BlockInfo) error
func (*FileEnumerator) TotalCount ¶
func (f *FileEnumerator) TotalCount() int
type FileEnumeratorWriter ¶
type FileEnumeratorWriter struct {
// contains filtered or unexported fields
}
func (*FileEnumeratorWriter) Count ¶
func (f *FileEnumeratorWriter) Count() uint
func (*FileEnumeratorWriter) Flush ¶
func (f *FileEnumeratorWriter) Flush() error
func (*FileEnumeratorWriter) Write ¶
func (f *FileEnumeratorWriter) Write(c cid.Cid) (int, error)
type MockCollector ¶
type MockCollector struct {
// contains filtered or unexported fields
}
func NewMockCollector ¶
func NewMockCollector(source *sync.Map) *MockCollector
type MockEnumerator ¶
type MockEnumerator struct {
// contains filtered or unexported fields
}
func (*MockEnumerator) CIDs ¶
func (m *MockEnumerator) CIDs(out chan<- BlockInfo) error
func (*MockEnumerator) TotalCount ¶
func (m *MockEnumerator) TotalCount() int
type NullProgressWriter ¶
type NullProgressWriter struct {
}
func NewNullProgressWriter ¶
func NewNullProgressWriter() *NullProgressWriter
func (*NullProgressWriter) Finish ¶
func (p *NullProgressWriter) Finish()
func (*NullProgressWriter) Increment ¶
func (p *NullProgressWriter) Increment() int
func (*NullProgressWriter) Prefix ¶
func (p *NullProgressWriter) Prefix(elem string)
func (*NullProgressWriter) SetTotal ¶
func (p *NullProgressWriter) SetTotal(total int)
type NullableFileEnumeratorWriter ¶
type NullableFileEnumeratorWriter struct {
// contains filtered or unexported fields
}
func (*NullableFileEnumeratorWriter) Count ¶
func (f *NullableFileEnumeratorWriter) Count() uint
func (*NullableFileEnumeratorWriter) Flush ¶
func (f *NullableFileEnumeratorWriter) Flush() error
func (*NullableFileEnumeratorWriter) Write ¶
func (f *NullableFileEnumeratorWriter) Write(c cid.Cid) (int, error)
type ProgressBarWriter ¶
type ProgressBarWriter struct {
// contains filtered or unexported fields
}
func NewProgressWriter ¶
func NewProgressWriter() *ProgressBarWriter
func (*ProgressBarWriter) Finish ¶
func (p *ProgressBarWriter) Finish()
func (*ProgressBarWriter) Increment ¶
func (p *ProgressBarWriter) Increment() int
func (*ProgressBarWriter) Prefix ¶
func (p *ProgressBarWriter) Prefix(elem string)
func (*ProgressBarWriter) SetTotal ¶
func (p *ProgressBarWriter) SetTotal(total int)
type ProgressWriter ¶
Source Files
¶
- coll_api.go
- coll_datastore.go
- coll_ds_badger.go
- coll_ds_flatfs.go
- coll_ds_s3.go
- drain_api.go
- drain_counted.go
- drain_datastore.go
- drain_ds_badger.go
- drain_ds_flatfs.go
- drain_ds_s3.go
- enum_api_pin.go
- enum_datastore.go
- enum_ds_badger.go
- enum_ds_flatfs.go
- enum_ds_s3.go
- enum_file.go
- enum_file_writer.go
- interfaces.go
- mocks.go
- pump.go
- pump_progress.go
Click to show internal directories.
Click to hide internal directories.