Versions in this module Expand all Collapse all v1 v1.0.1 Feb 21, 2024 v1.0.0 Feb 21, 2024 Changes in this version + const CheckpointPrefix + const ErrFailedRead + const ErrorsSuffix + const FuturesSuffix + const IndexLookupSuffix + const InputsSuffix + const OutputsSuffix + const TaskTemplateSuffix + func ConstructCheckpointPath(store storage.ReferenceConstructor, rawOutputPrefix storage.DataReference) storage.DataReference + func GenerateAlphabet(b []rune) []rune + func GenerateArabicNumerals(b []rune) []rune + func GetIndexLookupPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error) + func GetTaskTemplatePath(ctx context.Context, store storage.ReferenceConstructor, ...) (storage.DataReference, error) + func NewCachedInputReader(ctx context.Context, in io.InputReader) io.InputReader + func NewDeterministicUniqueRawOutputPath(ctx context.Context, ...) (io.RawOutputPaths, error) + func NewLazyUploadingTaskReader(baseTaskReader SimpleTaskReader, remotePath storage.DataReference, ...) pluginsCore.TaskReader + func NewRawOutputPaths(_ context.Context, rawOutputPrefix storage.DataReference) io.RawOutputPaths + func NewShardedDeterministicRawOutputPath(ctx context.Context, sharder ShardSelector, ...) (io.RawOutputPaths, error) + func NewShardedRawOutputPath(ctx context.Context, sharder ShardSelector, basePath storage.DataReference, ...) (io.RawOutputPaths, error) + func NewTaskIDRawOutputPath(ctx context.Context, rawOutputPrefix storage.DataReference, ...) (io.RawOutputPaths, error) + type BufferedOutputWriter struct + func NewBufferedOutputWriter(ctx context.Context, paths io.OutputFilePaths) *BufferedOutputWriter + func (o *BufferedOutputWriter) GetReader() io.OutputReader + func (o *BufferedOutputWriter) Put(ctx context.Context, reader io.OutputReader) error + type InMemoryOutputReader struct + DeckPath *storage.DataReference + func NewInMemoryOutputReader(literals *core.LiteralMap, DeckPath *storage.DataReference, ...) InMemoryOutputReader + func (r InMemoryOutputReader) DeckExists(_ context.Context) (bool, error) + func (r InMemoryOutputReader) Exists(_ context.Context) (bool, error) + func (r InMemoryOutputReader) IsError(ctx context.Context) (bool, error) + func (r InMemoryOutputReader) IsFile(_ context.Context) bool + func (r InMemoryOutputReader) Read(_ context.Context) (*core.LiteralMap, *io.ExecutionError, error) + func (r InMemoryOutputReader) ReadError(ctx context.Context) (io.ExecutionError, error) + type PrecomputedShardSelector struct + func (d *PrecomputedShardSelector) GetShardPrefix(_ context.Context, s []byte) (string, error) + type RemoteCheckpointPaths struct + func NewCheckpointRemoteFilePaths(ctx context.Context, store storage.ReferenceConstructor, ...) RemoteCheckpointPaths + func NewReadOnlyOutputFilePaths(ctx context.Context, store storage.ReferenceConstructor, ...) RemoteCheckpointPaths + func (r RemoteCheckpointPaths) GetCheckpointPrefix() storage.DataReference + func (r RemoteCheckpointPaths) GetPreviousCheckpointsPrefix() storage.DataReference + type RemoteFileInputReader struct + func NewRemoteFileInputReader(_ context.Context, store storage.ProtobufStore, inputPaths io.InputFilePaths) RemoteFileInputReader + func (r RemoteFileInputReader) Get(ctx context.Context) (*core.LiteralMap, error) + type RemoteFileOutputPaths struct + func NewRemoteFileOutputPaths(_ context.Context, store storage.ReferenceConstructor, ...) RemoteFileOutputPaths + func (w RemoteFileOutputPaths) GetDeckPath() storage.DataReference + func (w RemoteFileOutputPaths) GetErrorPath() storage.DataReference + func (w RemoteFileOutputPaths) GetFuturesPath() storage.DataReference + func (w RemoteFileOutputPaths) GetOutputPath() storage.DataReference + func (w RemoteFileOutputPaths) GetOutputPrefixPath() storage.DataReference + type RemoteFileOutputReader struct + func NewRemoteFileOutputReader(_ context.Context, store storage.ComposedProtobufStore, ...) RemoteFileOutputReader + func (r RemoteFileOutputReader) DeckExists(ctx context.Context) (bool, error) + func (r RemoteFileOutputReader) Exists(ctx context.Context) (bool, error) + func (r RemoteFileOutputReader) IsError(ctx context.Context) (bool, error) + func (r RemoteFileOutputReader) IsFile(ctx context.Context) bool + func (r RemoteFileOutputReader) Read(ctx context.Context) (*core.LiteralMap, *io.ExecutionError, error) + func (r RemoteFileOutputReader) ReadError(ctx context.Context) (io.ExecutionError, error) + type RemoteFileOutputWriter struct + func NewRemoteFileOutputWriter(_ context.Context, store storage.ProtobufStore, ...) RemoteFileOutputWriter + func (w RemoteFileOutputWriter) Put(ctx context.Context, reader io.OutputReader) error + type ShardSelector interface + GetShardPrefix func(ctx context.Context, s []byte) (string, error) + func NewBase36PrefixShardSelector(ctx context.Context) (ShardSelector, error) + func NewConstantShardSelector(shards []string) ShardSelector + type SimpleInputFilePath struct + func NewInputFilePaths(_ context.Context, store storage.ReferenceConstructor, ...) SimpleInputFilePath + func (s SimpleInputFilePath) GetInputPath() storage.DataReference + func (s SimpleInputFilePath) GetInputPrefixPath() storage.DataReference + type SimpleTaskReader interface + Read func(ctx context.Context) (*core.TaskTemplate, error)