ioutils

Documentation

Index

• Constants
• func GetErrorsPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error)
• func GetFuturesPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error)
• func GetIndexLookupPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error)
• func GetInputsPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error)
• func GetMasterOutputsPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error)
• func GetOutputsPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error)
• func GetPath(ctx context.Context, store storage.ReferenceConstructor, ...) (res storage.DataReference, err error)
• func NewCachedInputReader(ctx context.Context, in io.InputReader) io.InputReader
• type BufferedOutputWriter
  • func NewBufferedOutputWriter(ctx context.Context, paths io.OutputFilePaths) *BufferedOutputWriter
  • func (o *BufferedOutputWriter) GetReader() io.OutputReader
  • func (o *BufferedOutputWriter) Put(ctx context.Context, reader io.OutputReader) error
• type InMemoryOutputReader
  • func NewInMemoryOutputReader(literals *core.LiteralMap, err *io.ExecutionError) InMemoryOutputReader
  • func (r InMemoryOutputReader) Exists(ctx context.Context) (bool, error)
  • func (r InMemoryOutputReader) IsError(ctx context.Context) (bool, error)
  • func (r InMemoryOutputReader) IsFile(ctx context.Context) bool
  • func (r InMemoryOutputReader) Read(ctx context.Context) (*core.LiteralMap, *io.ExecutionError, error)
  • func (r InMemoryOutputReader) ReadError(ctx context.Context) (io.ExecutionError, error)
• type RemoteFileInputReader
  • func NewRemoteFileInputReader(_ context.Context, store storage.ProtobufStore, inputPaths io.InputFilePaths) RemoteFileInputReader
  • func (r RemoteFileInputReader) Get(ctx context.Context) (*core.LiteralMap, error)
• type RemoteFileOutputPaths
  • func NewRemoteFileOutputPaths(_ context.Context, store storage.ReferenceConstructor, ...) RemoteFileOutputPaths
  • func (w RemoteFileOutputPaths) GetErrorPath() storage.DataReference
  • func (w RemoteFileOutputPaths) GetFuturesPath() storage.DataReference
  • func (w RemoteFileOutputPaths) GetOutputPath() storage.DataReference
  • func (w RemoteFileOutputPaths) GetOutputPrefixPath() storage.DataReference
• type RemoteFileOutputReader
  • func NewRemoteFileOutputReader(_ context.Context, store storage.ComposedProtobufStore, ...) RemoteFileOutputReader
  • func (r RemoteFileOutputReader) Exists(ctx context.Context) (bool, error)
  • func (r RemoteFileOutputReader) IsError(ctx context.Context) (bool, error)
  • func (r RemoteFileOutputReader) IsFile(ctx context.Context) bool
  • func (r RemoteFileOutputReader) Read(ctx context.Context) (*core.LiteralMap, *io.ExecutionError, error)
  • func (r RemoteFileOutputReader) ReadError(ctx context.Context) (io.ExecutionError, error)
• type RemoteFileOutputWriter
  • func NewRemoteFileOutputWriter(_ context.Context, store storage.ProtobufStore, ...) RemoteFileOutputWriter
  • func (w RemoteFileOutputWriter) Put(ctx context.Context, reader io.OutputReader) error
• type SimpleInputFilePath
  • func NewInputFilePaths(_ context.Context, store storage.ReferenceConstructor, ...) SimpleInputFilePath
  • func (s SimpleInputFilePath) GetInputPath() storage.DataReference
  • func (s SimpleInputFilePath) GetInputPrefixPath() storage.DataReference

Constants

const (
	InputsSuffix      = "inputs.pb"
	FuturesSuffix     = "futures.pb"
	OutputsSuffix     = "outputs.pb"
	ErrorsSuffix      = "error.pb"
	IndexLookupSuffix = "indexlookup.pb"
)

const (
	ErrFailedRead string = "READ_FAILED"
)

Functions

func GetErrorsPath

func GetErrorsPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetFuturesPath

func GetFuturesPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetIndexLookupPath

func GetIndexLookupPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetInputsPath

func GetInputsPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetMasterOutputsPath

func GetMasterOutputsPath(ctx context.Context, store storage.ReferenceConstructor, output storage.DataReference) (res storage.DataReference, err error)

func GetOutputsPath

func GetOutputsPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetPath

func GetPath(ctx context.Context, store storage.ReferenceConstructor, root storage.DataReference, subNames ...string) (res storage.DataReference, err error)

func NewCachedInputReader

func NewCachedInputReader(ctx context.Context, in io.InputReader) io.InputReader

Creates a new Read-through cached Input Reader. the returned reader is not thread-safe It caches the inputs on a successful read from the underlying input reader

Types

type BufferedOutputWriter

type BufferedOutputWriter struct {
	io.OutputFilePaths
	// contains filtered or unexported fields
}

A Buffered outputWriter just records the io.OutputReader and can be accessed using special methods.

func NewBufferedOutputWriter

func NewBufferedOutputWriter(ctx context.Context, paths io.OutputFilePaths) *BufferedOutputWriter

Returns a new object of type BufferedOutputWriter

func (*BufferedOutputWriter) GetReader

func (o *BufferedOutputWriter) GetReader() io.OutputReader

func (*BufferedOutputWriter) Put

func (o *BufferedOutputWriter) Put(ctx context.Context, reader io.OutputReader) error

type InMemoryOutputReader

type InMemoryOutputReader struct {
	// contains filtered or unexported fields
}

func NewInMemoryOutputReader

func NewInMemoryOutputReader(literals *core.LiteralMap, err *io.ExecutionError) InMemoryOutputReader

func (InMemoryOutputReader) Exists

func (r InMemoryOutputReader) Exists(ctx context.Context) (bool, error)

func (InMemoryOutputReader) IsError

func (r InMemoryOutputReader) IsError(ctx context.Context) (bool, error)

func (InMemoryOutputReader) IsFile

func (r InMemoryOutputReader) IsFile(ctx context.Context) bool

func (InMemoryOutputReader) Read

func (r InMemoryOutputReader) Read(ctx context.Context) (*core.LiteralMap, *io.ExecutionError, error)

func (InMemoryOutputReader) ReadError

func (r InMemoryOutputReader) ReadError(ctx context.Context) (io.ExecutionError, error)

type RemoteFileInputReader

type RemoteFileInputReader struct {
	io.InputFilePaths
	// contains filtered or unexported fields
}

func NewRemoteFileInputReader

func NewRemoteFileInputReader(_ context.Context, store storage.ProtobufStore, inputPaths io.InputFilePaths) RemoteFileInputReader

func (RemoteFileInputReader) Get

func (r RemoteFileInputReader) Get(ctx context.Context) (*core.LiteralMap, error)

type RemoteFileOutputPaths

type RemoteFileOutputPaths struct {
	// contains filtered or unexported fields
}

func NewRemoteFileOutputPaths

func NewRemoteFileOutputPaths(_ context.Context, store storage.ReferenceConstructor, outputPrefix storage.DataReference) RemoteFileOutputPaths

func (RemoteFileOutputPaths) GetErrorPath

func (w RemoteFileOutputPaths) GetErrorPath() storage.DataReference

func (RemoteFileOutputPaths) GetFuturesPath

func (w RemoteFileOutputPaths) GetFuturesPath() storage.DataReference

func (RemoteFileOutputPaths) GetOutputPath

func (w RemoteFileOutputPaths) GetOutputPath() storage.DataReference

func (RemoteFileOutputPaths) GetOutputPrefixPath

func (w RemoteFileOutputPaths) GetOutputPrefixPath() storage.DataReference

type RemoteFileOutputReader

type RemoteFileOutputReader struct {
	// contains filtered or unexported fields
}

func NewRemoteFileOutputReader

func NewRemoteFileOutputReader(_ context.Context, store storage.ComposedProtobufStore, outPaths io.OutputFilePaths, maxDatasetSize int64) RemoteFileOutputReader

func (RemoteFileOutputReader) Exists

func (r RemoteFileOutputReader) Exists(ctx context.Context) (bool, error)

func (RemoteFileOutputReader) IsError

func (r RemoteFileOutputReader) IsError(ctx context.Context) (bool, error)

func (RemoteFileOutputReader) IsFile

func (r RemoteFileOutputReader) IsFile(ctx context.Context) bool

func (RemoteFileOutputReader) Read

func (r RemoteFileOutputReader) Read(ctx context.Context) (*core.LiteralMap, *io.ExecutionError, error)

func (RemoteFileOutputReader) ReadError

func (r RemoteFileOutputReader) ReadError(ctx context.Context) (io.ExecutionError, error)

type RemoteFileOutputWriter

type RemoteFileOutputWriter struct {
	io.OutputFilePaths
	// contains filtered or unexported fields
}

func NewRemoteFileOutputWriter

func NewRemoteFileOutputWriter(_ context.Context, store storage.ProtobufStore, outputFilePaths io.OutputFilePaths) RemoteFileOutputWriter

func (RemoteFileOutputWriter) Put

func (w RemoteFileOutputWriter) Put(ctx context.Context, reader io.OutputReader) error

type SimpleInputFilePath

type SimpleInputFilePath struct {
	// contains filtered or unexported fields
}

func NewInputFilePaths

func NewInputFilePaths(_ context.Context, store storage.ReferenceConstructor, inputPathPrefix storage.DataReference) SimpleInputFilePath

func (SimpleInputFilePath) GetInputPath

func (s SimpleInputFilePath) GetInputPath() storage.DataReference

func (SimpleInputFilePath) GetInputPrefixPath

func (s SimpleInputFilePath) GetInputPrefixPath() storage.DataReference