data

Documentation

Index

• Constants
• func CreateVector(ctx context.Context, engine storage.Engine, path *storage.URI, id ksuid.KSUID) error
• func DeleteVector(ctx context.Context, engine storage.Engine, path *storage.URI, id ksuid.KSUID) error
• func LookupSeekRange(ctx context.Context, engine storage.Engine, path *storage.URI, obj *Object, ...) ([]seekindex.Range, error)
• func RangeFromBitVector(ctx context.Context, engine storage.Engine, path *storage.URI, o *Object, ...) ([]seekindex.Range, error)
• func SeekIndexURI(path *storage.URI, id ksuid.KSUID) *storage.URI
• func SequenceURI(path *storage.URI, id ksuid.KSUID) *storage.URI
• func VectorURI(path *storage.URI, id ksuid.KSUID) *storage.URI
• type FileKind
  • func FileMatch(s string) (kind FileKind, id ksuid.KSUID, ok bool)
  • func (k FileKind) Description() string
• type Object
  • func NewObject() Object
  • func (o *Object) Equal(to *Object) bool
  • func (o Object) IsZero() bool
  • func (o *Object) NewReader(ctx context.Context, engine storage.Engine, path *storage.URI, ...) (*Reader, error)
  • func (o *Object) NewVectorWriter(ctx context.Context, engine storage.Engine, path *storage.URI) (*VectorWriter, error)
  • func (o *Object) NewWriter(ctx context.Context, engine storage.Engine, path *storage.URI, ...) (*Writer, error)
  • func (o Object) ObjectPrefix(path *storage.URI) *storage.URI
  • func (o Object) Remove(ctx context.Context, engine storage.Engine, path *storage.URI) error
  • func (o Object) SeekIndexURI(path *storage.URI) *storage.URI
  • func (o Object) SequenceURI(path *storage.URI) *storage.URI
  • func (o Object) Span(order order.Which) *extent.Generic
  • func (o Object) String() string
  • func (o Object) VectorURI(path *storage.URI) *storage.URI
• type Reader
• type VectorWriter
  • func NewVectorWriter(ctx context.Context, engine storage.Engine, path *storage.URI, id ksuid.KSUID) (*VectorWriter, error)
  • func (w *VectorWriter) Abort()
• type Writer
  • func (w *Writer) Abort()
  • func (w *Writer) BytesWritten() int64
  • func (w *Writer) Close(ctx context.Context) error
  • func (w *Writer) Object() *Object
  • func (w *Writer) RecordsWritten() uint64
  • func (w *Writer) Write(val zed.Value) error
  • func (w *Writer) WriteWithKey(key, val zed.Value) error

Constants

const (
	DefaultSeekStride = 64 * 1024
	DefaultThreshold  = 500 * 1024 * 1024
)

Functions

func CreateVector

func CreateVector(ctx context.Context, engine storage.Engine, path *storage.URI, id ksuid.KSUID) error

CreateVector writes the vectorized form of an existing Object in the VNG format.

func DeleteVector

func DeleteVector(ctx context.Context, engine storage.Engine, path *storage.URI, id ksuid.KSUID) error

func LookupSeekRange

func LookupSeekRange(ctx context.Context, engine storage.Engine, path *storage.URI,
	obj *Object, pruner expr.Evaluator) ([]seekindex.Range, error)

func RangeFromBitVector

func RangeFromBitVector(ctx context.Context, engine storage.Engine, path *storage.URI,
	o *Object, b *vector.Bool) ([]seekindex.Range, error)

func SeekIndexURI

func SeekIndexURI(path *storage.URI, id ksuid.KSUID) *storage.URI

func SequenceURI

func SequenceURI(path *storage.URI, id ksuid.KSUID) *storage.URI

func VectorURI

func VectorURI(path *storage.URI, id ksuid.KSUID) *storage.URI

Types

type FileKind

type FileKind string

A FileKind is the first part of a file name, used to differentiate files when they are listed from the archive's backing store.

const (
	FileKindUnknown  FileKind = ""
	FileKindData     FileKind = "data"
	FileKindMetadata FileKind = "meta"
	FileKindSeek     FileKind = "seek"
)

func FileMatch

func FileMatch(s string) (kind FileKind, id ksuid.KSUID, ok bool)

XXX this won't work right until we integrate segID

func (FileKind) Description

func (k FileKind) Description() string

type Object

type Object struct {
	ID    ksuid.KSUID `zed:"id"`
	Min   zed.Value   `zed:"min"`
	Max   zed.Value   `zed:"max"`
	Count uint64      `zed:"count"`
	Size  int64       `zed:"size"`
}

An Object represents a cloud object or file that holds an ordered sequence of Zed values sorted according to the pool's data order where From is the the first value in the sequence and To is the last value. Count is the number of values in the sequence and Size is total size in bytes of the Object as persisted to storage (i.e., its compressed size).

func NewObject

func NewObject() Object

func (*Object) Equal

func (o *Object) Equal(to *Object) bool

func (Object) IsZero

func (o Object) IsZero() bool

func (*Object) NewReader

func (o *Object) NewReader(ctx context.Context, engine storage.Engine, path *storage.URI, ranges []seekindex.Range) (*Reader, error)

NewReader returns a Reader for this data object. If the object has a seek index and if the provided span skips part of the object, the seek index will be used to limit the reading window of the returned reader.

func (*Object) NewVectorWriter

func (o *Object) NewVectorWriter(ctx context.Context, engine storage.Engine, path *storage.URI) (*VectorWriter, error)

func (*Object) NewWriter

func (o *Object) NewWriter(ctx context.Context, engine storage.Engine, path *storage.URI, sortKey order.SortKey, seekIndexStride int) (*Writer, error)

NewWriter returns a writer for writing the data of a zng-row storage object as well as optionally creating a seek index for the row object when the seekIndexStride is non-zero. We assume all records are non-volatile until Close as zed.Values from the various record bodies are referenced across calls to Write.

func (Object) ObjectPrefix

func (o Object) ObjectPrefix(path *storage.URI) *storage.URI

ObjectPrefix returns a prefix for the various objects that comprise a data object so they can all be deleted with the storage engine's DeleteByPrefix method.

func (Object) Remove

func (o Object) Remove(ctx context.Context, engine storage.Engine, path *storage.URI) error

Remove deletes the row object and its seek index. Any 'not found' errors are ignored.

func (Object) SeekIndexURI

func (o Object) SeekIndexURI(path *storage.URI) *storage.URI

func (Object) SequenceURI

func (o Object) SequenceURI(path *storage.URI) *storage.URI

func (Object) Span

func (o Object) Span(order order.Which) *extent.Generic

func (Object) String

func (o Object) String() string

func (Object) VectorURI

func (o Object) VectorURI(path *storage.URI) *storage.URI

type Reader

type Reader struct {
	io.Reader
	io.Closer
	TotalBytes int64
	ReadBytes  int64
}

type VectorWriter

type VectorWriter struct {
	*vng.Writer
	// contains filtered or unexported fields
}

func NewVectorWriter

func NewVectorWriter(ctx context.Context, engine storage.Engine, path *storage.URI, id ksuid.KSUID) (*VectorWriter, error)

func (*VectorWriter) Abort

func (w *VectorWriter) Abort()

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer is a zio.Writer that writes a stream of sorted records into a data object.

func (*Writer) Abort

func (w *Writer) Abort()

Abort is called when an error occurs during write. Errors are ignored because the write error will be more informative and should be returned.

func (*Writer) BytesWritten

func (w *Writer) BytesWritten() int64

func (*Writer) Close

func (w *Writer) Close(ctx context.Context) error

func (*Writer) Object

func (w *Writer) Object() *Object

Object returns the Object written by the writer. This is only valid after Close() has returned a nil error.

func (*Writer) RecordsWritten

func (w *Writer) RecordsWritten() uint64

func (*Writer) Write

func (w *Writer) Write(val zed.Value) error

func (*Writer) WriteWithKey

func (w *Writer) WriteWithKey(key, val zed.Value) error