vng

Documentation

Overview

Package vng implements the reading and writing of VNG storage objects to and from any Zed format. The VNG storage format is described at https://github.com/brimdata/zed/blob/main/docs/formats/vng.md.

A VNG storage object must be seekable (e.g., a local file or S3 object), so, unlike ZNG, streaming of VNG objects is not supported.

The vng/vector package handles reading and writing Zed sequence data to vectors, while the vng package comprises the API used to read and write VNG objects.

Index

• Constants
• func ReadIntVector(segments []vector.Segment, r io.ReaderAt) ([]int32, error)
• type FileMeta
• type Object
  • func NewObject(zctx *zed.Context, r io.ReaderAt, size int64) (*Object, error)
  • func NewObjectFromPath(ctx context.Context, zctx *zed.Context, engine storage.Engine, path string) (*Object, error)
  • func NewObjectFromStorageReader(zctx *zed.Context, r storage.Reader) (*Object, error)
  • func NewObjectFromStorageReaderNoCloser(zctx *zed.Context, r storage.Reader) (*Object, error)
  • func NewObjectFromURI(ctx context.Context, zctx *zed.Context, engine storage.Engine, ...) (*Object, error)
  • func (o *Object) Close() error
  • func (o *Object) FetchMetadata() ([]int32, []vector.Metadata, error)
  • func (o *Object) IsEmpty() bool
  • func (o *Object) NewReassemblyReader() *zngio.Reader
• type Reader
  • func NewReader(o *Object) (*Reader, error)
  • func (r *Reader) Read() (*zed.Value, error)
• type Writer
  • func NewWriter(w io.WriteCloser, skewThresh, segThresh int) (*Writer, error)
  • func (w *Writer) Close() error
  • func (w *Writer) Write(val *zed.Value) error

Constants

const (
	FileType = "vng"
	Version  = 3
)

const (
	MaxSegmentThresh = vector.MaxSegmentThresh
	MaxSkewThresh    = 512 * 1024 * 1024
)

Functions

func ReadIntVector

func ReadIntVector(segments []vector.Segment, r io.ReaderAt) ([]int32, error)

Types

type FileMeta

type FileMeta struct {
	SkewThresh    int `zed:"skew_thresh"`
	SegmentThresh int `zed:"segment_thresh"`
}

type Object

type Object struct {
	// contains filtered or unexported fields
}

func NewObject

func NewObject(zctx *zed.Context, r io.ReaderAt, size int64) (*Object, error)

func NewObjectFromPath

func NewObjectFromPath(ctx context.Context, zctx *zed.Context, engine storage.Engine, path string) (*Object, error)

func NewObjectFromStorageReader

func NewObjectFromStorageReader(zctx *zed.Context, r storage.Reader) (*Object, error)

func NewObjectFromStorageReaderNoCloser

func NewObjectFromStorageReaderNoCloser(zctx *zed.Context, r storage.Reader) (*Object, error)

func NewObjectFromURI

func NewObjectFromURI(ctx context.Context, zctx *zed.Context, engine storage.Engine, uri *storage.URI) (*Object, error)

func (*Object) Close

func (o *Object) Close() error

func (*Object) FetchMetadata

func (o *Object) FetchMetadata() ([]int32, []vector.Metadata, error)

func (*Object) IsEmpty

func (o *Object) IsEmpty() bool

func (*Object) NewReassemblyReader

func (o *Object) NewReassemblyReader() *zngio.Reader

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader implements zio.Reader for a VNG object.

func NewReader

func NewReader(o *Object) (*Reader, error)

NewReader returns a Reader for o.

func (*Reader) Read

func (r *Reader) Read() (*zed.Value, error)

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer implements the zio.Writer interface. A Writer creates a vector VNG object from a stream of zed.Records.

func NewWriter

func NewWriter(w io.WriteCloser, skewThresh, segThresh int) (*Writer, error)

func (*Writer) Close

func (w *Writer) Close() error

func (*Writer) Write

func (w *Writer) Write(val *zed.Value) error