zngio

Documentation

Overview

Package zngio provides an API for reading and writing zng values and directives in binary zng format. The Reader and Writer types implement the the zio.Reader and zio.Writer interfaces. Since these methods read and write only zed.Records, but the zng format includes additional functionality, other methods are available to read/write zng comments and include virtual channel numbers in the stream. Virtual channels provide a way to indicate which output of a flowgraph a result came from when a flowgraph computes multiple output channels. The zng values in this zng value are "machine format" as prescirbed by the ZNG spec. The vanilla zio.Reader and zio.Writer implementations ignore application-specific payloads (e.g., channel encodings).

Index

• Constants
• type AppMessage
• type Encoder
  • func NewEncoder() *Encoder
  • func (e *Encoder) Encode(dst []byte, external zed.Type) ([]byte, zed.Type, error)
  • func (e *Encoder) Lookup(external zed.Type) zed.Type
  • func (e *Encoder) Reset()
• type Reader
  • func NewReader(reader io.Reader, sctx *zed.Context) *Reader
  • func NewReaderWithOpts(reader io.Reader, sctx *zed.Context, opts ReaderOpts) *Reader
  • func (r *Reader) LastSOS() int64
  • func (r *Reader) NewScanner(ctx context.Context, filter zbuf.Filter) (zbuf.Scanner, error)
  • func (r *Reader) Position() int64
  • func (r *Reader) Read() (*zed.Value, error)
  • func (r *Reader) ReadByte() (byte, error)
  • func (r *Reader) ReadPayload() (*zed.Value, *AppMessage, error)
  • func (r *Reader) SkipStream() (*zed.Value, int64, error)
• type ReaderOpts
• type Writer
  • func NewWriter(w io.WriteCloser, opts WriterOpts) *Writer
  • func (w *Writer) Close() error
  • func (w *Writer) EndStream() error
  • func (w *Writer) Position() int64
  • func (w *Writer) Write(r *zed.Value) error
  • func (w *Writer) WriteControl(b []byte, encoding uint8) error
• type WriterOpts

Constants

const (
	ReadSize  = 512 * 1024
	MaxSize   = 10 * 1024 * 1024
	TypeLimit = 10000
)

const (
	// DefaultLZ4BlockSize is a reasonable default for WriterOpts.LZ4BlockSize.
	DefaultLZ4BlockSize = 16 * 1024
)

Types

type AppMessage

type AppMessage struct {
	Code     int
	Encoding int
	Bytes    []byte
}

type Encoder

type Encoder struct {
	// contains filtered or unexported fields
}

func NewEncoder

func NewEncoder() *Encoder

func (*Encoder) Encode

func (e *Encoder) Encode(dst []byte, external zed.Type) ([]byte, zed.Type, error)

Encode takes a type from outside this context and constructs a type from inside this context and emits ZNG typedefs for any type needed to construct the new type into the buffer provided.

func (*Encoder) Lookup

func (e *Encoder) Lookup(external zed.Type) zed.Type

func (*Encoder) Reset

func (e *Encoder) Reset()

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func NewReader

func NewReader(reader io.Reader, sctx *zed.Context) *Reader

func NewReaderWithOpts

func NewReaderWithOpts(reader io.Reader, sctx *zed.Context, opts ReaderOpts) *Reader

func (*Reader) LastSOS

func (r *Reader) LastSOS() int64

LastSOS returns the offset of the most recent Start-of-Stream

func (*Reader) NewScanner

func (r *Reader) NewScanner(ctx context.Context, filter zbuf.Filter) (zbuf.Scanner, error)

func (*Reader) Position

func (r *Reader) Position() int64

func (*Reader) Read

func (r *Reader) Read() (*zed.Value, error)

func (*Reader) ReadByte

func (r *Reader) ReadByte() (byte, error)

ReadByte implements io.ByteReader.ReadByte.

func (*Reader) ReadPayload

func (r *Reader) ReadPayload() (*zed.Value, *AppMessage, error)

func (*Reader) SkipStream

func (r *Reader) SkipStream() (*zed.Value, int64, error)

SkipStream skips over the records in the current stream and returns the first record of the next stream and the start-of-stream position of that record.

type ReaderOpts

type ReaderOpts struct {
	Validate bool
	Size     int
	Max      int
}

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func NewWriter

func NewWriter(w io.WriteCloser, opts WriterOpts) *Writer

func (*Writer) Close

func (w *Writer) Close() error

func (*Writer) EndStream

func (w *Writer) EndStream() error

func (*Writer) Position

func (w *Writer) Position() int64

func (*Writer) Write

func (w *Writer) Write(r *zed.Value) error

func (*Writer) WriteControl

func (w *Writer) WriteControl(b []byte, encoding uint8) error

type WriterOpts

type WriterOpts struct {
	LZ4BlockSize int
}