ocf

Documentation

Overview

Package container implements encoding and decoding of Avro Object Container Files as defined by the Avro specification.

See the Avro specification for an understanding of Avro: http://avro.apache.org/docs/current/

Index

• Variables
• type Codec
• type CodecName
• type Decoder
  • func NewDecoder(r io.Reader) (*Decoder, error)
  • func (d *Decoder) Decode(v interface{}) error
  • func (d *Decoder) Error() error
  • func (d *Decoder) HasNext() bool
• type DeflateCodec
  • func (*DeflateCodec) Decode(b []byte) ([]byte, error)
  • func (*DeflateCodec) Encode(b []byte) []byte
• type Encoder
  • func NewEncoder(s string, w io.Writer, opts ...EncoderFunc) (*Encoder, error)
  • func (e *Encoder) Close() error
  • func (e *Encoder) Encode(v interface{}) error
• type EncoderFunc
  • func WithBlockLength(length int) EncoderFunc
  • func WithCodec(codec CodecName) EncoderFunc
• type Header
• type NullCodec
  • func (*NullCodec) Decode(b []byte) ([]byte, error)
  • func (*NullCodec) Encode(b []byte) []byte
• type SnappyCodec
  • func (*SnappyCodec) Decode(b []byte) ([]byte, error)
  • func (*SnappyCodec) Encode(b []byte) []byte

Examples

• NewDecoder
• NewEncoder

Variables

var HeaderSchema = avro.MustParse(`{
	"type": "record", 
	"name": "org.apache.avro.file.Header",
	"fields": [
		{"name": "magic", "type": {"type": "fixed", "name": "Magic", "size": 4}},
		{"name": "meta", "type": {"type": "map", "values": "bytes"}},
		{"name": "sync", "type": {"type": "fixed", "name": "Sync", "size": 16}}
	]
}`)

HeaderSchema is the Avro schema of a container file header.

Types

type Codec

type Codec interface {
	Decode([]byte) ([]byte, error)
	Encode([]byte) []byte
}

type CodecName

type CodecName string

const (
	Null    CodecName = "null"
	Deflate CodecName = "deflate"
	Snappy  CodecName = "snappy"
)

type Decoder

type Decoder struct {
	// contains filtered or unexported fields
}

Decoder reads and decodes Avro values from a container file.

func NewDecoder

func NewDecoder(r io.Reader) (*Decoder, error)

NewDecoder returns a new decoder that reads from reader r.

Example

package main

import (
	"log"
	"os"

	"github.com/hamba/avro/ocf"
)

func main() {
	type SimpleRecord struct {
		A int64  `avro:"a"`
		B string `avro:"b"`
	}

	f, err := os.Open("/your/avro/file.avro")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	dec, err := ocf.NewDecoder(f)
	if err != nil {
		log.Fatal(err)
	}

	for dec.HasNext() {
		var record SimpleRecord
		err = dec.Decode(&record)
		if err != nil {
			log.Fatal(err)
		}

		// Do something with the data
	}

	if dec.Error() != nil {
		log.Fatal(err)
	}
}

func (*Decoder) Decode

func (d *Decoder) Decode(v interface{}) error

Decode reads the next Avro encoded value from its input and stores it in the value pointed to by v.

func (*Decoder) Error

func (d *Decoder) Error() error

Error returns the last reader error.

func (*Decoder) HasNext

func (d *Decoder) HasNext() bool

HasNext determines if there is another value to read.

type DeflateCodec

type DeflateCodec struct{}

func (*DeflateCodec) Decode

func (*DeflateCodec) Decode(b []byte) ([]byte, error)

func (*DeflateCodec) Encode

func (*DeflateCodec) Encode(b []byte) []byte

type Encoder

type Encoder struct {
	// contains filtered or unexported fields
}

Encoder writes Avro container file to an output stream.

func NewEncoder

func NewEncoder(s string, w io.Writer, opts ...EncoderFunc) (*Encoder, error)

NewEncoder returns a new encoder that writes to w using schema s.

Example

package main

import (
	"log"
	"os"

	"github.com/hamba/avro/ocf"
)

func main() {
	schema := `{
	    "type": "record",
	    "name": "simple",
	    "namespace": "org.hamba.avro",
	    "fields" : [
	        {"name": "a", "type": "long"},
	        {"name": "b", "type": "string"}
	    ]
	}`

	type SimpleRecord struct {
		A int64  `avro:"a"`
		B string `avro:"b"`
	}

	f, err := os.Open("/your/avro/file.avro")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	enc, err := ocf.NewEncoder(schema, f)
	if err != nil {
		log.Fatal(err)
	}

	var record SimpleRecord
	err = enc.Encode(record)
	if err != nil {
		log.Fatal(err)
	}

	if err := enc.Close(); err != nil {
		log.Fatal(err)
	}
}

func (*Encoder) Close

func (e *Encoder) Close() error

Close closes the encoder, flushing the writer.

func (*Encoder) Encode

func (e *Encoder) Encode(v interface{}) error

Encode writes the Avro encoding of v to the stream.

type EncoderFunc

type EncoderFunc func(cfg *encoderConfig)

EncoderFunc represents an configuration function for Encoder

func WithBlockLength

func WithBlockLength(length int) EncoderFunc

WithBlockLength sets the block length on the encoder.

func WithCodec

func WithCodec(codec CodecName) EncoderFunc

WithCodec sets the compression codec on the encoder.

type Header

type Header struct {
	Magic [4]byte           `avro:"magic"`
	Meta  map[string][]byte `avro:"meta"`
	Sync  [16]byte          `avro:"sync"`
}

Header represents an Avro container file header.

type NullCodec

type NullCodec struct{}

func (*NullCodec) Decode

func (*NullCodec) Decode(b []byte) ([]byte, error)

func (*NullCodec) Encode

func (*NullCodec) Encode(b []byte) []byte

type SnappyCodec

type SnappyCodec struct{}

func (*SnappyCodec) Decode

func (*SnappyCodec) Decode(b []byte) ([]byte, error)

func (*SnappyCodec) Encode

func (*SnappyCodec) Encode(b []byte) []byte