Documentation
¶
Overview ¶
Package ocf implements encoding and decoding of Avro Object Container Files as defined by the Avro specification.
See the Avro specification for an understanding of Avro: http://avro.apache.org/docs/current/
Index ¶
Examples ¶
Constants ¶
This section is empty.
Variables ¶
var HeaderSchema = avro.MustParse(`{
"type": "record",
"name": "org.apache.avro.file.Header",
"fields": [
{"name": "magic", "type": {"type": "fixed", "name": "Magic", "size": 4}},
{"name": "meta", "type": {"type": "map", "values": "bytes"}},
{"name": "sync", "type": {"type": "fixed", "name": "Sync", "size": 16}}
]
}`)
HeaderSchema is the Avro schema of a container file header.
Functions ¶
This section is empty.
Types ¶
type Codec ¶
type Codec interface {
// Decode decodes the given bytes.
Decode([]byte) ([]byte, error)
// Encode encodes the given bytes.
Encode([]byte) []byte
}
Codec represents a compression codec.
type Decoder ¶
type Decoder struct {
// contains filtered or unexported fields
}
Decoder reads and decodes Avro values from a container file.
func NewDecoder ¶
NewDecoder returns a new decoder that reads from reader r.
Example ¶
package main
import (
"log"
"os"
"github.com/hamba/avro/v2/ocf"
)
func main() {
type SimpleRecord struct {
A int64 `avro:"a"`
B string `avro:"b"`
}
f, err := os.Open("/your/avro/file.avro")
if err != nil {
log.Fatal(err)
}
defer f.Close()
dec, err := ocf.NewDecoder(f)
if err != nil {
log.Fatal(err)
}
for dec.HasNext() {
var record SimpleRecord
err = dec.Decode(&record)
if err != nil {
log.Fatal(err)
}
// Do something with the data
}
if dec.Error() != nil {
log.Fatal(err)
}
}
Output:
func (*Decoder) Decode ¶
Decode reads the next Avro encoded value from its input and stores it in the value pointed to by v.
type DeflateCodec ¶
type DeflateCodec struct {
// contains filtered or unexported fields
}
DeflateCodec is a flate compression codec.
func (*DeflateCodec) Decode ¶
func (c *DeflateCodec) Decode(b []byte) ([]byte, error)
Decode decodes the given bytes.
func (*DeflateCodec) Encode ¶
func (c *DeflateCodec) Encode(b []byte) []byte
Encode encodes the given bytes.
type Encoder ¶
type Encoder struct {
// contains filtered or unexported fields
}
Encoder writes Avro container file to an output stream.
func NewEncoder ¶
NewEncoder returns a new encoder that writes to w using schema s.
If the writer is an existing ocf file, it will append data using the existing schema.
Example ¶
package main
import (
"log"
"os"
"github.com/hamba/avro/v2/ocf"
)
func main() {
schema := `{
"type": "record",
"name": "simple",
"namespace": "org.hamba.avro",
"fields" : [
{"name": "a", "type": "long"},
{"name": "b", "type": "string"}
]
}`
type SimpleRecord struct {
A int64 `avro:"a"`
B string `avro:"b"`
}
f, err := os.Open("/your/avro/file.avro")
if err != nil {
log.Fatal(err)
}
defer f.Close()
enc, err := ocf.NewEncoder(schema, f)
if err != nil {
log.Fatal(err)
}
var record SimpleRecord
err = enc.Encode(record)
if err != nil {
log.Fatal(err)
}
if err := enc.Flush(); err != nil {
log.Fatal(err)
}
if err := f.Sync(); err != nil {
log.Fatal(err)
}
}
Output:
type EncoderFunc ¶
type EncoderFunc func(cfg *encoderConfig)
EncoderFunc represents an configuration function for Encoder.
func WithBlockLength ¶
func WithBlockLength(length int) EncoderFunc
WithBlockLength sets the block length on the encoder.
func WithCodec ¶
func WithCodec(codec CodecName) EncoderFunc
WithCodec sets the compression codec on the encoder.
func WithCompressionLevel ¶
func WithCompressionLevel(compLvl int) EncoderFunc
WithCompressionLevel sets the compression codec to deflate and the compression level on the encoder.
func WithEncodingConfig ¶
func WithEncodingConfig(wCfg avro.API) EncoderFunc
WithEncodingConfig sets the value encoder config on the OCF encoder.
func WithMetadata ¶
func WithMetadata(meta map[string][]byte) EncoderFunc
WithMetadata sets the metadata on the encoder header.
func WithSyncBlock ¶
func WithSyncBlock(sync [16]byte) EncoderFunc
WithSyncBlock sets the sync block.
type Header ¶
type Header struct {
Magic [4]byte `avro:"magic"`
Meta map[string][]byte `avro:"meta"`
Sync [16]byte `avro:"sync"`
}
Header represents an Avro container file header.
type NullCodec ¶
type NullCodec struct{}
NullCodec is a no op codec.
type SnappyCodec ¶
type SnappyCodec struct{}
SnappyCodec is a snappy compression codec.
func (*SnappyCodec) Decode ¶
func (*SnappyCodec) Decode(b []byte) ([]byte, error)
Decode decodes the given bytes.
func (*SnappyCodec) Encode ¶
func (*SnappyCodec) Encode(b []byte) []byte
Encode encodes the given bytes.
type ZStandardCodec ¶
type ZStandardCodec struct{}
ZStandardCodec is a zstandard compression codec.
func (*ZStandardCodec) Decode ¶
func (*ZStandardCodec) Decode(b []byte) ([]byte, error)
Decode decodes the given bytes.
func (*ZStandardCodec) Encode ¶
func (*ZStandardCodec) Encode(b []byte) []byte
Encode encodes the given bytes.
Source Files