hadoop

README

go-hadoop-io

Hadoop SequenceFile reader for Go. This project is still in early stage and may be subject to change.

Installation

Ubuntu

sudo apt-get install libbz2-dev liblz4-dev
go get github.com/eiiches/go-hadoop-io

CentOS

sudo yum install bzip2-devel
git clone https://github.com/Cyan4973/lz4
cd lz4
make
sudo make install
sudo ldconfig
go get github.com/eiiches/go-hadoop-io

Usage

Some examples are located in examples.

License

This project is licensed under the terms of the 3-clause BSD license. See LICENSE.txt for the full license text.

Documentation

Index

• Constants
• Variables
• func DecodeVIntSize(value byte) int
• func IsNegativeVInt(fst byte) bool
• func ReadBoolean(r io.Reader) (bool, error)
• func ReadBuffer(r io.Reader) ([]byte, error)
• func ReadByte(r io.Reader) (byte, error)
• func ReadInt(r io.Reader) (int32, error)
• func ReadVLong(r io.Reader) (int64, error)
• func WriteBoolean(w io.Writer, v bool) error
• func WriteBuffer(w io.Writer, buf []byte) (int, error)
• func WriteByte(w io.Writer, b byte) error
• func WriteInt(w io.Writer, v int32) error
• func WriteVLong(w io.Writer, i int64) (int, error)
• type BytesWritable
  • func (self *BytesWritable) Read(r io.Reader) error
  • func (self *BytesWritable) Write(w io.Writer) (int, error)
• type Bzip2Codec
  • func (c *Bzip2Codec) Compress(dst, src []byte) ([]byte, error)
  • func (c *Bzip2Codec) Uncompress(dst, src []byte) ([]byte, error)
• type Codec
• type IntWritable
  • func (self *IntWritable) Read(r io.Reader) error
  • func (self *IntWritable) Write(w io.Writer) (int, error)
• type LongWritable
  • func (self *LongWritable) Read(r io.Reader) error
  • func (self *LongWritable) Write(w io.Writer) (int, error)
• type Lz4Codec
  • func (c *Lz4Codec) Compress(dst, src []byte) ([]byte, error)
  • func (c *Lz4Codec) Uncompress(dst, src []byte) ([]byte, error)
• type SequenceFileReader
  • func NewSequenceFileReader(r io.Reader) (*SequenceFileReader, error)
  • func (self *SequenceFileReader) Close() error
  • func (self *SequenceFileReader) Read(key Writable, value Writable) error
• type SequenceFileWriter
  • func NewSequenceFileWriter(w io.Writer, opts *SequenceFileWriterOpts) (*SequenceFileWriter, error)
  • func (self *SequenceFileWriter) Close() error
  • func (self *SequenceFileWriter) Write(key Writable, value Writable) error
• type SequenceFileWriterOpts
• type SnappyCodec
• type TextWritable
  • func (self *TextWritable) Read(r io.Reader) error
  • func (self *TextWritable) Write(w io.Writer) (int, error)
• type Writable
• type ZlibCodec
  • func (c *ZlibCodec) Compress(dst, src []byte) ([]byte, error)
  • func (c *ZlibCodec) Uncompress(dst, src []byte) ([]byte, error)

Constants

const (
	VERSION_BLOCK_COMPRESS  = 4
	VERSION_CUSTOM_COMPRESS = 5
	VERSION_WITH_METADATA   = 6
)

const BLOCK_SIZE_MIN = 1 << 20 // Corresponds roughly to io.seqfile.compress.blocksize

const SYNC_HASH_SIZE = 16

Variables

var (
	Codecs map[string]Codec = map[string]Codec{
		"org.apache.hadoop.io.compress.DefaultCodec": &ZlibCodec{},
		"org.apache.hadoop.io.compress.Lz4Codec":     &Lz4Codec{},
		"org.apache.hadoop.io.compress.BZip2Codec":   &Bzip2Codec{},
	}
)

var SEQ_MAGIC = []byte("SEQ")

Functions

func DecodeVIntSize

func DecodeVIntSize(value byte) int

func IsNegativeVInt

func IsNegativeVInt(fst byte) bool

func ReadBoolean

func ReadBoolean(r io.Reader) (bool, error)

func ReadBuffer

func ReadBuffer(r io.Reader) ([]byte, error)

func ReadByte

func ReadByte(r io.Reader) (byte, error)

func ReadInt

func ReadInt(r io.Reader) (int32, error)

func ReadVLong

func ReadVLong(r io.Reader) (int64, error)

func WriteBoolean

func WriteBoolean(w io.Writer, v bool) error

func WriteBuffer

func WriteBuffer(w io.Writer, buf []byte) (int, error)

func WriteByte

func WriteByte(w io.Writer, b byte) error

func WriteInt

func WriteInt(w io.Writer, v int32) error

func WriteVLong

func WriteVLong(w io.Writer, i int64) (int, error)

Ported from https://hadoop.apache.org/docs/r2.6.2/api/src-html/org/apache/hadoop/io/WritableUtils.html#line.271

Types

type BytesWritable

type BytesWritable struct {
	Buf []byte
}

func (*BytesWritable) Read

func (self *BytesWritable) Read(r io.Reader) error

func (*BytesWritable) Write

func (self *BytesWritable) Write(w io.Writer) (int, error)

type Bzip2Codec

type Bzip2Codec struct {
}

func (*Bzip2Codec) Compress

func (c *Bzip2Codec) Compress(dst, src []byte) ([]byte, error)

func (*Bzip2Codec) Uncompress

func (c *Bzip2Codec) Uncompress(dst, src []byte) ([]byte, error)

type Codec

type Codec interface {
	Uncompress(dst, src []byte) ([]byte, error)
	Compress(dst, src []byte) ([]byte, error)
}

type IntWritable

type IntWritable int32

func (*IntWritable) Read

func (self *IntWritable) Read(r io.Reader) error

func (*IntWritable) Write

func (self *IntWritable) Write(w io.Writer) (int, error)

type LongWritable

type LongWritable int64

func (*LongWritable) Read

func (self *LongWritable) Read(r io.Reader) error

func (*LongWritable) Write

func (self *LongWritable) Write(w io.Writer) (int, error)

type Lz4Codec

type Lz4Codec struct {
}

func (*Lz4Codec) Compress

func (c *Lz4Codec) Compress(dst, src []byte) ([]byte, error)

func (*Lz4Codec) Uncompress

func (c *Lz4Codec) Uncompress(dst, src []byte) ([]byte, error)

type SequenceFileReader

type SequenceFileReader struct {
	// contains filtered or unexported fields
}

func NewSequenceFileReader

func NewSequenceFileReader(r io.Reader) (*SequenceFileReader, error)

func (*SequenceFileReader) Close

func (self *SequenceFileReader) Close() error

func (*SequenceFileReader) Read

func (self *SequenceFileReader) Read(key Writable, value Writable) error

type SequenceFileWriter

type SequenceFileWriter struct {
	// contains filtered or unexported fields
}

func NewSequenceFileWriter

func NewSequenceFileWriter(w io.Writer, opts *SequenceFileWriterOpts) (*SequenceFileWriter, error)

func (*SequenceFileWriter) Close

func (self *SequenceFileWriter) Close() error

func (*SequenceFileWriter) Write

func (self *SequenceFileWriter) Write(key Writable, value Writable) error

type SequenceFileWriterOpts

type SequenceFileWriterOpts struct {
	KeyClassName     string
	ValueClassName   string
	CompressionCodec string
}

type SnappyCodec

type SnappyCodec struct {
}

type TextWritable

type TextWritable struct {
	Buf []byte
}

func (*TextWritable) Read

func (self *TextWritable) Read(r io.Reader) error

func (*TextWritable) Write

func (self *TextWritable) Write(w io.Writer) (int, error)

type Writable

type Writable interface {
	Write(w io.Writer) (int, error)
	Read(r io.Reader) error
}

type ZlibCodec

type ZlibCodec struct{}

func (*ZlibCodec) Compress

func (c *ZlibCodec) Compress(dst, src []byte) ([]byte, error)

func (*ZlibCodec) Uncompress

func (c *ZlibCodec) Uncompress(dst, src []byte) ([]byte, error)