README
¶
libutils
import "github.com/cuberat/go-libutils/libutils"
Overview
The libutils package provides various utilities for working in Go.
Installation
go get github.com/cuberat/go-libutils/libutils
Index
- Constants
- Variables
- func AddCompressionLayer(w io.Writer, suffix string) (io.Writer, CloseFunc, error)
- func AddDecompressionLayer(r io.Reader, suffix string) (io.Reader, CloseFunc, error)
- func BytesToVILP(data []byte) []byte
- func CreateFile(outfile string) (io.WriteCloser, error)
- func CreateFileBuffered(outfile string, size int) (io.WriteCloser, error)
- func CreateFileSync(outfile string) (io.WriteCloser, error)
- func DecodeVarint(data_in []byte) (uint64, int, error)
- func EncodeVarint(int_in uint64) []byte
- func Errorf(fmt_str string, args ...interface{}) error
- func ErrorfLong(fmt_str string, args ...interface{}) error
- func NewPrefixLenScanner(r io.Reader) *bufio.Scanner
- func NewVILPScanner(r io.Reader) *bufio.Scanner
- func NewVILPScannerF(file_path string) (*bufio.Scanner, CloseFunc, error)
- func OpenFile(infile string) (io.ReadCloser, error)
- func OpenFileRO(infile string) (io.Reader, CloseFunc, error)
- func OpenFileW(outfile string) (io.Writer, CloseFunc, error)
- func OpenPipesToWriter(final_writer io.Writer, progs [][]string) (io.Writer, CloseFunc, error)
- func ScannerPrefixLenScan(data []byte, at_eof bool) (int, []byte, error)
- func ScannerVILPScan(data []byte, at_eof bool) (int, []byte, error)
- type CloseFunc
- type KeyedRecord
- func NewKeyedRecordFromBytes(raw_rec_bytes []byte, decoder KeyedRecordDecoder) *KeyedRecord
- func NewKeyedRecordFromKeyVal(key []byte, val interface{}) *KeyedRecord
- func (kr *KeyedRecord) Key() ([]byte, error)
- func (kr *KeyedRecord) KeyString() (string, error)
- func (kr *KeyedRecord) KeyVal() ([]byte, interface{}, error)
- func (kr *KeyedRecord) MarkDirty()
- func (kr *KeyedRecord) RecordBytesOut(encoder KeyedRecordEncoder) ([]byte, error)
- func (kr *KeyedRecord) SetDecoder(dec KeyedRecordDecoder)
- func (kr *KeyedRecord) SetEncoder(enc KeyedRecordEncoder)
- func (kr *KeyedRecord) Val() (interface{}, error)
- type KeyedRecordDecoder
- type KeyedRecordEncoder
- type KeyedRecordScanner
- type KeyedRecordWriter
- type PrefixLenWriter
- type ReaderCloser
- type TabColsKRCodec
- func NewTabColsKRCodec() *TabColsKRCodec
- func (krc *TabColsKRCodec) CodecSame() bool
- func (krc *TabColsKRCodec) JoinKV(key, val []byte) ([]byte, error)
- func (krc *TabColsKRCodec) MarshalVal(data interface{}) ([]byte, error)
- func (krc *TabColsKRCodec) SplitKV(wire_data []byte) ([]byte, []byte, error)
- func (krc *TabColsKRCodec) UnmarshalVal(val_bytes []byte) (interface{}, error)
- type TabColsKRScanner
- type TabColsKRWriter
- type VILPColsKRCodec
- func NewVILPColsKRCodec() *VILPColsKRCodec
- func (krc *VILPColsKRCodec) CodecSame() bool
- func (krc *VILPColsKRCodec) JoinKV(key, val []byte) ([]byte, error)
- func (krc *VILPColsKRCodec) MarshalVal(data interface{}) ([]byte, error)
- func (krc *VILPColsKRCodec) SplitKV(wire_data []byte) ([]byte, []byte, error)
- func (krc *VILPColsKRCodec) UnmarshalVal(val_bytes []byte) (interface{}, error)
- type VILPColsKRScanner
- type VILPColsKRWriter
- type VILPWriter
- type WriterCloser
Package files
io.go keyed_record.go libutils.go prefix_len.go tab_cols_kr.go varint.go vilp.go vilp_cols_kr.go
Constants
const (
Version = "1.03"
)
Variables
var (
UnknownSuffix error = errors.New("Unknown suffix")
VarintNotEnoughBytes error = errors.New("Not enough bytes in varint")
)
func AddCompressionLayer
func AddCompressionLayer(w io.Writer, suffix string) (io.Writer,
CloseFunc, error)
Adds compression to output written to writer w, if the suffix is supported.
Supported compression:
gzip (gz)
bzip2 (bz2) -- calls external program
xz (xz) -- calls external program
The CloseFunc object is a function that you must call to shutdown the compression layer properly.
func AddDecompressionLayer
func AddDecompressionLayer(r io.Reader, suffix string) (io.Reader,
CloseFunc, error)
Adds decompression to input read from reader r, if the suffix is supported.
Supported decompression:
gzip (gz)
bzip2 (bz2)
xz (xz) -- calls external program
The CloseFunc object is a function that you must call to shutdown the decompression layer properly.
func BytesToVILP
func BytesToVILP(data []byte) []byte
Returns a byte slice with a varint length prefix followed by the provided byte slice.
func CreateFile
func CreateFile(outfile string) (io.WriteCloser, error)
Shortcut for calling CreateFileBuffered()
with the default buffer size.
Equivalent to CreateFileBuffered()
with 0 as the size parameter.
func CreateFileBuffered
func CreateFileBuffered(outfile string, size int) (io.WriteCloser, error)
Open a file for writing (buffered). The size
argument indicates that the
underlying buffer should be at least size
bytes. If size
< 0, open the
file with no buffering. If size
== 0, a size of 16K is used. If the file
name ends in a supported compression suffix, output will be compressed in
that format.
Supported compression:
gzip (.gz)
bzip2 (.bz2) -- calls external program
xz (.xz) -- calls external program
Be sure to call Close()
explicitly to flush any buffers and properly shut
down any compression layers.
func CreateFileSync
func CreateFileSync(outfile string) (io.WriteCloser, error)
Non-buffered version of CreateFile()
and CreateFileBuffered()
. Equivalent
to CreateFileBuffered()
with -1 as the size parameter.
func DecodeVarint
func DecodeVarint(data_in []byte) (uint64, int, error)
Decodes a varint (as used in protobuffers) into a uint64. See https://developers.google.com/protocol-buffers/docs/encoding#varints for the specification.
func EncodeVarint
func EncodeVarint(int_in uint64) []byte
Encodes a uint64 as a varint (as used in protobuffers). See https://developers.google.com/protocol-buffers/docs/encoding#varints for the specification.
func Errorf
func Errorf(fmt_str string, args ...interface{}) error
Like fmt.Errorf(), except adds the (base) file name and line number to the
beginning of the error message in the format [%s:%d]
.
func ErrorfLong
func ErrorfLong(fmt_str string, args ...interface{}) error
Like fmt.Errorf(), except adds the full file name and line number to the
beginning of the error message in the format [%s:%d]
.
func NewPrefixLenScanner
func NewPrefixLenScanner(r io.Reader) *bufio.Scanner
Returns a bufio.Scanner that scans length-prefixed strings from the provided io.Reader.
Deprecated: use NewVILPScanner and varint length-prefixed files.
func NewVILPScanner
func NewVILPScanner(r io.Reader) *bufio.Scanner
Returns a bufio.Scanner that scans varint length-prefixed strings from the provided io.Reader.
func NewVILPScannerF
func NewVILPScannerF(file_path string) (*bufio.Scanner,
CloseFunc, error)
Returns a bufio.Scanner that scans varint length-prefixed strings from the provided file. Call close_func() to close the underlying file handle.
func OpenFile
func OpenFile(infile string) (io.ReadCloser, error)
Opens a file in read-only mode. If the file name ends in a supported compression suffix, input with be decompressed.
Supported decompression:
gzip (.gz)
bzip2 (.bz2)
xz (.xz) -- calls external program
Call Close() on the returned io.ReaderCloser to avoid leaking filehandles and to proplery shut down any compression layers.
func OpenFileRO
func OpenFileRO(infile string) (io.Reader, CloseFunc, error)
Deprecated Use OpenFile()
instead.
Opens a file in read-only mode. If the file name ends in a supported compression suffix, input with be decompressed.
Supported decompression:
gzip (.gz)
bzip2 (.bz2)
xz (.xz) -- calls external program
The CloseFunc object is a function that you must call to close the file properly.
func OpenFileW
func OpenFileW(outfile string) (io.Writer, CloseFunc, error)
Deprecated Use CreateFileSync()
or CreateFile()
instead.
Open a file for writing. If the file name ends in a supported compression suffix, output will be compressed in that format.
Supported compression:
gzip (.gz)
bzip2 (.bz2) -- calls external program
xz (.xz) -- calls external program
The CloseFunc object is a function that you must call to close the file properly.
func OpenPipesToWriter
func OpenPipesToWriter(final_writer io.Writer,
progs [][]string) (io.Writer, CloseFunc, error)
Runs the list of commands, piping the output of each one to the next. The output of the last command is sent to the final_writer passed in. Each command is represented as a slice of strings. The first element of the slice should be the full path to the program to run. The remaining elements of the slice should be the arguments to the program.
The writer returned writes to the standard input of the first program in the list. The CloseFunc should be called as a function when writing has been completed (and before final_writer has been closed).
func ScannerPrefixLenScan
func ScannerPrefixLenScan(data []byte, at_eof bool) (int, []byte, error)
A bufio.SplitFunc that reads length-prefixed strings from a reader
Deprecated: use NewVILPScanner and varint length-prefixed files.
func ScannerVILPScan
func ScannerVILPScan(data []byte, at_eof bool) (int, []byte, error)
A bufio.SplitFunc that reads length-prefixed strings from a reader.
type CloseFunc
type CloseFunc func()
Signature for Close() function return from OpenFileW and OpenFileRO. When ready to close the file, call the function to close and clean up.
type KeyedRecord
type KeyedRecord struct {
// contains filtered or unexported fields
}
KeyedRecord in this package supports the concept of records that consist of a (string) key and a (possible complex) value, along with lazy marshaling and unmarshaling of data.
Extracting records from files is considered separate from the records themselves. E.g., records might live in a tab-delimited file where each record is stored as a key and a JSON object separated by a tab character, with a newline character delimiting records, like so:
foo[tab]{"on": true}[newline]
bar[tab]{"on": false}[newline]
In this case, the newline is not part of the record. The KeyedRecordScanner and KeyedRecordWriter interfaces deal with reading and writing records. A KeyedRecord needs an decoder (KeyedRecordDecoder) in order to parse a record provided to it as a slice of bytes, and an encoder (KeyedRecordEncoder) to serialize the record to be written out. This allows for readily changing the output format, e.g., to variable integer length-prefixed records.
func NewKeyedRecordFromBytes
func NewKeyedRecordFromBytes(raw_rec_bytes []byte,
decoder KeyedRecordDecoder) *KeyedRecord
Parse the raw record from wire data, using the provided decoder. The decoder is stored internally for later use.
func NewKeyedRecordFromKeyVal
func NewKeyedRecordFromKeyVal(key []byte, val interface{}) *KeyedRecord
Create a new KeyedRecord object from a key and value.
func (*KeyedRecord) Key
func (kr *KeyedRecord) Key() ([]byte, error)
Parse out the key from the record (if necessary) and return it.
func (*KeyedRecord) KeyString
func (kr *KeyedRecord) KeyString() (string, error)
Parse out the key from the record (if necessary) and return it as a string.
func (*KeyedRecord) KeyVal
func (kr *KeyedRecord) KeyVal() ([]byte, interface{}, error)
Parse out (if necessary) the key and value, returning both.
func (*KeyedRecord) MarkDirty
func (kr *KeyedRecord) MarkDirty()
Mark the KeyedRecord dirty such that will reserialize the value data structure even if the encoder and encoder are the same and encoder.CodecSame() returns true.
func (*KeyedRecord) RecordBytesOut
func (kr *KeyedRecord) RecordBytesOut(encoder KeyedRecordEncoder) ([]byte, error)
Serialize the record into a slice of bytes using the provided encoder.
func (*KeyedRecord) SetDecoder
func (kr *KeyedRecord) SetDecoder(dec KeyedRecordDecoder)
Set the decoder object within the KeyedRecord for later use.
func (*KeyedRecord) SetEncoder
func (kr *KeyedRecord) SetEncoder(enc KeyedRecordEncoder)
Set the encoder object within the KeyedRecord for later use.
func (*KeyedRecord) Val
func (kr *KeyedRecord) Val() (interface{}, error)
Return the value of from the record as an interface{}. If you know what type the value should have, you can use an assertion to get to the underlying type, e.g.,
val, ok := kr.Val().(*MyStruct)
type KeyedRecordDecoder
type KeyedRecordDecoder interface {
// Splits the record, returning the key and the serialized value data
// structure.
SplitKV([]byte) ([]byte, []byte, error)
// Deserializes the value.
UnmarshalVal([]byte) (interface{}, error)
}
type KeyedRecordEncoder
type KeyedRecordEncoder interface {
// Joins the key and value bytes, returning the serialized record.
JoinKV(key []byte, val []byte) ([]byte, error)
// Serializes the value data structure.
MarshalVal(interface{}) ([]byte, error)
// If this object also implements the `KeyedRecordDecoder` interface, and
// the encoding is the same for both input and output, CodecSame() returns
// true. Otherwise, it returns false.
//
// This allows for lazy encoding. That is, if the raw record bytes that were
// read in do not need to change, they can be written back out as-is,
// instead of actually re-encoding.
CodecSame() bool
}
type KeyedRecordScanner
type KeyedRecordScanner interface {
// Advances the scanner to the next record. It returns false when the scan
// stops, either by reaching the end of the input or an error.
Scan() bool
// Returns the most recent serialized record generated by a call to Scan().
Record() *KeyedRecord
// Returns the first non-EOF error that was encountered by the Scanner.
Err() error
}
type KeyedRecordWriter
type KeyedRecordWriter interface {
// Writes the entire seralized record.
Write(*KeyedRecord) (int, error)
}
type PrefixLenWriter
type PrefixLenWriter struct {
// contains filtered or unexported fields
}
PrefixLenWriter is used to write length-prefixed strings to an io.Writer
Deprecated: use VILPWriter and its corresponding methods.
func NewPrefixLenWriter
func NewPrefixLenWriter(w io.Writer) *PrefixLenWriter
Returns a new PrefixLenWriter. PrefixLenWriter implements the io.Writer interface, in addition to the WriteString method.
Deprecated: use VILPWriter and its corresponding methods.
func (*PrefixLenWriter) Write
func (plw *PrefixLenWriter) Write(p []byte) (int, error)
Writes the provided bytes as a length-prefixed string to the underlying io.Writer. This uses 32-bit integers for the length prefix.
Deprecated: use VILPWriter and its corresponding methods.
func (*PrefixLenWriter) WriteString
func (plw *PrefixLenWriter) WriteString(s string) (int, error)
Writes the provided string as a length-prefixed string to the underlying io.Writer. This uses 32-bit integers for the length prefix.
Deprecated: use VILPWriter and its corresponding methods.
type ReaderCloser
type ReaderCloser struct {
// contains filtered or unexported fields
}
func (*ReaderCloser) Close
func (rc *ReaderCloser) Close() error
func (*ReaderCloser) Read
func (rc *ReaderCloser) Read(p []byte) (n int, err error)
type TabColsKRCodec
type TabColsKRCodec struct {
}
Implements the KeyedRecordEncoder and KeyedRecordDecoder interfaces specified
by github.com/cuberat/go-libutils/libutils
.
This codec serializes and deserializes keyed records where the key is the first tab-separated column, and the value is a slice of byte slices containing the remaining columns. This is not TSV, where tab bytes can be escaped, etc. It is a simple split on tab bytes. That is,
<key>[tab]<val1>[tab]<val2>...
func NewTabColsKRCodec
func NewTabColsKRCodec() *TabColsKRCodec
Returns a new TabColsKRCodec
func (*TabColsKRCodec) CodecSame
func (krc *TabColsKRCodec) CodecSame() bool
Returns true so that if this codec is used for both encoder and decoder, unnecessary re-serialization can be avoided.
This allows for lazy encoding. That is, if the raw record bytes that were read in do not need to change, they can be written back out as-is, instead of actually re-encoding.
func (*TabColsKRCodec) JoinKV
func (krc *TabColsKRCodec) JoinKV(key, val []byte) ([]byte, error)
Joins the key and value bytes, returning the serialized record.
func (*TabColsKRCodec) MarshalVal
func (krc *TabColsKRCodec) MarshalVal(data interface{}) ([]byte, error)
Serializes the value data structure.
func (*TabColsKRCodec) SplitKV
func (krc *TabColsKRCodec) SplitKV(wire_data []byte) ([]byte, []byte,
error)
Splits the record, returning the key and the serialized value data structure.
func (*TabColsKRCodec) UnmarshalVal
func (krc *TabColsKRCodec) UnmarshalVal(val_bytes []byte) (interface{},
error)
Deserializes the value.
type TabColsKRScanner
type TabColsKRScanner struct {
// contains filtered or unexported fields
}
Implements the libutils.KeyedRecordScanner
interface from
github.com/cuberat/go-libutils/libutils
.
This is a scanner for tab-delimited keyed records with column-based values.
Records are expected to look like
<key>[tab]<val1>[tab]<val2>...[newline]
func NewTabColsKRScanner
func NewTabColsKRScanner(r io.Reader) *TabColsKRScanner
Return a new TabColsKRScanner
func (*TabColsKRScanner) Err
func (krs *TabColsKRScanner) Err() error
Returns the first non-EOF error that was encountered by the Scanner.
func (*TabColsKRScanner) Record
func (krs *TabColsKRScanner) Record() *KeyedRecord
Returns the most recent serialized record generated by a call to Scan().
func (*TabColsKRScanner) Scan
func (krs *TabColsKRScanner) Scan() bool
Advances the scanner to the next record. It returns false when the scan stops, either by reaching the end of the input or an error.
type TabColsKRWriter
type TabColsKRWriter struct {
// contains filtered or unexported fields
}
Implements the libutils.KeyedRecordWriter
interface from
github.com/cuberat/go-libutils/libutils
. Records look like
<key>[tab]<val1>[tab]<val2>...[newline]
func NewTabColsKRWriter
func NewTabColsKRWriter(w io.Writer) *TabColsKRWriter
Returns a new TabColsKRWriter.
func (*TabColsKRWriter) Write
func (krw *TabColsKRWriter) Write(rec *KeyedRecord) (int, error)
The Write method for the KeyedRecordWriter interface
type VILPColsKRCodec
type VILPColsKRCodec struct {
}
Implements the KeyedRecordEncoder and KeyedRecordDecoder interfaces specified
by github.com/cuberat/go-libutils/libutils
.
This codec serializes and deserializes keyed records where the key is length-prefixed using a varint. The value immediately follows and consists of length-prefixed strings represented an array/list of strings. That is,
<key_len><key><val1_len><val1><val2_len><val2>...
func NewVILPColsKRCodec
func NewVILPColsKRCodec() *VILPColsKRCodec
Returns a new VILPColsKRCodec
func (*VILPColsKRCodec) CodecSame
func (krc *VILPColsKRCodec) CodecSame() bool
Returns true so that if this codec is used for both encoder and decoder, unnecessary re-serialization can be avoided.
This allows for lazy encoding. That is, if the raw record bytes that were read in do not need to change, they can be written back out as-is, instead of actually re-encoding.
func (*VILPColsKRCodec) JoinKV
func (krc *VILPColsKRCodec) JoinKV(key, val []byte) ([]byte, error)
Joins the key and value bytes, returning the serialized record.
func (*VILPColsKRCodec) MarshalVal
func (krc *VILPColsKRCodec) MarshalVal(data interface{}) ([]byte, error)
Serializes the value data structure.
func (*VILPColsKRCodec) SplitKV
func (krc *VILPColsKRCodec) SplitKV(wire_data []byte) ([]byte, []byte,
error)
Splits the record, returning the key and the serialized value data structure.
func (*VILPColsKRCodec) UnmarshalVal
func (krc *VILPColsKRCodec) UnmarshalVal(val_bytes []byte) (interface{},
error)
Deserializes the value.
type VILPColsKRScanner
type VILPColsKRScanner struct {
// contains filtered or unexported fields
}
Scanner for varint length-prefixed keyed records with column-based values. That is, the value is an array/list of strings, each one prefixed with a length.
func NewVILPColsKRScanner
func NewVILPColsKRScanner(r io.Reader) *VILPColsKRScanner
Return a new VILPColsKRScanner
func (*VILPColsKRScanner) Err
func (krs *VILPColsKRScanner) Err() error
Returns the first non-EOF error that was encountered by the Scanner.
func (*VILPColsKRScanner) Record
func (krs *VILPColsKRScanner) Record() *KeyedRecord
Returns the most recent serialized record generated by a call to Scan().
func (*VILPColsKRScanner) Scan
func (krs *VILPColsKRScanner) Scan() bool
Advances the scanner to the next record. It returns false when the scan stops, either by reaching the end of the input or an error.
type VILPColsKRWriter
type VILPColsKRWriter struct {
// contains filtered or unexported fields
}
Implements the libutils.KeyedRecordWriter
interface from
github.com/cuberat/go-libutils/libutils
.
func NewVILPColsKRWriter
func NewVILPColsKRWriter(w io.Writer) *VILPColsKRWriter
Returns a new VILPColsKRWriter.
func (*VILPColsKRWriter) Write
func (krw *VILPColsKRWriter) Write(rec *KeyedRecord) (int, error)
The Write method for the KeyedRecordWriter interface
type VILPWriter
type VILPWriter struct {
// contains filtered or unexported fields
}
VILPWriter is used to write length-prefixed strings to an io.Writer
func NewVILPWriter
func NewVILPWriter(w io.Writer) *VILPWriter
Returns a new VILPWriter. VILPWriter implements the io.Writer interface, in addition to the WriteString method.
func NewVILPWriterF
func NewVILPWriterF(file_path string) (*VILPWriter, CloseFunc,
error)
Opens the provided file and returns a *VILPWriter created using the resulting file handle. Call close_func() to close the underlying file handle.
func (*VILPWriter) Write
func (plw *VILPWriter) Write(p []byte) (int, error)
Writes the provided bytes as a length-prefixed string to the underlying io.Writer
func (*VILPWriter) WriteString
func (plw *VILPWriter) WriteString(s string) (int, error)
Writes the provided string as a length-prefixed string to the underlying io.Writer
type WriterCloser
type WriterCloser struct {
// contains filtered or unexported fields
}
func (*WriterCloser) Close
func (wc *WriterCloser) Close() error
func (*WriterCloser) Write
func (wc *WriterCloser) Write(p []byte) (n int, err error)
Generated by godoc2md