Documentation
¶
Index ¶
- Constants
- func ComputeSuffixArray(data []int, sa []int, fs int, n int, k int, isbwt bool) uint
- func GetBWTChunks(size int) int
- func GetName(functionType uint64) (string, error)
- func GetType(name string) (uint64, error)
- type AliasCodec
- type BWT
- type BWTBlockCodec
- type BWTS
- type ByteTransformSequence
- func (this *ByteTransformSequence) Forward(src, dst []byte) (uint, uint, error)
- func (this *ByteTransformSequence) Inverse(src, dst []byte) (uint, uint, error)
- func (this *ByteTransformSequence) Len() int
- func (this ByteTransformSequence) MaxEncodedLen(srcLen int) int
- func (this *ByteTransformSequence) SetSkipFlags(flags byte) bool
- func (this *ByteTransformSequence) SkipFlags() byte
- type DivSufSort
- type EXECodec
- type FSDCodec
- type LZCodec
- type LZPCodec
- type LZXCodec
- type NullTransform
- type RLT
- type ROLZCodec
- type SBRT
- type SRT
- type TextCodec
- type UTFCodec
- type ZRLT
Constants ¶
const ( // Up to 64 transforms can be declared (6 bit index) NONE_TYPE = uint64(0) // Copy BWT_TYPE = uint64(1) // Burrows Wheeler BWTS_TYPE = uint64(2) // Burrows Wheeler Scott LZ_TYPE = uint64(3) // Lempel Ziv SNAPPY_TYPE = uint64(4) // Snappy (obsolete) RLT_TYPE = uint64(5) // Run Length ZRLT_TYPE = uint64(6) // Zero Run Length MTFT_TYPE = uint64(7) // Move To Front RANK_TYPE = uint64(8) // Rank EXE_TYPE = uint64(9) // EXE codec DICT_TYPE = uint64(10) // Text codec ROLZ_TYPE = uint64(11) // ROLZ codec ROLZX_TYPE = uint64(12) // ROLZ Extra codec SRT_TYPE = uint64(13) // Sorted Rank LZP_TYPE = uint64(14) // Lempel Ziv Predict MM_TYPE = uint64(15) // Multimedia (FSD) codec LZX_TYPE = uint64(16) // Lempel Ziv Extra UTF_TYPE = uint64(17) // UTF codec PACK_TYPE = uint64(18) // Alias Codec RESERVED2 = uint64(19) // Reserved RESERVED3 = uint64(20) // Reserved RESERVED4 = uint64(21) // Reserved RESERVED5 = uint64(22) // Reserved )
const ( // SBRT_MODE_MTF mode MoveToFront SBRT_MODE_MTF = 1 // SBRT_MODE_RANK mode Rank SBRT_MODE_RANK = 2 // SBRT_MODE_TIMESTAMP mode TimeStamp SBRT_MODE_TIMESTAMP = 3 )
const ( // LF Line Feed symbol LF = byte(0x0A) // CR Carriage Return symbol CR = byte(0x0D) )
Variables ¶
This section is empty.
Functions ¶
func ComputeSuffixArray ¶
ComputeSuffixArray generates the suffix array for the given data and returns it in the 'sa' slice. Finds the suffix array sa of data[0..n-1] in {0..k-1}^n Returns the primary index if isbwt is true (0 otherwise)
func GetBWTChunks ¶
GetBWTChunks returns the number of chunks for a given block size
Types ¶
type AliasCodec ¶
type AliasCodec struct {
// contains filtered or unexported fields
}
AliasCodec is a simple codec replacing 2-byte symbols with 1-byte aliases whenever possible
func NewAliasCodec ¶
func NewAliasCodec() (*AliasCodec, error)
NewAliasCodec creates a new instance of AliasCodec
func NewAliasCodecWithCtx ¶
func NewAliasCodecWithCtx(ctx *map[string]interface{}) (*AliasCodec, error)
NewAliasCodecWithCtx creates a new instance of AliasCodec using a configuration map as parameter.
func (*AliasCodec) Forward ¶
func (this *AliasCodec) Forward(src, dst []byte) (uint, uint, error)
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*AliasCodec) Inverse ¶
func (this *AliasCodec) Inverse(src, dst []byte) (uint, uint, error)
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (AliasCodec) MaxEncodedLen ¶
func (this AliasCodec) MaxEncodedLen(srcLen int) int
MaxEncodedLen returns the max size required for the encoding output buffer
type BWT ¶
type BWT struct {
// contains filtered or unexported fields
}
BWT Burrows Wheeler Transform
func NewBWTWithCtx ¶
NewBWTWithCtx creates a new BWT instance. The number of jobs is extracted from the provided map or arguments.
func (*BWT) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*BWT) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (BWT) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
func (*BWT) PrimaryIndex ¶
PrimaryIndex returns the primary index for the n-th chunk
type BWTBlockCodec ¶
type BWTBlockCodec struct {
// contains filtered or unexported fields
}
BWTBlockCodec a codec that encapsulates a Burrows Wheeler Transform and takes care of encoding/decoding information about the primary indexes in a header.
func NewBWTBlockCodec ¶
func NewBWTBlockCodec() (*BWTBlockCodec, error)
NewBWTBlockCodec creates a new instance of BWTBlockCodec
func NewBWTBlockCodecWithCtx ¶
func NewBWTBlockCodecWithCtx(ctx *map[string]interface{}) (*BWTBlockCodec, error)
NewBWTBlockCodecWithCtx creates a new instance of BWTBlockCodec
func (*BWTBlockCodec) Forward ¶
func (this *BWTBlockCodec) Forward(src, dst []byte) (uint, uint, error)
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*BWTBlockCodec) Inverse ¶
func (this *BWTBlockCodec) Inverse(src, dst []byte) (uint, uint, error)
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (BWTBlockCodec) MaxEncodedLen ¶
func (this BWTBlockCodec) MaxEncodedLen(srcLen int) int
MaxEncodedLen returns the max size required for the encoding output buffer
type BWTS ¶
type BWTS struct {
// contains filtered or unexported fields
}
BWTS Bijective version of the Burrows-Wheeler Transform The main advantage over the regular BWT is that there is no need for a primary index (hence the bijectivity). BWTS is about 10% slower than BWT. Forward transform based on the code at https://code.google.com/p/mk-bwts/ by Neal Burns and DivSufSort (port of libDivSufSort by Yuta Mori)
func NewBWTSWithCtx ¶
NewBWTSWithCtx creates a new instance of BWTS using a configuration map as parameter.
func (*BWTS) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*BWTS) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (BWTS) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type ByteTransformSequence ¶
type ByteTransformSequence struct {
// contains filtered or unexported fields
}
ByteTransformSequence encapsulates a sequence of transforms or functions in a function
func New ¶
func New(ctx *map[string]interface{}, functionType uint64) (*ByteTransformSequence, error)
New creates a new instance of ByteTransformSequence based on the provided function type.
func NewByteTransformSequence ¶
func NewByteTransformSequence(transforms []kanzi.ByteTransform) (*ByteTransformSequence, error)
NewByteTransformSequence creates a new instance of NewByteTransformSequence containing the transforms provided as parameter.
func (*ByteTransformSequence) Forward ¶
func (this *ByteTransformSequence) Forward(src, dst []byte) (uint, uint, error)
Forward applies the function to the src and writes the result to the destination. Runs Forward on each transform in the sequence. Returns number of bytes read, number of bytes written and possibly an error.
func (*ByteTransformSequence) Inverse ¶
func (this *ByteTransformSequence) Inverse(src, dst []byte) (uint, uint, error)
Inverse applies the reverse function to the src and writes the result to the destination. Runs Inverse on each transform in the sequence. Returns number of bytes read, number of bytes written and possibly an error.
func (*ByteTransformSequence) Len ¶
func (this *ByteTransformSequence) Len() int
Len returns the number of functions in the sequence (in [0..8])
func (ByteTransformSequence) MaxEncodedLen ¶
func (this ByteTransformSequence) MaxEncodedLen(srcLen int) int
MaxEncodedLen returns the max size required for the encoding output buffer
func (*ByteTransformSequence) SetSkipFlags ¶
func (this *ByteTransformSequence) SetSkipFlags(flags byte) bool
SetSkipFlags sets the flags describing which function to skip
func (*ByteTransformSequence) SkipFlags ¶
func (this *ByteTransformSequence) SkipFlags() byte
SkipFlags returns the flags describing which function to skip (bit set to 1)
type DivSufSort ¶
type DivSufSort struct {
// contains filtered or unexported fields
}
DivSufSort main structure to compute suffix array or BWT using the algorithm developed by Yuta Mori.
func NewDivSufSort ¶
func NewDivSufSort() (*DivSufSort, error)
NewDivSufSort creates a new instance of DivSufSort
func (*DivSufSort) ComputeBWT ¶
func (this *DivSufSort) ComputeBWT(src, dst []byte, bwt []int32, indexes []uint, idxCount int) int32
ComputeBWT generates the BWT for the given data and return the primary index
func (*DivSufSort) ComputeSuffixArray ¶
func (this *DivSufSort) ComputeSuffixArray(src []byte, sa []int32)
ComputeSuffixArray generates the suffix array for the given data and returns it in the 'sa' slice.
type EXECodec ¶
type EXECodec struct {
// contains filtered or unexported fields
}
EXECodec a codec for x86 code
func NewEXECodec ¶
NewEXECodec creates a new instance of EXECodec
func NewEXECodecWithCtx ¶
NewEXECodecWithCtx creates a new instance of EXECodec using a configuration map as parameter.
func (*EXECodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error. If the source data does not represent X86 code, an error is returned.
func (*EXECodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (EXECodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type FSDCodec ¶
type FSDCodec struct {
// contains filtered or unexported fields
}
FSDCodec Fixed Step Delta codec is used to decorrelate values separated by a constant distance (step) and encode residuals
func NewFSDCodec ¶
NewFSDCodec creates a new instance of FSDCodec
func NewFSDCodecWithCtx ¶
NewFSDCodecWithCtx creates a new instance of FSDCodec using a configuration map as parameter.
func (*FSDCodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*FSDCodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*FSDCodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type LZCodec ¶
type LZCodec struct {
// contains filtered or unexported fields
}
LZCodec encapsulates an implementation of a Lempel-Ziv codec
func NewLZCodecWithCtx ¶
NewLZCodecWithCtx creates a new instance of LZCodec using a configuration map as parameter.
func (*LZCodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*LZCodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*LZCodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output mBuf
type LZPCodec ¶
type LZPCodec struct {
// contains filtered or unexported fields
}
LZPCodec an implementation of the Lempel Ziv Predict algorithm
func NewLZPCodec ¶
NewLZPCodec creates a new instance of LZXCodec
func NewLZPCodecWithCtx ¶
NewLZPCodecWithCtx creates a new instance of LZXCodec using a configuration map as parameter.
func (*LZPCodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*LZPCodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (LZPCodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type LZXCodec ¶
type LZXCodec struct {
// contains filtered or unexported fields
}
LZXCodec Simple byte oriented LZ77 implementation. It is a based on a heavily modified LZ4 with a bigger window, a bigger hash map, 3+n*8 bit literal lengths and 17 or 24 bit match lengths.
func NewLZXCodec ¶
NewLZXCodec creates a new instance of LZXCodec
func NewLZXCodecWithCtx ¶
NewLZXCodecWithCtx creates a new instance of LZXCodec using a configuration map as parameter.
func (*LZXCodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*LZXCodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (LZXCodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output mBuf
type NullTransform ¶
type NullTransform struct {
}
NullTransform is a pass through byte function
func NewNullTransform ¶
func NewNullTransform() (*NullTransform, error)
NewNullTransform creates a new instance of NullTransform
func NewNullTransformWithCtx ¶
func NewNullTransformWithCtx(ctx *map[string]interface{}) (*NullTransform, error)
NewNullTransformWithCtx creates a new instance of NullTransform using a configuration map as parameter.
func (*NullTransform) Forward ¶
func (this *NullTransform) Forward(src, dst []byte) (uint, uint, error)
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*NullTransform) Inverse ¶
func (this *NullTransform) Inverse(src, dst []byte) (uint, uint, error)
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (NullTransform) MaxEncodedLen ¶
func (this NullTransform) MaxEncodedLen(srcLen int) int
MaxEncodedLen returns the max size required for the encoding output buffer
type RLT ¶
type RLT struct {
// contains filtered or unexported fields
}
RLT a Run Length Transform with escape symbol
func NewRLTWithCtx ¶
NewRLTWithCtx creates a new instance of RLT using a configuration map as parameter.
func (*RLT) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*RLT) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (RLT) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type ROLZCodec ¶
type ROLZCodec struct {
// contains filtered or unexported fields
}
ROLZCodec Reduced Offset Lempel Ziv codec
func NewROLZCodec ¶
NewROLZCodec creates a new instance of ROLZCodec providing he log of the number of matches to check for during encoding.
func NewROLZCodecWithCtx ¶
NewROLZCodecWithCtx creates a new instance of ROLZCodec providing a context map. If the map contains a transform name set to "ROLZX" encode literals and matches using ANS. Otherwise encode literals and matches using CM and check more match positions.
func NewROLZCodecWithFlag ¶
NewROLZCodecWithFlag creates a new instance of ROLZCodec If the bool parameter is false, encode literals and matches using ANS. Otherwise encode literals and matches using CM and check more match positions.
func (*ROLZCodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*ROLZCodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*ROLZCodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type SBRT ¶
type SBRT struct {
// contains filtered or unexported fields
}
SBRT Sort By Rank Transform
func NewSBRTWithCtx ¶
NewSBRTWithCtx creates a new instance of SBRT using a configuration map as parameter.
func (*SBRT) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*SBRT) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (SBRT) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type SRT ¶
type SRT struct {
}
SRT Sorted Ranks Transform Sorted Ranks Transform is typically used after a BWT to reduce the variance of the data prior to entropy coding.
func NewSRTWithCtx ¶
NewSRTWithCtx creates a new instance of SRT using a configuration map as parameter.
func (*SRT) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*SRT) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (SRT) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type TextCodec ¶
type TextCodec struct {
// contains filtered or unexported fields
}
TextCodec is a simple one-pass text codec that replaces words with indexes. Uses a default (small) static dictionary. Generates a dynamic dictionary.
func NewTextCodec ¶
NewTextCodec creates a new instance of TextCodec
func NewTextCodecWithCtx ¶
NewTextCodecWithCtx creates a new instance of TextCodec using a configuration map as parameter.
func (*TextCodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*TextCodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*TextCodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type UTFCodec ¶
type UTFCodec struct {
// contains filtered or unexported fields
}
UTFCodec is a simple one-pass UTF8 codec that replaces code points with indexes.
func NewUTFCodec ¶
NewUTFCodec creates a new instance of UTFCodec
func NewUTFCodecWithCtx ¶
NewUTFCodecWithCtx creates a new instance of UTFCodec using a configuration map as parameter.
func (*UTFCodec) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*UTFCodec) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (UTFCodec) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
type ZRLT ¶
type ZRLT struct {
}
ZRLT Zero Run Length Transform Zero Length Encoding is a simple encoding algorithm by Wheeler closely related to Run Length Encoding. The main difference is that only runs of 0 values are processed. Also, the length is encoded in a different way (each digit in a different byte) This algorithm is well adapted to process post BWT/MTFT data
func NewZRLTWithCtx ¶
NewZRLTWithCtx creates a new instance of ZRLT using a configuration map as parameter.
func (*ZRLT) Forward ¶
Forward applies the function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (*ZRLT) Inverse ¶
Inverse applies the reverse function to the src and writes the result to the destination. Returns number of bytes read, number of bytes written and possibly an error.
func (ZRLT) MaxEncodedLen ¶
MaxEncodedLen returns the max size required for the encoding output buffer
Source Files