compact

package module
v0.0.0-...-933bb2f Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 20, 2023 License: MIT Imports: 19 Imported by: 1

README

compact

Compact binary encoding to minimize storage size and maximize read speed

Datatypes are reduced for datatypes 0 - 100 // 1 byte -- scores, type byte 0.0 - 1.0000 // 2 bytes (percent that goes to 1.000 @ 4 sig dig), type FP1 0.00 - 655.00 // 2 byte, type F16 lat, lon // 4 bytes, type float32

Presorted

The file is sorted by latitude, longitude ascending

Documentation

Index

Constants

View Source 
const F16_max = math.MaxUint16 / f16Mul
View Source 
const F32_max = math.MaxFloat32
View Source 
const F64_max = math.MaxFloat64
View Source 
const HolderHeaderSize = int(unsafe.Sizeof(HolderHeader{}))
View Source 
const MaxKeyCount = 65535
View Source 
const NX32_max = math.MaxInt32
View Source 
const NX64_max = math.MaxInt64

Variables

View Source 
var (
	ErrTooSmall = errors.New("buffer is too small")
	ErrExceeds  = errors.New("value exceeds max size")
)
View Source 
var ErrKeyOverload = fmt.Errorf("key cardinality exceeeds max size")
View Source 
var ErrNotFound = errors.New("not found")

Functions

func AbsPath 

func AbsPath(filename string) string

func AnalyzeFile 

func AnalyzeFile(filename string)

func EstimateRecordCount 

func EstimateRecordCount(filename string) (int, error)

EstimateRecordCount samples the file to estimate how many lines

func EvaluateFields 

func EvaluateFields(filename string) (int, error)

EvaluateFields establishs what type the columns represent

func Exists 

func Exists(filename string) bool

func FMe 

func FMe[T Number](b []byte, num T) error

func FetchByte 

func FetchByte(b []byte) (interface{}, error)

FetchByte reads a byte from the buffer and returns an int

func FetchF1 

func FetchF1(b []byte) (interface{}, error)

func FetchF16 

func FetchF16(b []byte) (interface{}, error)

func FetchF32 

func FetchF32(b []byte) (interface{}, error)

func FetchF64 

func FetchF64(b []byte) (interface{}, error)

func FetchI16 

func FetchI16(b []byte) (interface{}, error)

func FetchI32 

func FetchI32(b []byte) (interface{}, error)

func FetchI64 

func FetchI64(b []byte) (interface{}, error)

func FileSize 

func FileSize(filename string) (int64, error)

func FileWriter 

func FileWriter(filename string, bufSize int) (io.WriteCloser, error)

FileWriter creates in io.WriteCloser that buffers and optionally compresses the output

func GobDump 

func GobDump(filename string, obj interface{}) error

GobDump saves the object in a gzipped GOB encoded file

func GobLoad 

func GobLoad(filename string, obj interface{}) error

GobLoad populates the object from a gzipped GOB encoded file

func JDump 

func JDump(obj interface{})

func LineCount 

func LineCount(filename string) (int, error)

func LineReader 

func LineReader(r io.Reader, fn func(string) error) error

func LoadCSV 

func LoadCSV(filename string, fn func([]string) error) error

func LoadCSVHeaders 

func LoadCSVHeaders(filename string, skip int, fn func([]string) error) error

func LoadLines 

func LoadLines(filename string, fn func(string) error) error

func MagicHead 

func MagicHead() [4]byte

MagicHead is the 4 byte header prefix to identify the file type

func NewFileReader 

func NewFileReader(filename string) (io.ReadCloser, error)

func RoundUp 

func RoundUp(n int) int

RoundUp expects to be 32bit or less

func SerialByte 

func SerialByte(b []byte, x interface{}) error

SerialByte writes the value of x into a byte in b

func Serialized 

func Serialized[T ~int | ~float64](b []byte, x interface{}) error

Types

type AColumn 

type AColumn struct {
	Name       string
	Keys       map[string]int
	FMax, FMin float64
	IMax, IMin int64
}

type CSVWriter 

type CSVWriter interface {
	WriteCSV(io.Writer) error
}

type Column 

type Column struct {
	Name      string
	Width     int
	Precision int
}

type ColumnKey 

type ColumnKey struct {
	// contains filtered or unexported fields
}

func (*ColumnKey) Column 

func (ck *ColumnKey) Column() string

func (*ColumnKey) Evaluate 

func (ck *ColumnKey) Evaluate(s string) error

type Columnar 

type Columnar interface {
	Evaluate(string) error
	Column() string
}

type DataType 

type DataType int
const (
	DT_Unknown DataType = iota
	DT_Byte
	DT_F1
	DT_F16
	DT_F32
	DT_F64
	DT_X16
	DT_X32
	DT_X64
)

type Decipher 

type Decipher func([]byte) (interface{}, error)

func FetchString 

func FetchString(size int) Decipher

type Encipher 

type Encipher func([]byte, interface{}) error

type F16 

type F16 uint16

F16 allows up to 2 decimal places on a number < 655 (as we pack) it times 100 as an int NOTE: the number is expected to be *positive*

func F32ToF16 

func F32ToF16(v float32) F16

func (*F16) Decode 

func (f *F16) Decode(b []byte) error

func (F16) Encode 

func (f F16) Encode(b []byte) error

func (F16) Float32 

func (f F16) Float32() float32

func (*F16) FromFloat64 

func (f *F16) FromFloat64(v float64) error

func (*F16) Input 

func (f *F16) Input(s string) error

func (F16) Size 

func (f F16) Size() int

func (F16) String 

func (f F16) String() string

type F32 

type F32 float32

func (*F32) Decode 

func (f *F32) Decode(b []byte) error

func (F32) Encode 

func (f F32) Encode(b []byte) error

func (*F32) FromFloat64 

func (f *F32) FromFloat64(v float64) error

func (*F32) Input 

func (f *F32) Input(s string) error

func (F32) Size 

func (f F32) Size() int

func (F32) String 

func (f F32) String() string

type F64 

type F64 float64

func (*F64) Decode 

func (f *F64) Decode(b []byte) error

func (F64) Encode 

func (f F64) Encode(b []byte) error

func (*F64) FromFloat64 

func (f *F64) FromFloat64(v float64) error

func (*F64) Input 

func (f *F64) Input(s string) error

func (F64) Size 

func (f F64) Size() int

func (F64) String 

func (f F64) String() string

type FP1 

type FP1 uint16

FP1 allows up to 2 decimal places on a positive number < 655 Storage is 2 bytes

func F32ToFP1 

func F32ToFP1(v float32) FP1

func (*FP1) Decode 

func (f *FP1) Decode(b []byte) error

func (FP1) Encode 

func (f FP1) Encode(b []byte) error

func (FP1) Float32 

func (f FP1) Float32() float32

func (FP1) MarshalJSON 

func (f FP1) MarshalJSON() ([]byte, error)

func (FP1) Size 

func (f FP1) Size() int

func (FP1) String 

func (f FP1) String() string

type FWriter 

type FWriter struct {
	// contains filtered or unexported fields
}

func (*FWriter) Close 

func (fw *FWriter) Close() error

func (*FWriter) Write 

func (fw *FWriter) Write(b []byte) (int, error)

type FileBuf 

type FileBuf struct {
	// contains filtered or unexported fields
}

func NewFileBuf 

func NewFileBuf(filename string, size int) (*FileBuf, error)

func (*FileBuf) Close 

func (fb *FileBuf) Close() error

func (*FileBuf) Write 

func (fb *FileBuf) Write(p []byte) (int, error)

type FileInfo 

type FileInfo struct {
	Source    string // source of the data
	Bin       string // binary encoded version of the file
	Headers   int    // count of header lines in source
	Columns   []string
	SampleRow []string
	Stats     LineStats
}

func FileScan 

func FileScan(filename string) (FileInfo, error)

type FileReader 

type FileReader struct {
	// contains filtered or unexported fields
}

func (FileReader) Close 

func (br FileReader) Close() error

func (FileReader) Read 

func (br FileReader) Read(buf []byte) (int, error)

type Handler 

type Handler func([]byte) error

type HeadFlag 

type HeadFlag int
const (
	HeadsUnknown HeadFlag = iota
	Heads64      HeadFlag = 1 << iota
	HeadsLon              // Data is lon, lat
)

type Holder 

type Holder struct {
	RecordWidth int // binary record width
	Columns     []string
	// contains filtered or unexported fields
}

Holder manages a binary packed table layout on disk is ordered to do word alignment if possible

func NewHolder 

func NewHolder(columns ...string) (*Holder, error)

NewHolder takes columns with optional formatting/range info columnName:maxSize:precision if precision == "s" the field is a fixed with string of maxSize width

func (Holder) Importer 

func (h Holder) Importer(w io.Writer) func(ss ...string) error

Importer makes a function that persists a slice of strings to disk

func (Holder) MakeEncoder 

func (h Holder) MakeEncoder() func(b []byte) error

func (*Holder) NewRecord 

func (h *Holder) NewRecord() *RecordHolder

func (Holder) Show 

func (h Holder) Show()

func (*Holder) UnMarshal 

func (h *Holder) UnMarshal(b []byte) error

type HolderHeader 

type HolderHeader struct {
	Magic   [4]byte
	Version uint32
	Start   uint32
	Flags   HeadFlag
}

func (*HolderHeader) Decode 

func (hh *HolderHeader) Decode(b []byte) error

func (HolderHeader) Encode 

func (hh HolderHeader) Encode(b []byte) error
func (fp *FireJulySrc) Encode(buf []byte) error {
	if len(buf) < fp.Size() {
		log.Printf("buffer size: %d -- we need: %d", len(buf), fp.Size())
		return io.EOF
	}
	const off = 4
	idx := 0
	binary.LittleEndian.PutUint32(buf[idx:], math.Float32bits(float32(fp.Lat)))
	idx += 4
	binary.LittleEndian.PutUint32(buf[idx:], math.Float32bits(float32(fp.Lon)))
	idx += 4
	binary.LittleEndian.PutUint32(buf[idx:], fp.Score)

func (HolderHeader) Equal 

func (hh HolderHeader) Equal(x HolderHeader) error

type HoldsByte 

type HoldsByte byte

func (*HoldsByte) Decode 

func (h *HoldsByte) Decode(b []byte) error

func (HoldsByte) Encode 

func (h HoldsByte) Encode(b []byte) error

func (*HoldsByte) Input 

func (h *HoldsByte) Input(s string) error

func (HoldsByte) Size 

func (h HoldsByte) Size() int

func (HoldsByte) String 

func (h HoldsByte) String() string

type HoldsF16 

type HoldsF16 F16

type HoldsI32 

type HoldsI32 uint32

func (*HoldsI32) Decode 

func (h *HoldsI32) Decode(b []byte) error

func (HoldsI32) Encode 

func (h HoldsI32) Encode(b []byte) error

func (*HoldsI32) Input 

func (h *HoldsI32) Input(s string) error

type HoldsI64 

type HoldsI64 int64

func (*HoldsI64) Decode 

func (h *HoldsI64) Decode(b []byte) error

func (HoldsI64) Encode 

func (h HoldsI64) Encode(b []byte) error

func (*HoldsI64) Input 

func (h *HoldsI64) Input(s string) error

type HoldsText 

type HoldsText struct {
	// contains filtered or unexported fields
}

func NewText 

func NewText(size int) HoldsText

func (*HoldsText) Decode 

func (h *HoldsText) Decode(b []byte) error

func (HoldsText) Encode 

func (h HoldsText) Encode(b []byte) error

func (HoldsText) GoString 

func (h HoldsText) GoString() string

func (*HoldsText) Input 

func (h *HoldsText) Input(s string) error

func (HoldsText) String 

func (h HoldsText) String() string

String assumes that the text is padded with spaces

type Kind 

type Kind int
const (
	DTUnknown Kind = iota
	DTI64     Kind = 1 << iota
	DTI32
	DTI16
	DTF64
	DTF32
	DTF1
	DTF16
	DTByte
	DTOmitEmpty
)

type LineStats 

type LineStats struct {
	SampleSize int
	Min        int
	Max        int
	Avg        int
}

func LineLen 

func LineLen(r io.Reader, count int, headers int) (LineStats, error)

LineLen will give stats on CSV record size for estimating storage needs

type LiveHeader 

type LiveHeader struct {
	Head    HolderHeader
	Columns []Column
}

func MakeHeader 

func MakeHeader(columns ...Column) LiveHeader

func ReadHeader 

func ReadHeader(r io.Reader) (*LiveHeader, error)

type NX32 

type NX32 int32

func (*NX32) Decode 

func (f *NX32) Decode(b []byte) error

func (NX32) Encode 

func (f NX32) Encode(b []byte) error

func (*NX32) Input 

func (f *NX32) Input(s string) error

func (NX32) Size 

func (f NX32) Size() int

func (NX32) String 

func (f NX32) String() string

type NX64 

type NX64 int64

func (*NX64) Decode 

func (f *NX64) Decode(b []byte) error

func (NX64) Encode 

func (f NX64) Encode(b []byte) error

func (*NX64) Input 

func (f *NX64) Input(s string) error

func (NX64) Size 

func (f NX64) Size() int

func (NX64) String 

func (f NX64) String() string

type Number 

type Number interface {
	constraints.Integer | constraints.Float
}

type Odds 

type Odds uint16

Odds represents a number between 0.0 and 1.0 with accuracy of 4 decimal places It is able to do so in 2 bytes, for compactness

func SetOdds 

func SetOdds(v float32) Odds

func (*Odds) Decode 

func (oo *Odds) Decode(b []byte) error

func (*Odds) Encode 

func (oo *Odds) Encode(b []byte) error

func (Odds) Float32 

func (oo Odds) Float32() float32

func (*Odds) Get 

func (oo *Odds) Get() float32

func (*Odds) MarshalBinary 

func (oo *Odds) MarshalBinary() ([]byte, error)

func (*Odds) MarshalJSON 

func (oo *Odds) MarshalJSON() ([]byte, error)

func (*Odds) Set 

func (oo *Odds) Set(v float32)

func (*Odds) String 

func (oo *Odds) String() string

type RecFun 

type RecFun func() Record

var _ Record = (*HoldsF16)(nil) //.Record() var _ MyInterface = (*MyType)(nil)

type RecMux 

type RecMux interface {
	Mux() Record
}

type Record 

type Record interface {
	Decode([]byte) error
	Encode([]byte) error
	Size() int
	String() string
}

func Makers 

func Makers(in string) Record

func NewRig()

type RecordHolder 

type RecordHolder struct {
	// contains filtered or unexported fields
}

func (*RecordHolder) Decode 

func (rh *RecordHolder) Decode(b []byte) error

Decode converts a byte slice to a record

func (RecordHolder) MarshalJSON 

func (rh RecordHolder) MarshalJSON() ([]byte, error)

type Recorder 

type Recorder interface {
	GetRec() Record
}

Scan for header count, reset file

type Runner 

type Runner interface {
	Handle([]byte)
	Columns() []string
	MarshalJSON() ([]byte, error)
}

type Size 

type Size int

type Kind int

type Unknown 

type Unknown struct {
	Columns []string
	Rows    [][]string
	Index   map[string]int
}

func LoadUnknown 

func LoadUnknown(filename string, strip ...string) (Unknown, error)

func (Unknown) Show 

func (u Unknown) Show(columns ...string)

type What 

type What struct {
	// contains filtered or unexported fields
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.