internal

Documentation

Index

• Constants
• Variables
• func ComputeFirstOrderEntropy1024(blockLen int, histo []int) int
• func ComputeHistogram(block []byte, freqs []int, isOrder0, withTotal bool)
• func ComputeJobsPerTask(jobsPerTask []uint, jobs, tasks uint) ([]uint, error)
• func GetMagicType(src []byte) uint
• func IsDataCompressed(magic uint) bool
• func IsDataExecutable(magic uint) bool
• func IsDataMultimedia(magic uint) bool
• func Log2(x uint32) (uint32, error)
• func Log2NoCheck(x uint32) uint32
• func Log2ScaledBy1024(x uint32) (uint32, error)
• func Squash(d int) int
• type BufferStream
  • func NewBufferStream(args ...[]byte) *BufferStream
  • func (this *BufferStream) Available() int
  • func (this *BufferStream) Close() error
  • func (this *BufferStream) Len() int
  • func (this *BufferStream) Read(b []byte) (int, error)
  • func (this *BufferStream) Write(b []byte) (int, error)
• type DataType
  • func DetectSimpleType(count int, freqs0 []int) DataType
• type Magic

Constants

const (
	NO_MAGIC     = 0
	JPG_MAGIC    = 0xFFD8FFE0
	GIF_MAGIC    = 0x47494638
	PDF_MAGIC    = 0x25504446
	ZIP_MAGIC    = 0x504B0304 // Works for jar & office docs
	LZMA_MAGIC   = 0x377ABCAF // Works for 7z  37 7A BC AF 27 1C
	PNG_MAGIC    = 0x89504E47
	ELF_MAGIC    = 0x7F454C46
	MAC_MAGIC32  = 0xFEEDFACE
	MAC_CIGAM32  = 0xCEFAEDFE
	MAC_MAGIC64  = 0xFEEDFACF
	MAC_CIGAM64  = 0xCFFAEDFE
	ZSTD_MAGIC   = 0x28B52FFD
	BROTLI_MAGIC = 0x81CFB2CE
	RIFF_MAGIC   = 0x52494646 // WAV, AVI, WEBP
	CAB_MAGIC    = 0x4D534346
	FLAC_MAGIC   = 0x664C6143
	XZ_MAGIC     = 0xFD377A58 // FD 37 7A 58 5A 00
	KNZ_MAGIC    = 0x4B414E5A

	BZIP2_MAGIC   = 0x425A68
	MP3_ID3_MAGIC = 0x494433

	GZIP_MAGIC = 0x1F8B
	BMP_MAGIC  = 0x424D
	WIN_MAGIC  = 0x4D5A
	PBM_MAGIC  = 0x5034 // bin only
	PGM_MAGIC  = 0x5035 // bin only
	PPM_MAGIC  = 0x5036 // bin only

)

Variables

var (
	// LOG2 is an array with 256 elements: int(Math.log2(x-1))
	LOG2 = [...]uint32{}/* 256 elements not displayed */

	// LOG2_4096 is an array with 256 elements: 4096*Math.log2(x)
	LOG2_4096 = [...]uint32{}/* 257 elements not displayed */

	// SQUASH contains p = 1/(1 + exp(-d)), d scaled by 8 bits, p scaled by 12 bits
	SQUASH [4096]int

	// STRETCH is the inverse of squash. d = ln(p/(1-p)), d scaled by 8 bits, p by 12 bits.
	// d has range -2047 to 2047 representing -8 to 8. p in [0..4095].
	STRETCH [4096]int
)

Functions

func ComputeFirstOrderEntropy1024

func ComputeFirstOrderEntropy1024(blockLen int, histo []int) int

ComputeFirstOrderEntropy1024 computes the order 0 entropy of the block and scales the result by 1024 (result in the [0..1024] range) Fills in the histogram with order 0 frequencies. Incoming array size must be at least 256

func ComputeHistogram

func ComputeHistogram(block []byte, freqs []int, isOrder0, withTotal bool)

ComputeHistogram computes the order 0 or order 1 histogram for the input block and returns it in the 'freqs' slice. If withTotal is true, the last spot in each frequencies order 0 array is for the total (each order 0 frequency slice must be of length 257 in this case).

func ComputeJobsPerTask

func ComputeJobsPerTask(jobsPerTask []uint, jobs, tasks uint) ([]uint, error)

ComputeJobsPerTask computes the number of jobs associated with each task given a number of jobs available and a number of tasks to perform. The provided 'jobsPerTask' slice is returned as result.

func GetMagicType

func GetMagicType(src []byte) uint

GetMagicType checks the first bytes of the slice against a list of common magic values

func IsDataCompressed

func IsDataCompressed(magic uint) bool

IsDataCompressed return true if the provided magic parameter corresponds to a known compressed data type.

func IsDataExecutable

func IsDataExecutable(magic uint) bool

IsDataExecutable return true if the provided magic parameter corresponds to a known executable data type.

func IsDataMultimedia

func IsDataMultimedia(magic uint) bool

IsDataMultimedia return true if the provided magic parameter corresponds to a known multimedia data type.

func Log2

func Log2(x uint32) (uint32, error)

Log2 returns a fast, integer rounded value for log2(x)

func Log2NoCheck

func Log2NoCheck(x uint32) uint32

Log2NoCheck does the same as Log2() minus a null check on input value

func Log2ScaledBy1024

func Log2ScaledBy1024(x uint32) (uint32, error)

Log2ScaledBy1024 returns 1024 * log2(x). Max error is around 0.1%

func Squash

func Squash(d int) int

Squash returns p = 1/(1 + exp(-d)), d scaled by 8 bits, p scaled by 12 bits

Types

type BufferStream

type BufferStream struct {
	// contains filtered or unexported fields
}

BufferStream a closable read/write stream of bytes backed by a bytes.Buffer

func NewBufferStream

func NewBufferStream(args ...[]byte) *BufferStream

NewBufferStream creates a new instance of BufferStream

func (*BufferStream) Available

func (this *BufferStream) Available() int

Available returns the number of bytes available for read

func (*BufferStream) Close

func (this *BufferStream) Close() error

Close makes the stream unavailable for future reads or writes.

func (*BufferStream) Len

func (this *BufferStream) Len() int

Len returns the size of the stream

func (*BufferStream) Read

func (this *BufferStream) Read(b []byte) (int, error)

Read returns an error if the stream is closed, otherwise reads data from the internal buffer at the read offset position. Returns the number of bytes read.

func (*BufferStream) Write

func (this *BufferStream) Write(b []byte) (int, error)

Write returns an error if the stream is closed, otherwise writes the given data to the internal buffer (growing the buffer as needed). Returns the number of bytes written.

type DataType

type DataType int

DataType captures the type of input data

const (
	DT_UNDEFINED      DataType = 0
	DT_TEXT           DataType = 1
	DT_MULTIMEDIA     DataType = 2
	DT_EXE            DataType = 3
	DT_NUMERIC        DataType = 4
	DT_BASE64         DataType = 5
	DT_DNA            DataType = 6
	DT_BIN            DataType = 7
	DT_UTF8           DataType = 8
	DT_SMALL_ALPHABET DataType = 9
)

func DetectSimpleType

func DetectSimpleType(count int, freqs0 []int) DataType

type Magic

type Magic struct {
}

Magic is a utility to detect common header magic values