spm

Documentation

Index

• func AsBase64(key []byte) string
• func FromBase64(s string) ([]byte, error)
• func NmtNfkc() []byte
• func NormalizeMn(input string) string
• type Array
  • func Parse(precompiledCharsmap []byte) ([]byte, Array)
• type ArrayUnit
  • func (u ArrayUnit) HasLeaf() bool
  • func (u ArrayUnit) Label() uint
  • func (u ArrayUnit) Offset() uint
  • func (u ArrayUnit) Value() int
• type ArrayUnitTrait
• type DoubleArray
  • func NewDoubleArrayFrom(array Array) *DoubleArray
  • func (da *DoubleArray) CommonPrefixSearch(key []byte) []int
• type Precompiled
  • func NewPrecompiledFrom(data []byte) (*Precompiled, error)
  • func (m *Precompiled) NormalizeString(original string) string
  • func (m *Precompiled) Transform(chunk string) string

Functions

func AsBase64

func AsBase64(key []byte) string

func FromBase64

func FromBase64(s string) ([]byte, error)

func NmtNfkc

func NmtNfkc() []byte

func NormalizeMn

func NormalizeMn(input string) string

Types

type Array

type Array []ArrayUnit

func Parse

func Parse(precompiledCharsmap []byte) ([]byte, Array)

type ArrayUnit

type ArrayUnit struct {
	// contains filtered or unexported fields
}

func (ArrayUnit) HasLeaf

func (u ArrayUnit) HasLeaf() bool

Implement ArrayUnitTrait for ArrayUnit

func (ArrayUnit) Label

func (u ArrayUnit) Label() uint

func (ArrayUnit) Offset

func (u ArrayUnit) Offset() uint

func (ArrayUnit) Value

func (u ArrayUnit) Value() int

type ArrayUnitTrait

type ArrayUnitTrait interface {
	HasLeaf() bool
	Value() int   // isize
	Label() uint  // usize
	Offset() uint // usize
}

type DoubleArray

type DoubleArray struct {
	Array Array
}

func NewDoubleArrayFrom

func NewDoubleArrayFrom(array Array) *DoubleArray

func (*DoubleArray) CommonPrefixSearch

func (da *DoubleArray) CommonPrefixSearch(key []byte) []int

ref. https://github.com/huggingface/spm_precompiled/blob/81b911a362adef3ad3cc6d5835d2980690dbb871/src/lib.rs#L108

type Precompiled

type Precompiled struct {
	PrecompiledCharsmap []byte
	Normalized          string
	Trie                *DoubleArray
}

func NewPrecompiledFrom

func NewPrecompiledFrom(data []byte) (*Precompiled, error)

func (*Precompiled) NormalizeString

func (m *Precompiled) NormalizeString(original string) string

func (*Precompiled) Transform

func (m *Precompiled) Transform(chunk string) string