tokenizer

Aug 11, 2018

Changes in this version

type Tokenizer

+ func NewWithDicPath(p string) (Tokenizer, error)

Sep 7, 2017

Changes in this version

type Dic

+ func NewDicSimple(path string) (Dic, error)

+ func SysDicIPASimple() Dic

+ func SysDicSimple() Dic

+ func SysDicUniSimple() Dic

Mar 20, 2016

Changes in this version

+ type UserDicRecord struct

+ Pos string

+ Text string

+ Tokens []string

+ Yomi []string

+ type UserDicRecords []UserDicRecord

+ func NewUserDicRecords(r io.Reader) (UserDicRecords, error)

+ func (u UserDicRecords) Len() int

+ func (u UserDicRecords) Less(i, j int) bool

+ func (u UserDicRecords) NewUserDic() (UserDic, error)

+ func (u UserDicRecords) Swap(i, j int)

Mar 17, 2016

Changes in this version

type Token

+ func (t Token) Pos() string

Jan 12, 2016

Changes in this version

type Dic

+ func SysDicIPA() Dic

+ func SysDicUni() Dic

Dec 23, 2015

Changes in this version

type Tokenizer

+ func (t Tokenizer) AnalyzeGraph(input string, mode TokenizeMode, w io.Writer) (tokens []Token)

Sep 28, 2015

Changes in this version

+ type Dic struct

+ func NewDic(path string) (Dic, error)

+ func SysDic() Dic

+ type Token struct

+ Class TokenClass

+ End int

+ ID int

+ Start int

+ Surface string

+ func (t Token) Features() (features []string)

+ func (t Token) String() string

+ type TokenClass lattice.NodeClass

+ func (c TokenClass) String() string

+ type TokenizeMode int

+ const BosEosID

+ const Extended

+ const Normal

+ const Search

type Tokenizer

+ func New() (t Tokenizer)

+ func NewWithDic(d Dic) (t Tokenizer)

+ func (t *Tokenizer) SetDic(d Dic)

+ func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token)

+ func (t Tokenizer) Dot(input string, w io.Writer) (tokens []Token)

+ type UserDic struct

+ func NewUserDic(path string) (UserDic, error)

Jul 6, 2014

Changes in this version

+ func NewLattice() *lattice

type Morph

+ Class NodeClass

+ Id int

+ type NodeClass byte

+ const DUMMY

+ func (this NodeClass) String() string

+ type NodePool struct

+ func NewNodePool(a_size int) *NodePool

+ type Tokenizer struct

+ func NewTokenizer() *Tokenizer

+ func (this *Tokenizer) SetUserDic(a_udic *dic.UserDic)

+ func (this *Tokenizer) Tokenize(a_input string) (morphs []Morph, err error)

Jun 26, 2014

Changes in this version

+ const BOSEOS

+ type Lattice struct

+ func (this *Lattice) String() string

+ type Morph struct

+ End int

+ Start int

+ Surface string

+ func Tokenize(a_input string) (morphs []Morph, err error)

+ func (this Morph) Content() (content dic.Content, err error)

+ func (this Morph) String() string

+ type Node struct

+ func (this Node) String() string

+ type NodeType int

+ const INSERTED

+ const KNOWN

+ const UNKNOWN

+ const USER

+ func (this NodeType) String() string