tokenizer

Documentation

Index

• Constants
• type Lattice
  • func (this *Lattice) String() string
• type Morph
  • func Tokenize(a_input string) (morphs []Morph, err error)
  • func (this Morph) Content() (content dic.Content, err error)
  • func (this Morph) String() string
• type Node
  • func (this Node) String() string
• type NodeType
  • func (this NodeType) String() string

Constants

const (
	BOSEOS = -1
)

Types

type Lattice

type Lattice struct {
	// contains filtered or unexported fields
}

func (*Lattice) String

func (this *Lattice) String() string

type Morph

type Morph struct {
	Start, End int
	Surface    string
	// contains filtered or unexported fields
}

func Tokenize

func Tokenize(a_input string) (morphs []Morph, err error)

func (Morph) Content

func (this Morph) Content() (content dic.Content, err error)

func (Morph) String

func (this Morph) String() string

type Node

type Node struct {
	// contains filtered or unexported fields
}

func (Node) String

func (this Node) String() string

type NodeType

type NodeType int

const (
	KNOWN NodeType = iota
	UNKNOWN
	INSERTED
	USER
)

func (NodeType) String

func (this NodeType) String() string