tokenizer

Documentation

Overview

Package tokenizer is a japanese morphological analyzer library.

Index

• Constants
• type Dic
  • func NewDic(path string) (Dic, error)
  • func SysDic() Dic
  • func SysDicIPA() Dic
  • func SysDicUni() Dic
• type Token
  • func (t Token) Features() (features []string)
  • func (t Token) Pos() string
  • func (t Token) String() string
• type TokenClass
  • func (c TokenClass) String() string
• type TokenizeMode
• type Tokenizer
  • func New() (t Tokenizer)
  • func NewWithDic(d Dic) (t Tokenizer)
  • func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token)
  • func (t Tokenizer) AnalyzeGraph(input string, mode TokenizeMode, w io.Writer) (tokens []Token)
  • func (t Tokenizer) Dot(input string, w io.Writer) (tokens []Token)
  • func (t *Tokenizer) SetDic(d Dic)
  • func (t *Tokenizer) SetUserDic(d UserDic)
  • func (t Tokenizer) Tokenize(input string) []Token
• type UserDic
  • func NewUserDic(path string) (UserDic, error)
• type UserDicRecord
• type UserDicRecords
  • func NewUserDicRecords(r io.Reader) (UserDicRecords, error)
  • func (u UserDicRecords) Len() int
  • func (u UserDicRecords) Less(i, j int) bool
  • func (u UserDicRecords) NewUserDic() (UserDic, error)
  • func (u UserDicRecords) Swap(i, j int)

Constants

const (
	// DUMMY represents the dummy token.
	DUMMY = TokenClass(lattice.DUMMY)
	// KNOWN represents the token in the dictionary.
	KNOWN = TokenClass(lattice.KNOWN)
	// UNKNOWN represents the token which is not in the dictionary.
	UNKNOWN = TokenClass(lattice.UNKNOWN)
	// USER represents the token in the user dictionary.
	USER = TokenClass(lattice.USER)
)

Types

type Dic

type Dic struct {
	// contains filtered or unexported fields
}

Dic represents a dictionary.

func NewDic

func NewDic(path string) (Dic, error)

NewDic loads a dictionary from a file.

func SysDic

func SysDic() Dic

SysDic returns the system dictionary (IPA dictionary).

func SysDicIPA

func SysDicIPA() Dic

SysDicIPA returns the IPA dictionary as the system dictionary.

func SysDicUni

func SysDicUni() Dic

SysDicUni returns the UniDic dictionary as the system dictionary.

type Token

type Token struct {
	ID      int
	Class   TokenClass
	Start   int
	End     int
	Surface string
	// contains filtered or unexported fields
}

Token represents a morph of a sentence.

func (Token) Features

func (t Token) Features() (features []string)

Features returns contents of a token.

func (Token) Pos

func (t Token) Pos() string

Pos returns the first element of features.

func (Token) String

func (t Token) String() string

String returns a string representation of a token.

type TokenClass

type TokenClass lattice.NodeClass

TokenClass represents the token type.

func (TokenClass) String

func (c TokenClass) String() string

type TokenizeMode

type TokenizeMode int

TokenizeMode represents a mode of tokenize.

const (
	// Normal is the normal tokenize mode.
	Normal TokenizeMode = iota + 1
	// Search is the tokenize mode for search.
	Search
	// Extended is the experimental tokenize mode.
	Extended
	// BosEosID means the beginning a sentence or the end of a sentence.
	BosEosID = lattice.BosEosID
)

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

Tokenizer represents morphological analyzer.

func New

func New() (t Tokenizer)

New create a default tokenize.

func NewWithDic

func NewWithDic(d Dic) (t Tokenizer)

NewWithDic create a tokenizer with specified dictionary.

func (Tokenizer) Analyze

func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token)

Analyze tokenizes a sentence in the specified mode.

func (Tokenizer) AnalyzeGraph

func (t Tokenizer) AnalyzeGraph(input string, mode TokenizeMode, w io.Writer) (tokens []Token)

AnalyzeGraph returns morphs of a sentence and exports a lattice graph to dot format.

func (Tokenizer) Dot

func (t Tokenizer) Dot(input string, w io.Writer) (tokens []Token)

Dot returns morphs of a sentence and exports a lattice graph to dot format in standard tokenize mode.

func (*Tokenizer) SetDic

func (t *Tokenizer) SetDic(d Dic)

SetDic sets dictionary to dic.

func (*Tokenizer) SetUserDic

func (t *Tokenizer) SetUserDic(d UserDic)

SetUserDic sets user dictionary to udic.

func (Tokenizer) Tokenize

func (t Tokenizer) Tokenize(input string) []Token

Tokenize analyze a sentence in standard tokenize mode.

type UserDic

type UserDic struct {
	// contains filtered or unexported fields
}

UserDic represents a user dictionary.

func NewUserDic

func NewUserDic(path string) (UserDic, error)

NewUserDic build a user dictionary from a file.

type UserDicRecord

type UserDicRecord struct {
	Text   string   `json:"text"`
	Tokens []string `json:"tokens"`
	Yomi   []string `json:"yomi"`
	Pos    string   `json:"pos"`
}

UserDicRecord represents a record of the user dictionary file format.

type UserDicRecords

type UserDicRecords []UserDicRecord

UserDicRecords represents user dictionary data.

func NewUserDicRecords

func NewUserDicRecords(r io.Reader) (UserDicRecords, error)

NewUserDicRecords loads user dictionary data from io.Reader.

func (UserDicRecords) Len

func (u UserDicRecords) Len() int

func (UserDicRecords) Less

func (u UserDicRecords) Less(i, j int) bool

func (UserDicRecords) NewUserDic

func (u UserDicRecords) NewUserDic() (UserDic, error)

NewUserDic builds a user dictionary.

func (UserDicRecords) Swap

func (u UserDicRecords) Swap(i, j int)