wordseg

Documentation

Overview

******************************************************************************

• DANIEL'S ALGORITHM IMPLEMENTAIONS *
• /\ | _ _ ._ o _|_ |_ ._ _ _
• /--\ | (_| (_) | | |_ | | | | | _>
• _| *
• CHINESE WORD SEGMENTATION *
• Features:
• 1. based on Hidden Markov Model
• 2. definition the states in B,E,M,S
• 3. solve the hidden states with viterbi algorithm. *
• http://en.wikipedia.org/wiki/Hidden_Markov_model
• http://en.wikipedia.org/wiki/Viterbi_algorithm * *****************************************************************************

Index

• Constants
• Variables
• func Split(str string) []string
• type WordEP
• type Wordseg
  • func Default() *Wordseg
  • func NewWordseg() *Wordseg
  • func (ws *Wordseg) AddWord(word string, count uint32)
  • func (ws *Wordseg) LoadFile(path string)
  • func (ws *Wordseg) Split(str string) []string
  • func (ws *Wordseg) Update()

Constants

const (
	SINGLE = 0
	BEGIN  = 1
	MIDDLE = 2
	END    = 3
)

B,E,M,S

Variables

var (
	// the start probability of state
	// SP = [4]float64{0.23101714745, 0.76898285255, 0.0, 0.0}
	SP = [4]float64{0, 0, 0, 0}
	TP = [4][4]float64{
		{0, 0, 0, 0},
		{0, 0, 0, 0},
		{0, 0, 0, 0},
		{0, 0, 0, 0},
	}

	IGNORES = ",. ;。，；"
)

var (
	SPCount = [4]float64{0, 0, 0, 0}
	TPCount = [4][4]float64{
		{0, 0, 0, 0},
		{0, 0, 0, 0},
		{0, 0, 0, 0},
		{0, 0, 0, 0}}
)

Functions

func Split

func Split(str string) []string

---------------------------------------------------------- default word segmentation

Types

type WordEP

type WordEP struct {
	EP [4]float64 // the emission probability in each state
	SC [4]uint32  // the count of appearence in each state
}

------------------------------------------------------ 单个字的观察

type Wordseg

type Wordseg struct {
	// contains filtered or unexported fields
}

func Default

func Default() *Wordseg

---------------------------------------------------------- return default wordseg

func NewWordseg

func NewWordseg() *Wordseg

func (*Wordseg) AddWord

func (ws *Wordseg) AddWord(word string, count uint32)

func (*Wordseg) LoadFile

func (ws *Wordseg) LoadFile(path string)

func (*Wordseg) Split

func (ws *Wordseg) Split(str string) []string

------------------------------------------------------ words split

func (*Wordseg) Update

func (ws *Wordseg) Update()

----------------------------------------------------- should be called when finished adding words