kiwi

README

kiwigo

https://github.com/bab2min/Kiwi for go

Dependencies

• Install Kiwi

check out how to install Kiwi here

Example script (install using the helper script)

make install-kiwi

Example script (install from source)

git clone git@github.com:bab2min/Kiwi.git
cd Kiwi
git submodule sync
git submodule update --init --recursive
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ../
make
make install
ldconfig

Documentation

Overview

Package kiwi is a Go binding for Kiwi (https://github.com/bab2min/Kiwi) project.

Example

package main

import (
	"fmt"

	kiwi "github.com/sangx2/kiwigo"
)

func main() {
	kb := kiwi.NewBuilder("./ModelGenerator", 1 /*=numThread*/, kiwi.KIWI_BUILD_INTEGRATE_ALLOMORPH /*=options*/)
	kb.AddWord("코딩냄비", "NNP", 0)

	k := kb.Build()
	defer k.Close() // don't forget to Close()!

	results, _ := k.Analyze("안녕하세요 코딩냄비입니다. 부글부글.", 1 /*=topN*/, kiwi.KIWI_MATCH_ALL)
	fmt.Println(results)
}

Output:

[{[{0 NNG 안녕} {2 XSA 하} {4 EP 시} {3 EC 어요} {6 NNP 코딩냄비} {10 VCP 이} {11 EF ᆸ니다} {13 SF .} {15 NNP 부글부} {18 NNG 글} {19 SF .}] -69.74997}]

Index

• func KiwiClearError()
• func KiwiError() string
• func KiwiReaderImpl(lineNumber C.int, buffer *C.char, userData unsafe.Pointer) C.int
• func KiwiVersion() string
• type AnalyzeOption
• type BuildOption
• type Kiwi
  • func New(modelPath string, numThread int, options BuildOption) *Kiwi
  • func (k *Kiwi) Analyze(text string, topN int, options AnalyzeOption) ([]TokenResult, error)
  • func (k *Kiwi) Close() int
  • func (k *Kiwi) SplitSentence(text string, options AnalyzeOption) ([]SplitResult, error)
• type KiwiBuilder
  • func NewBuilder(modelPath string, numThread int, options BuildOption) *KiwiBuilder
  • func (kb *KiwiBuilder) AddWord(word string, pos POSType, score float32) int
  • func (kb *KiwiBuilder) Build() *Kiwi
  • func (kb *KiwiBuilder) Close() int
  • func (kb *KiwiBuilder) ExtractWords(readSeeker io.ReadSeeker, minCnt int, maxWordLen int, minScore float32, ...) ([]WordInfo, error)
  • func (kb *KiwiBuilder) LoadDict(dictPath string) int
• type POSType
  • func ParsePOSType(t string) (POSType, error)
• type SplitResult
• type TokenInfo
• type TokenResult
• type WordInfo

Examples

• Package

Functions

func KiwiClearError

func KiwiClearError()

KiwiClearError clear error.

func KiwiError

func KiwiError() string

KiwiError returns the Error messages.

func KiwiReaderImpl

func KiwiReaderImpl(lineNumber C.int, buffer *C.char, userData unsafe.Pointer) C.int

func KiwiVersion

func KiwiVersion() string

KiwiVersion returns the version of the kiwi library.

Types

type AnalyzeOption

type AnalyzeOption int

AnalyzeOption is a bitwise OR of the KiwiAnalyzeOption values.

const (
	KIWI_MATCH_URL                  AnalyzeOption = C.KIWI_MATCH_URL
	KIWI_MATCH_EMAIL                AnalyzeOption = C.KIWI_MATCH_EMAIL
	KIWI_MATCH_HASHTAG              AnalyzeOption = C.KIWI_MATCH_HASHTAG
	KIWI_MATCH_MENTION              AnalyzeOption = C.KIWI_MATCH_MENTION
	KIWI_MATCH_ALL                  AnalyzeOption = C.KIWI_MATCH_ALL
	KIWI_MATCH_NORMALIZE_CODA       AnalyzeOption = C.KIWI_MATCH_NORMALIZE_CODA
	KIWI_MATCH_ALL_WITH_NORMALIZING AnalyzeOption = C.KIWI_MATCH_ALL_WITH_NORMALIZING
)

type BuildOption

type BuildOption int

BuildOption is a bitwise OR of the KiwiBuildOption values.

const (
	KIWI_BUILD_LOAD_DEFAULT_DICT   BuildOption = C.KIWI_BUILD_LOAD_DEFAULT_DICT
	KIWI_BUILD_INTEGRATE_ALLOMORPH BuildOption = C.KIWI_BUILD_INTEGRATE_ALLOMORPH
	KIWI_BUILD_DEFAULT             BuildOption = C.KIWI_BUILD_DEFAULT
)

type Kiwi

type Kiwi struct {
	// contains filtered or unexported fields
}

Kiwi is a wrapper for the kiwi C library.

func New

func New(modelPath string, numThread int, options BuildOption) *Kiwi

New returns a new Kiwi instance. Don't forget to call Close after this.

func (*Kiwi) Analyze

func (k *Kiwi) Analyze(text string, topN int, options AnalyzeOption) ([]TokenResult, error)

Analyze returns the result of the analysis.

func (*Kiwi) Close

func (k *Kiwi) Close() int

Close frees the resource allocated for Kiwi and returns the exit status. This must be called after New. Returns 0 if successful. Safe to call multiple times.

func (*Kiwi) SplitSentence

func (k *Kiwi) SplitSentence(text string, options AnalyzeOption) ([]SplitResult, error)

SplitSentence returns the line of sentences.

type KiwiBuilder

type KiwiBuilder struct {
	// contains filtered or unexported fields
}

KiwiBuilder is a wrapper for the kiwi C library.

func NewBuilder

func NewBuilder(modelPath string, numThread int, options BuildOption) *KiwiBuilder

NewBuilder returns a new KiwiBuilder instance. Don't forget to call Close after this.

func (*KiwiBuilder) AddWord

func (kb *KiwiBuilder) AddWord(word string, pos POSType, score float32) int

AddWord set custom word with word, pos, score.

func (*KiwiBuilder) Build

func (kb *KiwiBuilder) Build() *Kiwi

Build creates kiwi instance with user word etc.

func (*KiwiBuilder) Close

func (kb *KiwiBuilder) Close() int

Close frees the resource allocated for KiwiBuilder and returns the exit status. This must be called after New but not need to called after Build. Returns 0 if successful. Safe to call multiple times.

func (*KiwiBuilder) ExtractWords

func (kb *KiwiBuilder) ExtractWords(readSeeker io.ReadSeeker, minCnt int, maxWordLen int, minScore float32, posThreshold float32) ([]WordInfo, error)

ExtractWords returns the result of extract word.

func (*KiwiBuilder) LoadDict

func (kb *KiwiBuilder) LoadDict(dictPath string) int

LoadDict loads user dict with dict file path.

type POSType

type POSType string

const (
	POS_UNKNOWN POSType = "UNKNOWN"

	POS_NNG POSType = "NNG"
	POS_NNP POSType = "NNP"
	POS_NNB POSType = "NNB"

	POS_VV POSType = "VV"
	POS_VA POSType = "VA"

	POS_MAG POSType = "MAG"

	POS_NR POSType = "NR"
	POS_NP POSType = "NP"

	POS_VX POSType = "VX"

	POS_MM  POSType = "MM"
	POS_MAJ POSType = "MAJ"

	POS_IC POSType = "IC"

	POS_XPN POSType = "XPN"
	POS_XSN POSType = "XSN"
	POS_XSV POSType = "XSV"
	POS_XSA POSType = "XSA"
	POS_XR  POSType = "XR"

	POS_VCP POSType = "VCP"
	POS_VCN POSType = "VCN"

	POS_SF POSType = "SF"
	POS_SP POSType = "SP"
	POS_SS POSType = "SS"
	POS_SE POSType = "SE"
	POS_SO POSType = "SO"
	POS_SW POSType = "SW"

	POS_SL POSType = "SL"
	POS_SH POSType = "SH"
	POS_SN POSType = "SN"

	POS_W_URL     POSType = "W_URL"
	POS_W_EMAIL   POSType = "W_EMAIL"
	POS_W_MENTION POSType = "W_MENTION"
	POS_W_HASHTAG POSType = "W_HASHTAG"

	POS_JKS POSType = "JKS"
	POS_JKC POSType = "JKC"
	POS_JKG POSType = "JKG"
	POS_JKO POSType = "JKO"
	POS_JKB POSType = "JKB"
	POS_JKV POSType = "JKV"
	POS_JKQ POSType = "JKQ"
	POS_JX  POSType = "JX"
	POS_JC  POSType = "JC"

	POS_EP  POSType = "EP"
	POS_EF  POSType = "EF"
	POS_EC  POSType = "EC"
	POS_ETN POSType = "ETN"
	POS_ETM POSType = "ETM"

	POS_V POSType = "V"

	POS_MAX POSType = "MAX"
)

NOTE: update isValid function when adding a new POSType.

func ParsePOSType

func ParsePOSType(t string) (POSType, error)

ParsePOSType return POS Tag for resault

type SplitResult

type SplitResult struct {
	Text  string
	Begin int
	End   int
}

SplitResult returns the Sentences.

type TokenInfo

type TokenInfo struct {
	// Position is the index of this token appears in the original text.
	Position int

	// Tag represents a type of this token (e.g. VV, NNG, ...).
	Tag POSType

	// Form is the actual string of this token.
	Form string
}

TokenInfo returns the token info for the given token(Str).

type TokenResult

type TokenResult struct {
	Tokens []TokenInfo
	Score  float32
}

TokenResult is a result for Analyze.

type WordInfo

type WordInfo struct {
	Form     string
	Freq     int
	POSScore float32
	Score    float32
}

WordInfo returns the token info for the given token(Str).