dartsclone

README

dartsclone : Double Array TRIE liblary

Port of Sudachi's dartsclone library to Go.

Build & Save

package main

import (
	"os"

	"github.com/ikawaha/dartsclone"
)

func main() {
	keys := []string{
		"電気",
		"電気通信",
		"電気通信大学",
		"電気通信大学大学院",
		"電気通信大学大学院大学",
	}

	// Build
	builder := dartsclone.NewBuilder(nil)
	if err := builder.Build(keys, nil); err != nil {
		panic(err)
	}
	// Save
	f, err := os.Create("my-double-array-file")
	if err != nil {
		panic(err)
	}
	builder.WriteTo(f)
	f.Close()
}

Load & Search

package main

import (
	"fmt"
	"github.com/ikawaha/dartsclone"
)

func main() {
	trie, err := dartsclone.Open("my-double-array-file")
	if err != nil {
		panic(err)
	}
	ret, err := trie.CommonPrefixSearch("電気通信大学大学院大学", 0)
	for i := 0; i < len(ret); i++ {
		fmt.Printf("id=%d, common prefix=%s\n", ret[i][0], "電気通信大学大学院大学"[0:ret[i][1]])
	}
}

outputs

id=0, common prefix=電気
id=1, common prefix=電気通信
id=2, common prefix=電気通信大学
id=3, common prefix=電気通信大学大学院
id=4, common prefix=電気通信大学大学院大学

Use memory mapping

• Build Tags : mmap
• Support OS : linux, osx, windows

$ go build -tags mmap ./...

package main

import (
	"fmt"
	"github.com/ikawaha/dartsclone"
)

func main() {
	trie, err := dartsclone.OpenMmaped("my-double-array-file") // ← ★
	if err != nil {
		panic(err)
	}
	defer trie.Close() // ← ★

	ret, err := trie.CommonPrefixSearch("電気通信大学大学院大学", 0)
	for i := 0; i < len(ids); i++ {
		fmt.Printf("id=%d, common prefix=%s\n", ret[i][0], "電気通信大学大学院大学"[0:ret[i][1]])
	}
}

Documentation

Index

• type Builder
  • func NewBuilder(progress ProgressFunction) *Builder
  • func (b Builder) WriteTo(w io.Writer) (int64, error)
• type ProgressFunction
• type Trie
  • func BuildTRIE(keys []string, values []uint32, progress ProgressFunction) (Trie, error)
  • func Open(name string) (Trie, error)

Types

type Builder

type Builder struct {
	*internal.DoubleArrayBuilder
}

Builder represents builder of the dartsclone TRIE.

func NewBuilder

func NewBuilder(progress ProgressFunction) *Builder

NewBuilder creates a builder of the dartsclone TRIE.

func (Builder) WriteTo

func (b Builder) WriteTo(w io.Writer) (int64, error)

WriteTo write to the serialize data of the dartsclone TRIE.

type ProgressFunction

type ProgressFunction interface {
	// SetMaximum sets the maximum of the progress bar.
	SetMaximum(int)
	// Increment with increase the current count on the progress bar.
	Increment()
}

ProgressFunction indicates progress bar of building double array.

type Trie

type Trie interface {
	// ExactMatchSearch searches TRIE by a given keyword and returns the id and it's length if found.
	ExactMatchSearch(key string) (id, size int, err error)
	// CommonPrefixSearch finds keywords sharing common prefix in an input and returns the array of pairs (id and it's length) if found.
	CommonPrefixSearch(key string, offset int) ([][2]int, error)
	// CommonPrefixSearchCallback finds keywords sharing common prefix in an input and callback with id and it's length.
	CommonPrefixSearchCallback(key string, offset int, callback func(id, size int)) error
}

Trie is the TRIE interface.

func BuildTRIE

func BuildTRIE(keys []string, values []uint32, progress ProgressFunction) (Trie, error)

BuildTRIE returns a dartsclone TRIE for keys and values.

func Open

func Open(name string) (Trie, error)

Open opens the named file of the double array.