tokenizerapi

Documentation

Index

• type TokenStream
• type Tokenizer

Types

type TokenStream

type TokenStream interface {
	Advance() bool
	Token() string
	Destroy()
}

type Tokenizer

type Tokenizer interface {
	NewTokenStream(text string) TokenStream
	Clone() (Tokenizer, error)
	Destroy()
}