golexer

package module
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 14, 2018 License: MIT Imports: 7 Imported by: 44

README

golexer

可自定义的词法解析器

C#版参考https://github.com/davyxu/SharpLexer

特性

  • 支持数值，字符串，注释，标识符等的内建匹配

  • 可自定义匹配器来拾取需要的token

  • 高性能并发匹配


// 自定义的token id
const (
	Token_Unknown = iota
	Token_Numeral
	Token_String
	Token_WhiteSpace
	Token_LineEnd
	Token_UnixStyleComment
	Token_Identifier
	Token_Go
	Token_Semicolon
)

// 使用lexer

	l := NewLexer()
	l.AddMatcher(NewNumeralMatcher(Token_Numeral))
	l.AddMatcher(NewStringMatcher(Token_String))

	l.AddIgnoreMatcher(NewWhiteSpaceMatcher(Token_WhiteSpace))
	l.AddIgnoreMatcher(NewLineEndMatcher(Token_LineEnd))
	l.AddIgnoreMatcher(NewUnixStyleCommentMatcher(Token_UnixStyleComment))

	l.AddMatcher(NewSignMatcher(Token_Semicolon, ";"))
	l.AddMatcher(NewSignMatcher(Token_Go, "go"))

	l.AddMatcher(NewIdentifierMatcher(Token_Identifier))

	l.AddMatcher(NewUnknownMatcher(Token_Unknown))

// 解析原文
	l.Start(`"a" 
	123.3;
	go
	_id # comment
	;
	'b'
	
	
	`)

	// 解析过程
	for {

		tk, err := l.Read()

		if err != nil {
			t.Error(err)
			break
		}

		if tk == nil {
			break
		}

		t.Log(tk.String())
	}

备注

感觉不错请star, 谢谢!

开源讨论群: 527430600

知乎: http://www.zhihu.com/people/sunicdavy

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var DefaultTokenPos = TokenPos{Line: 1, Col: 1}
View Source 
var EmptyToken = NewToken(nil, nil, "NIL", "")

Functions

func ErrorCatcher 

func ErrorCatcher(errFunc func(error))

func ParseKV 

func ParseKV(str string, callback func(string, interface{}) bool) (errRet error)

Types

type BackTicksMatcher 

type BackTicksMatcher struct {
	// contains filtered or unexported fields
}

字符串

func (*BackTicksMatcher) ID 

func (self *BackTicksMatcher) ID() int

func (*BackTicksMatcher) Match 

func (self *BackTicksMatcher) Match(tz *Tokenizer) (Token, error)

func (*BackTicksMatcher) String 

func (self *BackTicksMatcher) String() string

type CStyleCommentMatcher 

type CStyleCommentMatcher struct {
	// contains filtered or unexported fields
}

#开头的行注释

func (*CStyleCommentMatcher) ID 

func (self *CStyleCommentMatcher) ID() int

func (*CStyleCommentMatcher) Match 

func (self *CStyleCommentMatcher) Match(tz *Tokenizer) (Token, error)

func (*CStyleCommentMatcher) String 

func (self *CStyleCommentMatcher) String() string

type IdentifierMatcher 

type IdentifierMatcher struct {
	// contains filtered or unexported fields
}

标识符

func (*IdentifierMatcher) ID 

func (self *IdentifierMatcher) ID() int

func (*IdentifierMatcher) Match 

func (self *IdentifierMatcher) Match(tz *Tokenizer) (Token, error)

func (*IdentifierMatcher) String 

func (self *IdentifierMatcher) String() string

type KVPair 

type KVPair struct {
	// contains filtered or unexported fields
}

func NewKVPair 

func NewKVPair() *KVPair

func (*KVPair) ContainKey 

func (self *KVPair) ContainKey(key string) bool

func (*KVPair) ContainValue 

func (self *KVPair) ContainValue(key string, tgt string) bool

func (*KVPair) GetBool 

func (self *KVPair) GetBool(key string) bool

func (*KVPair) GetString 

func (self *KVPair) GetString(key string) string

func (*KVPair) Parse 

func (self *KVPair) Parse(str string) error

func (*KVPair) Raw 

func (self *KVPair) Raw() map[string]interface{}

func (*KVPair) SetString 

func (self *KVPair) SetString(key, value string)

func (*KVPair) String 

func (self *KVPair) String() string

type KeywordMatcher 

type KeywordMatcher struct {
	// contains filtered or unexported fields
}

下划线和字母(中文)+数字=关键字

func (*KeywordMatcher) ID 

func (self *KeywordMatcher) ID() int

func (*KeywordMatcher) Match 

func (self *KeywordMatcher) Match(tz *Tokenizer) (Token, error)

func (*KeywordMatcher) String 

func (self *KeywordMatcher) String() string

type Lexer 

type Lexer struct {
	// contains filtered or unexported fields
}

func NewLexer 

func NewLexer() *Lexer

func (*Lexer) AddIgnoreMatcher 

func (self *Lexer) AddIgnoreMatcher(m TokenMatcher)

添加一个匹配器，如果结果匹配，直接忽略匹配内容

func (*Lexer) AddMatcher 

func (self *Lexer) AddMatcher(m TokenMatcher)

添加一个匹配器，如果结果匹配，返回token

func (*Lexer) MatcherByID 

func (self *Lexer) MatcherByID(id int) TokenMatcher

func (*Lexer) MatcherString 

func (self *Lexer) MatcherString(id int) string

func (*Lexer) Read 

func (self *Lexer) Read() (Token, error)

func (*Lexer) Start 

func (self *Lexer) Start(src string)

func (*Lexer) VisitMatcher 

func (self *Lexer) VisitMatcher(callback func(TokenMatcher) bool)

type LineEndMatcher 

type LineEndMatcher struct {
	// contains filtered or unexported fields
}

行结束

func (*LineEndMatcher) ID 

func (self *LineEndMatcher) ID() int

func (*LineEndMatcher) Match 

func (self *LineEndMatcher) Match(tz *Tokenizer) (Token, error)

func (*LineEndMatcher) String 

func (self *LineEndMatcher) String() string

type NumeralMatcher 

type NumeralMatcher struct {
	// contains filtered or unexported fields
}

整形，浮点数

func (*NumeralMatcher) ID 

func (self *NumeralMatcher) ID() int

func (*NumeralMatcher) Match 

func (self *NumeralMatcher) Match(tz *Tokenizer) (Token, error)

func (*NumeralMatcher) String 

func (self *NumeralMatcher) String() string

type Parser 

type Parser struct {
	// contains filtered or unexported fields
}

func NewParser 

func NewParser(l *Lexer, srcName string) *Parser

func (*Parser) Expect 

func (self *Parser) Expect(id int) Token

func (*Parser) Lexer 

func (self *Parser) Lexer() *Lexer

func (*Parser) MatcherName 

func (self *Parser) MatcherName() string

func (*Parser) MatcherString 

func (self *Parser) MatcherString() string

func (*Parser) NextToken 

func (self *Parser) NextToken()

func (*Parser) PreTokenPos 

func (self *Parser) PreTokenPos() TokenPos

func (*Parser) RawToken 

func (self *Parser) RawToken() Token

func (*Parser) TokenID 

func (self *Parser) TokenID() int

func (*Parser) TokenPos 

func (self *Parser) TokenPos() TokenPos

func (*Parser) TokenRaw 

func (self *Parser) TokenRaw() string

func (*Parser) TokenValue 

func (self *Parser) TokenValue() string

type SignMatcher 

type SignMatcher struct {
	// contains filtered or unexported fields
}

操作符，分隔符

func (*SignMatcher) ID 

func (self *SignMatcher) ID() int

func (*SignMatcher) Match 

func (self *SignMatcher) Match(tz *Tokenizer) (Token, error)

func (*SignMatcher) String 

func (self *SignMatcher) String() string

type StringMatcher 

type StringMatcher struct {
	// contains filtered or unexported fields
}

字符串

func (*StringMatcher) ID 

func (self *StringMatcher) ID() int

func (*StringMatcher) Match 

func (self *StringMatcher) Match(tz *Tokenizer) (Token, error)

func (*StringMatcher) String 

func (self *StringMatcher) String() string

type Token 

type Token struct {
	// contains filtered or unexported fields
}

func NewToken 

func NewToken(m TokenMatcher, tz *Tokenizer, v string, raw string) Token

func (Token) Index 

func (self Token) Index() int

func (Token) Line 

func (self Token) Line() int

func (*Token) MatcherID 

func (self *Token) MatcherID() int

func (*Token) MatcherName 

func (self *Token) MatcherName() string

func (*Token) MatcherString 

func (self *Token) MatcherString() string

func (Token) Raw 

func (self Token) Raw() string

func (*Token) String 

func (self *Token) String() string

func (Token) ToFloat32 

func (self Token) ToFloat32() float32

func (Token) ToInt 

func (self Token) ToInt() int

func (Token) ToInt32 

func (self Token) ToInt32() int32

func (Token) ToInt64 

func (self Token) ToInt64() int64

func (Token) Value 

func (self Token) Value() string

type TokenMatcher 

type TokenMatcher interface {
	Match(*Tokenizer) (Token, error)
	ID() int
	String() string
}

func NewBackTicksMatcher 

func NewBackTicksMatcher(id int) TokenMatcher

func NewCStyleCommentMatcher 

func NewCStyleCommentMatcher(id int) TokenMatcher

func NewIdentifierMatcher 

func NewIdentifierMatcher(id int) TokenMatcher

func NewKeywordMatcher 

func NewKeywordMatcher(id int, word string) TokenMatcher

func NewLineEndMatcher 

func NewLineEndMatcher(id int) TokenMatcher

func NewNumeralMatcher 

func NewNumeralMatcher(id int) TokenMatcher

func NewPositiveNumeralMatcher 

func NewPositiveNumeralMatcher(id int) TokenMatcher

func NewSignMatcher 

func NewSignMatcher(id int, word string) TokenMatcher

func NewStringMatcher 

func NewStringMatcher(id int) TokenMatcher

func NewUnixStyleCommentMatcher 

func NewUnixStyleCommentMatcher(id int) TokenMatcher

func NewUnknownMatcher 

func NewUnknownMatcher(id int) TokenMatcher

func NewWhiteSpaceMatcher 

func NewWhiteSpaceMatcher(id int) TokenMatcher

type TokenPos 

type TokenPos struct {
	Line       int
	Col        int
	SourceName string
}

func (TokenPos) String 

func (self TokenPos) String() string

type Tokenizer 

type Tokenizer struct {
	// contains filtered or unexported fields
}

func NewTokenizer 

func NewTokenizer(s string, l *Lexer) *Tokenizer

func (*Tokenizer) ConsumeMulti 

func (self *Tokenizer) ConsumeMulti(count int)

func (*Tokenizer) ConsumeOne 

func (self *Tokenizer) ConsumeOne()

func (*Tokenizer) Count 

func (self *Tokenizer) Count() int

func (*Tokenizer) Current 

func (self *Tokenizer) Current() rune

func (*Tokenizer) EOF 

func (self *Tokenizer) EOF() bool

func (*Tokenizer) Index 

func (self *Tokenizer) Index() int

func (*Tokenizer) Line 

func (self *Tokenizer) Line() int

func (*Tokenizer) Peek 

func (self *Tokenizer) Peek(offset int) rune

func (*Tokenizer) StringRange 

func (self *Tokenizer) StringRange(begin, end int) string

type UnixStyleCommentMatcher 

type UnixStyleCommentMatcher struct {
	// contains filtered or unexported fields
}

#开头的行注释

func (*UnixStyleCommentMatcher) ID 

func (self *UnixStyleCommentMatcher) ID() int

func (*UnixStyleCommentMatcher) Match 

func (self *UnixStyleCommentMatcher) Match(tz *Tokenizer) (Token, error)

func (*UnixStyleCommentMatcher) String 

func (self *UnixStyleCommentMatcher) String() string

type UnknownMatcher 

type UnknownMatcher struct {
	// contains filtered or unexported fields
}

未知字符

func (*UnknownMatcher) ID 

func (self *UnknownMatcher) ID() int

func (*UnknownMatcher) Match 

func (self *UnknownMatcher) Match(tz *Tokenizer) (Token, error)

func (*UnknownMatcher) String 

func (self *UnknownMatcher) String() string

type WhiteSpaceMatcher 

type WhiteSpaceMatcher struct {
	// contains filtered or unexported fields
}

空白字符

func (*WhiteSpaceMatcher) ID 

func (self *WhiteSpaceMatcher) ID() int

func (*WhiteSpaceMatcher) Match 

func (self *WhiteSpaceMatcher) Match(tz *Tokenizer) (Token, error)

func (*WhiteSpaceMatcher) String 

func (self *WhiteSpaceMatcher) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.