parsly

README

Parsly - parsing utility.

This library is compatible with Go 1.11+

Please refer to CHANGELOG.md if you encounter breaking changes.

The goal of this project is to simplify implementing parsers with a tokenizer with a set of commonly use token matchers.

Usage

To build simple parser:

• define token lexical registry matchers

• create a tokenizer
• add parsing logic with MatchAfterWhitespace, MatchAny or Match

    var Whitespace = parsly.NewToken(0, "Whitespace", matcher.NewWhiteSpace())
    var Number = parsly.NewToken(1, "Number", matcher.NewNumber())
    var Term = parsly.NewToken(2, "Term", matcher.NewCharset("+-"))
    var Factor = parsly.NewToken(3, "Factor", matcher.NewCharset("*/"))
 
  
    func Parse(input []byte) (root *Expression, err error) {
        cursor := parsly.NewCursor("", input, 0)
        root = &Expression{}
        expression := root
        matched := cursor.MatchAfterOptional(Whitespace, Number)
        if matched.Code != Number.Code {
            return nil, cursor.NewError(Number)
        }
        value, _ := matched.Float(cursor)
        expression.LeftOp = NewValue(value)
    
        for ; ; {
            matched = cursor.MatchAfterOptional(Whitespace, Factor, Term)
            if matched.Code == parsly.EOF {
                break
            }
            operator := matched.Text(cursor)
            if expression.Operator != "" {
                expression.RightOp = &Operand{Expression: &Expression{LeftOp: expression.RightOp}}
                expression = expression.RightOp.Expression
            }
            expression.Operator = operator
    
            matched := cursor.MatchAfterOptional(Whitespace, Number)
            if matched.Code != Number.Code {
                return nil, cursor.NewError(Number)
            }
            value, _ := matched.Float(cursor)
            expression.RightOp = NewValue(value)
        }
        return root, nil
    }
    

See example basic arithmetic AST expression parser

Matchers

This project implements the following matchers:

• char to match individual rune
• charset to match set of runes
• digit to match a digit
• letter to match a letter
• fragment to match a fragment
• fragments to match a set of fragments
• number to match a number
• quote to match a quoted fragment
• terminator to match a byte terminated sequence

License

The source code is made available under the terms of the Apache License, Version 2, as stated in the file LICENSE.

Individual files may be made available under their own specific license, all compatible with Apache License, Version 2. Please see individual files for details.

Documentation

Index

• Constants
• Variables
• type Cursor
  • func NewCursor(path string, input []byte, offset int) *Cursor
  • func (c *Cursor) FindMatch(token *Token) *TokenMatch
  • func (c *Cursor) HasMore() bool
  • func (c *Cursor) MatchAfterOptional(optional *Token, candidates ...*Token) *TokenMatch
  • func (c *Cursor) MatchAny(candidates ...*Token) *TokenMatch
  • func (c *Cursor) MatchOne(token *Token) *TokenMatch
  • func (c *Cursor) NewError(expectedTokens ...*Token) error
  • func (c *Cursor) TokenMatch(token *Token, matchSize int) *TokenMatch
• type Matcher
• type Token
  • func NewToken(code int, name string, matcher Matcher) *Token
• type TokenMatch
  • func (m *TokenMatch) Bool(cursor *Cursor) (bool, error)
  • func (m *TokenMatch) Byte(cursor *Cursor) byte
  • func (m *TokenMatch) Bytes(cursor *Cursor) []byte
  • func (m *TokenMatch) ContainsRune(cursor *Cursor, r rune) bool
  • func (m *TokenMatch) Float(cursor *Cursor) (float64, error)
  • func (m *TokenMatch) Int(cursor *Cursor) (int64, error)
  • func (m *TokenMatch) SetToken(token *Token, cursor int, size int)
  • func (m *TokenMatch) Text(cursor *Cursor) string
  • func (m *TokenMatch) UnquotedText(cursor *Cursor) string

Constants

const (
	//EOF end of file token
	EOF = -1
	//Invalid invalid token
	Invalid = -2
)

Variables

var AsString = func(data []byte) string {
	return string(data)
}

AsString converts bytes to string

var AsZeroAllocString = func(data []byte) string {
	return string(data)
}

AsZeroAllocString converts bytes to string, In this package I do not any unsafe dependency But if you need zero allocation override AsZeroAllocString = usage.ByteSlice2String

var EOFToken = NewToken(EOF, "EOF", nil)

EOFToken

var InvalidToken = NewToken(Invalid, "Invalid", nil)

InvalidToken represents invalid token

Types

type Cursor

type Cursor struct {
	Path string

	Input     []byte
	InputSize int
	Pos       int

	OnError func(err error, cur *Cursor, destNode interface{}) error
	// contains filtered or unexported fields
}

Cursor represents a location

func NewCursor

func NewCursor(path string, input []byte, offset int) *Cursor

NewCursor creates a location

func (*Cursor) FindMatch

func (c *Cursor) FindMatch(token *Token) *TokenMatch

FindMatch tries to find a token match in the cursor

func (*Cursor) HasMore

func (c *Cursor) HasMore() bool

HasMore returns true if it has more

func (*Cursor) MatchAfterOptional

func (c *Cursor) MatchAfterOptional(optional *Token, candidates ...*Token) *TokenMatch

MatchAfterOptional matcher first candidate after optional token lastMatch

func (*Cursor) MatchAny

func (c *Cursor) MatchAny(candidates ...*Token) *TokenMatch

MatchAny matches the first of the candidates

func (*Cursor) MatchOne

func (c *Cursor) MatchOne(token *Token) *TokenMatch

MatchOne tries to lastMatch a candidate, it returns a lastMatch.

func (*Cursor) NewError

func (c *Cursor) NewError(expectedTokens ...*Token) error

NewError returns error for expected tokens

func (*Cursor) TokenMatch

func (c *Cursor) TokenMatch(token *Token, matchSize int) *TokenMatch

TokenMatch returns updated lastMatch

type Matcher

type Matcher interface {
	//TokenMatch matches input starting from offset, it return number of characters matched
	Match(cursor *Cursor) (matched int)
}

Matcher represents a matcher, that matches input from offset position, it returns number of bytes matched.

type Token

type Token struct {
	Code int
	Name string
	Matcher
}

Token represents token matcher

func NewToken

func NewToken(code int, name string, matcher Matcher) *Token

NewToken creates a token

type TokenMatch

type TokenMatch struct {
	Offset int
	Size   int
	*Token
}

TokenMatch represents a token match

func (*TokenMatch) Bool

func (m *TokenMatch) Bool(cursor *Cursor) (bool, error)

Bool return matched bool value

func (*TokenMatch) Byte

func (m *TokenMatch) Byte(cursor *Cursor) byte

Byte return matched byte

func (*TokenMatch) Bytes

func (m *TokenMatch) Bytes(cursor *Cursor) []byte

Matched return matched fragment

func (*TokenMatch) ContainsRune

func (m *TokenMatch) ContainsRune(cursor *Cursor, r rune) bool

Contains return true if lastMatch data contains rune

func (*TokenMatch) Float

func (m *TokenMatch) Float(cursor *Cursor) (float64, error)

Float return matched float value

func (*TokenMatch) Int

func (m *TokenMatch) Int(cursor *Cursor) (int64, error)

Int return matched int value

func (*TokenMatch) SetToken

func (m *TokenMatch) SetToken(token *Token, cursor int, size int)

SetToken sets token

func (*TokenMatch) Text

func (m *TokenMatch) Text(cursor *Cursor) string

Matched return matched fragment

func (*TokenMatch) UnquotedText

func (m *TokenMatch) UnquotedText(cursor *Cursor) string

MatchedUnquoted return matched unquoted fragment