ybase

README

ybase

Utilities to implement a lexer for goyacc.

Example

package main

import (
	"bytes"
	"fmt"
	"unicode"

	"github.com/berquerant/ybase"
)

func main() {
	input := "1 + 12 - (34-56)"
	s := ybase.NewLexer(ybase.NewScanner(ybase.NewReader(bytes.NewBufferString(input), nil), func(r ybase.Reader) int {
		r.DiscardWhile(unicode.IsSpace)
		top := r.Peek()
		switch {
		case unicode.IsDigit(top):
			r.NextWhile(unicode.IsDigit)
			return 901
		default:
			switch top {
			case '+':
				_ = r.Next()
				return 911
			case '-':
				_ = r.Next()
				return 912
			case '(':
				_ = r.Next()
				return 921
			case ')':
				_ = r.Next()
				return 922
			}
		}
		return ybase.EOF
	}))
	for s.DoLex(func(tok ybase.Token) { fmt.Printf("%d %s\n", tok.Type(), tok.Value()) }) != ybase.EOF {
	}
	if err := s.Err(); err != nil {
		panic(err)
	}
	// Output:
	// 901 1
	// 911 +
	// 901 12
	// 912 -
	// 921 (
	// 901 34
	// 912 -
	// 901 56
	// 922 )
}

Documentation

Index

• Constants
• Variables
• func NilDebugFunc(msg string, v ...any)
• type Adjacency
• type Bytes
  • func (b Bytes) Adjacency(line, column, count int) (*Adjacency, bool)
  • func (b Bytes) Context(line, count int) (*Context, bool)
  • func (b Bytes) LineColumn(offset int) (int, int, bool)
  • func (b Bytes) Offset(line, column int) (int, bool)
• type Context
• type ContextLine
• type DebugFunc
• type Lexer
  • func NewLexer(scanner Scanner) Lexer
• type Pos
  • func NewPos(line, col, offset int) Pos
• type Reader
  • func NewReader(rdr io.Reader, debugFunc DebugFunc) Reader
  • func NewReaderWithInitPos(rdr io.Reader, debugFunc DebugFunc, initPos Pos) Reader
• type ScanFunc
• type Scanner
  • func NewScanner(rdr Reader, scanFunc ScanFunc) Scanner
• type Token
  • func NewToken(t int, v string, start, end Pos) Token

Examples

• Lexer

Constants

const EOF = -1

Variables

var ErrYbase = errors.New("Ybase")

Functions

func NilDebugFunc

func NilDebugFunc(msg string, v ...any)

Types

type Adjacency

type Adjacency struct {
	Linum  int
	Column int
	Focus  rune
	String string
	Line   string
}

type Bytes

type Bytes []byte

func (Bytes) Adjacency

func (b Bytes) Adjacency(line, column, count int) (*Adjacency, bool)

Adjacency retrieves runes before and after a specified line and column.

func (Bytes) Context

func (b Bytes) Context(line, count int) (*Context, bool)

Context retrieves lines before and after a specified line number.

It returns the surrounding context, including the given number of lines before and after the specified line.

func (Bytes) LineColumn

func (b Bytes) LineColumn(offset int) (int, int, bool)

LineColumn calculates line number and column from offset.

func (Bytes) Offset

func (b Bytes) Offset(line, column int) (int, bool)

Offset calculates offset from line number and column.

type Context

type Context struct {
	Target *ContextLine
	Lines  []*ContextLine
}

type ContextLine

type ContextLine struct {
	Linum int
	Line  []byte
}

type DebugFunc

type DebugFunc func(msg string, v ...any)

DebugFunc outputs debug logs. Assuming a function like slog.Debug.

type Lexer

type Lexer interface {
	Scanner
	// DoLex runs the lexical analysis.
	// Returns EOF if EOF or an error occurs.
	DoLex(callback func(Token)) int
}

Lexer is an utility to implement yyLexer.

Recommendation: - Set level to yyDebug (YYDEBUG in yacc). - Set yyErrorVerbose to true (YYERROR_VERBOSE in yacc)

Implements yyLexer by Error(string) and Lex(*yySymType) int, e.g.

type ActualLexer struct {
  Lexer
}

func (a *ActualLexer) Lex(lval *yySymType) int {
  return a.DoLex(func(tok Token) {
    lval.token = tok  // declares in %union
  })
}

Example

package main

import (
	"bytes"
	"fmt"
	"unicode"

	"github.com/berquerant/ybase"
)

func main() {
	input := "1 + 12 - (34-56)"
	s := ybase.NewLexer(ybase.NewScanner(ybase.NewReader(bytes.NewBufferString(input), nil), func(r ybase.Reader) int {
		r.DiscardWhile(unicode.IsSpace)
		top := r.Peek()
		switch {
		case unicode.IsDigit(top):
			r.NextWhile(unicode.IsDigit)
			return 901
		default:
			switch top {
			case '+':
				_ = r.Next()
				return 911
			case '-':
				_ = r.Next()
				return 912
			case '(':
				_ = r.Next()
				return 921
			case ')':
				_ = r.Next()
				return 922
			}
		}
		return ybase.EOF
	}))
	for s.DoLex(func(tok ybase.Token) { fmt.Printf("%d %s\n", tok.Type(), tok.Value()) }) != ybase.EOF {
	}
	if err := s.Err(); err != nil {
		panic(err)
	}
}

Output:

901 1
911 +
901 12
912 -
921 (
901 34
912 -
901 56
922 )

func NewLexer

func NewLexer(scanner Scanner) Lexer

type Pos

type Pos interface {
	Line() int
	Column() int
	Offset() int
	Add(r rune) Pos
}

func NewPos

func NewPos(line, col, offset int) Pos

type Reader

type Reader interface {
	// ResetBuffer clears the buffer.
	ResetBuffer()
	// Buffer returns the read runes.
	Buffer() string
	// Next gets the next rune and advances the pos.
	Next() rune
	// Peek gets the next rune but keeps the pos.
	Peek() rune
	// Discard ignores the next rune.
	Discard() rune
	// Err returns an error during the reading.
	Err() error
	// Debugf outputs debug logs.
	Debugf(msg string, v ...any)
	// Errorf outputs logs and set an error.
	Errorf(err error, msg string, v ...any)
	// DiscardWhile calls Discard() while pred(Peek()).
	DiscardWhile(pred func(rune) bool)
	// NextWhile calls Next() while pred(Peek()).
	NextWhile(pred func(rune) bool)
	// Pos returns the current position.
	Pos() Pos
}

Reader represents a reader object for lex.

func NewReader

func NewReader(rdr io.Reader, debugFunc DebugFunc) Reader

func NewReaderWithInitPos

func NewReaderWithInitPos(rdr io.Reader, debugFunc DebugFunc, initPos Pos) Reader

type ScanFunc

type ScanFunc func(Reader) int

ScanFunc scans source and calculate token.

type Scanner

type Scanner interface {
	Reader
	Scan() int
	// Error consumes an error from yyLexer.
	Error(msg string)
}

func NewScanner

func NewScanner(rdr Reader, scanFunc ScanFunc) Scanner

type Token

type Token interface {
	Type() int
	Value() string
	Start() Pos
	End() Pos
}

func NewToken

func NewToken(t int, v string, start, end Pos) Token