lexer

Nov 14, 2024

Changes in this version

+ const EXCLAMATION

Sep 7, 2024

Changes in this version

+ type ShellLexer struct

+ Base *Lexer

+ DraftTokens []Token

+ func (sh *ShellLexer) AnalyzeToken() error

+ func (sh *ShellLexer) Comment() error

+ func (sh *ShellLexer) String(delim byte) error

Sep 3, 2024

Changes in this version

+ const CLOSE_PARAN

+ const OPEN_PARAN

type Comment

+ IssueNumber int

+ NotationEndIndex int

+ NotationStartIndex int

type TokenType

+ const TOKEN_CLOSE_PARAN

+ const TOKEN_HASH

+ const TOKEN_ISSUE_NUMBER

+ const TOKEN_OPEN_PARAN

Aug 25, 2024

Changes in this version

+ type Clexer struct

+ Base *Lexer

+ DraftTokens []Token

+ func (c *Clexer) AnalyzeToken() error

+ func (c *Clexer) Comment() error

+ func (c *Clexer) String(delim byte) error

type Comment

+ AnnotationPos []int

+ LineNumber int

+ TokenAnnotationIndex int

+ TokenEndIndex int

+ TokenStartIndex int

+ type CommentManager struct

+ Comments []Comment

+ func BuildComments(tokens []Token) CommentManager

type Lexer

+ Annotation []byte

+ FilePath string

+ Src []byte

+ type LexicalTokenizer interface

+ AnalyzeToken func() error

+ Comment func() error

+ String func(delim byte) error

+ func NewTargetLexer(base *Lexer) (LexicalTokenizer, error)

type Token

+ End int

+ Start int

+ Type TokenType

+ func NewToken(tokenType TokenType, lexeme []byte, lexer *Lexer) Token

type TokenType

+ const TOKEN_COMMENT_ANNOTATION

+ const TOKEN_COMMENT_DESCRIPTION

+ const TOKEN_COMMENT_TITLE

+ const TOKEN_EOF

+ const TOKEN_MULTI_LINE_COMMENT

+ const TOKEN_MULTI_LINE_COMMENT_END

+ const TOKEN_MULTI_LINE_COMMENT_START

+ const TOKEN_SINGLE_LINE_COMMENT

+ const TOKEN_SINGLE_LINE_COMMENT_END

+ const TOKEN_SINGLE_LINE_COMMENT_START

+ const TOKEN_UNKNOWN

+ type U8 uint8

+ const FLAG_PURGE

+ const FLAG_SCAN

May 19, 2024

Changes in this version

+ var ASTERISK byte = '*'

+ var BACKWARD_SLASH byte = '\\'

+ var BACK_TICK byte = '`'

+ var DOUBLE_QUOTE byte = '"'

+ var FORWARD_SLASH byte = '/'

+ var HASH byte = '#'

+ var NEWLINE byte = '\n'

+ var QUOTE byte = '\''

+ var TAB byte = '\t'

+ var WHITESPACE byte = ' '

type Comment

+ func (c *Comment) Push(comments *[]Comment, fileName string, tokenIndex int)

type Token

+ func (t *Token) ParseMultiLineCommentToken(annotation []byte, trim func(r rune) bool) Comment

+ func (t *Token) ParseSingleLineCommentToken(annotation []byte, trim func(r rune) bool) Comment

May 6, 2024

Changes in this version

+ const EOF

+ const MULTI_LINE_COMMENT

+ const SINGLE_LINE_COMMENT

+ const STRING

+ func IsAdoptedFromC(ext string) bool

+ type CLexer struct

+ func (cl *CLexer) AnalyzeToken(lex *Lexer) error

+ func (cl *CLexer) Comment(lex *Lexer) error

+ func (cl *CLexer) MultiLineComment(lex *Lexer) error

+ func (cl *CLexer) ParseCommentTokens(lex *Lexer, annotation []byte) ([]Comment, error)

+ func (cl *CLexer) SingleLineComment(lex *Lexer) error

+ func (cl *CLexer) String(lex *Lexer, delim byte) error

+ type Comment struct

+ Description []byte

+ Source []byte

+ SourceFileName string

+ Title []byte

+ TokenIndex int

+ func ParseMultiLineCommentToken(token *Token, annotation []byte) Comment

+ func ParseSingleLineCommentToken(token *Token, annotation []byte) Comment

+ func (c *Comment) Prepare(fileName string, index int)

+ func (c *Comment) Validate() bool

+ type Lexer struct

+ Current int

+ FileName string

+ Line int

+ Manager LexingManager

+ Source []byte

+ Start int

+ Tokens []Token

+ func NewLexer(src []byte, fileName string) (*Lexer, error)

+ func (l *Lexer) AnalyzeTokens() ([]Token, error)

+ type LexingManager interface

+ AnalyzeToken func(lexer *Lexer) error

+ Comment func(lexer *Lexer) error

+ ParseCommentTokens func(lexer *Lexer, annotation []byte) ([]Comment, error)

+ String func(lexer *Lexer, delim byte) error

+ func NewLexingManager(ext string) (LexingManager, error)

+ type Token struct

+ EndByteIndex int

+ Lexeme []byte

+ Line int

+ StartByteIndex int

+ TokenType TokenType

+ type TokenType = int