py_cst

package
v0.0.0-...-dd1d5c9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 1, 2023 License: BSD-3-Clause Imports: 4 Imported by: 0

README 

A half finished, comment perserving python concrete syntax tree (written in go)

Documentation

Index

Constants

View Source 
const (
	UnknownCollection = 0
	TupleCollection   = 1
	ListCollection    = 2
	SetCollection     = 3
	DictCollection    = 4
)
View Source 
const ADD = 57389
View Source 
const ADD_ASSIGN = 57390
View Source 
const AND = 57355
View Source 
const AND_ASSIGN = 57391
View Source 
const AND_OP = 57392
View Source 
const AS = 57356
View Source 
const ASSERT = 57357
View Source 
const ASSIGN = 57393
View Source 
const AT = 57394
View Source 
const BACK_QUOTE = 57395
View Source 
const BREAK = 57358
View Source 
const CLASS = 57359
View Source 
const COLON = 57396
View Source 
const COMMA = 57397
View Source 
const COMMENT_NEWLINE = 57352
View Source 
const CONTINUE = 57360
View Source 
const DEDENT = 57354
View Source 
const DEF = 57361
View Source 
const DEL = 57362
View Source 
const DIV = 57398
View Source 
const DIV_ASSIGN = 57399
View Source 
const DOT = 57400
View Source 
const ELIF = 57363
View Source 
const ELSE = 57364
View Source 
const EQUALS = 57401
View Source 
const EXCEPT = 57365
View Source 
const EXEC = 57366
View Source 
const FINALLY = 57367
View Source 
const FLOAT = 57346
View Source 
const FOR = 57368
View Source 
const FROM = 57369
View Source 
const GLOBAL = 57370
View Source 
const GREATER_THAN = 57402
View Source 
const GT_EQ = 57403
View Source 
const IDIV = 57404
View Source 
const IDIV_ASSIGN = 57405
View Source 
const IF = 57371
View Source 
const IMPORT = 57372
View Source 
const IN = 57373
View Source 
const INDENT = 57353
View Source 
const INTEGER = 57347
View Source 
const IS = 57374
View Source 
const IS_NOT = 57388
View Source 
const LAMBDA = 57375
View Source 
const LEFT_BRACE = 57406
View Source 
const LEFT_BRACKET = 57407
View Source 
const LEFT_PARENTHESIS = 57408
View Source 
const LEFT_SHIFT = 57409
View Source 
const LEFT_SHIFT_ASSIGN = 57410
View Source 
const LESS_THAN = 57411
View Source 
const LINE_CONTINUATION = 57351
View Source 
const LT_EQ = 57412
View Source 
const MINUS = 57413
View Source 
const MOD = 57414
View Source 
const MOD_ASSIGN = 57415
View Source 
const MULT_ASSIGN = 57416
View Source 
const NAME = 57349
View Source 
const NEWLINE = 57350
View Source 
const NONE = 57376
View Source 
const NOT = 57377
View Source 
const NOT_EQUAL = 57417
View Source 
const NOT_IN = 57387
View Source 
const NOT_OP = 57418
View Source 
const OR = 57378
View Source 
const OR_ASSIGN = 57419
View Source 
const OR_OP = 57420
View Source 
const PASS = 57379
View Source 
const POWER_ASSIGN = 57421
View Source 
const PRINT = 57380
View Source 
const RAISE = 57381
View Source 
const RETURN = 57382
View Source 
const RIGHT_BRACE = 57422
View Source 
const RIGHT_BRACKET = 57423
View Source 
const RIGHT_PARENTHESIS = 57424
View Source 
const RIGHT_SHIFT = 57425
View Source 
const RIGHT_SHIFT_ASSIGN = 57426
View Source 
const SEMI_COLON = 57427
View Source 
const STAR = 57428
View Source 
const STAR_STAR = 57429
View Source 
const STRING = 57348
View Source 
const SUB_ASSIGN = 57430
View Source 
const TRY = 57383
View Source 
const WHILE = 57384
View Source 
const WITH = 57385
View Source 
const XOR = 57431
View Source 
const XOR_ASSIGN = 57432
View Source 
const YIELD = 57386

Variables

This section is empty.

Functions

func Parse 

func Parse(ctx *Context) int

Types

type Argument 

type Argument struct {
	Node

	// Optional when used by CallExpr
	Name Expression

	// Optional when used by FuncDef / LambdaExpr
	Value Expression

	PositionVarArg bool // '*'
	KeywordVarArg  bool // '**'
}

func NewArgument 

func NewArgument(name Expression, assign *Token, value Expression) *Argument

func NewKeywordVarArg 

func NewKeywordVarArg(starStar *Token, value Expression) *Argument

**kwarg in the context of invocation.

func NewKeywordVarParam 

func NewKeywordVarParam(starStar *Token, name Expression) *Argument

**kwarg in the context of defintions.

func NewPositionVarArg 

func NewPositionVarArg(star *Token, value Expression) *Argument

*arg in the context of invocation.

func NewPositionVarParam 

func NewPositionVarParam(star *Token, name Expression) *Argument

*arg in the context of definitions.

func (*Argument) String 

func (a *Argument) String() string

type ArgumentList 

type ArgumentList struct {
	Node
	Args []*Argument
}

intermediate node. does not appear in parsed tree.

func NewArgumentList 

func NewArgumentList(
	leftParen *Token,
	args []*Argument,
	rightParen *Token) *ArgumentList

type AssertStmt 

type AssertStmt struct {
	Expr  Expression
	Debug Expression // Optional
	// contains filtered or unexported fields
}

func (AssertStmt) IsStatement 

func (AssertStmt) IsStatement()

func (*AssertStmt) String 

func (a *AssertStmt) String() string

type BinaryExpr 

type BinaryExpr struct {
	Left  Expression
	Op    string
	Right Expression
	// contains filtered or unexported fields
}

func NewBinaryExpr 

func NewBinaryExpr(lhs Expression, op *Token, rhs Expression) *BinaryExpr

func (BinaryExpr) IsExpression 

func (BinaryExpr) IsExpression()

func (*BinaryExpr) String 

func (b *BinaryExpr) String() string

type BreakStmt 

type BreakStmt struct {
	TokenStmt
}

func NewBreakStmt 

func NewBreakStmt(t *Token) *BreakStmt

func (BreakStmt) IsStatement 

func (BreakStmt) IsStatement()

type CallExpr 

type CallExpr struct {
	Method    Expression
	Arguments []*Argument
	// contains filtered or unexported fields
}

<method>(<args>)

func NewCallExpr 

func NewCallExpr(method Expression, args *ArgumentList) *CallExpr

func (CallExpr) IsExpression 

func (CallExpr) IsExpression()

func (*CallExpr) String 

func (i *CallExpr) String() string

type ClassDef 

type ClassDef struct {
	Decorators []*Decorator

	Name          *Token
	ParentClasses []Expression
	Statements    []Statement
	// contains filtered or unexported fields
}

TODO doc string ...

func (ClassDef) IsStatement 

func (ClassDef) IsStatement()

func (*ClassDef) SetDecorators 

func (c *ClassDef) SetDecorators(decorators []*Decorator)

func (*ClassDef) String 

func (c *ClassDef) String() string

type CollectionExpr 

type CollectionExpr struct {
	Type  CollectionType
	Items []Expression
	// contains filtered or unexported fields
}

func (CollectionExpr) IsExpression 

func (CollectionExpr) IsExpression()

func (*CollectionExpr) String 

func (c *CollectionExpr) String() string

type CollectionType 

type CollectionType int

type CommentsFormatting 

type CommentsFormatting struct {
	// Same idea as go's comment map:
	// A comment group g is associated with a node n if:
	// - g starts on the same line as n ends
	// - g starts on the line immediately following n, and there is
	//   at least one empty line after g and before the next node
	// - g starts before n and is not associated to the node before n
	//   via the previous rules
	LeadingCommentGroups     [][]string
	TrailingCommentGroups    [][]string
	TrailingNewLine          bool
	TrailingLineContinuation bool
}

type ComprehensionExpr 

type ComprehensionExpr struct {
	Type      ComprehensionType
	Key       Expression // Set by dict / set comprehension
	Value     Expression // Set by list / generator / dict comprehension
	Iterators []*Iterator
	// contains filtered or unexported fields
}

func (ComprehensionExpr) IsExpression 

func (ComprehensionExpr) IsExpression()

func (*ComprehensionExpr) String 

func (c *ComprehensionExpr) String() string

type ComprehensionType 

type ComprehensionType int
const (
	UnknownComprehension                     = 0
	GeneratorComprehension ComprehensionType = 1
	ListComprehension      ComprehensionType = 2
	SetComprehension       ComprehensionType = 3
	DictComprehension      ComprehensionType = 4
)

type ConditionClause 

type ConditionClause struct {
	// predicate for if/while stmt, exceptions for except clauses.
	// nil Matches in except clause means unconditional matching.
	Matches Expression

	Alias Expression // Optional.  Only used by except clauses

	Branch []Statement
}

func (*ConditionClause) String 

func (c *ConditionClause) String() string

type ConditionExpr 

type ConditionExpr struct {
	Predicate Expression
	True      Expression
	False     Expression
	// contains filtered or unexported fields
}

func (ConditionExpr) IsExpression 

func (ConditionExpr) IsExpression()

func (*ConditionExpr) String 

func (c *ConditionExpr) String() string

type ConditionStmt 

type ConditionStmt struct {
	If   *ConditionClause
	Elif []*ConditionClause // Optional
	Else []Statement        // Optional
	// contains filtered or unexported fields
}

func (ConditionStmt) IsStatement 

func (ConditionStmt) IsStatement()

func (*ConditionStmt) String 

func (c *ConditionStmt) String() string

type Context 

type Context struct {
	Statements []Statement

	ParseError error
	// contains filtered or unexported fields
}

func NewContext 

func NewContext(fileName string, reader io.Reader) (*Context, error)

NOTE: lex errors are returned immediately.

func (*Context) Error 

func (c *Context) Error(msg string)

func (*Context) Lex 

func (c *Context) Lex(lval *yySymType) int

hand rolling a lexer =...(

func (*Context) PrintTokens 

func (c *Context) PrintTokens()

type ContinueStmt 

type ContinueStmt struct {
	TokenStmt
}

func NewContinueStmt 

func NewContinueStmt(t *Token) *ContinueStmt

func (ContinueStmt) IsStatement 

func (ContinueStmt) IsStatement()

type Decorator 

type Decorator struct {
	Expression
}

Decorator is a pseudo-Expression

func NewDecorator 

func NewDecorator(at *Token, expr Expression, newline *Token) *Decorator

func (*Decorator) String 

func (d *Decorator) String() string

type DelStmt 

type DelStmt struct {
	List []Expression
	// contains filtered or unexported fields
}

func NewDelStmt 

func NewDelStmt(del *Token, list *ExprList) *DelStmt

func (DelStmt) IsStatement 

func (DelStmt) IsStatement()

func (*DelStmt) String 

func (d *DelStmt) String() string

type DotExpr 

type DotExpr struct {
	Parent Expression
	Field  *Token
	// contains filtered or unexported fields
}

<parent>.<field>

func (DotExpr) IsExpression 

func (DotExpr) IsExpression()

func (*DotExpr) String 

func (f *DotExpr) String() string

type EvalExpr 

type EvalExpr struct {
	Expression
}

func (*EvalExpr) String 

func (e *EvalExpr) String() string

type ExecStmt 

type ExecStmt struct {
	Expr   Expression
	Global Expression
	Local  Expression
	// contains filtered or unexported fields
}

func (ExecStmt) IsStatement 

func (ExecStmt) IsStatement()

func (*ExecStmt) String 

func (e *ExecStmt) String() string

type ExprList 

type ExprList struct {
	Expressions        []Expression
	ExplicitCollection bool
}

Intermediate node. Do not appear in the parsed tree. there are many ways to interpret exprlist / testlist / etc =...( (i) and (i,) have different meanings

func NewExprList 

func NewExprList(list []Expression) *ExprList

func (*ExprList) ConvertToExpr 

func (e *ExprList) ConvertToExpr() Expression

type ExprStmt 

type ExprStmt struct {
	Expression
	// contains filtered or unexported fields
}

func NewExprStmt 

func NewExprStmt(expr Expression) *ExprStmt

func (ExprStmt) IsStatement 

func (ExprStmt) IsStatement()

func (*ExprStmt) String 

func (e *ExprStmt) String() string

type Expression 

type Expression interface {
	NodeInfo

	IsExpression()
}

func DottedNameToExpr 

func DottedNameToExpr(names []*Token) Expression

func NewExpressionFromTrailers 

func NewExpressionFromTrailers(
	atom Expression,
	trailers []interface{}) Expression

func NewFactorExpr 

func NewFactorExpr(factorSigns []*Token, expr Expression) Expression

type ForStmt 

type ForStmt struct {
	Iterator *Iterator

	Loop []Statement
	Else []Statement // Optional
	// contains filtered or unexported fields
}

func (ForStmt) IsStatement 

func (ForStmt) IsStatement()

func (*ForStmt) String 

func (f *ForStmt) String() string

type FromStmt 

type FromStmt struct {
	DotPrefixCount int
	ModulePath     []*Token

	// Empty list implies "import *"
	Imports []*ImportClause
	// contains filtered or unexported fields
}

func (FromStmt) IsStatement 

func (FromStmt) IsStatement()

func (*FromStmt) String 

func (f *FromStmt) String() string

type FuncDef 

type FuncDef struct {
	Decorators []*Decorator

	Name       string
	Arguments  []*Argument
	Statements []Statement
	// contains filtered or unexported fields
}

TODO doc string ...

func NewFuncDef 

func NewFuncDef(
	def *Token,
	name *Token,
	args *ArgumentList,
	colon *Token,
	stmts []Statement) *FuncDef

func (FuncDef) IsStatement 

func (FuncDef) IsStatement()

func (*FuncDef) SetDecorators 

func (f *FuncDef) SetDecorators(decorators []*Decorator)

func (*FuncDef) String 

func (f *FuncDef) String() string

type GlobalStmt 

type GlobalStmt struct {
	Names []*Token
	// contains filtered or unexported fields
}

func (GlobalStmt) IsStatement 

func (GlobalStmt) IsStatement()

func (*GlobalStmt) String 

func (g *GlobalStmt) String() string

type Identifier 

type Identifier struct {
	TokenExpr
}

func NewIdentifier 

func NewIdentifier(t *Token) *Identifier

func (Identifier) IsExpression 

func (Identifier) IsExpression()

type ImportClause 

type ImportClause struct {
	Node

	Name  *Token
	Alias *Token
}

func (*ImportClause) String 

func (i *ImportClause) String() string

type ImportStmt 

type ImportStmt struct {
	ModulePath []*Token
	Alias      *Token // Optional
	// contains filtered or unexported fields
}

func (ImportStmt) IsStatement 

func (ImportStmt) IsStatement()

func (*ImportStmt) String 

func (i *ImportStmt) String() string

type Iterator 

type Iterator struct {
	Node

	BoundVariables Expression
	Source         Expression
	Filters        []Expression // Optional
}

Used by both for loops and comprehensions for <binded variables> in <source> if <filter>

func (*Iterator) String 

func (i *Iterator) String() string

type LambdaExpr 

type LambdaExpr struct {
	Arguments []*Argument
	Value     Expression
	// contains filtered or unexported fields
}

func (LambdaExpr) IsExpression 

func (LambdaExpr) IsExpression()

func (*LambdaExpr) String 

func (l *LambdaExpr) String() string

type Lexer 

type Lexer struct {
	// contains filtered or unexported fields
}

A "look ahead" lexer which handles line continuation, transforms ';' to newlines to simply parsing, and groups comments.

func NewLexer 

func NewLexer(fileName string, reader io.Reader) (*Lexer, error)

NOTE: lex errors are return immediately during initialization.

func (*Lexer) Lex 

func (l *Lexer) Lex(lval *yySymType) int

func (*Lexer) PrintTokens 

func (l *Lexer) PrintTokens()

func (*Lexer) ToError 

func (l *Lexer) ToError(msg string) error

type Location 

type Location struct {
	FileName string
	Line     int // one-base
	Column   int // zero-base
}

maybe track end line / end column?

func (Location) ShortString 

func (l Location) ShortString() string

func (Location) String 

func (l Location) String() string

type LocationReader 

type LocationReader struct {
	Location
	// contains filtered or unexported fields
}

Not efficient, but probably good enough

func NewLocationReader 

func NewLocationReader(fileName string, reader io.Reader) *LocationReader

func (*LocationReader) Peek 

func (r *LocationReader) Peek(n int) ([]byte, error)

func (*LocationReader) Read 

func (r *LocationReader) Read(buf []byte) (int, error)

func (*LocationReader) ReadByte 

func (r *LocationReader) ReadByte() (byte, error)

func (*LocationReader) ReadString 

func (r *LocationReader) ReadString(delim byte) (string, error)

type Node 

type Node struct {
	Location

	CommentsFormatting
}

func (*Node) MergeFrom 

func (n *Node) MergeFrom(other *Node)

func (*Node) MergeLeadingFrom 

func (n *Node) MergeLeadingFrom(other *Node)

func (*Node) MergeTrailingFrom 

func (n *Node) MergeTrailingFrom(other *Node)

func (*Node) NodeInfo 

func (n *Node) NodeInfo() *Node

func (*Node) String 

func (n *Node) String() string

type NodeInfo 

type NodeInfo interface {
	String() string

	NodeInfo() *Node
	// contains filtered or unexported methods
}

TODO: Add Loc, comment, extra

type None 

type None struct {
	TokenExpr
}

func NewNone 

func NewNone(t *Token) *None

func (None) IsExpression 

func (None) IsExpression()

type Number 

type Number struct {
	TokenExpr
}

func NewNumber 

func NewNumber(t *Token) *Number

func (Number) IsExpression 

func (Number) IsExpression()

type PassStmt 

type PassStmt struct {

	// newlines and comment blocks are implicitly pass statments
	IsImplicit bool
	// contains filtered or unexported fields
}

func NewPassStmt 

func NewPassStmt(token *Token, implicit bool) *PassStmt

func (PassStmt) IsStatement 

func (PassStmt) IsStatement()

func (*PassStmt) String 

func (p *PassStmt) String() string

type PrintStmt 

type PrintStmt struct {
	Output           Expression   // Optional
	Values           []Expression // Optional
	LineContinuation bool
	// contains filtered or unexported fields
}

func NewPrintStmt 

func NewPrintStmt(
	prnt *Token,
	shift *Token,
	out Expression,
	values *ExprList) *PrintStmt

func (PrintStmt) IsStatement 

func (PrintStmt) IsStatement()

func (*PrintStmt) String 

func (p *PrintStmt) String() string

type RaiseStmt 

type RaiseStmt struct {
	First  Expression
	Second Expression
	Third  Expression
	// contains filtered or unexported fields
}

func NewRaiseStmt 

func NewRaiseStmt(
	r *Token,
	first Expression,
	comma1 *Token,
	second Expression,
	comma2 *Token,
	third Expression) *RaiseStmt

func (RaiseStmt) IsStatement 

func (RaiseStmt) IsStatement()

func (*RaiseStmt) String 

func (r *RaiseStmt) String() string

type ReturnStmt 

type ReturnStmt struct {
	Value Expression
	// contains filtered or unexported fields
}

func NewReturnStmt 

func NewReturnStmt(ret *Token, value Expression) *ReturnStmt

func (ReturnStmt) IsStatement 

func (ReturnStmt) IsStatement()

func (*ReturnStmt) String 

func (r *ReturnStmt) String() string

type Statement 

type Statement interface {
	NodeInfo

	IsStatement()
}

type String 

type String struct {
	Pieces []*Token
	// contains filtered or unexported fields
}

func (String) IsExpression 

func (String) IsExpression()

func (*String) String 

func (s *String) String() string

type Subscript 

type Subscript struct {
	// dot dot dot
	Ellipsis bool // When true, Index/Left/Middle/Right must be nil

	// Exact index.
	Index Expression

	// <left>:<middle>:<right>
	Left   Expression
	Middle Expression
	Right  Expression
}

This is a "union" of Ellipsis, Index, and (Left:Middle:Right)

func (*Subscript) String 

func (s *Subscript) String() string

type SubscriptExpr 

type SubscriptExpr struct {
	Source     Expression
	Subscripts []*Subscript
	// contains filtered or unexported fields
}

func (SubscriptExpr) IsExpression 

func (SubscriptExpr) IsExpression()

func (*SubscriptExpr) String 

func (s *SubscriptExpr) String() string

type Token 

type Token struct {
	Node

	TokenType int
	Value     string
}

func (*Token) String 

func (t *Token) String() string

type TokenExpr 

type TokenExpr struct {
	DebugType string

	Value string
	// contains filtered or unexported fields
}

Base type for None, Number, and Identifer.

func NewTokenExpr 

func NewTokenExpr(t *Token, debugType string) TokenExpr

func (TokenExpr) IsExpression 

func (TokenExpr) IsExpression()

func (*TokenExpr) String 

func (t *TokenExpr) String() string

type TokenStmt 

type TokenStmt struct {
	DebugType string
	// contains filtered or unexported fields
}

Base type for break & continue

func NewTokenStmt 

func NewTokenStmt(t *Token, debugType string) TokenStmt

func (TokenStmt) IsStatement 

func (TokenStmt) IsStatement()

func (*TokenStmt) String 

func (t *TokenStmt) String() string

type Tokenizer 

type Tokenizer struct {
	// contains filtered or unexported fields
}

func (*Tokenizer) CurrentLocation 

func (c *Tokenizer) CurrentLocation() Location

func (*Tokenizer) Next 

func (c *Tokenizer) Next() (*Token, error)

hand rolling a tokenizer =...(

type TryStmt 

type TryStmt struct {
	Try     []Statement
	Except  []*ConditionClause // Optional
	Else    []Statement        // Optional
	Finally []Statement        // Optional
	// contains filtered or unexported fields
}

func (TryStmt) IsStatement 

func (TryStmt) IsStatement()

func (*TryStmt) String 

func (t *TryStmt) String() string

type UnaryExpr 

type UnaryExpr struct {
	Op    *Token
	Value Expression
	// contains filtered or unexported fields
}

func (UnaryExpr) IsExpression 

func (UnaryExpr) IsExpression()

func (*UnaryExpr) String 

func (u *UnaryExpr) String() string

type WhileStmt 

type WhileStmt struct {
	Loop *ConditionClause

	Else []Statement // Optional
	// contains filtered or unexported fields
}

func (WhileStmt) IsStatement 

func (WhileStmt) IsStatement()

func (*WhileStmt) String 

func (w *WhileStmt) String() string

type WithClause 

type WithClause struct {
	Value         Expression
	BoundVariable Expression
}

func (*WithClause) String 

func (w *WithClause) String() string

type WithStmt 

type WithStmt struct {
	WithClauses []*WithClause
	Statements  []Statement
	// contains filtered or unexported fields
}

func (WithStmt) IsStatement 

func (WithStmt) IsStatement()

func (*WithStmt) String 

func (w *WithStmt) String() string

type YieldExpr 

type YieldExpr struct {
	Expression // nil means implicit None
}

func (*YieldExpr) String 

func (y *YieldExpr) String() string

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.