normalizer

Documentation

Index

• Variables

Variables

var Annotations = []Mapping{}/* 394 elements not displayed */

Annotations is a list of individual transformations to annotate a native AST with roles.

var Native = Transformers([][]Transformer{

	{Mappings(Annotations...)},
	{

		RolesDedup(),
	},
}...)

Native is the of list `transformer.Transformer` to apply to a native AST. To learn more about the Transformers and to see the available ones, read https://godoc.org/github.com/bblfsh/sdk/uast/transformer.

var Normalize = Transformers([][]Transformer{
	{Mappings(

		Map(
			opMoveTrivias{Var("group")},
			Check(Has{uast.KeyType: String(typeGroup)}, Var("group")),
		),
	)},
	{Mappings(Normalizers...)},
}...)

var Normalizers = []Mapping{

	Map(
		Check(
			Has{
				uast.KeyType: String("IdentifierToken"),

				"Text":      String(""),
				"Value":     String(""),
				"ValueText": String(""),
			},
			Any(),
		),
		Is(nil),
	),

	MapSemantic("IdentifierToken", uast.Identifier{}, MapObj(
		Obj{
			"IsMissing": Bool(false),

			"Text": Any(),

			"Value":     Var("name"),
			"ValueText": Var("name"),
		}, Obj{
			"Name": Var("name"),
		},
	)),

	Map(
		Check(
			Has{
				uast.KeyType: String("IdentifierName"),
				"Identifier": Is(nil),
			},
			Any(),
		),
		Is(nil),
	),

	Map(
		Obj{
			uast.KeyType: String("IdentifierName"),
			uast.KeyPos:  Any(),

			"Identifier": Var("ident"),

			"Arity": Int(0),

			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),

			"IsUnmanaged": Any(),

			"IsVar": Any(),
		},
		Var("ident"),
	),

	MapSemantic("ArgListKeyword", uast.Identifier{}, MapObj(
		Obj{
			"IsMissing": Bool(false),

			"Text":      String("__arglist"),
			"Value":     String("__arglist"),
			"ValueText": String("__arglist"),
		}, Obj{
			"Name": String("__arglist"),
		},
	)),

	MapSemantic("StringLiteralExpression", uast.String{}, MapObj(
		Obj{
			"Token": Obj{
				uast.KeyType: String("StringLiteralToken"),
				uast.KeyPos:  Any(),

				"IsMissing": Bool(false),

				"Text": Any(),

				"Value":     Var("val"),
				"ValueText": Var("val"),
			},
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
		},
		Obj{
			"Value": Var("val"),
		},
	)),

	MapSemantic("InterpolatedStringTextToken", uast.String{}, MapObj(
		Obj{

			"LeadingTrivia":  Arr(),
			"TrailingTrivia": Arr(),

			"IsMissing": Bool(false),

			"Text": Any(),

			"Value":     Var("val"),
			"ValueText": Var("val"),
		},
		Obj{
			"Value": Var("val"),
		},
	)),

	MapSemantic("InterpolatedStringText", uast.String{}, MapObj(
		Obj{
			"TextToken": Obj{
				uast.KeyType: String(uast.TypeOf(uast.String{})),
				uast.KeyPos:  Any(),
				"Format":     String(""),
				"Value":      Var("val"),
			},
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
		},
		Obj{
			"Value": Var("val"),
		},
	)),

	MapSemantic("TrueLiteralExpression", uast.Bool{}, MapObj(
		Obj{
			"Token": Obj{
				uast.KeyType: String("TrueKeyword"),
				uast.KeyPos:  Any(),

				"Text":      String("true"),
				"Value":     Bool(true),
				"ValueText": String("true"),

				"IsMissing": Bool(false),
			},
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
		},
		Obj{
			"Value": Bool(true),
		},
	)),

	MapSemantic("FalseLiteralExpression", uast.Bool{}, MapObj(
		Obj{
			"Token": Obj{
				uast.KeyType: String("FalseKeyword"),
				uast.KeyPos:  Any(),

				"Text":      String("false"),
				"Value":     Bool(false),
				"ValueText": String("false"),

				"IsMissing": Bool(false),
			},
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
		},
		Obj{
			"Value": Bool(false),
		},
	)),

	MapSemantic("Block", uast.Block{}, MapObj(
		Obj{
			"Statements": Var("stmts"),

			"OpenBraceToken":     Any(),
			"CloseBraceToken":    Any(),
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
		},
		Obj{
			"Statements": Var("stmts"),
		},
	)),

	MapSemantic("SingleLineCommentTrivia", uast.Comment{}, MapObj(
		Obj{
			uast.KeyToken: CommentText([2]string{"//", ""}, "text"),
			"IsDirective": Bool(false),
		},
		CommentNode(false, "text", nil),
	)),

	MapSemantic("MultiLineCommentTrivia", uast.Comment{}, MapObj(
		Obj{
			uast.KeyToken: CommentText([2]string{"/*", "*/"}, uast.KeyToken),
			"IsDirective": Bool(false),
		},
		CommentNode(true, uast.KeyToken, nil),
	)),

	MapSemantic("SingleLineDocumentationCommentTrivia", uast.Comment{}, MapObj(
		Obj{
			uast.KeyToken: CommentText([2]string{"///", ""}, "text"),
			"IsDirective": Bool(false),
		},
		CommentNode(false, "text", nil),
	)),

	MapSemantic("UsingDirective", uast.Import{}, MapObj(
		Obj{
			"Name": Var("path"),

			"SemicolonToken": Any(),
			"UsingKeyword":   Any(),

			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
			"StaticKeyword": If("isStatic",
				Check(HasType("StaticKeyword"), Any()),
				Check(HasType("None"), Any()),
			),
			"Alias": If("isAlias",
				Obj{
					uast.KeyType:         String("NameEquals"),
					uast.KeyPos:          Any(),
					"EqualsToken":        Any(),
					"IsMissing":          Bool(false),
					"IsStructuredTrivia": Bool(false),
					"Name":               Var("alias"),
				},
				Is(nil),
			),
		},
		Obj{
			"Path": If("isAlias",
				UASTType(uast.Alias{}, Obj{
					"Name": Var("alias"),
					"Node": Var("path"),
				}),
				Var("path"),
			),
			"All": Bool(true),
			"Target": If("isStatic",
				Obj{"static": Bool(true)},
				Is(nil),
			),
		},
	)),

	MapSemantic("QualifiedName", uast.QualifiedIdentifier{}, MapObj(
		CasesObj("case",

			Obj{
				"Right":              Var("right"),
				"Arity":              Int(0),
				"DotToken":           Any(),
				"IsMissing":          Bool(false),
				"IsStructuredTrivia": Bool(false),
				"IsUnmanaged":        Bool(false),
				"IsVar":              Bool(false),
			},

			Objs{

				{
					"Left": Check(HasType(uast.Identifier{}), Var("left")),
				},

				{
					"Left": UASTType(uast.QualifiedIdentifier{}, Obj{

						uast.KeyPos: Any(),
						"Names":     Var("names"),
					}),
				},
			},
		),
		CasesObj("case", nil,
			Objs{

				{
					"Names": Arr(Var("left"), Var("right")),
				},

				{
					"Names": Append(Var("names"), Arr(Var("right"))),
				},
			},
		),
	)),

	MapSemantic("Parameter", uast.Argument{}, MapObj(
		Obj{
			"Identifier": Check(
				Has{
					uast.KeyType: String(uast.TypeOf(uast.Identifier{})),
					"Name":       String("__arglist"),
				}, Var("name")),
			"AttributeLists":     Arr(),
			"Default":            Var("def_init"),
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
			"Modifiers":          Arr(),
			"Type":               Var("type"),
		},
		Obj{
			"Name":        Var("name"),
			"Type":        Var("type"),
			"Init":        Var("def_init"),
			"Variadic":    Bool(true),
			"MapVariadic": Bool(false),
			"Receiver":    Bool(false),
		},
	)),

	MapSemantic("Parameter", uast.Argument{}, MapObj(
		Obj{
			"Identifier": Check(Has{
				uast.KeyType: String(uast.TypeOf(uast.Identifier{})),
			}, Var("name")),
			"AttributeLists":     Arr(),
			"Default":            Var("def_init"),
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Any(),
			"Modifiers": opArrHasKeyword{
				keyword: "ParamsKeyword",
				opHas:   Var("variadic"),
				opRest: opArrHasKeyword{
					keyword: "ThisKeyword",
					opHas:   Var("this"),
					opRest:  Var("rest"),
				},
			},
			"Type": Var("type"),
		},
		Obj{
			"Name": Var("name"),
			"Type": opArrToChain{
				opMods: Var("rest"),
				opType: Var("type"),
			},
			"Init":        Var("def_init"),
			"Variadic":    Var("variadic"),
			"MapVariadic": Bool(false),
			"Receiver":    Var("this"),
		},
	)),

	funcDefMap("MethodDeclaration", true,
		Obj{

			"Arity":                      Any(),
			"ExplicitInterfaceSpecifier": Is(nil),
			"ConstraintClauses": Cases("caseConstraint",
				Arr(),
				NotEmpty(Var("constraints")),
			),
			"TypeParameterList": Cases("caseTypeParams",
				Is(nil),
				Arr(),
				NotEmpty(Var("typeParams")),
			),
		},
		Cases("caseTypeParams",
			Is(nil),
			Is(nil),
			NotEmpty(Var("typeParams")),
		),
		Cases("caseConstraint",
			Is(nil),
			NotEmpty(Var("constraints")),
		),
	),

	MapSemantic("ConstructorDeclaration", uast.FunctionGroup{}, MapObj(
		Obj{

			"Identifier": Var("name"),
			"ParameterList": Obj{
				uast.KeyType:         String("ParameterList"),
				uast.KeyPos:          Any(),
				"OpenParenToken":     Any(),
				"CloseParenToken":    Any(),
				"IsMissing":          Bool(false),
				"IsStructuredTrivia": Bool(false),
				"Parameters":         Var("params"),
			},
			"IsMissing":          Bool(false),
			"IsStructuredTrivia": Bool(false),
			"SemicolonToken":     Any(),
			"AttributeLists": Cases("caseAttrs",
				Arr(),
				Check(OfKind(nodes.KindArray), Var("attr")),
				Check(OfKind(nodes.KindObject), Var("attr")),
			),

			"ExpressionBody": Is(nil),

			"Initializer": If("hasBaseInit",

				Check(HasType("BaseConstructorInitializer"), Var("baseInit")),

				Is(nil),
			),
			"Body": Part("bodyMap", Obj{
				"Statements": Var("stmts"),
			}),
			"Modifiers": Cases("caseMods",
				Arr(),
				NotEmpty(Var("modifiers")),
			),
		},
		Obj{
			"Nodes": Arr(
				Cases("caseAttrs",
					Is(nil),
					Var("attr"),
					Arr(Var("attr")),
				),
				Cases("caseMods",
					Is(nil),
					NotEmpty(Var("modifiers")),
				),
				UASTType(uast.Alias{}, Obj{
					"Name": Var("name"),
					"Node": UASTType(uast.Function{}, Obj{

						"Body": Part("bodyMap", Obj{
							"Statements": If("hasBaseInit",

								PrependOne(Var("baseInit"), Var("stmts")),

								Var("stmts"),
							),
						}),
						"Type": UASTType(uast.FunctionType{}, Obj{
							"Arguments": Var("params"),
						}),
					}),
				}),
			),
		},
	)),
	funcDefMap("DestructorDeclaration", false, Obj{
		"TildeToken": Any(),
	}),

	Map(
		opMergeGroups{Var("group")},
		Check(
			Has{uast.KeyType: In(
				nodes.String(typeFuncGroup),
				nodes.String(typeGroup),
			)},
			Var("group"),
		),
	),
}

Normalizers is the main block of normalization rules to convert native AST to semantic UAST.

var Preprocess = Transformers([][]Transformer{
	{Mappings(Preprocessors...)},
}...)

var PreprocessCode = []CodeTransformer{
	positioner.FromUTF16Offset(),
	positioner.TokenFromSource{
		Types: []string{
			"SingleLineCommentTrivia",
			"SingleLineDocumentationCommentTrivia",
			"MultiLineCommentTrivia",
		},
	},
}

PreprocessCode is a special block of transformations that are applied at the end and can access original source code file. It can be used to improve or fix positional information.

https://godoc.org/github.com/bblfsh/sdk/uast/transformer/positioner

var Preprocessors = []Mapping{

	Map(
		Obj{
			uast.KeyType: Check(
				In(
					nodes.String("WhitespaceTrivia"),
					nodes.String("EndOfLineTrivia"),
					nodes.String("SkippedTokensTrivia"),
				),
				Any(),
			),
			"FullSpan":    Any(),
			"Span":        Any(),
			"SpanStart":   Any(),
			"IsDirective": Bool(false),
		},

		Is(nil),
	),

	Map(
		Part("_", Obj{
			"LeadingTrivia": dropNils{Var("arr")},
		}),
		Part("_", Obj{
			"LeadingTrivia": Var("arr"),
		}),
	),
	Map(
		Part("_", Obj{
			"TrailingTrivia": dropNils{Var("arr")},
		}),
		Part("_", Obj{
			"TrailingTrivia": Var("arr"),
		}),
	),

	Map(
		Part("_", Obj{
			uast.KeyType: String("TextSpan"),
			"IsEmpty":    Any(),
		}),
		Part("_", Obj{
			uast.KeyType: String("TextSpan"),
		}),
	),

	Map(
		Part("_", Obj{
			"SpanStart": Any(),
		}),
		Part("_", Obj{}),
	),

	Map(
		Part("_", CasesObj("case", Obj{}, Objs{

			{
				uast.KeyType: Check(
					In(useFullSpan...),
					Var("typ"),
				),
				"FullSpan": Obj{
					uast.KeyType: String("TextSpan"),
					"Length":     Any(),
					"Start":      Var("start"),
					"End":        Var("end"),
				},

				"Span": Any(),
			},

			{
				uast.KeyType: Check(
					Not(In(useFullSpan...)),
					Var("typ"),
				),
				"Span": Obj{
					uast.KeyType: String("TextSpan"),
					"Length":     Any(),
					"Start":      Var("start"),
					"End":        Var("end"),
				},

				"FullSpan": Any(),
			},
		})),
		Part("_", CasesObj("case", Obj{

			"spanStart": Var("start"),
			"spanEnd":   Var("end"),
		}, Objs{

			{
				uast.KeyType: Check(
					In(useFullSpan...),
					Var("typ"),
				),
			},

			{
				uast.KeyType: Check(
					Not(In(useFullSpan...)),
					Var("typ"),
				),
			},
		})),
	),

	ObjectToNode{
		OffsetKey:    "spanStart",
		EndOffsetKey: "spanEnd",
	}.Mapping(),

	Map(
		Part("_", Obj{
			uast.KeyType: String("SingleLineCommentTrivia"),
		}),
		Part("_", Obj{
			uast.KeyType:  String("SingleLineCommentTrivia"),
			uast.KeyToken: String(""),
		}),
	),
	Map(
		Part("_", Obj{
			uast.KeyType: String("SingleLineDocumentationCommentTrivia"),
		}),
		Part("_", Obj{
			uast.KeyType:  String("SingleLineDocumentationCommentTrivia"),
			uast.KeyToken: String(""),
		}),
	),
	Map(
		Part("_", Obj{
			uast.KeyType: String("MultiLineCommentTrivia"),
		}),
		Part("_", Obj{
			uast.KeyType:  String("MultiLineCommentTrivia"),
			uast.KeyToken: String(""),
		}),
	),
}

Preprocessors is a block of AST preprocessing rules rules.

var Transforms = driver.Transforms{
	Namespace:      "csharp",
	Preprocess:     Preprocess,
	PreprocessCode: PreprocessCode,
	Normalize:      Normalize,
	Annotations:    Native,
}