lexers

Documentation

Index

• Variables
• type JSONFormatter
  • func (f *JSONFormatter) Filter(emit func(Token) error) func(Token) error

Variables

var CSS = Lexer{
	Name:      "css",
	MimeTypes: []string{"text/css"},
	Filenames: []string{"*.css"},
	States: StatesSpec{
		"root": {
			{Include: "whitespace"},
			{Include: "singleLineComment"},
			{Include: "multiLineComment"},
			{Include: "selector"},
			{Include: "declarationBlock"},
		},
		"selector": {
			{Regexp: `(\[)([^\]]+)(\])`,
				SubTypes: []TokenType{Punctuation, Attribute, Punctuation}},
			{Regexp: `(\.)([-a-zA-Z0-9]+)`,
				SubTypes: []TokenType{Punctuation, Attribute}},
			{Regexp: `@[-a-zA-Z0-9]+`, Type: Literal, State: "media"},
			{Regexp: `>`, Type: Punctuation},
			{Regexp: `\+`, Type: Punctuation},
			{Regexp: `:`, Type: Punctuation},
			{Regexp: `,`, Type: Punctuation},
			{Regexp: `[-a-zA-Z0-9]+`, Type: Attribute},
			{Regexp: `\*`, Type: Attribute},
		},
		"media": {
			{Regexp: ` and `, Type: Operator},
			{Regexp: `,`, Type: Punctuation},
			{Regexp: `[-a-zA-Z0-9]+`, Type: Attribute},
			{Regexp: `(\()` + `(\s*)` +
				`([-a-zA-Z0-9]+)` + `(:)` + `([^\)]+)` +
				`(\s*)` + `(\))`,
				SubTypes: []TokenType{Punctuation, Whitespace, Attribute, Assignment, Text, Whitespace, Punctuation}},
			{Include: "whitespace"},
			{Include: "singleLineComment"},
			{Include: "multiLineComment"},
			{Regexp: `{`, Type: Punctuation, State: "mediaBlock"},
		},
		"mediaBlock": {
			{Include: "whitespace"},
			{Include: "singleLineComment"},
			{Include: "multiLineComment"},
			{Include: "selector"},
			{Include: "declarationBlock"},
			{Regexp: `}`, Type: Punctuation, State: "#pop #pop"},
		},
		"ruleValue": {
			{Regexp: `;`, Type: "Punctuation", State: "#pop"},
			{Regexp: `.*`, Type: "Text"},
		},
		"declarationBlock": {
			{Regexp: `{`, Type: Punctuation, State: "declaration"},
		},
		"declaration": {
			{Include: "whitespace"},
			{Include: "singleLineComment"},
			{Include: "multiLineComment"},
			{Regexp: `([a-zA-Z0-9_-]+)(\w*)(:)`,
				SubTypes: []TokenType{Tag, Whitespace, Assignment},
				State:    "declarationValue"},
			{Regexp: `}`, Type: Punctuation, State: "#pop"},
			{Include: "selector"},
			{Include: "declarationBlock"},
		},
		"declarationValue": {
			{Regexp: `(")([^"]*)(")`,
				SubTypes: []TokenType{Punctuation, Text, Punctuation}},
			{Regexp: `(')([^']*)(')`,
				SubTypes: []TokenType{Punctuation, Text, Punctuation}},
			{Regexp: `[^;]+`, Type: Text},
			{Regexp: `,`, Type: Punctuation},
			{Regexp: `;`, Type: Punctuation, State: "#pop"},
		},
		"whitespace": {
			{Regexp: `[ \r\n\f\t]+`, Type: Whitespace},
		},
		"singleLineComment": {
			{Regexp: `\/\/.*`, Type: Comment},
		},
		"multiLineComment": {
			{Regexp: `\/\*`, Type: Comment, State: "multiLineCommentContents"},
		},
		"multiLineCommentContents": {
			{Regexp: `\*\/`, Type: Comment, State: "#pop"},
			{Regexp: `(.+?)(\*\/)`, Type: Comment, State: "#pop"},
			{Regexp: `.+`, Type: Comment},
		},
	},
}

var HTML = Lexer{
	Name:      "html",
	MimeTypes: []string{"text/html", "application/xhtml+xml"},
	Filenames: []string{"*.html", "*.htm", "*.xhtml"},
	States: StatesSpec{
		"root": {
			{Regexp: "[^<&]+", Type: Text},
			{Regexp: "&\\S+?;", Type: Tag},
			{Regexp: "<!--", Type: Comment, State: "comment"},
			{Regexp: "(<)(![^>]*)(>)",
				SubTypes: []TokenType{Punctuation, Tag, Punctuation}},
			{Regexp: "(</?)([\\w-]*:?[\\w-]+)(\\s*)(>)",
				SubTypes: []TokenType{Punctuation, Tag, Text, Punctuation}},
			{Regexp: "(<)([\\w-]*:?[\\w-]+)(\\s*)",
				SubTypes: []TokenType{Punctuation, Tag, Text},
				State:    "tag"},
		},
		"comment": {
			{Regexp: "-->", Type: Comment, State: "#pop"},
			{Regexp: "[^-]+", Type: Comment},
		},
		"tag": {
			{Regexp: "([\\w-]+)(=)(\\s*)",
				SubTypes: []TokenType{Attribute, Operator, Text},
				State:    "tagAttr"},
			{Regexp: "[\\w-]+\\s*", Type: Attribute},
			{Regexp: "\\s+", Type: Tag},
			{Regexp: "(/?)(\\s*)(>)",
				SubTypes: []TokenType{Punctuation, Tag, Punctuation},
				State:    "#pop",
			},
		},
		"tagAttr": {
			{Regexp: "\"[^\"]*\"", Type: String, State: "#pop"},
			{Regexp: "'[^']*'", Type: String, State: "#pop"},
			{Regexp: "\\w+", Type: String, State: "#pop"},
		},
	},
}

var HTTP = Lexer{
	Name: "http",
	States: StatesSpec{
		"root": {

			{Regexp: `^([a-zA-Z]+)( )([^ ]+)( )(HTTP)(/)([0-9\.]+)(\r\n)$`,
				SubTypes: []TokenType{Tag, Whitespace, String, Whitespace,
					Tag, Punctuation, Tag, Whitespace},
				State: "headers"},

			{Regexp: `^(HTTP)(/)([0-9\.]+)( )([0-9]+)(.*)(\r\n)$`,
				SubTypes: []TokenType{Tag, Punctuation, Tag,
					Whitespace, Number, Whitespace, String, Whitespace},
				State: "headers"},
		},
		"headers": {
			{Regexp: `^(.*?)(:)(\s*)`,
				SubTypes: []TokenType{Attribute, Assignment, Whitespace},
				State:    "headerValue"},
			{Regexp: `^\r\n$`, State: "#pop #pop"},
		},
		"headerValue": {
			{Regexp: `\r\n$`, State: "#pop", Type: Whitespace},
			{Regexp: `[^;]+?`, Type: Text},
			{Regexp: `;`, Type: Punctuation},
			{Regexp: `\r\n$`, State: "#pop"},
		},
	},
	Filters: []Filter{},
}

var JSON = Lexer{
	Name:      "json",
	MimeTypes: []string{"application/json"},
	Filenames: []string{"*.json"},
	States: StatesSpec{
		"root": {
			{Include: "value"},
		},
		"whitespace": {
			{Regexp: "\\s+", Type: Whitespace},
		},

		"literal": {
			{Regexp: "(true|false|null)", Type: Literal},
		},

		"number": {

			{Regexp: "-?[0-9]+\\.?[0-9]*[eE][\\+\\-]?[0-9]+", Type: Number},

			{Regexp: "-?[0-9]+\\.[0-9]+", Type: Number},

			{Regexp: "-?[0-9]+", Type: Number},
		},

		"string": {
			{Regexp: `(")(")`,
				SubTypes: []TokenType{Punctuation, Punctuation}},
			{Regexp: `(")((?:\\\"|[^\"])*?)(")`,
				SubTypes: []TokenType{Punctuation, String, Punctuation}},
		},

		"value": {
			{Include: "whitespace"},
			{Include: "literal"},
			{Include: "number"},
			{Include: "string"},
			{Include: "array"},
			{Include: "object"},
		},

		"object": {
			{Regexp: "{", Type: Punctuation, State: "objectKey"},
		},

		"objectKey": {
			{Include: "whitespace"},
			{Regexp: `(")((?:\\\"|[^\"])*?)(")(\s*)(:)`,
				SubTypes: []TokenType{Punctuation, Attribute, Punctuation,
					Whitespace, Assignment},
				State: "objectValue"},
			{Regexp: "}", Type: Punctuation, State: "#pop"},
		},

		"objectValue": {
			{Include: "whitespace"},
			{Include: "value"},
			{Regexp: ",", Type: Punctuation, State: "#pop"},
			{Regexp: "}", Type: Punctuation, State: "#pop #pop"},
		},

		"array": {
			{Regexp: "\\[", Type: Punctuation, State: "arrayValue"},
		},

		"arrayValue": {
			{Include: "whitespace"},
			{Include: "value"},
			{Regexp: ",", Type: Punctuation},
			{Regexp: "\\]", Type: Punctuation, State: "#pop"},
		},
	},
	Filters: []Filter{
		RemoveEmptiesFilter,
	},
	Formatter: &JSONFormatter{Indent: "  "},
}

Types

type JSONFormatter

type JSONFormatter struct {
	Indent string
}

JSONFormatter consumes a series of JSON tokens and emits additional tokens to produce indented, formatted output.

func (*JSONFormatter) Filter

func (f *JSONFormatter) Filter(emit func(Token) error) func(
	Token) error