tokenization

Documentation

Index

• Variables
• type Token
  • func (t *Token) Bytes() []byte
  • func (t *Token) Len() int
  • func (t *Token) Next() *Token
  • func (t *Token) NextNCollection(n int) (nexts *TokenSliceCollection, foundAll bool)
  • func (t *Token) NextNTypesCollection(types []TokenType) (nexts *TokenSliceCollection, foundAll bool)
  • func (t *Token) NextOfType(y TokenType) *Token
  • func (t *Token) NextOfTypes(potentialTypes ...TokenType) *Token
  • func (t *Token) NextsCollectionUntilEndOfPotentialTypes(potentialTypes ...TokenType) (nexts *TokenSliceCollection, foundAny bool)
  • func (t *Token) NextsCollectionUntilMeetType(y TokenType) (nexts *TokenSliceCollection, foundAny bool)
  • func (t *Token) NextsCollectionUntilStartOfPotentialTypes(potentialTypes ...TokenType) (nexts *TokenSliceCollection, foundAny bool)
  • func (t *Token) Prev() *Token
  • func (t *Token) PrevNCollection(n int) (prevs *TokenSliceCollection, foundAll bool)
  • func (t *Token) PrevNTypesCollection(types []TokenType) (prevs *TokenSliceCollection, foundAll bool)
  • func (t *Token) PrevOfType(y TokenType) *Token
  • func (t *Token) PrevsCollectionUntilEndOfPotentialTypes(potentialTypes ...TokenType) (prevs *TokenSliceCollection, foundAny bool)
  • func (t *Token) PrevsCollectionUntilMeetToken(stopToken *Token) (prevs *TokenSliceCollection, foundAny bool)
  • func (t *Token) PrevsCollectionUntilStartOfPotentialTypes(potentialTypes ...TokenType) (prevs *TokenSliceCollection, foundAny bool)
  • func (t *Token) SimpleCloneForClosingTag() *Token
  • func (t *Token) String() string
  • func (t *Token) TypeDatum() *TokenTypeDatum
• type TokenListCollection
  • func TokenListCollectionNew(input []byte) *TokenListCollection
  • func (c *TokenListCollection) HTML() template.HTML
  • func (c *TokenListCollection) InsertAfter(referenceToken *Token, tokens ...*Token)
  • func (c *TokenListCollection) InsertBefore(referenceToken *Token, tokens ...*Token)
  • func (c *TokenListCollection) InsertNewAfter(referenceToken *Token, y TokenType, inputStartIndex int, inputEndIndex int) *Token
  • func (c *TokenListCollection) InsertNewBefore(referenceToken *Token, y TokenType, inputStartIndex int, inputEndIndex int) *Token
  • func (c *TokenListCollection) InsertNewEmptyAfter(referenceToken *Token, y TokenType) *Token
  • func (c *TokenListCollection) InsertNewEmptyBefore(referenceToken *Token, y TokenType) *Token
  • func (c *TokenListCollection) InsertNewSingleAfter(referenceToken *Token, y TokenType, inputStartIndex int) *Token
  • func (c *TokenListCollection) InsertNewSingleBefore(referenceToken *Token, y TokenType, inputStartIndex int) *Token
  • func (c *TokenListCollection) Len() int
  • func (c *TokenListCollection) Peek() *Token
  • func (c *TokenListCollection) Pop() *Token
  • func (c *TokenListCollection) Push(tokens ...*Token)
  • func (c *TokenListCollection) PushNew(y TokenType, inputStartIndex int, inputEndIndex int) *Token
  • func (c *TokenListCollection) PushNewEmpty(y TokenType) *Token
  • func (c *TokenListCollection) PushNewSingle(y TokenType, inputStartIndex int) *Token
  • func (c *TokenListCollection) Remove(tokens ...*Token)
  • func (c *TokenListCollection) String() string
• type TokenSliceCollection
  • func TokenSliceCollectionNew() *TokenSliceCollection
  • func (c *TokenSliceCollection) ContainsType(y TokenType) bool
  • func (c *TokenSliceCollection) Get(index int) *Token
  • func (t *TokenSliceCollection) Len() int
  • func (c *TokenSliceCollection) Peek() *Token
  • func (c *TokenSliceCollection) Pop() *Token
  • func (c *TokenSliceCollection) Push(tokens ...*Token)
  • func (c *TokenSliceCollection) SetAllTokenTypes(y TokenType)
  • func (c *TokenSliceCollection) SetAllTokenTypesToEmpty()
• type TokenType
  • func (t TokenType) ClassName() string
  • func (t TokenType) String() string
• type TokenTypeDatum

Variables

var (
	TokenTypeListLinkSegmentText = []TokenType{
		TokenTypeTextGroup,
		TokenTypeSpaceGroup,
		TokenTypeTabGroup,
		TokenTypeBackslash,
		TokenTypeAsterisk,
		TokenTypeAsteriskDouble,
		TokenTypeAsteriskTriple,
		TokenTypeUnderscore,
		TokenTypeUnderscoreDouble,
		TokenTypeUnderscoreTriple,
		TokenTypeHyphen,
		TokenTypeHyphenDouble,
		TokenTypeHyphenTriple,
		TokenTypeEqualsDouble,
		TokenTypeBacktick,
		TokenTypeExclamation,
		TokenTypeParenthesisOpen,
		TokenTypeParenthesisClose,
	}
	TokenTypeListLinkSegmentLink = []TokenType{
		TokenTypeTextGroup,
		TokenTypeAsterisk,
		TokenTypeAsteriskDouble,
		TokenTypeUnderscore,
		TokenTypeUnderscoreDouble,
	}
	TokenTypeListLinkSegmentTitle = []TokenType{
		TokenTypeTextGroup,
		TokenTypeAsterisk,
		TokenTypeAsteriskDouble,
		TokenTypeUnderscore,
		TokenTypeUnderscoreDouble,
		TokenTypeSpaceGroup,
	}
)

var (
	TokenTypeListImageSegmentText  = TokenTypeListLinkSegmentText[:]
	TokenTypeListImageSegmentLink  = TokenTypeListLinkSegmentLink[:]
	TokenTypeListImageSegmentTitle = TokenTypeListLinkSegmentTitle[:]
)

Types

type Token

type Token struct {
	RawPrev         *Token
	RawNext         *Token
	ListCollection  *TokenListCollection
	Attributes      map[string]string
	HTML            []byte
	InputStartIndex int
	InputEndIndex   int
	Type            TokenType
	Indent          int
}

func (*Token) Bytes

func (t *Token) Bytes() []byte

func (*Token) Len

func (t *Token) Len() int

func (*Token) Next

func (t *Token) Next() *Token

func (*Token) NextNCollection

func (t *Token) NextNCollection(n int) (nexts *TokenSliceCollection, foundAll bool)

func (*Token) NextNTypesCollection

func (t *Token) NextNTypesCollection(types []TokenType) (nexts *TokenSliceCollection, foundAll bool)

func (*Token) NextOfType

func (t *Token) NextOfType(y TokenType) *Token

func (*Token) NextOfTypes

func (t *Token) NextOfTypes(potentialTypes ...TokenType) *Token

func (*Token) NextsCollectionUntilEndOfPotentialTypes

func (t *Token) NextsCollectionUntilEndOfPotentialTypes(potentialTypes ...TokenType) (nexts *TokenSliceCollection, foundAny bool)

func (*Token) NextsCollectionUntilMeetType

func (t *Token) NextsCollectionUntilMeetType(y TokenType) (nexts *TokenSliceCollection, foundAny bool)

func (*Token) NextsCollectionUntilStartOfPotentialTypes

func (t *Token) NextsCollectionUntilStartOfPotentialTypes(potentialTypes ...TokenType) (nexts *TokenSliceCollection, foundAny bool)

func (*Token) Prev

func (t *Token) Prev() *Token

func (*Token) PrevNCollection

func (t *Token) PrevNCollection(n int) (prevs *TokenSliceCollection, foundAll bool)

func (*Token) PrevNTypesCollection

func (t *Token) PrevNTypesCollection(types []TokenType) (prevs *TokenSliceCollection, foundAll bool)

func (*Token) PrevOfType

func (t *Token) PrevOfType(y TokenType) *Token

func (*Token) PrevsCollectionUntilEndOfPotentialTypes

func (t *Token) PrevsCollectionUntilEndOfPotentialTypes(potentialTypes ...TokenType) (prevs *TokenSliceCollection, foundAny bool)

func (*Token) PrevsCollectionUntilMeetToken

func (t *Token) PrevsCollectionUntilMeetToken(stopToken *Token) (prevs *TokenSliceCollection, foundAny bool)

func (*Token) PrevsCollectionUntilStartOfPotentialTypes

func (t *Token) PrevsCollectionUntilStartOfPotentialTypes(potentialTypes ...TokenType) (prevs *TokenSliceCollection, foundAny bool)

func (*Token) SimpleCloneForClosingTag

func (t *Token) SimpleCloneForClosingTag() *Token

func (*Token) String

func (t *Token) String() string

func (*Token) TypeDatum

func (t *Token) TypeDatum() *TokenTypeDatum

type TokenListCollection

type TokenListCollection struct {
	HeadToken *Token
	TailToken *Token

	TagPairCleanData [][2]*Token
	Input            []byte
	// contains filtered or unexported fields
}

func TokenListCollectionNew

func TokenListCollectionNew(input []byte) *TokenListCollection

func (*TokenListCollection) HTML

func (c *TokenListCollection) HTML() template.HTML

func (*TokenListCollection) InsertAfter

func (c *TokenListCollection) InsertAfter(referenceToken *Token, tokens ...*Token)

func (*TokenListCollection) InsertBefore

func (c *TokenListCollection) InsertBefore(referenceToken *Token, tokens ...*Token)

func (*TokenListCollection) InsertNewAfter

func (c *TokenListCollection) InsertNewAfter(referenceToken *Token, y TokenType, inputStartIndex int, inputEndIndex int) *Token

func (*TokenListCollection) InsertNewBefore

func (c *TokenListCollection) InsertNewBefore(referenceToken *Token, y TokenType, inputStartIndex int, inputEndIndex int) *Token

func (*TokenListCollection) InsertNewEmptyAfter

func (c *TokenListCollection) InsertNewEmptyAfter(referenceToken *Token, y TokenType) *Token

func (*TokenListCollection) InsertNewEmptyBefore

func (c *TokenListCollection) InsertNewEmptyBefore(referenceToken *Token, y TokenType) *Token

func (*TokenListCollection) InsertNewSingleAfter

func (c *TokenListCollection) InsertNewSingleAfter(referenceToken *Token, y TokenType, inputStartIndex int) *Token

func (*TokenListCollection) InsertNewSingleBefore

func (c *TokenListCollection) InsertNewSingleBefore(referenceToken *Token, y TokenType, inputStartIndex int) *Token

func (*TokenListCollection) Len

func (c *TokenListCollection) Len() int

func (*TokenListCollection) Peek

func (c *TokenListCollection) Peek() *Token

func (*TokenListCollection) Pop

func (c *TokenListCollection) Pop() *Token

func (*TokenListCollection) Push

func (c *TokenListCollection) Push(tokens ...*Token)

func (*TokenListCollection) PushNew

func (c *TokenListCollection) PushNew(y TokenType, inputStartIndex int, inputEndIndex int) *Token

func (*TokenListCollection) PushNewEmpty

func (c *TokenListCollection) PushNewEmpty(y TokenType) *Token

func (*TokenListCollection) PushNewSingle

func (c *TokenListCollection) PushNewSingle(y TokenType, inputStartIndex int) *Token

func (*TokenListCollection) Remove

func (c *TokenListCollection) Remove(tokens ...*Token)

func (*TokenListCollection) String

func (c *TokenListCollection) String() string

type TokenSliceCollection

type TokenSliceCollection struct {
	Tokens []*Token
}

func TokenSliceCollectionNew

func TokenSliceCollectionNew() *TokenSliceCollection

func (*TokenSliceCollection) ContainsType

func (c *TokenSliceCollection) ContainsType(y TokenType) bool

func (*TokenSliceCollection) Get

func (c *TokenSliceCollection) Get(index int) *Token

func (*TokenSliceCollection) Len

func (t *TokenSliceCollection) Len() int

func (*TokenSliceCollection) Peek

func (c *TokenSliceCollection) Peek() *Token

func (*TokenSliceCollection) Pop

func (c *TokenSliceCollection) Pop() *Token

func (*TokenSliceCollection) Push

func (c *TokenSliceCollection) Push(tokens ...*Token)

func (*TokenSliceCollection) SetAllTokenTypes

func (c *TokenSliceCollection) SetAllTokenTypes(y TokenType)

func (*TokenSliceCollection) SetAllTokenTypesToEmpty

func (c *TokenSliceCollection) SetAllTokenTypesToEmpty()

type TokenType

type TokenType uint8

const (
	TokenTypeEmpty TokenType = iota
	TokenTypeStart
	TokenTypeEnd
	TokenTypeDocumentDoctype
	TokenTypeDocumentHTMLBound
	TokenTypeDocumentHeadBound
	TokenTypeDocumentBodyBound
	TokenTypeParagraphBound
	TokenTypeBlockquoteBound
	TokenTypeHeading1Bound
	TokenTypeHeading2Bound
	TokenTypeHeading3Bound
	TokenTypeHeading4Bound
	TokenTypeHeading5Bound
	TokenTypeHeading6Bound
	TokenTypeLineBreak
	TokenTypeTextGroup
	TokenTypeSpaceGroup
	TokenTypeSpaceHair
	TokenTypeTabGroup
	TokenTypeCarriageReturn
	TokenTypeHorizontalRule
	TokenTypeBackslash
	TokenTypeAsterisk
	TokenTypeAsteriskDouble
	TokenTypeAsteriskTriple
	TokenTypeUnderscore
	TokenTypeUnderscoreDouble
	TokenTypeUnderscoreTriple
	TokenTypeHash
	TokenTypeHashDouble
	TokenTypeHashTriple
	TokenTypeHashQuadruple
	TokenTypeHashQuintuple
	TokenTypeHashSextuple
	TokenTypeHyphen
	TokenTypeHyphenDouble
	TokenTypeHyphenTriple
	TokenTypeDashEm
	TokenTypeDashEn
	TokenTypeEqualsDouble
	TokenTypeBacktick
	TokenTypeBacktickDouble
	TokenTypeExclamation
	TokenTypeParenthesisOpen
	TokenTypeParenthesisClose
	TokenTypeSquareBracketOpen
	TokenTypeSquareBracketClose
	TokenTypeAngleBracketOpen
	TokenTypeAngleBracketClose
	TokenTypeLinkBound
	TokenTypeImageBound
	TokenTypeUnorderedListBound
	TokenTypeListItemBound
)

func (TokenType) ClassName

func (t TokenType) ClassName() string

func (TokenType) String

func (t TokenType) String() string

type TokenTypeDatum

type TokenTypeDatum struct {
	Tags        []string
	SelfClosing bool
}