lex

Documentation

Index

• Constants
• Variables
• func FeatureValues(pairs []string, withType bool) []string
• func ParseMSR(msr string, add_suf bool) (string, string, map[string]string, string, error)
• func ParseMSRSuffix(hostPOS, msr string) (string, string, map[string]string, string, error)
• type AnalyzedToken
  • func ProcessAnalyzedPrefix(analysis string) (*AnalyzedToken, error)
  • func ProcessAnalyzedToken(analysis string) (*AnalyzedToken, error)
  • func Read(input io.Reader, format string) ([]*AnalyzedToken, error)
  • func ReadFile(filename string, format string) ([]*AnalyzedToken, error)
  • func (a *AnalyzedToken) NumMorphemes() (num int)
• type LexReader

Constants

const (
	APPROX_LEX_SIZE         = 100000
	SEPARATOR               = " "
	MSR_SEPARATOR           = ":"
	FEATURE_SEPARATOR       = "-"
	PREFIX_SEPARATOR        = "^"
	PREFIX_MSR_SEPARATOR    = "+"
	FEATURE_PAIR_SEPARATOR  = "|"
	FEATURE_VALUE_SEPARATOR = "="
)

Variables

var (
	ADD_NNP_NO_FEATS = false
	HEBREW_XLITER8   = &xliter8.Hebrew{}
	LOG_FAILURES     = false
	SKIP_BINYAN      = true
	SKIP_POLAR       = true
	SUFFIX_ONLY_CPOS = map[string]bool{
		"NN":       true,
		"DT":       true,
		"EX":       true,
		"PRP":      true,
		"PRP-REF":  true,
		"PRP-PERS": true,
		"QW":       true,
	}
	MSR_TYPE_FROM_VALUE = map[string]string{
		"1":              "per=1",
		"2":              "per=2",
		"3":              "per=3",
		"A":              "per=A",
		"BEINONI":        "tense=BEINONI",
		"D":              "num=D",
		"DP":             "num=D|num=P",
		"F":              "gen=F",
		"FUTURE":         "tense=FUTURE",
		"IMPERATIVE":     "tense=IMPERATIVE",
		"M":              "gen=M",
		"MF":             "gen=F|gen=M",
		"SP":             "num=s|num=P",
		"NEGATIVE":       "polar=neg",
		"P":              "num=P",
		"PAST":           "tense=PAST",
		"POSITIVE":       "polar=pos",
		"S":              "num=S",
		"PERS":           "type=PERS",
		"DEM":            "type=DEM",
		"REF":            "type=REF",
		"IMP":            "type=IMP",
		"INT":            "type=INT",
		"HIFIL":          "binyan=HIFIL",
		"PAAL":           "binyan=PAAL",
		"NIFAL":          "binyan=NIFAL",
		"HITPAEL":        "binyan=HITPAEL",
		"PIEL":           "binyan=PIEL",
		"PUAL":           "binyan=PUAL",
		"HUFAL":          "binyan=HUFAL",
		"TOINFINITIVE":   "type=TOINFINITIVE",
		"BAREINFINITIVE": "type=BAREINFINITIVE",
		"COORD":          "type=COORD",
		"SUB":            "type=SUB",
		"REL":            "type=REL",
		"SUBCONJ":        "type=SUBCONJ",
	}
	SKIP_ALL_TYPE bool = true
	SKIP_TYPES         = map[string]bool{
		"COORD": true,
	}
	PP_FROM_MSR      map[string][]string
	PP_FROM_MSR_DATA = []string{

		"gen=MF|num=P|per=1:אנחנו",
		"gen=MF|num=S|per=1:אני",
		"gen=F|num=S|per=2:את",
		"gen=M|num=S|per=2:אתה",
		"gen=M|num=P|per=2:אתם",
		"gen=F|num=P|per=2:אתן",
		"gen=M|num=S|per=3:הוא",
		"gen=F|num=S|per=3:היא",
		"gen=M|num=P|per=3:הם",
		"gen=F|num=P|per=3:הן",
	}
	PP_BRIDGE = map[string]string{
		"CD":   "של",
		"NN":   "של",
		"VB":   "את",
		"BN":   "את",
		"IN":   "",
		"INTJ": "",
		"RB":   "",
	}
)

Functions

func FeatureValues

func FeatureValues(pairs []string, withType bool) []string

func ParseMSR

func ParseMSR(msr string, add_suf bool) (string, string, map[string]string, string, error)

func ParseMSRSuffix

func ParseMSRSuffix(hostPOS, msr string) (string, string, map[string]string, string, error)

Types

type AnalyzedToken

type AnalyzedToken struct {
	Token     string
	Morphemes []types.BasicMorphemes
}

func ProcessAnalyzedPrefix

func ProcessAnalyzedPrefix(analysis string) (*AnalyzedToken, error)

func ProcessAnalyzedToken

func ProcessAnalyzedToken(analysis string) (*AnalyzedToken, error)

func Read

func Read(input io.Reader, format string) ([]*AnalyzedToken, error)

func ReadFile

func ReadFile(filename string, format string) ([]*AnalyzedToken, error)

func (*AnalyzedToken) NumMorphemes

func (a *AnalyzedToken) NumMorphemes() (num int)

type LexReader

type LexReader func(string) (*AnalyzedToken, error)