lex

package

v1.2.1 Latest Latest Go to latest Published: Jul 30, 2017 License: Apache-2.0 Imports: 12 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/OnlpLab/yap

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func FeatureValues(pairs []string, withType bool) []string
func ParseMSR(msr string, add_suf bool) (string, string, map[string]string, string, error)
func ParseMSRSuffix(hostPOS, msr string) (string, string, map[string]string, string, error)
type AnalyzedToken
- func (a *AnalyzedToken) NumMorphemes() (num int)
type LexReader

Constants ¶

View Source

const (
	APPROX_LEX_SIZE         = 100000
	SEPARATOR               = " "
	MSR_SEPARATOR           = ":"
	FEATURE_SEPARATOR       = "-"
	PREFIX_SEPARATOR        = "^"
	PREFIX_MSR_SEPARATOR    = "+"
	FEATURE_PAIR_SEPARATOR  = "|"
	FEATURE_VALUE_SEPARATOR = "="
)

Variables ¶

View Source

var (
	ADD_NNP_NO_FEATS       = false
	STRIP_ALL_NNP_OF_FEATS = false
	HEBREW_XLITER8         = &xliter8.Hebrew{}
	LOG_FAILURES           = false
	SKIP_BINYAN            = true
	SKIP_POLAR             = true
	SUFFIX_ONLY_CPOS       = map[string]bool{
		"NN":       true,
		"DT":       true,
		"EX":       true,
		"PRP":      true,
		"PRP-REF":  true,
		"PRP-PERS": true,
		"QW":       true,
	}
	MSR_TYPE_FROM_VALUE = map[string]string{
		"1":              "per=1",
		"2":              "per=2",
		"3":              "per=3",
		"A":              "per=A",
		"BEINONI":        "tense=BEINONI",
		"D":              "num=D",
		"DP":             "num=D|num=P",
		"F":              "gen=F",
		"FUTURE":         "tense=FUTURE",
		"IMPERATIVE":     "tense=IMPERATIVE",
		"M":              "gen=M",
		"MF":             "gen=F|gen=M",
		"SP":             "num=S|num=P",
		"NEGATIVE":       "polar=neg",
		"P":              "num=P",
		"PAST":           "tense=PAST",
		"POSITIVE":       "polar=pos",
		"S":              "num=S",
		"PERS":           "type=PERS",
		"DEM":            "type=DEM",
		"REF":            "type=REF",
		"IMP":            "type=IMP",
		"INT":            "type=INT",
		"HIFIL":          "binyan=HIFIL",
		"PAAL":           "binyan=PAAL",
		"NIFAL":          "binyan=NIFAL",
		"HITPAEL":        "binyan=HITPAEL",
		"PIEL":           "binyan=PIEL",
		"PUAL":           "binyan=PUAL",
		"HUFAL":          "binyan=HUFAL",
		"TOINFINITIVE":   "type=TOINFINITIVE",
		"BAREINFINITIVE": "type=BAREINFINITIVE",
		"COORD":          "type=COORD",
		"SUB":            "type=SUB",
		"REL":            "type=REL",
		"SUBCONJ":        "type=SUBCONJ",
	}
	SKIP_ALL_TYPE bool = true
	SKIP_TYPES         = map[string]bool{
		"COORD": true,
	}
	PP_FROM_MSR      map[string][]string
	PP_FROM_MSR_DATA = []string{

		"gen=MF|num=P|per=1:אנחנו",
		"gen=MF|num=S|per=1:אני",
		"gen=F|num=S|per=2:את",
		"gen=M|num=S|per=2:אתה",
		"gen=M|num=P|per=2:אתם",
		"gen=F|num=P|per=2:אתן",
		"gen=M|num=S|per=3:הוא",
		"gen=F|num=S|per=3:היא",
		"gen=M|num=P|per=3:הם",
		"gen=F|num=P|per=3:הן",
	}
	PP_BRIDGE = map[string]string{
		"CD":   "של",
		"NN":   "של",
		"VB":   "את",
		"BN":   "את",
		"IN":   "",
		"INTJ": "",
		"RB":   "",
	}
)

Functions ¶

func FeatureValues ¶

func FeatureValues(pairs []string, withType bool) []string

func ParseMSR ¶

func ParseMSR(msr string, add_suf bool) (string, string, map[string]string, string, error)

func ParseMSRSuffix ¶

func ParseMSRSuffix(hostPOS, msr string) (string, string, map[string]string, string, error)

Types ¶

type AnalyzedToken ¶

type AnalyzedToken struct {
	Token     string
	Morphemes []types.BasicMorphemes
}

func ProcessAnalyzedPrefix ¶

func ProcessAnalyzedPrefix(analysis string) (*AnalyzedToken, error)

func ProcessAnalyzedToken ¶

func ProcessAnalyzedToken(analysis string) (*AnalyzedToken, error)

func ProcessUDAnalyzedPrefix ¶ added in v1.2.0

func ProcessUDAnalyzedPrefix(analysis string) (*AnalyzedToken, error)

func ProcessUDAnalyzedToken ¶ added in v1.2.0

func ProcessUDAnalyzedToken(analysis string) (*AnalyzedToken, error)

func Read ¶

func Read(input io.Reader, format string, maType string) ([]*AnalyzedToken, error)

func ReadFile ¶

func ReadFile(filename string, format string, maType string) ([]*AnalyzedToken, error)

func (*AnalyzedToken) NumMorphemes ¶

func (a *AnalyzedToken) NumMorphemes() (num int)

type LexReader ¶

type LexReader func(string) (*AnalyzedToken, error)

Source Files ¶

View all Source files

lex.go

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL