Versions in this module Expand all Collapse all v0 v0.1.1 Oct 2, 2020 v0.1.0 Jul 6, 2020 Changes in this version + func FindRawTokenType(value, operators, separators string) text.TokenType + func JoinMultiCharSymbols(l *list.List, multiChar string) *list.List + func JoinQuote(l *list.List) *list.List + func RegroupSymbols(l *list.List, symbols string) []string + func SplitKeepSeparators(s, sep string) *list.List + type SourceReader struct + func NewSourceReader(reader io.Reader, separators, sourceName string) *SourceReader + func NewSourceReaderFromFile(filePath, separators string) *SourceReader + func (source *SourceReader) FilterLine(line *text.SourceLine, options TokenMatchingOptions) []text.Token + func (source *SourceReader) HasNext() bool + func (source *SourceReader) LineNumber() uint + func (source *SourceReader) NextLine() *list.List + func (source *SourceReader) NormalizeLine(line *list.List, lineNumber uint, symbols string) *text.SourceLine + func (source *SourceReader) SourceName() string + type TokenMatchingOptions struct + func NewMatchingOptions(operators, separators, multiCharSymbols, lineCommentChar string) TokenMatchingOptions + func (opt TokenMatchingOptions) GetAllSeparators() string + func (opt TokenMatchingOptions) GetMultiCharSymbols() string + func (opt TokenMatchingOptions) GetOperators() string + func (opt TokenMatchingOptions) LineToTokens(l *text.SourceLine, lineNumber uint, fileName string) []text.Token