analysis

type Analyzer interface {
	TokenStreamForReader(string, io.RuneReader) (TokenStream, error)
	// Returns a TokenStream suitable for fieldName, tokenizing the
	// contents of text.
	//
	// This method uses createComponents(string, Reader) to obtain an
	// instance of TokenStreamComponents. It returns the sink of the
	// components and stores the components internally. Subsequent
	// calls to this method will reuse the previously stored components
	// after resetting them through TokenStreamComponents.SetReader(Reader).
	//
	// NOTE: After calling this method, the consumer must follow the
	// workflow described in TokenStream to propperly consume its
	// contents. See the Analysis package documentation for some
	// examples demonstrating this.
	TokenStreamForString(fieldName, text string) (TokenStream, error)
	PositionIncrementGap(string) int
	OffsetGap(string) int
}

type AnalyzerImpl struct {
	Spi AnalyzerSPI
	// contains filtered or unexported fields
}

type AnalyzerSPI interface {
	// Creates a new TokenStreamComponents instance for this analyzer.
	CreateComponents(fieldName string, reader io.RuneReader) *TokenStreamComponents
	// Override this if you want to add a CharFilter chain.
	//
	// The default implementation returns reader unchanged.
	InitReader(fieldName string, reader io.RuneReader) io.RuneReader
}

type CachingTokenFilter struct {
	*TokenFilter
	// contains filtered or unexported fields
}

type CharFilterService interface {
	// Chains the corrected offset through the input CharFilter(s).
	CorrectOffset(int) int
}

type GlobalReuseStrategy struct {
	*ReuseStrategyImpl
}

type PerFieldReuseStrategy struct {
}

type ReusableStringReader struct {
	// contains filtered or unexported fields
}

type ReuseStrategy interface {
	// Gets the reusable TokenStreamComponents for the field with the
	// given name.
	ReusableComponents(*AnalyzerImpl, string) *TokenStreamComponents
	// Stores the given TokenStreamComponents as the reusable
	// components for the field with the given name.
	SetReusableComponents(*AnalyzerImpl, string, *TokenStreamComponents)
}

type ReuseStrategyImpl struct {
}

type TokenFilter struct {
	*TokenStreamImpl
	// contains filtered or unexported fields
}

type TokenStream interface {
	// Releases resouces associated with this stream.
	//
	// If you override this method, always call TokenStreamImpl.Close(),
	// otherwise some internal state will not be correctly reset (e.g.,
	// Tokenizer will panic on reuse).
	io.Closer
	Attributes() *util.AttributeSource
	// Consumers (i.e., IndexWriter) use this method to advance the
	// stream to the next token. Implementing classes must implement
	// this method and update the appropriate AttributeImpls with the
	// attributes of he next token.
	//
	// The producer must make no assumptions about the attributes after
	// the method has been returned: the caller may arbitrarily change
	// it. If the producer needs to preserve the state for subsequent
	// calls, it can use captureState to create a copy of the current
	// attribute state.
	//
	// This method is called for every token of a docuent, so an
	// efficient implementation is crucial for good performance.l To
	// avoid calls to AddAttribute(clas) and GetAttribute(Class),
	// references to all AttributeImpls that this stream uses should be
	// retrived during instantiation.
	//
	// To ensure that filters and consumers know which attributes are
	// available, the attributes must be added during instantiation.
	// Filters and consumers are not required to check for availability
	// of attribute in IncrementToken().
	IncrementToken() (bool, error)
	// This method is called by the consumer after the last token has
	// been consumed, after IncrementToken() returned false (using the
	// new TokenSream API). Streams implementing the old API should
	// upgrade to use this feature.
	//
	// This method can be used to perform any end-of-stream operations,
	// such as setting the final offset of a stream. The final offset
	// of a stream might difer from the offset of the last token eg in
	// case one or more whitespaces followed after the last token, but
	// a WhitespaceTokenizer was used.
	//
	// Additionally any skipped positions (such as those removed by a
	// stopFilter) can be applied to the position increment, or any
	// adjustment or other attributes where the end-of-stream value may
	// be important.
	//
	// If you override this method, alwasy call TokenStreamImpl.End().
	End() error
	// This method is called by a consumer before it begins consumption
	// using IncrementToken().
	//
	// Resets this stream to a clean state. Stateful implementation
	// must implement this method so that they can be reused, just as
	// if they had been created fresh.
	//
	// If you override this method, alwasy call TokenStreamImpl.Reset(),
	// otherwise some internal state will not be correctly reset (e.g.,
	// Tokenizer will panic on further usage).
	Reset() error
}

type TokenStreamComponents struct {

	// Resets the encapculated components with the given reader. If the
	// components canno be reset, an error should be returned.
	SetReader func(io.RuneReader) error
	// contains filtered or unexported fields
}

type TokenStreamImpl struct {
	// contains filtered or unexported fields
}

type Tokenizer struct {
	*TokenStreamImpl
	// The text source for this Tokenizer
	Input io.RuneReader
	// contains filtered or unexported fields
}