Documentation
¶
Index ¶
- Constants
- type Model
- func (m *Model) Download(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)
- func (m *Model) DownloadTokenizer(tokenizer Tokenizer, downloaderArgs downloadermodel.Args) (success bool, warnings []string, err error)
- func (m *Model) DownloadedOnDevice(useBasePath bool) (bool, error)
- func (m *Model) FromDownloaderModel(dlModel downloadermodel.Model)
- func (m *Model) GenClass() *codegen.Class
- func (m *Model) GenFile() *codegen.File
- func (m *Model) GenImports() []codegen.Import
- func (m *Model) GenInitParamsWithModule() []codegen.Parameter
- func (m *Model) GenModelPath() string
- func (m *Model) GenSuperInitParamsWithModule() []codegen.FunctionCallParameter
- func (m *Model) GetAccessToken() (string, error)
- func (m *Model) GetBasePath() string
- func (m *Model) GetConfig(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)
- func (m *Model) GetFormattedModelName() string
- func (m *Model) GetHuggingFaceClassImport() string
- func (m *Model) GetModelDirectory() (path string, err error)
- func (m *Model) GetModuleAutoPipelineClassName() string
- func (m *Model) GetSDKClassNameWithModule() string
- func (m *Model) GetTokenizersNotDownloadedOnDevice() Tokenizers
- func (m *Model) SaveAccessToken(accessToken string) error
- func (m *Model) TidyConfiguredModel(accessToken string) (warnings []string, success bool, clean bool, err error)
- func (m *Model) Update(yes bool, accessToken string) (warnings []string, success bool, err error)
- func (m *Model) UpdatePaths()
- type Models
- func (m Models) ContainsByName(name string) bool
- func (m Models) Difference(slice Models) Models
- func (m Models) Empty() bool
- func (m Models) FilterWithAddToBinaryFileTrue() Models
- func (m Models) FilterWithIsDownloadedOrAddToBinaryFileTrue() Models
- func (m Models) FilterWithIsDownloadedTrue() Models
- func (m Models) FilterWithNames(namesSlice []string) Models
- func (m Models) FilterWithSourceHuggingface() Models
- func (m Models) GetNames() []string
- func (m Models) Map() map[string]Model
- func (m Models) Union(slice Models) Models
- type Tokenizer
- type Tokenizers
Constants ¶
const ( HUGGING_FACE = "hugging_face" CUSTOM = "custom" )
Sources
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Model ¶
type Model struct {
Name string
Path string
Module huggingface.Module
Class string
Options map[string]string
Tokenizers Tokenizers
PipelineTag huggingface.PipelineTag
Source string
AddToBinaryFile bool
IsDownloaded bool
Version string
AccessToken string
}
func FromHuggingfaceModel ¶
func FromHuggingfaceModel(huggingfaceModel huggingface.Model) Model
FromHuggingfaceModel map the Huggingface API huggingface.Model to a Model
func (*Model) Download ¶
func (m *Model) Download(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)
Download attempts to download the model
func (*Model) DownloadTokenizer ¶
func (m *Model) DownloadTokenizer(tokenizer Tokenizer, downloaderArgs downloadermodel.Args) (success bool, warnings []string, err error)
DownloadTokenizer attempts to download the tokenizer
func (*Model) DownloadedOnDevice ¶
DownloadedOnDevice returns true if the model is physically present on the device.
func (*Model) FromDownloaderModel ¶
func (m *Model) FromDownloaderModel(dlModel downloadermodel.Model)
FromDownloaderModel maps data from downloadermodel.Model to Model and keeps unchanged properties of Model.
func (*Model) GenImports ¶
GenImports generate the imports for the given model
func (*Model) GenInitParamsWithModule ¶
GenInitParamsWithModule generate the init params for the given model
func (*Model) GenModelPath ¶ added in v0.0.2
GenModelPath returns the model path to be used in the code generation
func (*Model) GenSuperInitParamsWithModule ¶
func (m *Model) GenSuperInitParamsWithModule() []codegen.FunctionCallParameter
GenSuperInitParamsWithModule generate the init params for the super class
func (*Model) GetAccessToken ¶
GetAccessToken gets the access token from the .env file
func (*Model) GetBasePath ¶
GetBasePath return the base path to the model
func (*Model) GetConfig ¶
func (m *Model) GetConfig(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)
GetConfig attempts to get the model's configuration
func (*Model) GetFormattedModelName ¶
GetFormattedModelName format the model name in CapsWord start with "Model"
func (*Model) GetHuggingFaceClassImport ¶
GetHuggingFaceClassImport Get hugging face class for the import
func (*Model) GetModelDirectory ¶ added in v0.0.2
GetModelDirectory returns the directory path leading up to the 'models' directory
func (*Model) GetModuleAutoPipelineClassName ¶
GetModuleAutoPipelineClassName return the auto pipeline for the given model
func (*Model) GetSDKClassNameWithModule ¶
GetSDKClassNameWithModule return the sdk model for the given model
func (*Model) GetTokenizersNotDownloadedOnDevice ¶
func (m *Model) GetTokenizersNotDownloadedOnDevice() Tokenizers
GetTokenizersNotDownloadedOnDevice returns the list of tokenizers that should but are not physically present on the device.
func (*Model) SaveAccessToken ¶
SaveAccessToken saves the access token in the .env file
func (*Model) TidyConfiguredModel ¶
func (m *Model) TidyConfiguredModel(accessToken string) (warnings []string, success bool, clean bool, err error)
TidyConfiguredModel downloads the missing elements that were configured first bool is true if success, second bool is true if model was clean from the start
func (*Model) UpdatePaths ¶
func (m *Model) UpdatePaths()
UpdatePaths to update the model's path to elements accordingly to its configuration.
type Models ¶
type Models []Model
func BuildModelsFromDevice ¶
BuildModelsFromDevice builds a slice of models recovered from the device folders.
func (Models) ContainsByName ¶
ContainsByName checks if a models slice contains the requested model name
func (Models) Difference ¶
Difference returns the models in that are not present in `slice`
func (Models) FilterWithAddToBinaryFileTrue ¶
FilterWithAddToBinaryFileTrue return a sub-slice of models with AddToBinaryFile to true.
func (Models) FilterWithIsDownloadedOrAddToBinaryFileTrue ¶ added in v0.0.2
FilterWithIsDownloadedOrAddToBinaryFileTrue return a sub-slice of models with IsDownloaded or AddToBinaryFile to true.
func (Models) FilterWithIsDownloadedTrue ¶
FilterWithIsDownloadedTrue return a sub-slice of models with IsDownloaded to true.
func (Models) FilterWithNames ¶
FilterWithNames retrieves the models by their names given an input slice.
func (Models) FilterWithSourceHuggingface ¶
FilterWithSourceHuggingface return a sub-slice of models sourcing from huggingface.
type Tokenizer ¶
func (*Tokenizer) DownloadedOnDevice ¶
DownloadedOnDevice returns true if the tokenizer is physically present on the device.
type Tokenizers ¶
type Tokenizers []Tokenizer
func (Tokenizers) ContainsByClass ¶
func (t Tokenizers) ContainsByClass(class string) bool
ContainsByClass checks if a tokenizers slice contains the requested tokenizers name
func (Tokenizers) Difference ¶
func (t Tokenizers) Difference(slice Tokenizers) Tokenizers
Difference returns the models in that are not present in `slice`
func (Tokenizers) FilterWithClass ¶
func (t Tokenizers) FilterWithClass(namesSlice []string) Tokenizers
FilterWithClass retrieves the tokenizers by their class given an input slice.
func (Tokenizers) GetNames ¶
func (t Tokenizers) GetNames() []string
GetNames retrieves the names from the tokenizers.
func (Tokenizers) Map ¶
func (t Tokenizers) Map() map[string]Tokenizer
Map creates a map from tokenizers for faster lookup.
Source Files