model

Documentation

Index

• Constants
• type Model
  • func FromHuggingfaceModel(huggingfaceModel huggingface.Model) Model
  • func (m *Model) Download(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)
  • func (m *Model) DownloadTokenizer(tokenizer Tokenizer, downloaderArgs downloadermodel.Args) (success bool, warnings []string, err error)
  • func (m *Model) DownloadedOnDevice(useBasePath bool) (bool, error)
  • func (m *Model) FromDownloaderModel(dlModel downloadermodel.Model)
  • func (m *Model) GenClass() *codegen.Class
  • func (m *Model) GenFile() *codegen.File
  • func (m *Model) GenImports() []codegen.Import
  • func (m *Model) GenInitParamsWithModule() []codegen.Parameter
  • func (m *Model) GenModelPath() string
  • func (m *Model) GenSuperInitParamsWithModule() []codegen.FunctionCallParameter
  • func (m *Model) GetAccessToken() (string, error)
  • func (m *Model) GetBasePath() string
  • func (m *Model) GetConfig(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)
  • func (m *Model) GetFormattedModelName() string
  • func (m *Model) GetHuggingFaceClassImport() string
  • func (m *Model) GetModelDirectory() (path string, err error)
  • func (m *Model) GetModuleAutoPipelineClassName() string
  • func (m *Model) GetSDKClassNameWithModule() string
  • func (m *Model) GetTokenizersNotDownloadedOnDevice() Tokenizers
  • func (m *Model) SaveAccessToken(accessToken string) error
  • func (m *Model) TidyConfiguredModel(accessToken string) (warnings []string, success bool, clean bool, err error)
  • func (m *Model) Update(yes bool, accessToken string) (warnings []string, success bool, err error)
  • func (m *Model) UpdatePaths()
• type Models
  • func BuildModelsFromDevice(accessToken string) Models
  • func (m Models) ContainsByName(name string) bool
  • func (m Models) Difference(slice Models) Models
  • func (m Models) Empty() bool
  • func (m Models) FilterWithAddToBinaryFileTrue() Models
  • func (m Models) FilterWithIsDownloadedOrAddToBinaryFileTrue() Models
  • func (m Models) FilterWithIsDownloadedTrue() Models
  • func (m Models) FilterWithNames(namesSlice []string) Models
  • func (m Models) FilterWithSourceHuggingface() Models
  • func (m Models) GetNames() []string
  • func (m Models) Map() map[string]Model
  • func (m Models) Union(slice Models) Models
• type Tokenizer
  • func (t *Tokenizer) DownloadedOnDevice() (bool, error)
• type Tokenizers
  • func (t Tokenizers) ContainsByClass(class string) bool
  • func (t Tokenizers) Difference(slice Tokenizers) Tokenizers
  • func (t Tokenizers) FilterWithClass(namesSlice []string) Tokenizers
  • func (t Tokenizers) GetNames() []string
  • func (t Tokenizers) Map() map[string]Tokenizer

Constants

const (
	HUGGING_FACE = "hugging_face"
	CUSTOM       = "custom"
)

Sources

Types

type Model

type Model struct {
	Name            string
	Path            string
	Module          huggingface.Module
	Class           string
	Options         map[string]string
	Tokenizers      Tokenizers
	PipelineTag     huggingface.PipelineTag
	Source          string
	AddToBinaryFile bool
	IsDownloaded    bool
	Version         string
	AccessToken     string
}

func FromHuggingfaceModel

func FromHuggingfaceModel(huggingfaceModel huggingface.Model) Model

FromHuggingfaceModel map the Huggingface API huggingface.Model to a Model

func (*Model) Download

func (m *Model) Download(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)

Download attempts to download the model

func (*Model) DownloadTokenizer

func (m *Model) DownloadTokenizer(tokenizer Tokenizer, downloaderArgs downloadermodel.Args) (success bool, warnings []string, err error)

DownloadTokenizer attempts to download the tokenizer

func (*Model) DownloadedOnDevice

func (m *Model) DownloadedOnDevice(useBasePath bool) (bool, error)

DownloadedOnDevice returns true if the model is physically present on the device.

func (*Model) FromDownloaderModel

func (m *Model) FromDownloaderModel(dlModel downloadermodel.Model)

FromDownloaderModel maps data from downloadermodel.Model to Model and keeps unchanged properties of Model.

func (*Model) GenClass

func (m *Model) GenClass() *codegen.Class

func (*Model) GenFile

func (m *Model) GenFile() *codegen.File

GenFile generates a single python file for the given model

func (*Model) GenImports

func (m *Model) GenImports() []codegen.Import

GenImports generate the imports for the given model

func (*Model) GenInitParamsWithModule

func (m *Model) GenInitParamsWithModule() []codegen.Parameter

GenInitParamsWithModule generate the init params for the given model

func (*Model) GenModelPath

func (m *Model) GenModelPath() string

GenModelPath returns the model path to be used in the code generation

func (*Model) GenSuperInitParamsWithModule

func (m *Model) GenSuperInitParamsWithModule() []codegen.FunctionCallParameter

GenSuperInitParamsWithModule generate the init params for the super class

func (*Model) GetAccessToken

func (m *Model) GetAccessToken() (string, error)

GetAccessToken gets the access token from the .env file

func (*Model) GetBasePath

func (m *Model) GetBasePath() string

GetBasePath return the base path to the model

func (*Model) GetConfig

func (m *Model) GetConfig(downloaderArgs downloadermodel.Args) (succeeded bool, warnings []string, err error)

GetConfig attempts to get the model's configuration

func (*Model) GetFormattedModelName

func (m *Model) GetFormattedModelName() string

GetFormattedModelName format the model name in CapsWord start with "Model"

func (*Model) GetHuggingFaceClassImport

func (m *Model) GetHuggingFaceClassImport() string

GetHuggingFaceClassImport Get hugging face class for the import

func (*Model) GetModelDirectory

func (m *Model) GetModelDirectory() (path string, err error)

GetModelDirectory returns the directory path leading up to the 'models' directory

func (*Model) GetModuleAutoPipelineClassName

func (m *Model) GetModuleAutoPipelineClassName() string

GetModuleAutoPipelineClassName return the auto pipeline for the given model

func (*Model) GetSDKClassNameWithModule

func (m *Model) GetSDKClassNameWithModule() string

GetSDKClassNameWithModule return the sdk model for the given model

func (*Model) GetTokenizersNotDownloadedOnDevice

func (m *Model) GetTokenizersNotDownloadedOnDevice() Tokenizers

GetTokenizersNotDownloadedOnDevice returns the list of tokenizers that should but are not physically present on the device.

func (*Model) SaveAccessToken

func (m *Model) SaveAccessToken(accessToken string) error

SaveAccessToken saves the access token in the .env file

func (*Model) TidyConfiguredModel

func (m *Model) TidyConfiguredModel(accessToken string) (warnings []string, success bool, clean bool, err error)

TidyConfiguredModel downloads the missing elements that were configured first bool is true if success, second bool is true if model was clean from the start

func (*Model) Update

func (m *Model) Update(yes bool, accessToken string) (warnings []string, success bool, err error)

Update attempts to update the model

func (*Model) UpdatePaths

func (m *Model) UpdatePaths()

UpdatePaths to update the model's path to elements accordingly to its configuration.

type Models

type Models []Model

func BuildModelsFromDevice

func BuildModelsFromDevice(accessToken string) Models

BuildModelsFromDevice builds a slice of models recovered from the device folders.

func (Models) ContainsByName

func (m Models) ContainsByName(name string) bool

ContainsByName checks if a models slice contains the requested model name

func (Models) Difference

func (m Models) Difference(slice Models) Models

Difference returns the models in that are not present in `slice`

func (Models) Empty

func (m Models) Empty() bool

Empty checks if the models slice is empty.

func (Models) FilterWithAddToBinaryFileTrue

func (m Models) FilterWithAddToBinaryFileTrue() Models

FilterWithAddToBinaryFileTrue return a sub-slice of models with AddToBinaryFile to true.

func (Models) FilterWithIsDownloadedOrAddToBinaryFileTrue

func (m Models) FilterWithIsDownloadedOrAddToBinaryFileTrue() Models

FilterWithIsDownloadedOrAddToBinaryFileTrue return a sub-slice of models with IsDownloaded or AddToBinaryFile to true.

func (Models) FilterWithIsDownloadedTrue

func (m Models) FilterWithIsDownloadedTrue() Models

FilterWithIsDownloadedTrue return a sub-slice of models with IsDownloaded to true.

func (Models) FilterWithNames

func (m Models) FilterWithNames(namesSlice []string) Models

FilterWithNames retrieves the models by their names given an input slice.

func (Models) FilterWithSourceHuggingface

func (m Models) FilterWithSourceHuggingface() Models

FilterWithSourceHuggingface return a sub-slice of models sourcing from huggingface.

func (Models) GetNames

func (m Models) GetNames() []string

GetNames retrieves the names from the models.

func (Models) Map

func (m Models) Map() map[string]Model

Map creates a map from models for faster lookup.

func (Models) Union

func (m Models) Union(slice Models) Models

Union returns the models present in `slice` as well

type Tokenizer

type Tokenizer struct {
	Path    string
	Class   string
	Options map[string]string
}

func (*Tokenizer) DownloadedOnDevice

func (t *Tokenizer) DownloadedOnDevice() (bool, error)

DownloadedOnDevice returns true if the tokenizer is physically present on the device.

type Tokenizers

type Tokenizers []Tokenizer

func (Tokenizers) ContainsByClass

func (t Tokenizers) ContainsByClass(class string) bool

ContainsByClass checks if a tokenizers slice contains the requested tokenizers name

func (Tokenizers) Difference

func (t Tokenizers) Difference(slice Tokenizers) Tokenizers

Difference returns the models in that are not present in `slice`

func (Tokenizers) FilterWithClass

func (t Tokenizers) FilterWithClass(namesSlice []string) Tokenizers

FilterWithClass retrieves the tokenizers by their class given an input slice.

func (Tokenizers) GetNames

func (t Tokenizers) GetNames() []string

GetNames retrieves the names from the tokenizers.

func (Tokenizers) Map

func (t Tokenizers) Map() map[string]Tokenizer

Map creates a map from tokenizers for faster lookup.