huggingface

Documentation

Index

• Constants
• type Author
• type Data
• type Discussion
  • func (d Discussion) GetCreatedAt() string
  • func (d Discussion) GetDiscussionPath() string
  • func (d Discussion) GetGitPath() string
  • func (d Discussion) GetID() string
  • func (d Discussion) GetRepo() string
  • func (d Discussion) GetTitle() string
• type DiscussionList
• type Event
  • func (e Event) GetAuthor() string
  • func (e Event) GetCreatedAt() string
  • func (e Event) GetID() string
• type HFClient
  • func NewHFClient(baseURL, apiKey string, timeout time.Duration) *HFClient
  • func (c *HFClient) GetDiscussionByID(ctx context.Context, repoInfo repoInfo, discussionID string) (Discussion, error)
  • func (c *HFClient) GetRepo(ctx context.Context, repoName string, resourceType string) (Repo, error)
  • func (c *HFClient) ListDiscussions(ctx context.Context, repoInfo repoInfo) (DiscussionList, error)
  • func (c *HFClient) ListReposByAuthor(ctx context.Context, resourceType string, author string) ([]Repo, error)
• type Latest
• type Repo
• type RepoData
• type Source
  • func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, ...) error
  • func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, ...) error
  • func (s *Source) JobID() sources.JobID
  • func (s *Source) SourceID() sources.SourceID
  • func (s *Source) Type() sourcespb.SourceType
  • func (s *Source) WithCustomContentWriter()

Constants

const (
	SourceType       = sourcespb.SourceType_SOURCE_TYPE_HUGGINGFACE
	DatasetsRoute    = "datasets"
	SpacesRoute      = "spaces"
	ModelsAPIRoute   = "models"
	DiscussionsRoute = "discussions"
	APIRoute         = "api"
	DATASET          = "dataset"
	MODEL            = "model"
	SPACE            = "space"
)

Types

type Author

type Author struct {
	Username string `json:"name"`
}

type Data

type Data struct {
	Latest Latest `json:"latest"`
}

type Discussion

type Discussion struct {
	ID        int      `json:"num"`
	IsPR      bool     `json:"isPullRequest"`
	CreatedAt string   `json:"createdAt"`
	Title     string   `json:"title"`
	Events    []Event  `json:"events"`
	Repo      RepoData `json:"repo"`
}

func (Discussion) GetCreatedAt

func (d Discussion) GetCreatedAt() string

func (Discussion) GetDiscussionPath

func (d Discussion) GetDiscussionPath() string

GetDiscussionPath returns the path (ex: "/models/user/repo/discussions/1") for the discussion

func (Discussion) GetGitPath

func (d Discussion) GetGitPath() string

GetGitPath returns the path (ex: "/models/user/repo.git") for the repo's git directory

func (Discussion) GetID

func (d Discussion) GetID() string

func (Discussion) GetRepo

func (d Discussion) GetRepo() string

func (Discussion) GetTitle

func (d Discussion) GetTitle() string

type DiscussionList

type DiscussionList struct {
	Discussions []Discussion `json:"discussions"`
}

type Event

type Event struct {
	Type      string `json:"type"`
	Author    Author `json:"author"`
	CreatedAt string `json:"createdAt"`
	Data      Data   `json:"data"`
	ID        string `json:"id"`
}

func (Event) GetAuthor

func (e Event) GetAuthor() string

func (Event) GetCreatedAt

func (e Event) GetCreatedAt() string

func (Event) GetID

func (e Event) GetID() string

type HFClient

type HFClient struct {
	BaseURL    string
	APIKey     string
	HTTPClient *http.Client
}

func NewHFClient

func NewHFClient(baseURL, apiKey string, timeout time.Duration) *HFClient

NewClient creates a new API client

func (*HFClient) GetDiscussionByID

func (c *HFClient) GetDiscussionByID(ctx context.Context, repoInfo repoInfo, discussionID string) (Discussion, error)

func (*HFClient) GetRepo

func (c *HFClient) GetRepo(ctx context.Context, repoName string, resourceType string) (Repo, error)

GetRepo retrieves repo from the Hugging Face API

func (*HFClient) ListDiscussions

func (c *HFClient) ListDiscussions(ctx context.Context, repoInfo repoInfo) (DiscussionList, error)

ListDiscussions retrieves discussions from the Hugging Face API

func (*HFClient) ListReposByAuthor

func (c *HFClient) ListReposByAuthor(ctx context.Context, resourceType string, author string) ([]Repo, error)

ListReposByAuthor retrieves repos from the Hugging Face API by author (user or org) Note: not addressing pagination b/c allow by default 1000 results, which should be enough for 99.99% of cases

type Latest

type Latest struct {
	Raw string `json:"raw"`
}

type Repo

type Repo struct {
	IsPrivate bool   `json:"private"`
	Owner     string `json:"author"`
	RepoID    string `json:"id"`
}

type RepoData

type RepoData struct {
	FullName     string `json:"name"`
	ResourceType string `json:"type"`
}

type Source

type Source struct {
	sources.Progress
	sources.CommonSourceUnitUnmarshaller
	// contains filtered or unexported fields
}

func (*Source) Chunks

func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error

Chunks emits chunks of bytes over a channel.

func (*Source) Init

func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error

Init returns an initialized HuggingFace source.

func (*Source) JobID

func (s *Source) JobID() sources.JobID

func (*Source) SourceID

func (s *Source) SourceID() sources.SourceID

func (*Source) Type

func (s *Source) Type() sourcespb.SourceType

Type returns the type of source. It is used for matching source types in configuration and job input.

func (*Source) WithCustomContentWriter

func (s *Source) WithCustomContentWriter()

WithCustomContentWriter sets the useCustomContentWriter flag on the source.