model

package
v0.0.0-...-4b22ae8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 13, 2024 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ResourceTypeLocalFile = ResourceType("local")
	ResourceTypeS3        = ResourceType("s3")
	ResourceTypeGCS       = ResourceType("gs")
	ResourceTypeNone      = ResourceType("none")
)

Define all supported resource types.

Variables

View Source 
var DefaultConfig = Config{
	Local: LocalFileConfig{BaseDir: ""},
	S3: S3Config{
		S3BackendOptions: brStorage.S3BackendOptions{
			ForcePathStyle: true,
		},
		Bucket: "",
		Prefix: "",
	},
	GCS: GCSConfig{
		Bucket: "",
		Prefix: "",
	},
}

DefaultConfig defines the default configuration for external storage

View Source 
var ResourceUpdateColumns = []string{
	"updated_at",
	"project_id",
	"tenant_id",
	"id",
	"job_id",
	"worker_id",
	"executor_id",
	"deleted",
}

ResourceUpdateColumns is used in gorm update

Functions

func DecodeResourceName 

func DecodeResourceName(resName ResourceName) (string, error)

DecodeResourceName decodes resource name to raw resource name.

func ParseResourceID 

func ParseResourceID(rpath ResourceID) (ResourceType, ResourceName, error)

ParseResourceID returns the ResourceType and the path suffix.

func ToResourceRequirement 

func ToResourceRequirement(jobID JobID, resourceIDs ...ResourceID) []*pb.ResourceKey

ToResourceRequirement return the resource requirement of pb

Types

type Config 

type Config struct {
	Local LocalFileConfig `json:"local" toml:"local"`
	S3    S3Config        `json:"s3" toml:"s3"`
	GCS   GCSConfig       `json:"gcs" toml:"gcs"`
}

Config defines configurations for an external storage resource

func (*Config) Adjust 

func (c *Config) Adjust(executorID ExecutorID)

Adjust adjusts the configuration

func (Config) GCSEnabled 

func (c Config) GCSEnabled() bool

GCSEnabled returns true if the GCS storage is enabled

func (Config) LocalEnabled 

func (c Config) LocalEnabled() bool

LocalEnabled returns true if the local storage is enabled

func (Config) S3Enabled 

func (c Config) S3Enabled() bool

S3Enabled returns true if the S3 storage is enabled

func (Config) ToBrBackendOptions 

func (c Config) ToBrBackendOptions() (opts *brStorage.BackendOptions, bucket, prefix string, tp ResourceType)

ToBrBackendOptions return BackendOptions for brStorage Make sure the Config is a valid config

func (Config) Validate 

func (c Config) Validate() error

Validate implements the validation.Validatable interface

type ExecutorID 

type ExecutorID = model.ExecutorID

ExecutorID alias model.ExecutorID

type GCSConfig 

type GCSConfig struct {
	brStorage.GCSBackendOptions
	Bucket string `json:"bucket" toml:"bucket"`
	Prefix string `json:"prefix" toml:"prefix"`
}

GCSConfig defines configurations for gcs based resources

type JobID 

type JobID = model.JobID

JobID alias job id string

type LocalFileConfig 

type LocalFileConfig struct {
	BaseDir string `json:"base-dir" toml:"base-dir"`
}

LocalFileConfig defines configurations for a local file based resource

func (*LocalFileConfig) Adjust 

func (c *LocalFileConfig) Adjust(executorID ExecutorID)

Adjust adjusts the local file config

type ResourceID 

type ResourceID = string

ResourceID should be in the form of `/<type>/<unique-name>`, currently only local type is available.

func BuildResourceID 

func BuildResourceID(rtype ResourceType, resName ResourceName) ResourceID

BuildResourceID returns an ResourceID based on given ResourceType and ResourceName.

type ResourceKey 

type ResourceKey struct {
	JobID JobID
	ID    ResourceID
}

ResourceKey is the unique identifier for the resource

func ToResourceKeys 

func ToResourceKeys(requires []*pb.ResourceKey) []ResourceKey

ToResourceKeys converts resource requirements in pb to resource keys

type ResourceMeta 

type ResourceMeta struct {
	ormModel.Model
	ProjectID tenant.ProjectID `json:"project-id" gorm:"column:project_id;type:varchar(128) not null;"`
	TenantID  tenant.Tenant    `json:"tenant-id" gorm:"column:tenant_id;type:varchar(128) not null;"`
	ID        ResourceID       `json:"id" gorm:"column:id;type:varchar(128) not null;uniqueIndex:uidx_rid,priority:2;index:idx_rei,priority:2"`
	Job       JobID            `json:"job" gorm:"column:job_id;type:varchar(128) not null;uniqueIndex:uidx_rid,priority:1"`
	Worker    WorkerID         `json:"worker" gorm:"column:worker_id;type:varchar(128) not null"`
	Executor  ExecutorID       `json:"executor" gorm:"column:executor_id;type:varchar(128) not null;index:idx_rei,priority:1"`
	GCPending bool             `json:"gc-pending" gorm:"column:gc_pending;type:BOOLEAN"`

	// TODO soft delete has not be implemented, because it requires modifying too many
	// unit tests in engine/pkg/orm
	Deleted bool `json:"deleted" gorm:"column:deleted;type:BOOLEAN"`
}

ResourceMeta is the records stored in the metastore.

func (*ResourceMeta) GetID 

func (m *ResourceMeta) GetID() string

GetID implements dataset.DataEntry

func (*ResourceMeta) Map 

func (m *ResourceMeta) Map() map[string]interface{}

Map is used in gorm update

func (*ResourceMeta) ToQueryResourceResponse 

func (m *ResourceMeta) ToQueryResourceResponse() *pb.QueryResourceResponse

ToQueryResourceResponse converts the ResourceMeta to pb.QueryResourceResponse

type ResourceName 

type ResourceName = string

ResourceName is a string encoding raw resource name in hexadecimal. The raw resource name is the ResourceID with its type prefix removed. For example, the raw resource name of `/local/resource-1` is `resource-1`.

func EncodeResourceName 

func EncodeResourceName(rawResName string) ResourceName

EncodeResourceName encodes raw resource name to a valid resource name.

type ResourceType 

type ResourceType string

ResourceType represents the type of the resource

func (ResourceType) BuildPrefix 

func (r ResourceType) BuildPrefix() string

BuildPrefix returns the prefix of the resource type.

type S3Config 

type S3Config struct {
	brStorage.S3BackendOptions
	Bucket string `json:"bucket" toml:"bucket"`
	Prefix string `json:"prefix" toml:"prefix"`
}

S3Config defines configurations for s3 based resources

type WorkerID 

type WorkerID = string

WorkerID alias worker id string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.