model

Documentation

Index

• Constants
• Variables
• func DecodeResourceName(resName ResourceName) (string, error)
• func ParseResourceID(rpath ResourceID) (ResourceType, ResourceName, error)
• func ToResourceRequirement(jobID JobID, resourceIDs ...ResourceID) []*pb.ResourceKey
• type Config
  • func (c *Config) Adjust(executorID ExecutorID)
  • func (c Config) GCSEnabled() bool
  • func (c Config) LocalEnabled() bool
  • func (c Config) S3Enabled() bool
  • func (c Config) ToBrBackendOptions() (opts *brStorage.BackendOptions, bucket, prefix string, tp ResourceType)
  • func (c Config) Validate() error
• type ExecutorID
• type GCSConfig
• type JobID
• type LocalFileConfig
  • func (c *LocalFileConfig) Adjust(executorID ExecutorID)
• type ResourceID
  • func BuildResourceID(rtype ResourceType, resName ResourceName) ResourceID
• type ResourceKey
  • func ToResourceKeys(requires []*pb.ResourceKey) []ResourceKey
• type ResourceMeta
  • func (m *ResourceMeta) GetID() string
  • func (m *ResourceMeta) Map() map[string]interface{}
  • func (m *ResourceMeta) ToQueryResourceResponse() *pb.QueryResourceResponse
• type ResourceName
  • func EncodeResourceName(rawResName string) ResourceName
• type ResourceType
  • func (r ResourceType) BuildPrefix() string
• type S3Config
• type WorkerID

Constants

const (
	ResourceTypeLocalFile = ResourceType("local")
	ResourceTypeS3        = ResourceType("s3")
	ResourceTypeGCS       = ResourceType("gs")
	ResourceTypeNone      = ResourceType("none")
)

Define all supported resource types.

Variables

var DefaultConfig = Config{
	Local: LocalFileConfig{BaseDir: ""},
	S3: S3Config{
		S3BackendOptions: brStorage.S3BackendOptions{
			ForcePathStyle: true,
		},
		Bucket: "",
		Prefix: "",
	},
	GCS: GCSConfig{
		Bucket: "",
		Prefix: "",
	},
}

DefaultConfig defines the default configuration for external storage

var ResourceUpdateColumns = []string{
	"updated_at",
	"project_id",
	"tenant_id",
	"id",
	"job_id",
	"worker_id",
	"executor_id",
	"deleted",
}

ResourceUpdateColumns is used in gorm update

Functions

func DecodeResourceName

func DecodeResourceName(resName ResourceName) (string, error)

DecodeResourceName decodes resource name to raw resource name.

func ParseResourceID

func ParseResourceID(rpath ResourceID) (ResourceType, ResourceName, error)

ParseResourceID returns the ResourceType and the path suffix.

func ToResourceRequirement

func ToResourceRequirement(jobID JobID, resourceIDs ...ResourceID) []*pb.ResourceKey

ToResourceRequirement return the resource requirement of pb

Types

type Config

type Config struct {
	Local LocalFileConfig `json:"local" toml:"local"`
	S3    S3Config        `json:"s3" toml:"s3"`
	GCS   GCSConfig       `json:"gcs" toml:"gcs"`
}

Config defines configurations for an external storage resource

func (*Config) Adjust

func (c *Config) Adjust(executorID ExecutorID)

Adjust adjusts the configuration

func (Config) GCSEnabled

func (c Config) GCSEnabled() bool

GCSEnabled returns true if the GCS storage is enabled

func (Config) LocalEnabled

func (c Config) LocalEnabled() bool

LocalEnabled returns true if the local storage is enabled

func (Config) S3Enabled

func (c Config) S3Enabled() bool

S3Enabled returns true if the S3 storage is enabled

func (Config) ToBrBackendOptions

func (c Config) ToBrBackendOptions() (opts *brStorage.BackendOptions, bucket, prefix string, tp ResourceType)

ToBrBackendOptions return BackendOptions for brStorage Make sure the Config is a valid config

func (Config) Validate

func (c Config) Validate() error

Validate implements the validation.Validatable interface

type ExecutorID

type ExecutorID = model.ExecutorID

ExecutorID alias model.ExecutorID

type GCSConfig

type GCSConfig struct {
	brStorage.GCSBackendOptions
	Bucket string `json:"bucket" toml:"bucket"`
	Prefix string `json:"prefix" toml:"prefix"`
}

GCSConfig defines configurations for gcs based resources

type JobID

type JobID = model.JobID

JobID alias job id string

type LocalFileConfig

type LocalFileConfig struct {
	BaseDir string `json:"base-dir" toml:"base-dir"`
}

LocalFileConfig defines configurations for a local file based resource

func (*LocalFileConfig) Adjust

func (c *LocalFileConfig) Adjust(executorID ExecutorID)

Adjust adjusts the local file config

type ResourceID

type ResourceID = string

ResourceID should be in the form of `/<type>/<unique-name>`, currently only local type is available.

func BuildResourceID

func BuildResourceID(rtype ResourceType, resName ResourceName) ResourceID

BuildResourceID returns an ResourceID based on given ResourceType and ResourceName.

type ResourceKey

type ResourceKey struct {
	JobID JobID
	ID    ResourceID
}

ResourceKey is the unique identifier for the resource

func ToResourceKeys

func ToResourceKeys(requires []*pb.ResourceKey) []ResourceKey

ToResourceKeys converts resource requirements in pb to resource keys

type ResourceMeta

type ResourceMeta struct {
	ormModel.Model
	ProjectID tenant.ProjectID `json:"project-id" gorm:"column:project_id;type:varchar(128) not null;"`
	TenantID  tenant.Tenant    `json:"tenant-id" gorm:"column:tenant_id;type:varchar(128) not null;"`
	ID        ResourceID       `json:"id" gorm:"column:id;type:varchar(128) not null;uniqueIndex:uidx_rid,priority:2;index:idx_rei,priority:2"`
	Job       JobID            `json:"job" gorm:"column:job_id;type:varchar(128) not null;uniqueIndex:uidx_rid,priority:1"`
	Worker    WorkerID         `json:"worker" gorm:"column:worker_id;type:varchar(128) not null"`
	Executor  ExecutorID       `json:"executor" gorm:"column:executor_id;type:varchar(128) not null;index:idx_rei,priority:1"`
	GCPending bool             `json:"gc-pending" gorm:"column:gc_pending;type:BOOLEAN"`

	// TODO soft delete has not be implemented, because it requires modifying too many
	// unit tests in engine/pkg/orm
	Deleted bool `json:"deleted" gorm:"column:deleted;type:BOOLEAN"`
}

ResourceMeta is the records stored in the metastore.

func (*ResourceMeta) GetID

func (m *ResourceMeta) GetID() string

GetID implements dataset.DataEntry

func (*ResourceMeta) Map

func (m *ResourceMeta) Map() map[string]interface{}

Map is used in gorm update

func (*ResourceMeta) ToQueryResourceResponse

func (m *ResourceMeta) ToQueryResourceResponse() *pb.QueryResourceResponse

ToQueryResourceResponse converts the ResourceMeta to pb.QueryResourceResponse

type ResourceName

type ResourceName = string

ResourceName is a string encoding raw resource name in hexadecimal. The raw resource name is the ResourceID with its type prefix removed. For example, the raw resource name of `/local/resource-1` is `resource-1`.

func EncodeResourceName

func EncodeResourceName(rawResName string) ResourceName

EncodeResourceName encodes raw resource name to a valid resource name.

type ResourceType

type ResourceType string

ResourceType represents the type of the resource

func (ResourceType) BuildPrefix

func (r ResourceType) BuildPrefix() string

BuildPrefix returns the prefix of the resource type.

type S3Config

type S3Config struct {
	brStorage.S3BackendOptions
	Bucket string `json:"bucket" toml:"bucket"`
	Prefix string `json:"prefix" toml:"prefix"`
}

S3Config defines configurations for s3 based resources

type WorkerID

type WorkerID = string

WorkerID alias worker id string