jindo

package
v1.0.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 17, 2024 License: Apache-2.0 Imports: 42 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	SUMMARY_PREFIX_TOTAL_CAPACITY = "Total Capacity: "

	SUMMARY_PREFIX_USED_CAPACITY = "Used Capacity: "

	METADATA_SYNC_NOT_DONE_MSG = "[Calculating]"

	CHECK_METADATA_SYNC_DONE_TIMEOUT_MILLISEC = 500

	HADOOP_CONF_HDFS_SITE_FILENAME = "hdfs-site.xml"

	HADOOP_CONF_CORE_SITE_FILENAME = "core-site.xml"

	JINDO_MASTERNUM_DEFAULT = 1
	JINDO_HA_MASTERNUM      = 3

	DEFAULT_WORKER_RPC_PORT = 6101
	DEFAULT_RAFT_RPC_PORT   = 8103

	DEFAULT_JINDO_RUNTIME_IMAGE = "registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.8.0"
)

Variables

This section is empty.

Functions

func Build 

func Build(id string, ctx cruntime.ReconcileRequestContext) (base.Engine, error)

func GetReservedPorts added in v0.6.0 

func GetReservedPorts(client client.Client) (ports []int, err error)

GetReservedPorts defines restoration logic for JindoRuntime

Types

type Fuse 

type Fuse struct {
	Args              []string                  `yaml:"args"`
	HostPath          string                    `yaml:"hostPath"`
	NodeSelector      map[string]string         `yaml:"nodeSelector,omitempty"`
	FuseProperties    map[string]string         `yaml:"properties"`
	Global            bool                      `yaml:"global,omitempty"`
	RunAs             string                    `yaml:"runAs,omitempty"`
	Tolerations       []v1.Toleration           `yaml:"tolerations,omitempty"`
	Labels            map[string]string         `yaml:"labels,omitempty"`
	CriticalPod       bool                      `yaml:"criticalPod,omitempty"`
	Resources         Resources                 `yaml:"resources,omitempty"`
	MountPath         string                    `yaml:"mountPath,omitempty"`
	VirtualFuseDevice bool                      `yaml:"virtualFuseDevice"`
	HostPID           bool                      `json:"hostPID,omitempty"`
	ImagePullSecrets  []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

type HadoopConfig added in v0.6.0 

type HadoopConfig struct {
	ConfigMap       string `yaml:"configMap"`
	IncludeHdfsSite bool   `yaml:"includeHdfsSite"`
	IncludeCoreSite bool   `yaml:"includeCoreSite"`
}

type Jindo 

type Jindo struct {
	Image            string                    `yaml:"image"`
	ImageTag         string                    `yaml:"imageTag"`
	ImagePullPolicy  string                    `yaml:"imagePullPolicy"`
	FuseImage        string                    `yaml:"fuseImage"`
	FuseImageTag     string                    `yaml:"fuseImageTag"`
	User             int                       `yaml:"user"`
	Group            int                       `yaml:"group"`
	FsGroup          int                       `yaml:"fsGroup"`
	UseHostNetwork   bool                      `yaml:"useHostNetwork"`
	Properties       map[string]string         `yaml:"properties"`
	Master           Master                    `yaml:"master"`
	Worker           Worker                    `yaml:"worker"`
	Fuse             Fuse                      `yaml:"fuse"`
	Mounts           Mounts                    `yaml:"mounts"`
	HadoopConfig     HadoopConfig              `yaml:"hadoopConfig,omitempty"`
	Secret           string                    `yaml:"secret,omitempty"`
	Tolerations      []v1.Toleration           `yaml:"tolerations,omitempty"`
	InitPortCheck    common.InitPortCheck      `yaml:"initPortCheck,omitempty"`
	Labels           map[string]string         `yaml:"labels,omitempty"`
	LogConfig        map[string]string         `yaml:"logConfig,omitempty"`
	PlacementMode    string                    `yaml:"placement,omitempty"`
	Owner            *common.OwnerReference    `yaml:"owner,omitempty"`
	RuntimeIdentity  common.RuntimeIdentity    `yaml:"runtimeIdentity"`
	ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

type JindoEngine 

type JindoEngine struct {
	Log logr.Logger
	client.Client

	MetadataSyncDoneCh chan base.MetadataSyncResult

	Recorder record.EventRecorder
	*ctrl.Helper
	// contains filtered or unexported fields
}

func (*JindoEngine) BindToDataset 

func (e *JindoEngine) BindToDataset() (err error)

func (*JindoEngine) CheckAndUpdateRuntimeStatus 

func (e *JindoEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)

CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.

func (*JindoEngine) CheckMasterReady 

func (e *JindoEngine) CheckMasterReady() (ready bool, err error)

func (*JindoEngine) CheckRuntimeHealthy 

func (e *JindoEngine) CheckRuntimeHealthy() (err error)

func (*JindoEngine) CheckRuntimeReady added in v0.6.0 

func (e *JindoEngine) CheckRuntimeReady() (ready bool)

func (*JindoEngine) CheckWorkersReady 

func (e *JindoEngine) CheckWorkersReady() (ready bool, err error)

CheckWorkersReady checks if the workers are ready

func (*JindoEngine) CreateVolume 

func (e *JindoEngine) CreateVolume() (err error)

CreateVolume creates volume

func (*JindoEngine) DeleteVolume 

func (e *JindoEngine) DeleteVolume() (err error)

DeleteVolume

func (*JindoEngine) FreeStorageBytes 

func (e *JindoEngine) FreeStorageBytes() (value int64, err error)

FreeStorageBytes returns free storage size of Jindo in bytes

func (*JindoEngine) GetDataOperationValueFile added in v0.9.0 

func (e *JindoEngine) GetDataOperationValueFile(ctx cruntime.ReconcileRequestContext, operation dataoperation.OperationInterface) (valueFileName string, err error)

func (*JindoEngine) GetReportSummary added in v0.6.0 

func (e *JindoEngine) GetReportSummary() (summary string, err error)

report jindo summary

func (*JindoEngine) HasDeprecatedCommonLabelname added in v0.6.0 

func (e *JindoEngine) HasDeprecatedCommonLabelname() (deprecated bool, err error)

func (*JindoEngine) PrepareUFS 

func (e *JindoEngine) PrepareUFS() (err error)

PrepareUFS do all the UFS preparations

func (*JindoEngine) SetupMaster 

func (e *JindoEngine) SetupMaster() (err error)

SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered

func (*JindoEngine) SetupWorkers 

func (e *JindoEngine) SetupWorkers() (err error)

SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.

func (*JindoEngine) ShouldCheckUFS 

func (e *JindoEngine) ShouldCheckUFS() (should bool, err error)

ShouldCheckUFS checks if it requires checking UFS

func (*JindoEngine) ShouldSetupMaster 

func (e *JindoEngine) ShouldSetupMaster() (should bool, err error)

ShouldSetupMaster checks if we need setup the master

func (*JindoEngine) ShouldSetupWorkers 

func (e *JindoEngine) ShouldSetupWorkers() (should bool, err error)

ShouldSetupWorkers checks if we need setup the workers

func (*JindoEngine) ShouldUpdateUFS added in v0.6.0 

func (e *JindoEngine) ShouldUpdateUFS() (ufsToUpdate *utils.UFSToUpdate)

JindoEngine hasn't support UpdateOnUFSChange

func (*JindoEngine) Shutdown 

func (e *JindoEngine) Shutdown() (err error)

Shutdown shuts down the Jindo engine

func (*JindoEngine) SyncMetadata 

func (e *JindoEngine) SyncMetadata() (err error)

func (JindoEngine) SyncReplicas 

func (e JindoEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)

func (*JindoEngine) SyncRuntime added in v0.8.0 

func (e *JindoEngine) SyncRuntime(ctx cruntime.ReconcileRequestContext) (changed bool, err error)

SyncRuntime syncs the runtime spec

func (*JindoEngine) SyncScheduleInfoToCacheNodes added in v0.7.0 

func (e *JindoEngine) SyncScheduleInfoToCacheNodes() (err error)

SyncScheduleInfoToCacheNodes syncs the cache info of the nodes by labeling the nodes And the Application pod can leverage such info for scheduling

func (*JindoEngine) TotalFileNums 

func (e *JindoEngine) TotalFileNums() (value int64, err error)

return the total num of files in Jindo

func (*JindoEngine) TotalJindoStorageBytes 

func (e *JindoEngine) TotalJindoStorageBytes(useStsSecret bool) (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoEngine) TotalStorageBytes 

func (e *JindoEngine) TotalStorageBytes() (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoEngine) UpdateCacheOfDataset 

func (e *JindoEngine) UpdateCacheOfDataset() (err error)

func (*JindoEngine) UpdateDatasetStatus 

func (e *JindoEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)

func (*JindoEngine) UpdateOnUFSChange added in v0.6.0 

func (e *JindoEngine) UpdateOnUFSChange(*utils.UFSToUpdate) (updateReady bool, err error)

func (*JindoEngine) UsedStorageBytes 

func (e *JindoEngine) UsedStorageBytes() (value int64, err error)

UsedStorageBytes returns used storage size of Jindo in bytes

func (*JindoEngine) Validate added in v1.0.2 

func (e *JindoEngine) Validate(ctx cruntime.ReconcileRequestContext) (err error)

type Master 

type Master struct {
	ReplicaCount     int                       `yaml:"replicaCount"`
	Resources        Resources                 `yaml:"resources"`
	NodeSelector     map[string]string         `yaml:"nodeSelector,omitempty"`
	MasterProperties map[string]string         `yaml:"properties"`
	TokenProperties  map[string]string         `yaml:"secretProperties"`
	Port             Ports                     `yaml:"ports,omitempty"`
	OssKey           string                    `yaml:"osskey,omitempty"`
	OssSecret        string                    `yaml:"osssecret,omitempty"`
	Tolerations      []v1.Toleration           `yaml:"tolerations,omitempty"`
	DnsServer        string                    `yaml:"dnsServer,omitempty"`
	NameSpace        string                    `yaml:"namespace,omitempty"`
	Labels           map[string]string         `yaml:"labels,omitempty"`
	ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

type Mounts 

type Mounts struct {
	Master            map[string]string `yaml:"master"`
	WorkersAndClients map[string]string `yaml:"workersAndClients"`
}

type Ports added in v0.6.0 

type Ports struct {
	Rpc  int `yaml:"rpc,omitempty"`
	Raft int `yaml:"raft,omitempty"`
}

type Resource 

type Resource struct {
	CPU    string `yaml:"cpu"`
	Memory string `yaml:"memory"`
}

type Resources 

type Resources struct {
	Limits   Resource `yaml:"limits"`
	Requests Resource `yaml:"requests"`
}

type Worker 

type Worker struct {
	Resources        Resources         `yaml:"resources,omitempty"`
	NodeSelector     map[string]string `yaml:"nodeSelector,omitempty"`
	WorkerProperties map[string]string `yaml:"properties"`
	Port             Ports             `yaml:"ports,omitempty"`
	Tolerations      []v1.Toleration   `yaml:"tolerations,omitempty"`
	// Affinity         v1.Affinity       `yaml:"affinity,omitempty"`
	Labels           map[string]string         `yaml:"labels,omitempty"`
	ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.