jindofsx

package
v0.8.7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 24, 2023 License: Apache-2.0 Imports: 41 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CSI_DRIVER = "fuse.csi.fluid.io"

	Mount_TYPE = "mount_type"

	SUMMARY_PREFIX_TOTAL_CAPACITY = "Total Disk Capacity: "

	SUMMARY_PREFIX_USED_CAPACITY = "Used Disk Capacity: "

	SUMMARY_PREFIX_TOTAL_MEM_CAPACITY = "Total MEM Capacity: "

	SUMMARY_PREFIX_USED_MEM_CAPACITY = "Used MEM Capacity: "

	METADATA_SYNC_NOT_DONE_MSG = "[Calculating]"

	CHECK_METADATA_SYNC_DONE_TIMEOUT_MILLISEC = 500

	HADOOP_CONF_HDFS_SITE_FILENAME = "hdfs-site.xml"

	HADOOP_CONF_CORE_SITE_FILENAME = "core-site.xml"

	JINDO_MASTERNUM_DEFAULT = 1
	JINDO_HA_MASTERNUM      = 3

	DEFAULT_MASTER_RPC_PORT = 8101
	DEFAULT_WORKER_RPC_PORT = 6101
	DEFAULT_RAFT_RPC_PORT   = 8103

	PodRoleType = "role"

	WOKRER_POD_ROLE = "jindo-worker"

	JINDO_FUSE_MONNTPATH = "/jfs/jindofs-fuse"

	DEFAULT_JINDOFSX_RUNTIME_IMAGE = "registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.1"

	ENGINE_TYPE = "jindofsx"

	FuseOnly = "fuseOnly"

	QueryUfsTotal = "QUERY_UFS_TOTAL"
)

Variables

This section is empty.

Functions

func Build 

func Build(id string, ctx cruntime.ReconcileRequestContext) (base.Engine, error)

func GetReservedPorts 

func GetReservedPorts(client client.Client) (ports []int, err error)

GetReservedPorts defines restoration logic for JindoRuntime

func Precheck 

func Precheck(client client.Client, key types.NamespacedName) (found bool, err error)

Precheck checks if the given key can be found in the current runtime types

Types

type Fuse 

type Fuse struct {
	Args           []string          `yaml:"args"`
	HostPath       string            `yaml:"hostPath"`
	NodeSelector   map[string]string `yaml:"nodeSelector,omitempty"`
	FuseProperties map[string]string `yaml:"properties"`
	Global         bool              `yaml:"global,omitempty"`
	RunAs          string            `yaml:"runAs,omitempty"`
	Tolerations    []v1.Toleration   `yaml:"tolerations,omitempty"`
	Labels         map[string]string `yaml:"labels,omitempty"`
	Annotations    map[string]string `yaml:"annotations,omitempty"`
	CriticalPod    bool              `yaml:"criticalPod,omitempty"`
	Resources      Resources         `yaml:"resources,omitempty"`
	MountPath      string            `yaml:"mountPath,omitempty"`
	Mode           string            `yaml:"mode,omitempty"`
}

type HadoopConfig 

type HadoopConfig struct {
	ConfigMap       string `yaml:"configMap"`
	IncludeHdfsSite bool   `yaml:"includeHdfsSite"`
	IncludeCoreSite bool   `yaml:"includeCoreSite"`
}

type Jindo 

type Jindo struct {
	Image           string                 `yaml:"image"`
	ImageTag        string                 `yaml:"imageTag"`
	ImagePullPolicy string                 `yaml:"imagePullPolicy"`
	FuseImage       string                 `yaml:"fuseImage"`
	FuseImageTag    string                 `yaml:"fuseImageTag"`
	User            int                    `yaml:"user"`
	Group           int                    `yaml:"group"`
	FsGroup         int                    `yaml:"fsGroup"`
	UseHostNetwork  bool                   `yaml:"useHostNetwork"`
	UseHostPID      bool                   `yaml:"useHostPID"`
	Properties      map[string]string      `yaml:"properties"`
	Master          Master                 `yaml:"master"`
	Worker          Worker                 `yaml:"worker"`
	Fuse            Fuse                   `yaml:"fuse"`
	Mounts          Mounts                 `yaml:"mounts"`
	HadoopConfig    HadoopConfig           `yaml:"hadoopConfig,omitempty"`
	Secret          string                 `yaml:"secret,omitempty"`
	Tolerations     []v1.Toleration        `yaml:"tolerations,omitempty"`
	InitPortCheck   common.InitPortCheck   `yaml:"initPortCheck,omitempty"`
	LogConfig       map[string]string      `yaml:"logConfig,omitempty"`
	PlacementMode   string                 `yaml:"placement,omitempty"`
	Owner           *common.OwnerReference `yaml:"owner,omitempty"`
	RuntimeIdentity common.RuntimeIdentity `yaml:"runtimeIdentity"`
}

type JindoFSxEngine 

type JindoFSxEngine struct {
	Log logr.Logger
	client.Client

	MetadataSyncDoneCh chan MetadataSyncResult

	Recorder record.EventRecorder
	*ctrl.Helper
	// contains filtered or unexported fields
}

func (*JindoFSxEngine) AssignNodesToCache 

func (e *JindoFSxEngine) AssignNodesToCache(desiredNum int32) (currentScheduleNum int32, err error)

func (*JindoFSxEngine) BindToDataset 

func (e *JindoFSxEngine) BindToDataset() (err error)

func (*JindoFSxEngine) CheckAndUpdateRuntimeStatus 

func (e *JindoFSxEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)

CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.

func (*JindoFSxEngine) CheckExistenceOfPath 

func (e *JindoFSxEngine) CheckExistenceOfPath(targetDataload datav1alpha1.DataLoad) (notExist bool, err error)

func (*JindoFSxEngine) CheckMasterReady 

func (e *JindoFSxEngine) CheckMasterReady() (ready bool, err error)

func (*JindoFSxEngine) CheckRuntimeHealthy 

func (e *JindoFSxEngine) CheckRuntimeHealthy() (err error)

func (*JindoFSxEngine) CheckRuntimeReady 

func (e *JindoFSxEngine) CheckRuntimeReady() (ready bool)

func (*JindoFSxEngine) CheckWorkersReady 

func (e *JindoFSxEngine) CheckWorkersReady() (ready bool, err error)

CheckWorkersReady checks if the workers are ready

func (*JindoFSxEngine) CreateDataLoadJob 

func (e *JindoFSxEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext, targetDataload datav1alpha1.DataLoad) (err error)

CreateDataLoadJob creates the job to load data

func (*JindoFSxEngine) CreateVolume 

func (e *JindoFSxEngine) CreateVolume() (err error)

CreateVolume creates volume

func (*JindoFSxEngine) DeleteVolume 

func (e *JindoFSxEngine) DeleteVolume() (err error)

DeleteVolume

func (*JindoFSxEngine) FreeStorageBytes 

func (e *JindoFSxEngine) FreeStorageBytes() (value int64, err error)

FreeStorageBytes returns free storage size of Jindo in bytes

func (*JindoFSxEngine) GetHCFSStatus 

func (e *JindoFSxEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)

Query the hcfs status

func (*JindoFSxEngine) GetHCFSStatusWithoutMaster 

func (e *JindoFSxEngine) GetHCFSStatusWithoutMaster() (status *datav1alpha1.HCFSStatus, err error)

func (*JindoFSxEngine) GetReportSummary 

func (e *JindoFSxEngine) GetReportSummary() (summary string, err error)

report jindo summary

func (*JindoFSxEngine) HasDeprecatedCommonLabelname 

func (e *JindoFSxEngine) HasDeprecatedCommonLabelname() (deprecated bool, err error)

func (*JindoFSxEngine) PrepareUFS 

func (e *JindoFSxEngine) PrepareUFS() (err error)

PrepareUFS do all the UFS preparations

func (*JindoFSxEngine) SetupMaster 

func (e *JindoFSxEngine) SetupMaster() (err error)

SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered

func (*JindoFSxEngine) SetupWorkers 

func (e *JindoFSxEngine) SetupWorkers() (err error)

SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.

func (*JindoFSxEngine) ShouldCheckUFS 

func (e *JindoFSxEngine) ShouldCheckUFS() (should bool, err error)

ShouldCheckUFS checks if it requires checking UFS

func (*JindoFSxEngine) ShouldSetupMaster 

func (e *JindoFSxEngine) ShouldSetupMaster() (should bool, err error)

ShouldSetupMaster checks if we need setup the master

func (*JindoFSxEngine) ShouldSetupWorkers 

func (e *JindoFSxEngine) ShouldSetupWorkers() (should bool, err error)

ShouldSetupWorkers checks if we need setup the workers

func (*JindoFSxEngine) ShouldUpdateUFS 

func (e *JindoFSxEngine) ShouldUpdateUFS() (ufsToUpdate *utils.UFSToUpdate)

JindoFSxEngine hasn't support UpdateOnUFSChange

func (*JindoFSxEngine) Shutdown 

func (e *JindoFSxEngine) Shutdown() (err error)

Shutdown shuts down the Jindo engine

func (*JindoFSxEngine) SyncMetadata 

func (e *JindoFSxEngine) SyncMetadata() (err error)

func (JindoFSxEngine) SyncReplicas 

func (e JindoFSxEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)

func (*JindoFSxEngine) SyncRuntime 

func (e *JindoFSxEngine) SyncRuntime(ctx cruntime.ReconcileRequestContext) (changed bool, err error)

SyncRuntime syncs the runtime spec

func (*JindoFSxEngine) SyncScheduleInfoToCacheNodes 

func (e *JindoFSxEngine) SyncScheduleInfoToCacheNodes() (err error)

SyncScheduleInfoToCacheNodes syncs the cache info of the nodes by labeling the nodes And the Application pod can leverage such info for scheduling

func (*JindoFSxEngine) TotalFileNums 

func (e *JindoFSxEngine) TotalFileNums() (value int64, err error)

return the total num of files in Jindo

func (*JindoFSxEngine) TotalJindoStorageBytes 

func (e *JindoFSxEngine) TotalJindoStorageBytes() (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoFSxEngine) TotalStorageBytes 

func (e *JindoFSxEngine) TotalStorageBytes() (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoFSxEngine) UpdateCacheOfDataset 

func (e *JindoFSxEngine) UpdateCacheOfDataset() (err error)

func (*JindoFSxEngine) UpdateDatasetStatus 

func (e *JindoFSxEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)

func (*JindoFSxEngine) UpdateOnUFSChange 

func (e *JindoFSxEngine) UpdateOnUFSChange(*utils.UFSToUpdate) (updateReady bool, err error)

func (*JindoFSxEngine) UsedStorageBytes 

func (e *JindoFSxEngine) UsedStorageBytes() (value int64, err error)

UsedStorageBytes returns used storage size of Jindo in bytes

type Level 

type Level struct {
	Path       string `yaml:"path,omitempty"`
	Type       string `yaml:"type,omitempty"`
	MediumType string `yaml:"mediumType,omitempty"`
	Quota      string `yaml:"quota,omitempty"`
}

type Master 

type Master struct {
	ReplicaCount        int               `yaml:"replicaCount"`
	Resources           Resources         `yaml:"resources"`
	NodeSelector        map[string]string `yaml:"nodeSelector,omitempty"`
	MasterProperties    map[string]string `yaml:"properties"`
	FileStoreProperties map[string]string `yaml:"fileStoreProperties"`
	TokenProperties     map[string]string `yaml:"secretProperties"`
	Port                Ports             `yaml:"ports,omitempty"`
	OssKey              string            `yaml:"osskey,omitempty"`
	OssSecret           string            `yaml:"osssecret,omitempty"`
	Tolerations         []v1.Toleration   `yaml:"tolerations,omitempty"`
	DnsServer           string            `yaml:"dnsServer,omitempty"`
	NameSpace           string            `yaml:"namespace,omitempty"`
	Labels              map[string]string `yaml:"labels,omitempty"`
	Annotations         map[string]string `yaml:"annotations,omitempty"`
	ServiceCount        int               `yaml:"svccount"`
}

type MetadataSyncResult 

type MetadataSyncResult struct {
	Done      bool
	StartTime time.Time
	UfsTotal  string
	Err       error
}

MetadataSyncResult describes result for asynchronous metadata sync

type Mounts 

type Mounts struct {
	Master            map[string]*Level `yaml:"master"`
	WorkersAndClients map[string]*Level `yaml:"workersAndClients"`
}

type Ports 

type Ports struct {
	Rpc  int `yaml:"rpc,omitempty"`
	Raft int `yaml:"raft,omitempty"`
}

type Resource 

type Resource struct {
	CPU    string `yaml:"cpu"`
	Memory string `yaml:"memory"`
}

type Resources 

type Resources struct {
	Limits   Resource `yaml:"limits"`
	Requests Resource `yaml:"requests"`
}

type Worker 

type Worker struct {
	ReplicaCount     int               `yaml:"replicaCount"`
	Resources        Resources         `yaml:"resources,omitempty"`
	NodeSelector     map[string]string `yaml:"nodeSelector,omitempty"`
	WorkerProperties map[string]string `yaml:"properties"`
	Port             Ports             `yaml:"ports,omitempty"`
	Tolerations      []v1.Toleration   `yaml:"tolerations,omitempty"`
	// Affinity         v1.Affinity       `yaml:"affinity,omitempty"`
	Labels      map[string]string `yaml:"labels,omitempty"`
	Annotations map[string]string `yaml:"annotations,omitempty"`
	Path        string            `yaml:"dataPath"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.