jindocache

package
v1.0.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 17, 2024 License: Apache-2.0 Imports: 44 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CSI_DRIVER = "fuse.csi.fluid.io"

	Mount_TYPE = "mount_type"

	SUMMARY_PREFIX_TOTAL_CAPACITY = "Total Disk Capacity: "

	SUMMARY_PREFIX_USED_CAPACITY = "Used Disk Capacity: "

	SUMMARY_PREFIX_TOTAL_MEM_CAPACITY = "Total MEM Capacity: "

	SUMMARY_PREFIX_USED_MEM_CAPACITY = "Used MEM Capacity: "

	METADATA_SYNC_NOT_DONE_MSG = "[Calculating]"

	CHECK_METADATA_SYNC_DONE_TIMEOUT_MILLISEC = 500

	HADOOP_CONF_HDFS_SITE_FILENAME = "hdfs-site.xml"

	HADOOP_CONF_CORE_SITE_FILENAME = "core-site.xml"

	JINDO_MASTERNUM_DEFAULT = 1
	JINDO_HA_MASTERNUM      = 3

	DEFAULT_MASTER_RPC_PORT   = 8101
	DEFAULT_WORKER_RPC_PORT   = 6101
	DEFAULT_RAFT_RPC_PORT     = 8103
	DEFAULT_FUSE_METRICS_PORT = 10001

	FuseOnly = "fuseOnly"

	QueryUfsTotal = "QUERY_UFS_TOTAL"

	// Write Policy
	WriteAround  = "WRITE_AROUND"
	WriteThrough = "WRITE_THROUGH"
	CacheOnly    = "CACHE_ONLY"
)

Variables

This section is empty.

Functions

func Build 

func Build(id string, ctx cruntime.ReconcileRequestContext) (base.Engine, error)

func GetReservedPorts 

func GetReservedPorts(client client.Client) (ports []int, err error)

GetReservedPorts defines restoration logic for JindoRuntime

func Precheck 

func Precheck(client client.Client, key types.NamespacedName) (found bool, err error)

Precheck checks if the given key can be found in the current runtime types

Types

type CacheSet 

type CacheSet struct {
	Name              string `json:"name"`
	Path              string `json:"path,omitempty"`
	CacheStrategy     string `json:"cacheStrategy"`
	MetaPolicy        string `json:"metaPolicy"`
	ReadPolicy        string `json:"readPolicy"`
	WritePolicy       string `json:"writePolicy"`
	ReadCacheReplica  int    `json:"readCacheReplica"`
	WriteCacheReplica int    `json:"writeCacheReplica"`
}

type Fuse 

type Fuse struct {
	Args             []string                      `json:"args"`
	HostPath         string                        `json:"hostPath"`
	NodeSelector     map[string]string             `json:"nodeSelector,omitempty"`
	FuseProperties   map[string]string             `json:"properties"`
	Global           bool                          `json:"global,omitempty"`
	RunAs            string                        `json:"runAs,omitempty"`
	Tolerations      []corev1.Toleration           `json:"tolerations,omitempty"`
	Labels           map[string]string             `json:"labels,omitempty"`
	Annotations      map[string]string             `json:"annotations,omitempty"`
	CriticalPod      bool                          `json:"criticalPod,omitempty"`
	Resources        Resources                     `json:"resources,omitempty"`
	MountPath        string                        `json:"mountPath,omitempty"`
	Mode             string                        `json:"mode,omitempty"`
	Env              map[string]string             `json:"env,omitempty"`
	HostPID          bool                          `json:"hostPID,omitempty"`
	MetricsPort      int                           `json:"metricsPort,omitempty"`
	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

type HadoopConfig 

type HadoopConfig struct {
	ConfigMap       string `json:"configMap"`
	IncludeHdfsSite bool   `json:"includeHdfsSite"`
	IncludeCoreSite bool   `json:"includeCoreSite"`
}

type Jindo 

type Jindo struct {
	FullnameOverride    string                        `json:"fullnameOverride"`
	Image               string                        `json:"image"`
	ImageTag            string                        `json:"imageTag"`
	ImagePullPolicy     string                        `json:"imagePullPolicy"`
	FuseImage           string                        `json:"fuseImage"`
	FuseImageTag        string                        `json:"fuseImageTag"`
	FuseImagePullPolicy string                        `json:"fuseImagePullPolicy"`
	User                int                           `json:"user"`
	Group               int                           `json:"group"`
	UseHostNetwork      bool                          `json:"useHostNetwork"`
	Properties          map[string]string             `json:"properties"`
	Master              Master                        `json:"master"`
	Worker              Worker                        `json:"worker"`
	Fuse                Fuse                          `json:"fuse"`
	Mounts              Mounts                        `json:"mounts"`
	HadoopConfig        HadoopConfig                  `json:"hadoopConfig,omitempty"`
	Secret              string                        `json:"secret,omitempty"`
	Tolerations         []corev1.Toleration           `json:"tolerations,omitempty"`
	InitPortCheck       common.InitPortCheck          `json:"initPortCheck,omitempty"`
	LogConfig           map[string]string             `json:"logConfig,omitempty"`
	FuseLogConfig       map[string]string             `json:"fuseLogConfig,omitempty"`
	PlacementMode       string                        `json:"placement,omitempty"`
	Owner               *common.OwnerReference        `json:"owner,omitempty"`
	RuntimeIdentity     common.RuntimeIdentity        `json:"runtimeIdentity"`
	ClusterDomain       string                        `json:"clusterDomain,omitempty"`
	UFSVolumes          []UFSVolume                   `json:"ufsVolumes,omitempty"`
	SecretKey           string                        `json:"secretKey,omitempty"`
	SecretValue         string                        `json:"secretValue,omitempty"`
	UseStsToken         bool                          `json:"UseStsToken"`
	MountType           string                        `json:"mountType,omitempty"`
	ImagePullSecrets    []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

type JindoCacheEngine 

type JindoCacheEngine struct {
	Log logr.Logger
	client.Client

	MetadataSyncDoneCh chan base.MetadataSyncResult

	Recorder record.EventRecorder
	*ctrl.Helper
	// contains filtered or unexported fields
}

func (*JindoCacheEngine) BindToDataset 

func (e *JindoCacheEngine) BindToDataset() (err error)

func (*JindoCacheEngine) CheckAndUpdateRuntimeStatus 

func (e *JindoCacheEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)

CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.

func (*JindoCacheEngine) CheckMasterReady 

func (e *JindoCacheEngine) CheckMasterReady() (ready bool, err error)

func (*JindoCacheEngine) CheckRuntimeHealthy 

func (e *JindoCacheEngine) CheckRuntimeHealthy() (err error)

func (*JindoCacheEngine) CheckRuntimeReady 

func (e *JindoCacheEngine) CheckRuntimeReady() (ready bool)

func (*JindoCacheEngine) CheckWorkersReady 

func (e *JindoCacheEngine) CheckWorkersReady() (ready bool, err error)

CheckWorkersReady checks if the workers are ready

func (*JindoCacheEngine) CreateVolume 

func (e *JindoCacheEngine) CreateVolume() (err error)

CreateVolume creates volume

func (*JindoCacheEngine) DeleteVolume 

func (e *JindoCacheEngine) DeleteVolume() (err error)

DeleteVolume

func (*JindoCacheEngine) FreeStorageBytes 

func (e *JindoCacheEngine) FreeStorageBytes() (value int64, err error)

FreeStorageBytes returns free storage size of Jindo in bytes

func (*JindoCacheEngine) GetDataOperationValueFile 

func (e *JindoCacheEngine) GetDataOperationValueFile(ctx cruntime.ReconcileRequestContext, operation dataoperation.OperationInterface) (valueFileName string, err error)

func (*JindoCacheEngine) GetHCFSStatus 

func (e *JindoCacheEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)

Query the hcfs status

func (*JindoCacheEngine) GetHCFSStatusWithoutMaster 

func (e *JindoCacheEngine) GetHCFSStatusWithoutMaster() (status *datav1alpha1.HCFSStatus, err error)

func (*JindoCacheEngine) GetReportSummary 

func (e *JindoCacheEngine) GetReportSummary() (summary string, err error)

report jindo summary

func (*JindoCacheEngine) HasDeprecatedCommonLabelname 

func (e *JindoCacheEngine) HasDeprecatedCommonLabelname() (deprecated bool, err error)

func (*JindoCacheEngine) PrepareUFS 

func (e *JindoCacheEngine) PrepareUFS() (err error)

PrepareUFS do all the UFS preparations

func (*JindoCacheEngine) RefreshCacheSet 

func (e *JindoCacheEngine) RefreshCacheSet() (err error)

func (*JindoCacheEngine) SetupMaster 

func (e *JindoCacheEngine) SetupMaster() (err error)

SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered

func (*JindoCacheEngine) SetupWorkers 

func (e *JindoCacheEngine) SetupWorkers() (err error)

SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.

func (*JindoCacheEngine) ShouldCheckUFS 

func (e *JindoCacheEngine) ShouldCheckUFS() (should bool, err error)

ShouldCheckUFS checks if it requires checking UFS

func (*JindoCacheEngine) ShouldRefreshCacheSet 

func (e *JindoCacheEngine) ShouldRefreshCacheSet() (shouldRefresh bool, err error)

func (*JindoCacheEngine) ShouldSetupMaster 

func (e *JindoCacheEngine) ShouldSetupMaster() (should bool, err error)

ShouldSetupMaster checks if we need setup the master

func (*JindoCacheEngine) ShouldSetupWorkers 

func (e *JindoCacheEngine) ShouldSetupWorkers() (should bool, err error)

ShouldSetupWorkers checks if we need setup the workers

func (*JindoCacheEngine) ShouldUpdateUFS 

func (e *JindoCacheEngine) ShouldUpdateUFS() (ufsToUpdate *utils.UFSToUpdate)

JindoCacheEngine hasn't support UpdateOnUFSChange

func (*JindoCacheEngine) Shutdown 

func (e *JindoCacheEngine) Shutdown() (err error)

Shutdown shuts down the Jindo engine

func (*JindoCacheEngine) SyncMetadata 

func (e *JindoCacheEngine) SyncMetadata() (err error)

func (JindoCacheEngine) SyncReplicas 

func (e JindoCacheEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)

func (*JindoCacheEngine) SyncRuntime 

func (e *JindoCacheEngine) SyncRuntime(ctx cruntime.ReconcileRequestContext) (changed bool, err error)

SyncRuntime syncs the runtime spec

func (*JindoCacheEngine) SyncScheduleInfoToCacheNodes 

func (e *JindoCacheEngine) SyncScheduleInfoToCacheNodes() (err error)

SyncScheduleInfoToCacheNodes syncs the cache info of the nodes by labeling the nodes And the Application pod can leverage such info for scheduling

func (*JindoCacheEngine) TotalFileNums 

func (e *JindoCacheEngine) TotalFileNums() (value int64, err error)

return the total num of files in Jindo

func (*JindoCacheEngine) TotalJindoStorageBytes 

func (e *JindoCacheEngine) TotalJindoStorageBytes() (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoCacheEngine) TotalStorageBytes 

func (e *JindoCacheEngine) TotalStorageBytes() (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoCacheEngine) UpdateCacheOfDataset 

func (e *JindoCacheEngine) UpdateCacheOfDataset() (err error)

func (*JindoCacheEngine) UpdateDatasetStatus 

func (e *JindoCacheEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)

func (*JindoCacheEngine) UpdateOnUFSChange 

func (e *JindoCacheEngine) UpdateOnUFSChange(*utils.UFSToUpdate) (updateReady bool, err error)

func (*JindoCacheEngine) UsedStorageBytes 

func (e *JindoCacheEngine) UsedStorageBytes() (value int64, err error)

UsedStorageBytes returns used storage size of Jindo in bytes

func (*JindoCacheEngine) Validate added in v1.0.2 

func (e *JindoCacheEngine) Validate(ctx cruntime.ReconcileRequestContext) (err error)

type Level 

type Level struct {
	Path       string `json:"path,omitempty"`
	Type       string `json:"type,omitempty"`
	MediumType string `json:"mediumType,omitempty"`
	Quota      string `json:"quota,omitempty"`
}

type Master 

type Master struct {
	ReplicaCount        int                           `json:"replicaCount"`
	Resources           Resources                     `json:"resources"`
	NodeSelector        map[string]string             `json:"nodeSelector,omitempty"`
	MasterProperties    map[string]string             `json:"properties"`
	FileStoreProperties map[string]string             `json:"fileStoreProperties"`
	TokenProperties     map[string]string             `json:"secretProperties"`
	Port                Ports                         `json:"ports,omitempty"`
	OssKey              string                        `json:"osskey,omitempty"`
	OssSecret           string                        `json:"osssecret,omitempty"`
	Tolerations         []corev1.Toleration           `json:"tolerations,omitempty"`
	DnsServer           string                        `json:"dnsServer,omitempty"`
	NameSpace           string                        `json:"namespace,omitempty"`
	Labels              map[string]string             `json:"labels,omitempty"`
	Annotations         map[string]string             `json:"annotations,omitempty"`
	ServiceCount        int                           `json:"svccount"`
	Env                 map[string]string             `json:"env,omitempty"`
	CacheSets           map[string]*CacheSet          `json:"cachesets,omitempty"`
	VolumeMounts        []corev1.VolumeMount          `json:"volumeMounts,omitempty"`
	Volumes             []corev1.Volume               `json:"volumes,omitempty"`
	ImagePullSecrets    []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

type Mounts 

type Mounts struct {
	Master            map[string]*Level `json:"master"`
	WorkersAndClients map[string]*Level `json:"workersAndClients"`
}

type Ports 

type Ports struct {
	Rpc  int `json:"rpc,omitempty"`
	Raft int `json:"raft,omitempty"`
}

type Resource 

type Resource struct {
	CPU    string `json:"cpu"`
	Memory string `json:"memory"`
}

type Resources 

type Resources struct {
	Limits   Resource `json:"limits"`
	Requests Resource `json:"requests"`
}

type UFSVolume 

type UFSVolume struct {
	Name          string `json:"name"`
	SubPath       string `json:"subPath,omitempty"`
	ContainerPath string `json:"containerPath"`
	ReadOnly      bool   `json:"readOnly"`
}

type Worker 

type Worker struct {
	ReplicaCount     int                 `json:"replicaCount"`
	Resources        Resources           `json:"resources,omitempty"`
	NodeSelector     map[string]string   `json:"nodeSelector,omitempty"`
	WorkerProperties map[string]string   `json:"properties"`
	Port             Ports               `json:"ports,omitempty"`
	Tolerations      []corev1.Toleration `json:"tolerations,omitempty"`
	// Affinity         corev1.Affinity       `json:"affinity,omitempty"`
	Labels           map[string]string             `json:"labels,omitempty"`
	Annotations      map[string]string             `json:"annotations,omitempty"`
	Path             string                        `json:"dataPath"`
	Env              map[string]string             `json:"env,omitempty"`
	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.