goosefs

package
v1.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 28, 2024 License: Apache-2.0 Imports: 47 Imported by: 0

Documentation

Index

Constants

View Source 
const (

	// goosefsUser string = "fluid"
	METRICS_PREFIX_BYTES_READ_LOCAL = "Cluster.BytesReadLocal "

	METRICS_PREFIX_BYTES_READ_REMOTE = "Cluster.BytesReadRemote "

	METRICS_PREFIX_BYTES_READ_UFS_ALL = "Cluster.BytesReadUfsAll "

	METRICS_PREFIX_BYTES_READ_LOCAL_THROUGHPUT = "Cluster.BytesReadLocalThroughput "

	METRICS_PREFIX_BYTES_READ_REMOTE_THROUGHPUT = "Cluster.BytesReadRemoteThroughput "

	METRICS_PREFIX_BYTES_READ_UFS_THROUGHPUT = "Cluster.BytesReadUfsThroughput "

	SUMMARY_PREFIX_TOTAL_CAPACITY = "Total Capacity: "

	SUMMARY_PREFIX_USED_CAPACITY = "Used Capacity: "

	METADATA_SYNC_NOT_DONE_MSG = "[Calculating]"

	GOOSEFS_RUNTIME_METRICS_LABEL = "goosefs_runtime_metrics"

	CHECK_METADATA_SYNC_DONE_TIMEOUT_MILLISEC = 500

	AUTO_SELECT_PORT_MIN = 20000
	AUTO_SELECT_PORT_MAX = 30000

	PORT_NUM = 9

	CACHE_HIT_QUERY_INTERVAL_MIN = 1

	HADOOP_CONF_HDFS_SITE_FILENAME = "hdfs-site.xml"

	HADOOP_CONF_CORE_SITE_FILENAME = "core-site.xml"

	HADOOP_CONF_MOUNT_PATH = "/hdfs-config"

	WOKRER_POD_ROLE = "goosefs-worker"
)

Variables

This section is empty.

Functions

func Build 

func Build(id string, ctx cruntime.ReconcileRequestContext) (base.Engine, error)

Build function builds the GooseFS Engine

func ContainsString added in v0.7.0 

func ContainsString(s []string, v string) bool

ContainsString returns true if a string is present in a iteratee.

func GetReservedPorts 

func GetReservedPorts(client client.Client) (ports []int, err error)

GetReservedPorts defines restoration logic for goosefsRuntime

func Precheck added in v0.8.0 

func Precheck(client client.Client, key types.NamespacedName) (found bool, err error)

Precheck checks if the given key can be found in the current runtime types

Types

type APIGateway 

type APIGateway struct {
	Enabled bool  `yaml:"enabled,omitempty"`
	Ports   Ports `yaml:"ports,omitempty"`
}

type Affinity 

type Affinity struct {
	NodeAffinity *NodeAffinity `yaml:"nodeAffinity"`
}

type Fuse 

type Fuse struct {
	Image              string            `yaml:"image,omitempty"`
	NodeSelector       map[string]string `yaml:"nodeSelector,omitempty"`
	ImageTag           string            `yaml:"imageTag,omitempty"`
	ImagePullPolicy    string            `yaml:"imagePullPolicy,omitempty"`
	Properties         map[string]string `yaml:"properties,omitempty"`
	Env                map[string]string `yaml:"env,omitempty"`
	JvmOptions         []string          `yaml:"jvmOptions,omitempty"`
	MountPath          string            `yaml:"mountPath,omitempty"`
	ShortCircuitPolicy string            `yaml:"shortCircuitPolicy,omitempty"`
	Args               []string          `yaml:"args,omitempty"`
	HostNetwork        bool              `yaml:"hostNetwork,omitempty"`
	HostPID            bool              `json:"hostPID,omitempty"`
	Enabled            bool              `yaml:"enabled,omitempty"`
	Resources          common.Resources  `yaml:"resources,omitempty"`
	Global             bool              `yaml:"global,omitempty"`
	CriticalPod        bool              `yaml:"criticalPod,omitempty"`
	Annotations        map[string]string `json:"annotations,omitempty"`
}

type GooseFS 

type GooseFS struct {
	FullnameOverride string `yaml:"fullnameOverride"`

	common.ImageInfo `yaml:",inline"`
	common.UserInfo  `yaml:",inline"`

	NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
	JvmOptions   []string          `yaml:"jvmOptions,omitempty"`

	Properties map[string]string `yaml:"properties,omitempty"`

	Master Master `yaml:"master,omitempty"`

	JobMaster JobMaster `yaml:"jobMaster,omitempty"`

	Worker Worker `yaml:"worker,omitempty"`

	JobWorker JobWorker `yaml:"jobWorker,omitempty"`

	Fuse Fuse `yaml:"fuse,omitempty"`

	APIGateway APIGateway `yaml:"apiGateway,omitempty"`

	Tieredstore Tieredstore `yaml:"tieredstore,omitempty"`

	Metastore Metastore `yaml:"metastore,omitempty"`

	Journal Journal `yaml:"journal,omitempty"`

	ShortCircuit ShortCircuit `yaml:"shortCircuit,omitempty"`

	UFSPaths []UFSPath `yaml:"ufsPaths,omitempty"`

	UFSVolumes []UFSVolume `yaml:"ufsVolumes,omitempty"`

	InitUsers common.InitUsers `yaml:"initUsers,omitempty"`

	Monitoring string `yaml:"monitoring,omitempty"`

	HadoopConfig HadoopConfig `yaml:"hadoopConfig,omitempty"`

	Tolerations []v1.Toleration `yaml:"tolerations,omitempty"`

	PlacementMode string `yaml:"placement,omitempty"`
}

The value yaml file

type GooseFSEngine 

type GooseFSEngine struct {
	Log logr.Logger
	client.Client

	MetadataSyncDoneCh chan base.MetadataSyncResult

	UnitTest bool

	*ctrl.Helper
	Recorder record.EventRecorder
	// contains filtered or unexported fields
}

GooseFSEngine implements the Engine interface.

func (*GooseFSEngine) BindToDataset 

func (e *GooseFSEngine) BindToDataset() (err error)

func (*GooseFSEngine) CheckAndUpdateRuntimeStatus 

func (e *GooseFSEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)

CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.

func (*GooseFSEngine) CheckMasterReady 

func (e *GooseFSEngine) CheckMasterReady() (ready bool, err error)

CheckMasterReady checks if the master is ready

func (*GooseFSEngine) CheckRuntimeHealthy 

func (e *GooseFSEngine) CheckRuntimeHealthy() (err error)

CheckRuntimeHealthy checks the healthy of the runtime

func (*GooseFSEngine) CheckRuntimeReady 

func (e *GooseFSEngine) CheckRuntimeReady() (ready bool)

func (*GooseFSEngine) CheckWorkersReady 

func (e *GooseFSEngine) CheckWorkersReady() (ready bool, err error)

CheckWorkersReady check the workers ready.

func (*GooseFSEngine) CreateVolume 

func (e *GooseFSEngine) CreateVolume() (err error)

CreateVolume creates volume

func (*GooseFSEngine) DeleteVolume 

func (e *GooseFSEngine) DeleteVolume() (err error)

DeleteVolume creates volume

func (*GooseFSEngine) FreeStorageBytes 

func (e *GooseFSEngine) FreeStorageBytes() (value int64, err error)

FreeStorageBytes returns free storage size of GooseFS in bytes

func (*GooseFSEngine) GetAPIGatewayStatus 

func (e *GooseFSEngine) GetAPIGatewayStatus() (status *datav1alpha1.APIGatewayStatus, err error)

Query the API Gateway status

func (*GooseFSEngine) GetCacheHitStates 

func (e *GooseFSEngine) GetCacheHitStates() (cacheHitStates cacheHitStates)

GetCacheHitStates gets cache hit related info by parsing GooseFS metrics

func (*GooseFSEngine) GetDataOperationValueFile added in v0.9.0 

func (e *GooseFSEngine) GetDataOperationValueFile(ctx cruntime.ReconcileRequestContext, operation dataoperation.OperationInterface) (valueFileName string, err error)

func (*GooseFSEngine) GetHCFSStatus 

func (e *GooseFSEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)

Query the hcfs status

func (*GooseFSEngine) GetMetadataFileName 

func (e *GooseFSEngine) GetMetadataFileName() string

func (*GooseFSEngine) GetMetadataInfoFile 

func (e *GooseFSEngine) GetMetadataInfoFile() string

func (*GooseFSEngine) GetMetadataInfoFileName 

func (e *GooseFSEngine) GetMetadataInfoFileName() string

func (*GooseFSEngine) GetReportMetrics 

func (e *GooseFSEngine) GetReportMetrics() (summary string, err error)

reportMetrics reports goosefs metrics

func (*GooseFSEngine) GetReportSummary 

func (e *GooseFSEngine) GetReportSummary() (summary string, err error)

reportSummary reports goosefs summary

func (*GooseFSEngine) GetWorkerUsedCapacity 

func (e *GooseFSEngine) GetWorkerUsedCapacity() (map[string]int64, error)

GetWorkerUsedCapacity gets cache capacity usage for each worker as a map. It parses result from stdout when executing `goosefs fsadmin report capacity` command and extracts worker name(IP or hostname) along with used capacity for that worker

func (*GooseFSEngine) HasDeprecatedCommonLabelname 

func (e *GooseFSEngine) HasDeprecatedCommonLabelname() (deprecated bool, err error)

func (GooseFSEngine) ParseReportMetric 

func (e GooseFSEngine) ParseReportMetric(metrics string, cacheHitStates, lastCacheHitStates *cacheHitStates)

parse goosefs report metric to cacheHitStates

func (GooseFSEngine) ParseReportSummary 

func (e GooseFSEngine) ParseReportSummary(s string) cacheStates

parse goosefs report summary to cacheStates

func (*GooseFSEngine) PrepareUFS 

func (e *GooseFSEngine) PrepareUFS() (err error)

PrepareUFS does all the UFS preparations

func (*GooseFSEngine) RestoreMetadataInternal 

func (e *GooseFSEngine) RestoreMetadataInternal() (err error)

RestoreMetadataInternal restore metadata from backup there are three kinds of data to be restored 1. metadata of dataset 2. ufsTotal info of dataset 3. fileNum info of dataset if 1 fails, the goosefs master will fail directly, if 2 or 3 fails, fluid will get the info from goosefs again

func (*GooseFSEngine) SetupMaster 

func (e *GooseFSEngine) SetupMaster() (err error)

SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered

func (*GooseFSEngine) SetupWorkers 

func (e *GooseFSEngine) SetupWorkers() (err error)

SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.

func (*GooseFSEngine) ShouldCheckUFS 

func (e *GooseFSEngine) ShouldCheckUFS() (should bool, err error)

ShouldCheckUFS checks if it requires checking UFS

func (*GooseFSEngine) ShouldSetupMaster 

func (e *GooseFSEngine) ShouldSetupMaster() (should bool, err error)

ShouldSetupMaster checks if we need setup the master

func (*GooseFSEngine) ShouldSetupWorkers 

func (e *GooseFSEngine) ShouldSetupWorkers() (should bool, err error)

ShouldSetupWorkers checks if we need setup the workers

func (*GooseFSEngine) ShouldUpdateUFS 

func (e *GooseFSEngine) ShouldUpdateUFS() (ufsToUpdate *utils.UFSToUpdate)

func (*GooseFSEngine) Shutdown 

func (e *GooseFSEngine) Shutdown() (err error)

shut down the GooseFS engine

func (*GooseFSEngine) SyncMetadata 

func (e *GooseFSEngine) SyncMetadata() (err error)

SyncMetadata syncs metadata if necessary For GooseFS Engine, metadata sync is an asynchronous operation, which means you should call this function periodically to make sure the function actually takes effect.

func (*GooseFSEngine) SyncReplicas 

func (e *GooseFSEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)

SyncReplicas syncs the replicas

func (*GooseFSEngine) SyncRuntime added in v0.8.0 

func (e *GooseFSEngine) SyncRuntime(ctx cruntime.ReconcileRequestContext) (changed bool, err error)

SyncRuntime syncs the runtime spec

func (*GooseFSEngine) SyncScheduleInfoToCacheNodes added in v0.7.0 

func (e *GooseFSEngine) SyncScheduleInfoToCacheNodes() (err error)

SyncScheduleInfoToCacheNodes syncs schedule info to nodes

func (*GooseFSEngine) TotalFileNums 

func (e *GooseFSEngine) TotalFileNums() (value int64, err error)

TotalFileNums returns the total num of files in GooseFS

func (*GooseFSEngine) TotalStorageBytes 

func (e *GooseFSEngine) TotalStorageBytes() (value int64, err error)

TotalStorageBytes return total storage size of GooseFS in bytes

func (*GooseFSEngine) UpdateCacheOfDataset 

func (e *GooseFSEngine) UpdateCacheOfDataset() (err error)

UpdateCacheOfDataset updates the CacheStates and Runtimes of the dataset.

func (*GooseFSEngine) UpdateDatasetStatus 

func (e *GooseFSEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)

UpdateDatasetStatus updates the status of the dataset

func (*GooseFSEngine) UpdateOnUFSChange 

func (e *GooseFSEngine) UpdateOnUFSChange(ufsToUpdate *utils.UFSToUpdate) (updateReady bool, err error)

func (*GooseFSEngine) UsedStorageBytes 

func (e *GooseFSEngine) UsedStorageBytes() (value int64, err error)

UsedStorageBytes returns used storage size of GooseFS in bytes

func (*GooseFSEngine) Validate added in v1.0.2 

func (e *GooseFSEngine) Validate(ctx cruntime.ReconcileRequestContext) (err error)

type HadoopConfig 

type HadoopConfig struct {
	ConfigMap       string `yaml:"configMap"`
	IncludeHdfsSite bool   `yaml:"includeHdfsSite"`
	IncludeCoreSite bool   `yaml:"includeCoreSite"`
}

type JobMaster 

type JobMaster struct {
	Ports     Ports            `yaml:"ports,omitempty"`
	Resources common.Resources `yaml:"resources,omitempty"`
}

type JobWorker 

type JobWorker struct {
	Ports     Ports            `yaml:"ports,omitempty"`
	Resources common.Resources `yaml:"resources,omitempty"`
}

type Journal 

type Journal struct {
	VolumeType string `yaml:"volumeType,omitempty"`
	Size       string `yaml:"size,omitempty"`
}

type Level 

type Level struct {
	Alias      string `yaml:"alias,omitempty"`
	Level      int    `yaml:"level"`
	Mediumtype string `yaml:"mediumtype,omitempty"`
	Type       string `yaml:"type,omitempty"`
	Path       string `yaml:"path,omitempty"`
	Quota      string `yaml:"quota,omitempty"`
	High       string `yaml:"high,omitempty"`
	Low        string `yaml:"low,omitempty"`
}

type Master 

type Master struct {
	JvmOptions   []string          `yaml:"jvmOptions,omitempty"`
	Env          map[string]string `yaml:"env,omitempty"`
	Affinity     Affinity          `yaml:"affinity"`
	NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
	Properties   map[string]string `yaml:"properties,omitempty"`
	Replicas     int32             `yaml:"replicaCount,omitempty"`
	HostNetwork  bool              `yaml:"hostNetwork,omitempty"`
	Resources    common.Resources  `yaml:"resources,omitempty"`
	Ports        Ports             `yaml:"ports,omitempty"`
	BackupPath   string            `yaml:"backupPath,omitempty"`
	Restore      Restore           `yaml:"restore,omitempty"`
	Annotations  map[string]string `json:"annotations,omitempty"`
}

type Metastore 

type Metastore struct {
	VolumeType string `yaml:"volumeType,omitempty"`
	Size       string `yaml:"size,omitempty"`
}

type NodeAffinity 

type NodeAffinity struct {

	// If the affinity requirements specified by this field are not met at
	// scheduling time, the pod will not be scheduled onto the node.
	// If the affinity requirements specified by this field cease to be met
	// at some point during pod execution (e.g. due to an update), the system
	// may or may not try to eventually evict the pod from its node.
	// +optional
	RequiredDuringSchedulingIgnoredDuringExecution *NodeSelector `yaml:"requiredDuringSchedulingIgnoredDuringExecution"`
}

type NodeSelector 

type NodeSelector struct {
	//Required. A list of node selector terms.
	NodeSelectorTerms []NodeSelectorTerm `yaml:"nodeSelectorTerms"`
}

NodeSelector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.

type NodeSelectorRequirement 

type NodeSelectorRequirement struct {
	// The label key that the selector applies to.
	Key string `yaml:"key,omitempty"`
	// Represents a key's relationship to a set of values.
	// Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
	Operator string `yaml:"operator,omitempty"`
	// +optional
	Values []string `yaml:"values,omitempty"`
}

NodeSelectorRequirement is a selector that contains values, a key, and an operator that relates the key and values.

type NodeSelectorTerm 

type NodeSelectorTerm struct {
	// A list of node selector requirements by node's labels.
	MatchExpressions []NodeSelectorRequirement `yaml:"matchExpressions"`
}

NodeSelectorTerm represents expressions and fields required to select nodes. A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.

type Ports 

type Ports struct {
	Rpc      int `yaml:"rpc,omitempty"`
	Web      int `yaml:"web,omitempty"`
	Embedded int `yaml:"embedded,omitempty"`
	Data     int `yaml:"data,omitempty"`
	Rest     int `yaml:"rest,omitempty"`
}

type Restore 

type Restore struct {
	Enabled bool   `yaml:"enabled,omitempty"`
	Path    string `yaml:"path,omitempty"`
	PVCName string `yaml:"pvcName,omitempty"`
}

type ShortCircuit 

type ShortCircuit struct {
	Enable     bool   `yaml:"enable,omitempty"`
	Policy     string `yaml:"policy,omitempty"`
	VolumeType string `yaml:"volumeType,omitempty"`
}

type Tieredstore 

type Tieredstore struct {
	Levels []Level `yaml:"levels,omitempty"`
}

type UFSPath 

type UFSPath struct {
	HostPath  string `yaml:"hostPath"`
	UFSVolume `yaml:",inline"`
}

type UFSVolume 

type UFSVolume struct {
	Name          string `yaml:"name"`
	SubPath       string `yaml:"subPath,omitempty"`
	ContainerPath string `yaml:"containerPath"`
}

type Worker 

type Worker struct {
	JvmOptions   []string          `yaml:"jvmOptions,omitempty"`
	Env          map[string]string `yaml:"env,omitempty"`
	NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
	Properties   map[string]string `yaml:"properties,omitempty"`
	HostNetwork  bool              `yaml:"hostNetwork,omitempty"`
	Resources    common.Resources  `yaml:"resources,omitempty"`
	Ports        Ports             `yaml:"ports,omitempty"`
	Annotations  map[string]string `json:"annotations,omitempty"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.