Documentation ¶
Index ¶
- Constants
- Variables
- func GetSeedPeerClusterConfigBySeedPeer(seedPeer *managerv2.SeedPeer) (types.SeedPeerClusterConfig, error)
- type CacheConfig
- type CertSpec
- type Config
- type DatabaseConfig
- type DownloadTinyConfig
- type DownloadTinyTLSClientConfig
- type DynConfig
- type DynconfigData
- type DynconfigInterface
- type DynconfigOption
- type GCConfig
- type HostConfig
- type JobConfig
- type KeepAliveConfig
- type ManagerConfig
- type MetricsConfig
- type NetworkConfig
- type NetworkTopologyConfig
- type Observer
- type ProbeConfig
- type RedisConfig
- type ResourceConfig
- type SchedulerConfig
- type SecurityConfig
- type SeedPeerConfig
- type ServerConfig
- type StorageConfig
- type TaskConfig
- type TrainerConfig
Constants ¶
View Source
const ( // DefaultSeedPeerConcurrentUploadLimit is default number for seed peer concurrent upload limit. DefaultSeedPeerConcurrentUploadLimit = 2000 // DefaultPeerConcurrentUploadLimit is default number for peer concurrent upload limit. DefaultPeerConcurrentUploadLimit = 200 // DefaultSchedulerCandidateParentLimit is default limit the number of candidate parent. DefaultSchedulerCandidateParentLimit = 4 // DefaultSchedulerFilterParentLimit is default limit the number for filter parent. DefaultSchedulerFilterParentLimit = 15 )
View Source
const ( // DefaultServerPort is default port for server. DefaultServerPort = 8002 // DefaultServerAdvertisePort is default advertise port for server. DefaultServerAdvertisePort = 8002 )
View Source
const ( // DefaultRedisBrokerDB is default db for redis broker. DefaultRedisBrokerDB = 1 // DefaultRedisBackendDB is default db for redis backend. DefaultRedisBackendDB = 2 // DefaultNetworkTopologyDB is default db for network topology. DefaultNetworkTopologyDB = 3 )
View Source
const ( // DefaultSchedulerAlgorithm is default algorithm for scheduler. DefaultSchedulerAlgorithm = "default" // DefaultSchedulerBackToSourceCount is default back-to-source count for scheduler. DefaultSchedulerBackToSourceCount = 200 // DefaultSchedulerRetryBackToSourceLimit is default retry back-to-source limit for scheduler. DefaultSchedulerRetryBackToSourceLimit = 4 // DefaultSchedulerRetryLimit is default retry limit for scheduler. DefaultSchedulerRetryLimit = 5 // DefaultSchedulerRetryInterval is default retry interval for scheduler. DefaultSchedulerRetryInterval = 500 * time.Millisecond // DefaultSchedulerPieceDownloadTimeout is default timeout of downloading piece. DefaultSchedulerPieceDownloadTimeout = 30 * time.Minute // DefaultSchedulerPeerGCInterval is default interval for peer gc. DefaultSchedulerPeerGCInterval = 10 * time.Second // DefaultSchedulerPeerTTL is default ttl for peer. DefaultSchedulerPeerTTL = 24 * time.Hour // DefaultSchedulerTaskGCInterval is default interval for task gc. DefaultSchedulerTaskGCInterval = 30 * time.Minute // DefaultSchedulerHostGCInterval is default interval for host gc. DefaultSchedulerHostGCInterval = 5 * time.Minute // DefaultSchedulerHostTTL is default ttl for host. DefaultSchedulerHostTTL = 1 * time.Hour // DefaultRefreshModelInterval is model refresh interval. DefaultRefreshModelInterval = 168 * time.Hour // DefaultCPU is default cpu usage. DefaultCPU = 1 // NetworkTopologyAlgorithm is a scheduling algorithm based on rules and network topology. NetworkTopologyAlgorithm = "nt" // DefaultNetworkTopologyCollectInterval is the default interval of collecting network topology. DefaultSchedulerNetworkTopologyCollectInterval = 2 * time.Hour // DefaultNetworkTopologyCacheInterval is the default cache cleanup interval. DefaultSchedulerNetworkTopologyCacheInterval = 5 * time.Minute // DefaultNetworkTopologyCacheTLL is the default ttl of networkTopology cache. DefaultSchedulerNetworkTopologyCacheTLL = 5 * time.Minute // DefaultProbeQueueLength is the default length of probe queue. DefaultSchedulerNetworkTopologyProbeQueueLength = 5 // DefaultProbeCount is the default number of probing hosts. DefaultSchedulerNetworkTopologyProbeCount = 5 )
View Source
const ( // DefaultResourceTaskDownloadTinyScheme is default scheme of downloading tiny task. DefaultResourceTaskDownloadTinyScheme = "http" // DefaultResourceTaskDownloadTinyTimeout is default timeout of downloading tiny task. DefaultResourceTaskDownloadTinyTimeout = 1 * time.Minute )
View Source
const ( // DefaultManagerSchedulerClusterID is default id for scheduler cluster. DefaultManagerSchedulerClusterID = 1 // DefaultManagerKeepAliveInterval is default interval for keepalive. DefaultManagerKeepAliveInterval = 5 * time.Second )
View Source
const ( // DefaultJobGlobalWorkerNum is default global worker number for job. DefaultJobGlobalWorkerNum = 500 // DefaultJobSchedulerWorkerNum is default scheduler worker number for job. DefaultJobSchedulerWorkerNum = 500 // DefaultJobGlobalWorkerNum is default local worker number for job. DefaultJobLocalWorkerNum = 1000 // DefaultJobRedisBrokerDB is default db for redis broker. DefaultJobRedisBrokerDB = 1 // DefaultJobRedisBackendDB is default db for redis backend. DefaultJobRedisBackendDB = 2 )
View Source
const ( // DefaultStorageMaxSize is the default maximum size of record file. DefaultStorageMaxSize = 100 // DefaultStorageMaxBackups is the default maximum count of backup. DefaultStorageMaxBackups = 10 // DefaultStorageBufferSize is the default size of buffer container. DefaultStorageBufferSize = 100 )
View Source
const ( // DefaultTrainerAddr is the default address of trainer. DefaultTrainerAddr = "127.0.0.1:9090" // DefaultTrainerInterval is the default interval of training. DefaultTrainerInterval = 7 * 24 * time.Hour // DefaultTrainerUploadTimeout is the default timeout of uploading dataset to trainer. DefaultTrainerUploadTimeout = 1 * time.Hour )
View Source
const ( // DefaultLogRotateMaxSize is the default maximum size in megabytes of log files before rotation. DefaultLogRotateMaxSize = 1024 // DefaultLogRotateMaxAge is the default number of days to retain old log files. DefaultLogRotateMaxAge = 7 // DefaultLogRotateMaxBackups is the default number of old log files to keep. DefaultLogRotateMaxBackups = 20 )
View Source
const ( // DefaultDynConfigRefreshInterval is default refresh interval for dynamic configuration. DefaultDynConfigRefreshInterval = 1 * time.Minute )
View Source
const (
// DefaultMetricsAddr is default address for metrics server.
DefaultMetricsAddr = ":8000"
)
View Source
const ( // DefaultSeedTaskDownloadTimeout is default timeout of downloading task by seed peer. DefaultSeedPeerTaskDownloadTimeout = 10 * time.Hour )
Variables ¶
View Source
var ( // DefaultCertIPAddresses is default ip addresses of certificate. DefaultCertIPAddresses = []net.IP{ip.IPv4, ip.IPv6} // DefaultCertDNSNames is default dns names of certificate. DefaultCertDNSNames = []string{"dragonfly-scheduler", "dragonfly-scheduler.dragonfly-system.svc", "dragonfly-scheduler.dragonfly-system.svc.cluster.local"} // DefaultCertValidityPeriod is default validity period of certificate. DefaultCertValidityPeriod = 180 * 24 * time.Hour )
View Source
var ( // DefaultNetworkEnableIPv6 is default value of enableIPv6. DefaultNetworkEnableIPv6 = false )
Functions ¶
func GetSeedPeerClusterConfigBySeedPeer ¶ added in v2.0.9
func GetSeedPeerClusterConfigBySeedPeer(seedPeer *managerv2.SeedPeer) (types.SeedPeerClusterConfig, error)
GetSeedPeerClusterConfigBySeedPeer returns the seed peer cluster config by seed peer.
Types ¶
type CacheConfig ¶ added in v2.1.28
type CertSpec ¶ added in v2.0.6
type CertSpec struct { // DNSNames is a list of dns names be set on the certificate. DNSNames []string `mapstructure:"dnsNames" yaml:"dnsNames"` // IPAddresses is a list of ip addresses be set on the certificate. IPAddresses []net.IP `mapstructure:"ipAddresses" yaml:"ipAddresses"` // ValidityPeriod is the validity period of certificate. ValidityPeriod time.Duration `mapstructure:"validityPeriod" yaml:"validityPeriod"` }
type Config ¶
type Config struct { // Base options. base.Options `yaml:",inline" mapstructure:",squash"` // Server configuration. Server ServerConfig `yaml:"server" mapstructure:"server"` // Scheduler configuration. Scheduler SchedulerConfig `yaml:"scheduler" mapstructure:"scheduler"` // Database configuration. Database DatabaseConfig `yaml:"database" mapstructure:"database"` // Resource configuration. Resource ResourceConfig `yaml:"resource" mapstructure:"resource"` // Dynconfig configuration. DynConfig DynConfig `yaml:"dynConfig" mapstructure:"dynConfig"` // Manager configuration. Manager ManagerConfig `yaml:"manager" mapstructure:"manager"` // SeedPeer configuration. SeedPeer SeedPeerConfig `yaml:"seedPeer" mapstructure:"seedPeer"` // Host configuration. Host HostConfig `yaml:"host" mapstructure:"host"` // Job configuration. Job JobConfig `yaml:"job" mapstructure:"job"` // Storage configuration. Storage StorageConfig `yaml:"storage" mapstructure:"storage"` // Metrics configuration. Metrics MetricsConfig `yaml:"metrics" mapstructure:"metrics"` // Security configuration. Security SecurityConfig `yaml:"security" mapstructure:"security"` // Network configuration. Network NetworkConfig `yaml:"network" mapstructure:"network"` // Trainer configuration. Trainer TrainerConfig `yaml:"trainer" mapstructure:"trainer"` }
type DatabaseConfig ¶ added in v2.0.30
type DatabaseConfig struct { // Redis configuration. Redis RedisConfig `yaml:"redis" mapstructure:"redis"` }
type DownloadTinyConfig ¶ added in v2.0.30
type DownloadTinyConfig struct { // Scheme is download tiny task scheme. Scheme string `yaml:"scheme" mapstructure:"scheme"` // Timeout is http request timeout. Timeout time.Duration `yaml:"timeout" mapstructure:"timeout"` // TLS is download tiny task TLS configuration. TLS DownloadTinyTLSClientConfig `yaml:"tls" mapstructure:"tls"` }
type DownloadTinyTLSClientConfig ¶ added in v2.0.30
type DownloadTinyTLSClientConfig struct { // InsecureSkipVerify controls whether a client verifies the // server's certificate chain and host name. InsecureSkipVerify bool `yaml:"insecureSkipVerify" mapstructure:"insecureSkipVerify"` }
type DynconfigData ¶
type DynconfigData struct { Scheduler *managerv2.Scheduler Applications []*managerv2.Application }
type DynconfigInterface ¶
type DynconfigInterface interface { // GetResolveSeedPeerAddrs returns the dynamic schedulers resolve addrs. GetResolveSeedPeerAddrs() ([]resolver.Address, error) // GetScheduler returns the scheduler config from manager. GetScheduler() (*managerv2.Scheduler, error) // GetApplications returns the applications config from manager. GetApplications() ([]*managerv2.Application, error) // GetSeedPeers returns the dynamic seed peers config from manager. GetSeedPeers() ([]*managerv2.SeedPeer, error) // GetSeedPeerClusterConfig returns the seed peer cluster config. GetSeedPeerClusterConfig() (types.SeedPeerClusterConfig, error) // GetSchedulerCluster returns the scheduler cluster config from manager. GetSchedulerCluster() (*managerv2.SchedulerCluster, error) // GetSchedulerClusterConfig returns the scheduler cluster config. GetSchedulerClusterConfig() (types.SchedulerClusterConfig, error) // GetSchedulerClusterClientConfig returns the client config. GetSchedulerClusterClientConfig() (types.SchedulerClusterClientConfig, error) // Get returns the dynamic config from manager. Get() (*DynconfigData, error) // Refresh refreshes dynconfig in cache. Refresh() error // Register allows an instance to register itself to listen/observe events. Register(Observer) // Deregister allows an instance to remove itself from the collection of observers/listeners. Deregister(Observer) // Notify publishes new events to listeners. Notify() error // Serve the dynconfig listening service. Serve() error // Stop the dynconfig listening service. Stop() error }
func NewDynconfig ¶
func NewDynconfig(rawManagerClient managerclient.V2, cacheDir string, cfg *Config, options ...DynconfigOption) (DynconfigInterface, error)
NewDynconfig returns a new dynconfig instance.
type DynconfigOption ¶ added in v2.0.9
type DynconfigOption func(d *dynconfig) error
DynconfigOption is a functional option for configuring the dynconfig.
func WithTransportCredentials ¶ added in v2.0.9
func WithTransportCredentials(creds credentials.TransportCredentials) DynconfigOption
WithTransportCredentials returns a DialOption which configures a connection level security credentials (e.g., TLS/SSL).
type GCConfig ¶
type GCConfig struct { // PieceDownloadTimeout is timeout of downloading piece. PieceDownloadTimeout time.Duration `yaml:"pieceDownloadTimeout" mapstructure:"pieceDownloadTimeout"` // PeerGCInterval is interval of peer gc. PeerGCInterval time.Duration `yaml:"peerGCInterval" mapstructure:"peerGCInterval"` // PeerTTL is time to live of peer. If the peer has been downloaded by other peers, // then PeerTTL will be reset. PeerTTL time.Duration `yaml:"peerTTL" mapstructure:"peerTTL"` // TaskGCInterval is interval of task gc. If all the peers have been reclaimed in the task, // then the task will also be reclaimed. TaskGCInterval time.Duration `yaml:"taskGCInterval" mapstructure:"taskGCInterval"` // HostGCInterval is interval of host gc. HostGCInterval time.Duration `yaml:"hostGCInterval" mapstructure:"hostGCInterval"` // HostTTL is time to live of host. If host announces message to scheduler, // then HostTTl will be reset. HostTTL time.Duration `yaml:"hostTTL" mapstructure:"hostTTL"` }
type HostConfig ¶
type JobConfig ¶
type JobConfig struct { // Enable job service. Enable bool `yaml:"enable" mapstructure:"enable"` // Number of workers in global queue. GlobalWorkerNum uint `yaml:"globalWorkerNum" mapstructure:"globalWorkerNum"` // Number of workers in scheduler queue. SchedulerWorkerNum uint `yaml:"schedulerWorkerNum" mapstructure:"schedulerWorkerNum"` // Number of workers in local queue. LocalWorkerNum uint `yaml:"localWorkerNum" mapstructure:"localWorkerNum"` // DEPRECATED: Please use the `database.redis` field instead. Redis RedisConfig `yaml:"redis" mapstructure:"redis"` }
type KeepAliveConfig ¶
type ManagerConfig ¶
type ManagerConfig struct { // Addr is manager address. Addr string `yaml:"addr" mapstructure:"addr"` // SchedulerClusterID is scheduler cluster id. SchedulerClusterID uint `yaml:"schedulerClusterID" mapstructure:"schedulerClusterID"` // KeepAlive configuration. KeepAlive KeepAliveConfig `yaml:"keepAlive" mapstructure:"keepAlive"` }
type MetricsConfig ¶ added in v2.0.1
type NetworkConfig ¶ added in v2.0.7
type NetworkConfig struct { // EnableIPv6 enables ipv6 for server. EnableIPv6 bool `mapstructure:"enableIPv6" yaml:"enableIPv6"` }
type NetworkTopologyConfig ¶ added in v2.0.9
type NetworkTopologyConfig struct { // CollectInterval is the interval of collecting network topology. CollectInterval time.Duration `mapstructure:"collectInterval" yaml:"collectInterval"` // Probe is the configuration of probe. Probe ProbeConfig `yaml:"probe" mapstructure:"probe"` // Cache is the configuration of cache. Cache CacheConfig `yaml:"cache" mapstructure:"cache"` }
type Observer ¶
type Observer interface { // OnNotify allows an event to be published to interface implementations. OnNotify(*DynconfigData) }
type ProbeConfig ¶ added in v2.0.9
type RedisConfig ¶
type RedisConfig struct { // DEPRECATED: Please use the `addrs` field instead. Host string `yaml:"host" mapstructure:"host"` // DEPRECATED: Please use the `addrs` field instead. Port int `yaml:"port" mapstructure:"port"` // Addrs is server addresses. Addrs []string `yaml:"addrs" mapstructure:"addrs"` // MasterName is the sentinel master name. MasterName string `yaml:"masterName" mapstructure:"masterName"` // Username is server username. Username string `yaml:"username" mapstructure:"username"` // Password is server password. Password string `yaml:"password" mapstructure:"password"` // BrokerDB is broker database name. BrokerDB int `yaml:"brokerDB" mapstructure:"brokerDB"` // BackendDB is backend database name. BackendDB int `yaml:"backendDB" mapstructure:"backendDB"` // NetworkTopologyDB is network topology database name. NetworkTopologyDB int `yaml:"networkTopologyDB" mapstructure:"networkTopologyDB"` }
type ResourceConfig ¶ added in v2.0.30
type ResourceConfig struct { // Task resource configuration. Task TaskConfig `yaml:"task" mapstructure:"task"` }
type SchedulerConfig ¶
type SchedulerConfig struct { // Algorithm is scheduling algorithm used by the scheduler. Algorithm string `yaml:"algorithm" mapstructure:"algorithm"` // BackToSourceCount is single task allows the peer to back-to-source count. BackToSourceCount int `yaml:"backToSourceCount" mapstructure:"backToSourceCount"` // RetryBackToSourceLimit reaches the limit, then the peer back-to-source. RetryBackToSourceLimit int `yaml:"retryBackToSourceLimit" mapstructure:"retryBackToSourceLimit"` // RetryLimit reaches the limit, then scheduler returns scheduling failed. RetryLimit int `yaml:"retryLimit" mapstructure:"retryLimit"` // RetryInterval is scheduling interval. RetryInterval time.Duration `yaml:"retryInterval" mapstructure:"retryInterval"` // GC configuration. GC GCConfig `yaml:"gc" mapstructure:"gc"` // NetworkTopology configuration. NetworkTopology NetworkTopologyConfig `yaml:"networkTopology" mapstructure:"networkTopology"` }
type SecurityConfig ¶ added in v2.0.6
type SecurityConfig struct { // AutoIssueCert indicates to issue client certificates for all grpc call // if AutoIssueCert is false, any other option in Security will be ignored. AutoIssueCert bool `mapstructure:"autoIssueCert" yaml:"autoIssueCert"` // CACert is the root CA certificate for all grpc tls handshake, it can be path or PEM format string. CACert types.PEMContent `mapstructure:"caCert" yaml:"caCert"` // TLSVerify indicates to verify client certificates. TLSVerify bool `mapstructure:"tlsVerify" yaml:"tlsVerify"` // TLSPolicy controls the grpc shandshake behaviors: // force: both ClientHandshake and ServerHandshake are only support tls. // prefer: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support tls. // default: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support insecure (non-tls). TLSPolicy string `mapstructure:"tlsPolicy" yaml:"tlsPolicy"` // CertSpec is the desired state of certificate. CertSpec CertSpec `mapstructure:"certSpec" yaml:"certSpec"` }
type SeedPeerConfig ¶ added in v2.0.3
type ServerConfig ¶
type ServerConfig struct { // AdvertiseIP is advertise ip. AdvertiseIP net.IP `yaml:"advertiseIP" mapstructure:"advertiseIP"` // AdvertisePort is advertise port. AdvertisePort int `yaml:"advertisePort" mapstructure:"advertisePort"` // ListenIP is listen ip, like: 0.0.0.0, 192.168.0.1. ListenIP net.IP `yaml:"listenIP" mapstructure:"listenIP"` // Server port. Port int `yaml:"port" mapstructure:"port"` // Server hostname. Host string `yaml:"host" mapstructure:"host"` // Server work directory. WorkHome string `yaml:"workHome" mapstructure:"workHome"` // Server dynamic config cache directory. CacheDir string `yaml:"cacheDir" mapstructure:"cacheDir"` // Server log directory. LogDir string `yaml:"logDir" mapstructure:"logDir"` // Maximum size in megabytes of log files before rotation (default: 1024) LogMaxSize int `yaml:"logMaxSize" mapstructure:"logMaxSize"` // Maximum number of days to retain old log files (default: 7) LogMaxAge int `yaml:"logMaxAge" mapstructure:"logMaxAge"` // Maximum number of old log files to keep (default: 20) LogMaxBackups int `yaml:"logMaxBackups" mapstructure:"logMaxBackups"` // Server plugin directory. PluginDir string `yaml:"pluginDir" mapstructure:"pluginDir"` // Server storage data directory. DataDir string `yaml:"dataDir" mapstructure:"dataDir"` }
type StorageConfig ¶ added in v2.0.3
type StorageConfig struct { // MaxSize sets the maximum size in megabytes of storage file. MaxSize int `yaml:"maxSize" mapstructure:"maxSize"` // MaxBackups sets the maximum number of storage files to retain. MaxBackups int `yaml:"maxBackups" mapstructure:"maxBackups"` // BufferSize sets the size of buffer container, // if the buffer is full, write all the records in the buffer to the file. BufferSize int `yaml:"bufferSize" mapstructure:"bufferSize"` }
type TaskConfig ¶ added in v2.0.30
type TaskConfig struct { // Download tiny task configuration. DownloadTiny DownloadTinyConfig `yaml:"downloadTiny" mapstructure:"downloadTiny"` }
type TrainerConfig ¶ added in v2.0.10
type TrainerConfig struct { // Enable trainer service. Enable bool `yaml:"enable" mapstructure:"enable"` // Addr is trainer service address. Addr string `yaml:"addr" mapstructure:"addr"` // Interval is the interval of training. Interval time.Duration `yaml:"interval" mapstructure:"interval"` // UploadTimeout is the timeout of uploading dataset to trainer. UploadTimeout time.Duration `yaml:"uploadTimeout" mapstructure:"uploadTimeout"` }
Click to show internal directories.
Click to hide internal directories.