Documentation ¶
Overview ¶
Package config holds all DaemonConfig of dfget.
Package config holds all options of peerhost.
Index ¶
- Constants
- Variables
- func ConvertCacheDelete(cfg *CacheOption, args []string) error
- func ConvertCacheExport(cfg *CacheOption, args []string) error
- func ConvertCacheStat(cfg *CacheOption, args []string) error
- func GetCertificate(certifyClient *certify.Certify) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error)
- func MkdirAll(dir string, perm os.FileMode, uid, gid int) error
- func ValidateCacheDelete(cfg *CacheOption) error
- func ValidateCacheExport(cfg *CacheOption) error
- type AnnouncerOption
- type BasicAuth
- type CacheOption
- type CertPool
- type CertSpec
- type ClientOption
- type ConcurrentOption
- type DaemonConfig
- type DaemonOption
- type DfcacheConfig
- type DfgetConfig
- type DfstoreConfig
- type DownloadOption
- type DurationValue
- type Dynconfig
- type DynconfigData
- type DynconfigOption
- type GlobalSecurityOption
- type HealthOption
- type HijackConfig
- type HijackHost
- type HostOption
- type KeepAliveOption
- type ListenOption
- type ManagerOption
- type NetAddrsValue
- type NetworkOption
- type NetworkTopologyOption
- type ObjectStorageOption
- type Observer
- type PeerExchangeOption
- type ProbeOption
- type ProxyOption
- type ProxyRule
- type RateLimitValue
- type RecursiveConcurrent
- type Regexp
- type RegistryMirror
- type ReloadOption
- type ResourceClientsOption
- type SchedulerOption
- type SecurityOption
- type SeedPeerOption
- type SourceType
- type StorageOption
- type StoreStrategy
- type SupernodesValue
- type TCPListenOption
- type TCPListenPortRange
- type TLSConfig
- type TelemetryOption
- type TransportOption
- type URL
- type UnixListenOption
- type UploadOption
- type WhiteList
Constants ¶
const ( DefaultPerPeerDownloadLimit = 512 * unit.MB DefaultTotalDownloadLimit = 1024 * unit.MB DefaultUploadLimit = 1024 * unit.MB DefaultMinRate = 20 * unit.MB )
Download limit.
const ( DefaultTaskExpireTime = 6 * time.Hour DefaultGCInterval = 1 * time.Minute DefaultDaemonAliveTime = 5 * time.Minute DefaultScheduleTimeout = 5 * time.Minute DefaultSchedulerIP = "127.0.0.1" DefaultSchedulerPort = 8002 DefaultPieceDispatcherRandomRatio = 0.1 DefaultObjectMaxReplicas = 3 )
Others.
const ( SimpleLocalTaskStoreStrategy = StoreStrategy("io.d7y.storage.v2.simple") AdvanceLocalTaskStoreStrategy = StoreStrategy("io.d7y.storage.v2.advance") )
Store strategy.
const ( CmdStat = "stat" CmdImport = "import" CmdExport = "export" CmdDelete = "delete" )
Dfcache subcommand names.
const ( DefaultEndPort = 65535 DefaultPeerStartPort = 65000 DefaultUploadStartPort = 65002 DefaultObjectStorageStartPort = 65004 DefaultHealthyStartPort = 40901 )
Service default port of listening.
const ( // DefaultLogRotateMaxSize is the default maximum size in megabytes of log files before rotation. DefaultLogRotateMaxSize = 1024 // DefaultLogRotateMaxAge is the default number of days to retain old log files. DefaultLogRotateMaxAge = 7 // DefaultLogRotateMaxBackups is the default number of old log files to keep. DefaultLogRotateMaxBackups = 20 )
const ( AttributePeerHost = attribute.Key("d7y.peer.host") AttributeTaskID = attribute.Key("d7y.peer.task.id") AttributeTaskContentLength = attribute.Key("d7y.peer.task.content_length") AttributePeerID = attribute.Key("d7y.peer.id") AttributeTargetPeerID = attribute.Key("d7y.peer.target.id") AttributeReusePeerID = attribute.Key("d7y.peer.reuse.id") AttributeReuseRange = attribute.Key("d7y.peer.reuse.range") AttributeTargetPeerAddr = attribute.Key("d7y.peer.target.addr") AttributeMainPeer = attribute.Key("d7y.peer.task.main_peer") AttributePeerPacketCode = attribute.Key("d7y.peer.packet.code") AttributePeerTaskSizeScope = attribute.Key("d7y.peer.size.scope") AttributePeerTaskSize = attribute.Key("d7y.peer.size") AttributePeerTaskSuccess = attribute.Key("d7y.peer.task.success") AttributePeerTaskCode = attribute.Key("d7y.peer.task.code") AttributePeerTaskMessage = attribute.Key("d7y.peer.task.message") AttributePeerTaskCost = attribute.Key("d7y.peer.task.cost") AttributePiece = attribute.Key("d7y.peer.piece") AttributePieceSize = attribute.Key("d7y.peer.piece.size") AttributePieceWorker = attribute.Key("d7y.peer.piece.worker") AttributePieceSuccess = attribute.Key("d7y.peer.piece.success") AttributeGetPieceStartNum = attribute.Key("d7y.peer.piece.start") AttributeGetPieceLimit = attribute.Key("d7y.peer.piece.limit") AttributeGetPieceCount = attribute.Key("d7y.peer.piece.count") AttributeGetPieceRetry = attribute.Key("d7y.peer.piece.retry") AttributeWritePieceSuccess = attribute.Key("d7y.peer.piece.write.success") AttributeSeedTaskSuccess = attribute.Key("d7y.seed.task.success") SpanFileTask = "file-task" SpanStreamTask = "stream-task" SpanSeedTask = "seed-task" SpanPeerTask = "peer-task" SpanDownload = "download" SpanRecursiveDownload = "recursive-download" SpanTransport = "transport" SpanReusePeerTask = "reuse-peer-task" SpanRegisterTask = "register" SpanReportPeerResult = "report-peer-result" SpanReportPieceResult = "report-piece-result" SpanBackSource = "client-back-source" SpanFirstSchedule = "schedule-#1" SpanGetPieceTasks = "get-piece-tasks" SpanSyncPieceTasks = "sync-piece-tasks" SpanDownloadPiece = "download-piece-#%d" SpanProxy = "proxy" SpanWritePiece = "write-piece" SpanWriteBackPiece = "write-back-piece" SpanWaitPieceLimit = "wait-limit" SpanPeerGC = "peer-gc" )
const ( // LocalSourceType represents read configuration from local file. LocalSourceType = "local" // ManagerSourceType represents pulling configuration from manager. ManagerSourceType = "manager" )
const ( HeaderDragonflyFilter = "X-Dragonfly-Filter" HeaderDragonflyPeer = "X-Dragonfly-Peer" HeaderDragonflyTask = "X-Dragonfly-Task" HeaderDragonflyRange = "X-Dragonfly-Range" // HeaderDragonflyTag different HeaderDragonflyTag for the same url will be divided into different P2P overlay HeaderDragonflyTag = "X-Dragonfly-Tag" // HeaderDragonflyApplication is used for statistics and traffic control HeaderDragonflyApplication = "X-Dragonfly-Application" // HeaderDragonflyPriority scheduler will schedule tasks according to priority HeaderDragonflyPriority = "X-Dragonfly-Priority" // HeaderDragonflyRegistry is used for dynamic registry mirrors. HeaderDragonflyRegistry = "X-Dragonfly-Registry" // HeaderDragonflyObjectMetaDigest is used for digest of object storage. HeaderDragonflyObjectMetaDigest = "X-Dragonfly-Object-Meta-Digest" // HeaderDragonflyObjectMetaLastModifiedTime is used for last modified time of object storage. HeaderDragonflyObjectMetaLastModifiedTime = "X-Dragonfly-Object-Meta-Last-Modified-Time" // HeaderDragonflyObjectMetaStorageClass is used for storage class of object storage. HeaderDragonflyObjectMetaStorageClass = "X-Dragonfly-Object-Meta-Storage-Class" // HeaderDragonflyObjectOperation is used for object storage operation. HeaderDragonflyObjectOperation = "X-Dragonfly-Object-Operation" )
const ( // DefaultProbeInterval is the default interval of probing host. DefaultProbeInterval = 20 * time.Minute )
Variables ¶
var ( // DefaultCertIPAddresses is default ip addresses of certificate. DefaultCertIPAddresses = []net.IP{ip.IPv4, ip.IPv6} // DefaultCertDNSNames is default dns names of certificate. DefaultCertDNSNames = []string{"dragonfly-peer", "dragonfly-peer.dragonfly-system.svc", "dragonfly-peer.dragonfly-system.svc.cluster.local", "dragonfly-seed-peer", "dragonfly-seed-peer.dragonfly-system.svc", "dragonfly-seed-peer.dragonfly-system.svc.cluster.local", "dragonfly-proxy", "dragonfly-proxy.dragonfly-system.svc", "dragonfly-proxy.dragonfly-system.svc.cluster.local", "dragonfly-dfdaemon", "dragonfly-dfdaemon.dragonfly-system.svc", "dragonfly-dfdaemon.dragonfly-system.svc.cluster.local", } // DefaultCertValidityPeriod is default validity period of certificate. DefaultCertValidityPeriod = 180 * 24 * time.Hour )
var ( // DefaultAnnouncerSchedulerInterval is default interface of announcing scheduler. DefaultAnnouncerSchedulerInterval = 30 * time.Second )
var DefaultSupernodesValue = &SupernodesValue{ Nodes: []string{ fmt.Sprintf("%s:%d", DefaultSchedulerIP, DefaultSchedulerPort), }, }
var (
ErrUnimplemented = errors.New("operation is not implemented")
)
Functions ¶
func ConvertCacheDelete ¶ added in v2.0.3
func ConvertCacheDelete(cfg *CacheOption, args []string) error
func ConvertCacheExport ¶ added in v2.0.3
func ConvertCacheExport(cfg *CacheOption, args []string) error
func ConvertCacheStat ¶ added in v2.0.3
func ConvertCacheStat(cfg *CacheOption, args []string) error
func GetCertificate ¶ added in v2.0.6
func GetCertificate(certifyClient *certify.Certify) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error)
func MkdirAll ¶
MkdirAll make directories recursive, and changes uid, gid to the latest directory. For example: the path /data/x exists, uid=1, gid=1 when call MkdirAll("/data/x/y/z", 0755, 2, 2) MkdirAll creates /data/x/y and change owner to 2:2, creates /data/x/y/z and change owner to 2:2
func ValidateCacheDelete ¶ added in v2.0.3
func ValidateCacheDelete(cfg *CacheOption) error
func ValidateCacheExport ¶ added in v2.0.3
func ValidateCacheExport(cfg *CacheOption) error
Types ¶
type AnnouncerOption ¶ added in v2.0.8
type CacheOption ¶ added in v2.0.3
type CacheOption struct { base.Options `yaml:",inline" mapstructure:",squash"` // Cid content/cache ID Cid string `yaml:"cid,omitempty" mapstructure:"cid,omitempty"` // Tag identify task Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"` // Timeout operation timeout(second). Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"` // LogDir is log directory of dfcache. LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"` // Maximum size in megabytes of log files before rotation (default: 1024) LogMaxSize int `yaml:"logMaxSize" mapstructure:"logMaxSize"` // Maximum number of days to retain old log files (default: 7) LogMaxAge int `yaml:"logMaxAge" mapstructure:"logMaxAge"` // Maximum number of old log files to keep (default: 20) LogMaxBackups int `yaml:"logMaxBackups" mapstructure:"logMaxBackups"` // WorkHome is working directory of dfcache. WorkHome string `yaml:"workHome,omitempty" mapstructure:"workHome,omitempty"` // DaemonSock is socket path of dfdaemon to connect. DaemonSock string `yaml:"daemonSocket,omitempty" mapstructure:"daemon-sock,omitempty"` // Output full output path for export task Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"` // Path full input path for import task // TODO: change to Input Path string `yaml:"path,omitempty" mapstructure:"path,omitempty"` // RateLimit limits export task RateLimit rate.Limit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"` // LocalOnly indicates check local cache only LocalOnly bool `yaml:"localOnly,omitempty" mapstructure:"localOnly,omitempty"` }
CacheOption holds all the runtime config information.
func NewDfcacheConfig ¶ added in v2.0.3
func NewDfcacheConfig() *CacheOption
func (*CacheOption) Convert ¶ added in v2.0.3
func (cfg *CacheOption) Convert(cmd string, args []string) error
func (*CacheOption) String ¶ added in v2.0.3
func (cfg *CacheOption) String() string
func (*CacheOption) Validate ¶ added in v2.0.3
func (cfg *CacheOption) Validate(cmd string) error
type CertPool ¶
CertPool is a wrapper around x509.CertPool, which can be unmarshalled and constructed from a list of filenames.
func (*CertPool) MarshalJSON ¶
MarshalJSON implements json.Marshaller to print the cert pool.
func (*CertPool) MarshalYAML ¶
MarshalYAML implements yaml.Marshaller to print the cert pool.
func (*CertPool) UnmarshalJSON ¶
UnmarshalJSON implements json.Unmarshaler.
type CertSpec ¶ added in v2.0.6
type CertSpec struct { // DNSNames is a list of dns names be set on the certificate. DNSNames []string `mapstructure:"dnsNames" yaml:"dnsNames"` // IPAddresses is a list of ip addresses be set on the certificate. IPAddresses []net.IP `mapstructure:"ipAddresses" yaml:"ipAddresses"` // ValidityPeriod is the validity period of certificate. ValidityPeriod time.Duration `mapstructure:"validityPeriod" yaml:"validityPeriod"` }
type ClientOption ¶
type ClientOption struct { base.Options `yaml:",inline" mapstructure:",squash"` // URL download URL. URL string `yaml:"url,omitempty" mapstructure:"url,omitempty"` // Output full output path. Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"` // Timeout download timeout(second). Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"` BenchmarkRate unit.Bytes `yaml:"benchmarkRate,omitempty" mapstructure:"benchmarkRate,omitempty"` // Md5 expected file md5. // Deprecated: Md5 is deprecated, use DigestMethod with DigestValue instead Md5 string `yaml:"md5,omitempty" mapstructure:"md5,omitempty"` Digest string `yaml:"digest,omitempty" mapstructure:"digest,omitempty"` // DigestMethod indicates digest method, like md5, sha256 DigestMethod string `yaml:"digestMethod,omitempty" mapstructure:"digestMethod,omitempty"` // DigestValue indicates digest value DigestValue string `yaml:"digestValue,omitempty" mapstructure:"digestValue,omitempty"` // Tag identify download task, it is available merely when md5 param not exist. Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"` // Application name that executes dfget. Application string `yaml:"application,omitempty" mapstructure:"application,omitempty"` // DaemonSock is daemon download socket path. DaemonSock string `yaml:"daemonSock,omitempty" mapstructure:"daemon-sock,omitempty"` // Priority scheduler will schedule tasks according to priority Priority int32 `yaml:"priority,omitempty" mapstructure:"priority,omitempty"` // CA certificate to verify when supernode interact with the source. Cacerts []string `yaml:"cacert,omitempty" mapstructure:"cacert,omitempty"` // Filter filter some query params of url, use char '&' to separate different params. // eg: -f 'key&sign' will filter 'key' and 'sign' query param. // in this way, different urls correspond one same download task that can use p2p mode. Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"` // Header of http request. // eg: --header='Accept: *' --header='Host: abc'. Header []string `yaml:"header,omitempty" mapstructure:"header,omitempty"` // DisableBackSource indicates whether to not back source to download when p2p fails. DisableBackSource bool `yaml:"disableBackSource,omitempty" mapstructure:"disable-back-source,omitempty"` // Insecure indicates whether skip secure verify when supernode interact with the source. Insecure bool `yaml:"insecure,omitempty" mapstructure:"insecure,omitempty"` // ShowProgress shows progress bar, it's conflict with `--console`. ShowProgress bool `yaml:"show-progress,omitempty" mapstructure:"show-progress,omitempty"` // LogDir is log directory of dfget. LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"` // Maximum size in megabytes of log files before rotation (default: 1024) LogMaxSize int `yaml:"logMaxSize" mapstructure:"logMaxSize"` // Maximum number of days to retain old log files (default: 7) LogMaxAge int `yaml:"logMaxAge" mapstructure:"logMaxAge"` // Maximum number of old log files to keep (default: 20) LogMaxBackups int `yaml:"logMaxBackups" mapstructure:"logMaxBackups"` // DataDir is data directory of dfget. DataDir string `yaml:"dataDir,omitempty" mapstructure:"dataDir,omitempty"` // CacheDir is cache directory of dfget. CacheDir string `yaml:"cacheDir,omitempty" mapstructure:"cacheDir,omitempty"` // WorkHome is working directory of dfget. WorkHome string `yaml:"workHome,omitempty" mapstructure:"workHome,omitempty"` RateLimit util.RateLimit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"` // MoreDaemonOptions indicates more options passed to daemon by command line. MoreDaemonOptions string `yaml:"moreDaemonOptions,omitempty" mapstructure:"moreDaemonOptions,omitempty"` // Recursive indicates to download all resources in target url, the target source client must support list action Recursive bool `yaml:"recursive,omitempty" mapstructure:"recursive,omitempty"` // RecursiveList indicates to list all resources in target url, the target source client must support list action RecursiveList bool `yaml:"recursiveList,omitempty" mapstructure:"list,omitempty"` // RecursiveLevel indicates to the maximum number of subdirectories that dfget will recurse into RecursiveLevel uint `yaml:"recursiveLevel,omitempty" mapstructure:"level,omitempty"` RecursiveAcceptRegex string `yaml:"acceptRegex,omitempty" mapstructure:"accept-regex,omitempty"` RecursiveRejectRegex string `yaml:"rejectRegex,omitempty" mapstructure:"reject-regex,omitempty"` KeepOriginalOffset bool `yaml:"keepOriginalOffset,omitempty" mapstructure:"original-offset,omitempty"` // Range stands download range for url, like: 0-9, will download 10 bytes from 0 to 9 ([0:9]) Range string `yaml:"range,omitempty" mapstructure:"range,omitempty"` }
ClientOption holds all the runtime config information.
func NewDfgetConfig ¶
func NewDfgetConfig() *ClientOption
func (*ClientOption) Convert ¶
func (cfg *ClientOption) Convert(args []string) error
func (*ClientOption) String ¶
func (cfg *ClientOption) String() string
func (*ClientOption) Validate ¶
func (cfg *ClientOption) Validate() error
type ConcurrentOption ¶ added in v2.0.5
type ConcurrentOption struct { // ThresholdSize indicates the threshold to download pieces concurrently ThresholdSize util.Size `mapstructure:"thresholdSize" yaml:"thresholdSize"` // ThresholdSpeed indicates the threshold download speed to download pieces concurrently ThresholdSpeed unit.Bytes `mapstructure:"thresholdSpeed" yaml:"thresholdSpeed"` // GoroutineCount indicates the concurrent goroutine count for every task GoroutineCount int `mapstructure:"goroutineCount" yaml:"goroutineCount"` // InitBackoff second for every piece failed, default: 0.5 InitBackoff float64 `mapstructure:"initBackoff" yaml:"initBackoff"` // MaxBackoff second for every piece failed, default: 3 MaxBackoff float64 `mapstructure:"maxBackoff" yaml:"maxBackoff"` // MaxAttempts for every piece failed,default: 3 MaxAttempts int `mapstructure:"maxAttempts" yaml:"maxAttempts"` }
type DaemonConfig ¶
type DaemonConfig = DaemonOption
type DaemonOption ¶
type DaemonOption struct { base.Options `yaml:",inline" mapstructure:",squash"` // AliveTime indicates alive duration for which daemon keeps no accessing by any uploading and download requests, // after this period daemon will automatically exit // when AliveTime == 0, will run infinitely AliveTime util.Duration `mapstructure:"aliveTime" yaml:"aliveTime"` GCInterval util.Duration `mapstructure:"gcInterval" yaml:"gcInterval"` Metrics string `mapstructure:"metrics" yaml:"metrics"` // MaxThreads indicates to invoke debug.SetMaxThreads to set max threads that golang runtime can create, // it's useful for large-scale downloading resource with none native golang source code, like call cgo to invoke c++ // one cgo call will create a standalone os tread, the default 10000 limit is to small. // Normally, we can also set seed concurrent to limit for large-scale downloading at same time. MaxThreads int `mapstructure:"maxThreads" yaml:"maxThreads"` WorkHome string `mapstructure:"workHome" yaml:"workHome"` WorkHomeMode uint32 `mapstructure:"workHomeMode" yaml:"workHomeMode"` CacheDir string `mapstructure:"cacheDir" yaml:"cacheDir"` CacheDirMode uint32 `mapstructure:"cacheDirMode" yaml:"cacheDirMode"` LogDir string `mapstructure:"logDir" yaml:"logDir"` LogMaxSize int `yaml:"logMaxSize" mapstructure:"logMaxSize"` LogMaxAge int `yaml:"logMaxAge" mapstructure:"logMaxAge"` LogMaxBackups int `yaml:"logMaxBackups" mapstructure:"logMaxBackups"` PluginDir string `mapstructure:"pluginDir" yaml:"pluginDir"` DataDir string `mapstructure:"dataDir" yaml:"dataDir"` DataDirMode uint32 `mapstructure:"dataDirMode" yaml:"dataDirMode"` KeepStorage bool `mapstructure:"keepStorage" yaml:"keepStorage"` Security GlobalSecurityOption `mapstructure:"security" yaml:"security"` Scheduler SchedulerOption `mapstructure:"scheduler" yaml:"scheduler"` Host HostOption `mapstructure:"host" yaml:"host"` Download DownloadOption `mapstructure:"download" yaml:"download"` Proxy *ProxyOption `mapstructure:"proxy" yaml:"proxy"` Upload UploadOption `mapstructure:"upload" yaml:"upload"` ObjectStorage ObjectStorageOption `mapstructure:"objectStorage" yaml:"objectStorage"` Storage StorageOption `mapstructure:"storage" yaml:"storage"` Health *HealthOption `mapstructure:"health" yaml:"health"` Reload ReloadOption `mapstructure:"reload" yaml:"reload"` Network *NetworkOption `mapstructure:"network" yaml:"network"` Announcer AnnouncerOption `mapstructure:"announcer" yaml:"announcer"` NetworkTopology NetworkTopologyOption `mapstructure:"networkTopology" yaml:"networkTopology"` PeerExchange PeerExchangeOption `mapstructure:"peerExchange" yaml:"peerExchange"` }
func NewDaemonConfig ¶
func NewDaemonConfig() *DaemonOption
func (*DaemonOption) Convert ¶
func (p *DaemonOption) Convert() error
func (*DaemonOption) IsSupportPeerExchange ¶ added in v2.1.40
func (p *DaemonOption) IsSupportPeerExchange() bool
func (*DaemonOption) Load ¶
func (p *DaemonOption) Load(path string) error
func (*DaemonOption) Validate ¶
func (p *DaemonOption) Validate() error
type DfcacheConfig ¶ added in v2.0.3
type DfcacheConfig = CacheOption
type DfgetConfig ¶
type DfgetConfig = ClientOption
type DfstoreConfig ¶ added in v2.0.5
type DfstoreConfig struct { // Address of the object storage service. Endpoint string `yaml:"endpoint,omitempty" mapstructure:"endpoint,omitempty"` // Filter is used to generate a unique Task ID by // filtering unnecessary query params in the URL, // it is separated by & character. Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"` // Mode is the mode in which the backend is written, // including WriteBack and AsyncWriteBack. Mode int `yaml:"mode,omitempty" mapstructure:"mode,omitempty"` // MaxReplicas is the maximum number of // replicas of an object cache in seed peers. MaxReplicas int `yaml:"maxReplicas,omitempty" mapstructure:"mode,maxReplicas"` }
func (*DfstoreConfig) Validate ¶ added in v2.0.5
func (cfg *DfstoreConfig) Validate() error
type DownloadOption ¶
type DownloadOption struct { TotalRateLimit util.RateLimit `mapstructure:"totalRateLimit" yaml:"totalRateLimit"` PerPeerRateLimit util.RateLimit `mapstructure:"perPeerRateLimit" yaml:"perPeerRateLimit"` TrafficShaperType string `mapstructure:"trafficShaperType" yaml:"trafficShaperType"` PieceDownloadTimeout time.Duration `mapstructure:"pieceDownloadTimeout" yaml:"pieceDownloadTimeout"` GRPCDialTimeout time.Duration `mapstructure:"grpcDialTimeout" yaml:"grpcDialTimeout"` DownloadGRPC ListenOption `mapstructure:"downloadGRPC" yaml:"downloadGRPC"` PeerGRPC ListenOption `mapstructure:"peerGRPC" yaml:"peerGRPC"` CalculateDigest bool `mapstructure:"calculateDigest" yaml:"calculateDigest"` Transport *TransportOption `mapstructure:"transportOption" yaml:"transportOption"` GetPiecesMaxRetry int `mapstructure:"getPiecesMaxRetry" yaml:"getPiecesMaxRetry"` Prefetch bool `mapstructure:"prefetch" yaml:"prefetch"` WatchdogTimeout time.Duration `mapstructure:"watchdogTimeout" yaml:"watchdogTimeout"` Concurrent *ConcurrentOption `mapstructure:"concurrent" yaml:"concurrent"` SyncPieceViaHTTPS bool `mapstructure:"syncPieceViaHTTPS" yaml:"syncPieceViaHTTPS"` SplitRunningTasks bool `mapstructure:"splitRunningTasks" yaml:"splitRunningTasks"` // resource clients option ResourceClients ResourceClientsOption `mapstructure:"resourceClients" yaml:"resourceClients"` RecursiveConcurrent RecursiveConcurrent `mapstructure:"recursiveConcurrent" yaml:"recursiveConcurrent"` SeedConcurrent int64 `mapstructure:"seedConcurrent" yaml:"seedConcurrent"` CacheRecursiveMetadata time.Duration `mapstructure:"cacheRecursiveMetadata" yaml:"cacheRecursiveMetadata"` }
type DurationValue ¶
DurationValue supports time.Duration format like 30s, 1m30s, 1h and also treat integer as seconds
func NewDurationValue ¶
func NewDurationValue(p *time.Duration) *DurationValue
func (*DurationValue) Set ¶
func (d *DurationValue) Set(s string) error
func (*DurationValue) String ¶
func (d *DurationValue) String() string
func (*DurationValue) Type ¶
func (d *DurationValue) Type() string
type Dynconfig ¶ added in v2.0.1
type Dynconfig interface { // Get the dynamic seed peers config. GetSeedPeers() ([]*managerv1.SeedPeer, error) // Get the dynamic schedulers resolve addrs. GetResolveSchedulerAddrs() ([]resolver.Address, error) // Get the dynamic schedulers config. GetSchedulers() ([]*managerv1.Scheduler, error) // Get the dynamic schedulers cluster id. GetSchedulerClusterID() uint64 // Get the dynamic object storage config. GetObjectStorage() (*managerv1.ObjectStorage, error) // Get the dynamic config. Get() (*DynconfigData, error) // Refresh refreshes dynconfig in cache. Refresh() error // Register allows an instance to register itself to listen/observe events. Register(Observer) // Deregister allows an instance to remove itself from the collection of observers/listeners. Deregister(Observer) // Notify publishes new events to listeners. Notify() error // OnNotify allows an event to be published to the dynconfig. // Used for listening changes of the local configuration. OnNotify(*DaemonOption) // Serve the dynconfig listening service. Serve() error // Stop the dynconfig listening service. Stop() error }
func NewDynconfig ¶ added in v2.0.1
func NewDynconfig(sourceType SourceType, cfg *DaemonOption, options ...DynconfigOption) (Dynconfig, error)
New returns a new dynconfig interface.
type DynconfigData ¶ added in v2.0.1
type DynconfigOption ¶ added in v2.0.9
type DynconfigOption func(d *dynconfig) error
DynconfigOption is a functional option for configuring the dynconfig.
func WithCacheDir ¶ added in v2.0.6
func WithCacheDir(dir string) DynconfigOption
WithCacheDir set the cache dir.
func WithManagerClient ¶ added in v2.0.6
func WithManagerClient(c managerclient.V1) DynconfigOption
WithManagerClient set the manager client.
func WithTransportCredentials ¶ added in v2.0.9
func WithTransportCredentials(creds credentials.TransportCredentials) DynconfigOption
WithTransportCredentials returns a DialOption which configures a connection level security credentials (e.g., TLS/SSL).
type GlobalSecurityOption ¶ added in v2.0.6
type GlobalSecurityOption struct { // AutoIssueCert indicates to issue client certificates for all grpc call // if AutoIssueCert is false, any other option in Security will be ignored AutoIssueCert bool `mapstructure:"autoIssueCert" yaml:"autoIssueCert"` // CACert is the root CA certificate for all grpc tls handshake, it can be path or PEM format string CACert types.PEMContent `mapstructure:"caCert" yaml:"caCert"` // TLSVerify indicates to verify client certificates. TLSVerify bool `mapstructure:"tlsVerify" yaml:"tlsVerify"` // TLSPolicy controls the grpc shandshake behaviors: // force: both ClientHandshake and ServerHandshake are only support tls // prefer: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support tls // default: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support insecure (non-tls) TLSPolicy string `mapstructure:"tlsPolicy" yaml:"tlsPolicy"` // CertSpec is the desired state of certificate. CertSpec *CertSpec `mapstructure:"certSpec" yaml:"certSpec"` }
type HealthOption ¶ added in v2.0.2
type HealthOption struct { ListenOption `yaml:",inline" mapstructure:",squash"` Path string `mapstructure:"path" yaml:"path"` }
type HijackConfig ¶
type HijackConfig struct { Cert types.PEMContent `yaml:"cert" mapstructure:"cert"` Key types.PEMContent `yaml:"key" mapstructure:"key"` Hosts []*HijackHost `yaml:"hosts" mapstructure:"hosts"` SNI []*TCPListenOption `yaml:"sni" mapstructure:"sni"` }
HijackConfig represents how dfdaemon hijacks http requests.
type HijackHost ¶
type HijackHost struct { Regx *Regexp `yaml:"regx" mapstructure:"regx"` Insecure bool `yaml:"insecure" mapstructure:"insecure"` Certs *CertPool `yaml:"certs" mapstructure:"certs"` }
HijackHost is a hijack rule for the hosts that matches Regx.
type HostOption ¶
type HostOption struct { // IDC for scheduler IDC string `mapstructure:"idc" yaml:"idc"` // Location for scheduler Location string `mapstructure:"location" yaml:"location"` // Hostname is daemon host name Hostname string `mapstructure:"hostname" yaml:"hostname"` // The ip report to scheduler, normal same with listen ip AdvertiseIP net.IP `mapstructure:"advertiseIP" yaml:"advertiseIP"` }
type KeepAliveOption ¶ added in v2.0.3
type ListenOption ¶
type ListenOption struct { Security SecurityOption `mapstructure:"security" yaml:"security"` TCPListen *TCPListenOption `mapstructure:"tcpListen,omitempty" yaml:"tcpListen,omitempty"` UnixListen *UnixListenOption `mapstructure:"unixListen,omitempty" yaml:"unixListen,omitempty"` }
type ManagerOption ¶ added in v2.0.1
type ManagerOption struct { // Enable get configuration from manager. Enable bool `mapstructure:"enable" yaml:"enable"` // NetAddrs is manager addresses. NetAddrs []dfnet.NetAddr `mapstructure:"netAddrs" yaml:"netAddrs"` // RefreshInterval is the refresh interval. RefreshInterval time.Duration `mapstructure:"refreshInterval" yaml:"refreshInterval"` // SeedPeer configuration. SeedPeer SeedPeerOption `mapstructure:"seedPeer" yaml:"seedPeer"` }
type NetAddrsValue ¶
type NetAddrsValue struct {
// contains filtered or unexported fields
}
func NewNetAddrsValue ¶
func NewNetAddrsValue(n *[]dfnet.NetAddr) *NetAddrsValue
func (*NetAddrsValue) Set ¶
func (nv *NetAddrsValue) Set(value string) error
func (*NetAddrsValue) String ¶
func (nv *NetAddrsValue) String() string
func (*NetAddrsValue) Type ¶
func (nv *NetAddrsValue) Type() string
type NetworkOption ¶ added in v2.0.7
type NetworkOption struct { // EnableIPv6 enables ipv6 for server. EnableIPv6 bool `mapstructure:"enableIPv6" yaml:"enableIPv6"` }
type NetworkTopologyOption ¶ added in v2.0.30
type NetworkTopologyOption struct { // Enable network topology service. Enable bool `mapstructure:"enable" yaml:"enable"` // Probe is the configuration of probe. Probe ProbeOption `mapstructure:"probe" yaml:"probe"` }
type ObjectStorageOption ¶ added in v2.0.4
type ObjectStorageOption struct { // Enable object storage. Enable bool `mapstructure:"enable" yaml:"enable"` // Filter is used to generate a unique Task ID by // filtering unnecessary query params in the URL, // it is separated by & character. Filter string `mapstructure:"filter" yaml:"filter"` // MaxReplicas is the maximum number of replicas of an object cache in seed peers. MaxReplicas int `mapstructure:"maxReplicas" yaml:"maxReplicas"` // ListenOption is object storage service listener. ListenOption `yaml:",inline" mapstructure:",squash"` }
type Observer ¶ added in v2.0.1
type Observer interface { // OnNotify allows an event to be published to interface implementations. OnNotify(*DynconfigData) }
type PeerExchangeOption ¶ added in v2.1.40
type PeerExchangeOption struct { // Enable peer exchange service. Enable bool `mapstructure:"enable" yaml:"enable"` // InitialInterval is the initial retry interval when start and join gossip. InitialInterval time.Duration `mapstructure:"initialInterval" yaml:"initialInterval"` // InitialBroadcastDelay is the initial broadcast delay when daemon start due to we can only broadcast peers after join gossip with all daemons, it should be less than TaskExpireTime InitialBroadcastDelay time.Duration `mapstructure:"initialBroadcastDelay" yaml:"initialBroadcastDelay"` // ReSyncInterval is the re-sync interval for check running gossip members. ReSyncInterval time.Duration `mapstructure:"reSyncInterval" yaml:"reSyncInterval"` // ReplicaThreshold is used for keeping replicas in all peers is not bigger than threshold to save storage ReplicaThreshold int `mapstructure:"replicaThreshold" yaml:"replicaThreshold"` // ReplicaCleanPercentage is percentage probability to clean local replica when reach threshold, available values: [0, 100] ReplicaCleanPercentage int32 `mapstructure:"replicaCleanPercentage" yaml:"replicaCleanPercentage"` }
type ProbeOption ¶ added in v2.0.30
type ProxyOption ¶
type ProxyOption struct { // WARNING: when add more option, please update ProxyOption.unmarshal function ListenOption `mapstructure:",squash" yaml:",inline"` BasicAuth *BasicAuth `mapstructure:"basicAuth" yaml:"basicAuth"` DefaultFilter string `mapstructure:"defaultFilter" yaml:"defaultFilter"` DefaultTag string `mapstructure:"defaultTag" yaml:"defaultTag"` DefaultApplication string `mapstructure:"defaultApplication" yaml:"defaultApplication"` DefaultPriority commonv1.Priority `mapstructure:"defaultPriority" yaml:"defaultPriority"` MaxConcurrency int64 `mapstructure:"maxConcurrency" yaml:"maxConcurrency"` RegistryMirror *RegistryMirror `mapstructure:"registryMirror" yaml:"registryMirror"` WhiteList []*WhiteList `mapstructure:"whiteList" yaml:"whiteList"` ProxyRules []*ProxyRule `mapstructure:"proxies" yaml:"proxies"` HijackHTTPS *HijackConfig `mapstructure:"hijackHTTPS" yaml:"hijackHTTPS"` DumpHTTPContent bool `mapstructure:"dumpHTTPContent" yaml:"dumpHTTPContent"` // ExtraRegistryMirrors add more mirrors for different ports ExtraRegistryMirrors []*RegistryMirror `mapstructure:"extraRegistryMirrors" yaml:"extraRegistryMirrors"` }
func (*ProxyOption) UnmarshalJSON ¶
func (p *ProxyOption) UnmarshalJSON(b []byte) error
func (*ProxyOption) UnmarshalYAML ¶
func (p *ProxyOption) UnmarshalYAML(node *yaml.Node) error
type ProxyRule ¶ added in v2.0.2
type ProxyRule struct { Regx *Regexp `yaml:"regx" mapstructure:"regx"` UseHTTPS bool `yaml:"useHTTPS" mapstructure:"useHTTPS"` Direct bool `yaml:"direct" mapstructure:"direct"` // Redirect is the host to redirect to, if not empty Redirect string `yaml:"redirect" mapstructure:"redirect"` }
ProxyRule describes a regular expression matching rule for how to proxy a request.
func NewProxyRule ¶ added in v2.0.2
type RateLimitValue ¶
type RateLimitValue struct {
// contains filtered or unexported fields
}
func NewLimitRateValue ¶
func NewLimitRateValue(rate *util.RateLimit) *RateLimitValue
func (*RateLimitValue) Set ¶
func (r *RateLimitValue) Set(s string) error
func (*RateLimitValue) String ¶
func (r *RateLimitValue) String() string
func (*RateLimitValue) Type ¶
func (r *RateLimitValue) Type() string
type RecursiveConcurrent ¶ added in v2.0.7
type RecursiveConcurrent struct { // GoroutineCount indicates the concurrent goroutine count for every recursive task GoroutineCount int `mapstructure:"goroutineCount" yaml:"goroutineCount"` }
type Regexp ¶
Regexp is a simple wrapper around regexp. Regexp to make it unmarshallable from a string.
func (*Regexp) MarshalJSON ¶
MarshalJSON implements json.Marshaller to print the regexp.
func (*Regexp) MarshalYAML ¶
MarshalYAML implements yaml.Marshaller to print the regexp.
func (*Regexp) UnmarshalJSON ¶
UnmarshalJSON implements json.Unmarshaler.
type RegistryMirror ¶
type RegistryMirror struct { // Remote url for the registry mirror, default is https://index.docker.io Remote *URL `yaml:"url" mapstructure:"url"` // DynamicRemote indicates using header "X-Dragonfly-Registry" for remote instead of Remote // if header "X-Dragonfly-Registry" does not exist, use Remote by default DynamicRemote bool `yaml:"dynamic" mapstructure:"dynamic"` // Optional certificates if the mirror uses self-signed certificates Certs *CertPool `yaml:"certs" mapstructure:"certs"` // Whether to ignore certificates errors for the registry Insecure bool `yaml:"insecure" mapstructure:"insecure"` // Request the remote registry directly. Direct bool `yaml:"direct" mapstructure:"direct"` // Whether to use proxies to decide when to use dragonfly UseProxies bool `yaml:"useProxies" mapstructure:"useProxies"` }
RegistryMirror configures the mirror of the official docker registry
func (*RegistryMirror) TLSConfig ¶
func (r *RegistryMirror) TLSConfig() *tls.Config
TLSConfig returns the tls.Config used to communicate with the mirror.
type ReloadOption ¶ added in v2.0.2
type ResourceClientsOption ¶ added in v2.0.9
type SchedulerOption ¶
type SchedulerOption struct { // Manager is to get the scheduler configuration remotely. Manager ManagerOption `mapstructure:"manager" yaml:"manager"` // NetAddrs is scheduler addresses. NetAddrs []dfnet.NetAddr `mapstructure:"netAddrs" yaml:"netAddrs"` // ScheduleTimeout is request timeout. ScheduleTimeout util.Duration `mapstructure:"scheduleTimeout" yaml:"scheduleTimeout"` // DisableAutoBackSource indicates not back source normally, only scheduler says back source. DisableAutoBackSource bool `mapstructure:"disableAutoBackSource" yaml:"disableAutoBackSource"` }
type SecurityOption ¶
type SecurityOption struct { // Insecure indicate enable tls or not Insecure bool `mapstructure:"insecure" yaml:"insecure"` CACert types.PEMContent `mapstructure:"caCert" yaml:"caCert"` Cert types.PEMContent `mapstructure:"cert" yaml:"cert"` Key types.PEMContent `mapstructure:"key" yaml:"key"` TLSVerify bool `mapstructure:"tlsVerify" yaml:"tlsVerify"` TLSConfig *tls.Config `mapstructure:"tlsConfig" yaml:"tlsConfig"` }
type SeedPeerOption ¶ added in v2.0.3
type SeedPeerOption struct { // Enable seed peer mode. Enable bool `mapstructure:"enable" yaml:"enable"` // Type is seed peer type. Type string `mapstructure:"type" yaml:"type"` // ClusterID is seed peer cluster id. ClusterID uint `mapstructure:"clusterID" yaml:"clusterID"` // KeepAlive configuration. KeepAlive KeepAliveOption `yaml:"keepAlive" mapstructure:"keepAlive"` }
type SourceType ¶ added in v2.0.6
type SourceType string
type StorageOption ¶
type StorageOption struct { // DataPath indicates directory which stores temporary files for p2p uploading DataPath string `mapstructure:"dataPath" yaml:"dataPath"` // TaskExpireTime indicates caching duration for which cached file keeps no accessed by any process, // after this period cache file will be gc TaskExpireTime util.Duration `mapstructure:"taskExpireTime" yaml:"taskExpireTime"` // DiskGCThreshold indicates the threshold to gc the oldest tasks DiskGCThreshold unit.Bytes `mapstructure:"diskGCThreshold" yaml:"diskGCThreshold"` // DiskGCThresholdPercent indicates the threshold to gc the oldest tasks according the disk usage // Eg, DiskGCThresholdPercent=80, when the disk usage is above 80%, start to gc the oldest tasks DiskGCThresholdPercent float64 `mapstructure:"diskGCThresholdPercent" yaml:"diskGCThresholdPercent"` // Multiplex indicates reusing underlying storage for same task id Multiplex bool `mapstructure:"multiplex" yaml:"multiplex"` StoreStrategy StoreStrategy `mapstructure:"strategy" yaml:"strategy"` // WriteBufferSize indicates the buffer size when read from source, same usage with io.Copy // for some resource plugins, bigger buffer size with better performance, on the other hand, bigger buffer size cost huge memory WriteBufferSize unit.Bytes `mapstructure:"writeBufferSize" yaml:"writeBufferSize"` // ReloadGoroutineCount indicates concurrent goroutine count when daemon load cache data ReloadGoroutineCount int `mapstructure:"reloadGoroutineCount" yaml:"reloadGoroutineCount"` }
type StoreStrategy ¶
type StoreStrategy string
type SupernodesValue ¶
type SupernodesValue struct {
Nodes []string
}
func (*SupernodesValue) Set ¶
func (sv *SupernodesValue) Set(value string) error
Set implements the pflag.Value interface.
func (*SupernodesValue) String ¶
func (sv *SupernodesValue) String() string
String implements the pflag.Value interface.
func (*SupernodesValue) Type ¶
func (sv *SupernodesValue) Type() string
Type implements the pflag.Value interface.
type TCPListenOption ¶
type TCPListenOption struct { // Listen stands listen interface, like: 0.0.0.0, 192.168.0.1 Listen string `mapstructure:"listen" yaml:"listen"` // PortRange stands listen port // yaml example 1: // port: 12345 // yaml example 2: // port: // start: 12345 // end: 12346 PortRange TCPListenPortRange `mapstructure:"port" yaml:"port"` // Namespace stands the linux net namespace, like /proc/1/ns/net // It's useful for running daemon in pod with ip allocated and listen in host Namespace string `mapstructure:"namespace" yaml:"namespace"` }
type TCPListenPortRange ¶
func (*TCPListenPortRange) UnmarshalJSON ¶
func (t *TCPListenPortRange) UnmarshalJSON(b []byte) error
func (*TCPListenPortRange) UnmarshalYAML ¶
func (t *TCPListenPortRange) UnmarshalYAML(node *yaml.Node) error
type TLSConfig ¶
func (*TLSConfig) UnmarshalJSON ¶
func (*TLSConfig) UnmarshalYAML ¶ added in v2.0.6
type TelemetryOption ¶
type TelemetryOption struct {
Jaeger string `yaml:"jaeger" mapstructure:"jaeger"`
}
TelemetryOption is the option for telemetry
type TransportOption ¶ added in v2.0.1
type TransportOption struct { DialTimeout time.Duration `mapstructure:"dialTimeout" yaml:"dialTimeout"` KeepAlive time.Duration `mapstructure:"keepAlive" yaml:"keepAlive"` MaxIdleConns int `mapstructure:"maxIdleConns" yaml:"maxIdleConns"` IdleConnTimeout time.Duration `mapstructure:"idleConnTimeout" yaml:"idleConnTimeout"` ResponseHeaderTimeout time.Duration `mapstructure:"responseHeaderTimeout" yaml:"responseHeaderTimeout"` TLSHandshakeTimeout time.Duration `mapstructure:"tlsHandshakeTimeout" yaml:"tlsHandshakeTimeout"` ExpectContinueTimeout time.Duration `mapstructure:"expectContinueTimeout" yaml:"expectContinueTimeout"` }
type URL ¶
URL is simple wrapper around url.URL to make it unmarshallable from a string.
func (*URL) MarshalJSON ¶
MarshalJSON implements json.Marshaller to print the url.
func (*URL) MarshalYAML ¶
MarshalYAML implements yaml.Marshaller to print the url.
func (*URL) UnmarshalJSON ¶
UnmarshalJSON implements json.Unmarshaler.
type UnixListenOption ¶
type UnixListenOption struct {
Socket string `mapstructure:"socket" yaml:"socket"`
}
type UploadOption ¶
type UploadOption struct { ListenOption `yaml:",inline" mapstructure:",squash"` RateLimit util.RateLimit `mapstructure:"rateLimit" yaml:"rateLimit"` }