config

package
v2.1.67 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 23, 2024 License: Apache-2.0 Imports: 46 Imported by: 0

Documentation

Overview

Package config holds all DaemonConfig of dfget.

Package config holds all options of peerhost.

Index

Constants

View Source 
const (
	DefaultPerPeerDownloadLimit = 512 * unit.MB
	DefaultTotalDownloadLimit   = 1024 * unit.MB
	DefaultUploadLimit          = 1024 * unit.MB
	DefaultMinRate              = 20 * unit.MB
)

Download limit.

View Source 
const (
	DefaultTaskExpireTime  = 6 * time.Hour
	DefaultGCInterval      = 1 * time.Minute
	DefaultDaemonAliveTime = 5 * time.Minute
	DefaultScheduleTimeout = 5 * time.Minute

	DefaultSchedulerIP   = "127.0.0.1"
	DefaultSchedulerPort = 8002

	DefaultPieceDispatcherRandomRatio = 0.1
	DefaultObjectMaxReplicas          = 3
)

Others.

View Source 
const (
	SimpleLocalTaskStoreStrategy  = StoreStrategy("io.d7y.storage.v2.simple")
	AdvanceLocalTaskStoreStrategy = StoreStrategy("io.d7y.storage.v2.advance")
)

Store strategy.

View Source 
const (
	CmdStat   = "stat"
	CmdImport = "import"
	CmdExport = "export"
	CmdDelete = "delete"
)

Dfcache subcommand names.

View Source 
const (
	DefaultEndPort                = 65535
	DefaultPeerStartPort          = 65000
	DefaultUploadStartPort        = 65002
	DefaultObjectStorageStartPort = 65004
	DefaultHealthyStartPort       = 40901
)

Service default port of listening.

View Source 
const (
	// DefaultLogRotateMaxSize is the default maximum size in megabytes of log files before rotation.
	DefaultLogRotateMaxSize = 1024

	// DefaultLogRotateMaxAge is the default number of days to retain old log files.
	DefaultLogRotateMaxAge = 7

	// DefaultLogRotateMaxBackups is the default number of old log files to keep.
	DefaultLogRotateMaxBackups = 20
)
View Source 
const (
	AttributePeerHost          = attribute.Key("d7y.peer.host")
	AttributeTaskID            = attribute.Key("d7y.peer.task.id")
	AttributeTaskContentLength = attribute.Key("d7y.peer.task.content_length")
	AttributePeerID            = attribute.Key("d7y.peer.id")
	AttributeTargetPeerID      = attribute.Key("d7y.peer.target.id")
	AttributeReusePeerID       = attribute.Key("d7y.peer.reuse.id")
	AttributeReuseRange        = attribute.Key("d7y.peer.reuse.range")
	AttributeTargetPeerAddr    = attribute.Key("d7y.peer.target.addr")
	AttributeMainPeer          = attribute.Key("d7y.peer.task.main_peer")
	AttributePeerPacketCode    = attribute.Key("d7y.peer.packet.code")
	AttributePeerTaskSizeScope = attribute.Key("d7y.peer.size.scope")
	AttributePeerTaskSize      = attribute.Key("d7y.peer.size")
	AttributePeerTaskSuccess   = attribute.Key("d7y.peer.task.success")
	AttributePeerTaskCode      = attribute.Key("d7y.peer.task.code")
	AttributePeerTaskMessage   = attribute.Key("d7y.peer.task.message")
	AttributePeerTaskCost      = attribute.Key("d7y.peer.task.cost")
	AttributePiece             = attribute.Key("d7y.peer.piece")
	AttributePieceSize         = attribute.Key("d7y.peer.piece.size")
	AttributePieceWorker       = attribute.Key("d7y.peer.piece.worker")
	AttributePieceSuccess      = attribute.Key("d7y.peer.piece.success")
	AttributeGetPieceStartNum  = attribute.Key("d7y.peer.piece.start")
	AttributeGetPieceLimit     = attribute.Key("d7y.peer.piece.limit")
	AttributeGetPieceCount     = attribute.Key("d7y.peer.piece.count")
	AttributeGetPieceRetry     = attribute.Key("d7y.peer.piece.retry")
	AttributeWritePieceSuccess = attribute.Key("d7y.peer.piece.write.success")
	AttributeSeedTaskSuccess   = attribute.Key("d7y.seed.task.success")

	SpanFileTask          = "file-task"
	SpanStreamTask        = "stream-task"
	SpanSeedTask          = "seed-task"
	SpanPeerTask          = "peer-task"
	SpanDownload          = "download"
	SpanRecursiveDownload = "recursive-download"
	SpanTransport         = "transport"
	SpanReusePeerTask     = "reuse-peer-task"
	SpanRegisterTask      = "register"
	SpanReportPeerResult  = "report-peer-result"
	SpanReportPieceResult = "report-piece-result"
	SpanBackSource        = "client-back-source"
	SpanFirstSchedule     = "schedule-#1"
	SpanGetPieceTasks     = "get-piece-tasks"
	SpanSyncPieceTasks    = "sync-piece-tasks"
	SpanDownloadPiece     = "download-piece-#%d"
	SpanProxy             = "proxy"
	SpanWritePiece        = "write-piece"
	SpanWriteBackPiece    = "write-back-piece"
	SpanWaitPieceLimit    = "wait-limit"
	SpanPeerGC            = "peer-gc"
)
View Source 
const (
	// LocalSourceType represents read configuration from local file.
	LocalSourceType = "local"

	// ManagerSourceType represents pulling configuration from manager.
	ManagerSourceType = "manager"
)
View Source 
const (
	HeaderDragonflyFilter = "X-Dragonfly-Filter"
	HeaderDragonflyPeer   = "X-Dragonfly-Peer"
	HeaderDragonflyTask   = "X-Dragonfly-Task"
	HeaderDragonflyRange  = "X-Dragonfly-Range"
	// HeaderDragonflyTag different HeaderDragonflyTag for the same url will be divided into different P2P overlay
	HeaderDragonflyTag = "X-Dragonfly-Tag"
	// HeaderDragonflyApplication is used for statistics and traffic control
	HeaderDragonflyApplication = "X-Dragonfly-Application"
	// HeaderDragonflyPriority scheduler will schedule tasks according to priority
	HeaderDragonflyPriority = "X-Dragonfly-Priority"
	// HeaderDragonflyRegistry is used for dynamic registry mirrors.
	HeaderDragonflyRegistry = "X-Dragonfly-Registry"
	// HeaderDragonflyObjectMetaDigest is used for digest of object storage.
	HeaderDragonflyObjectMetaDigest = "X-Dragonfly-Object-Meta-Digest"
	// HeaderDragonflyObjectMetaLastModifiedTime is used for last modified time of object storage.
	HeaderDragonflyObjectMetaLastModifiedTime = "X-Dragonfly-Object-Meta-Last-Modified-Time"
	// HeaderDragonflyObjectMetaStorageClass is used for storage class of object storage.
	HeaderDragonflyObjectMetaStorageClass = "X-Dragonfly-Object-Meta-Storage-Class"
	// HeaderDragonflyObjectOperation is used for object storage operation.
	HeaderDragonflyObjectOperation = "X-Dragonfly-Object-Operation"
)

Variables

View Source 
var (
	// DefaultCertIPAddresses is default ip addresses of certificate.
	DefaultCertIPAddresses = []net.IP{ip.IPv4, ip.IPv6}

	// DefaultCertDNSNames is default dns names of certificate.
	DefaultCertDNSNames = []string{"dragonfly-peer", "dragonfly-peer.dragonfly-system.svc", "dragonfly-peer.dragonfly-system.svc.cluster.local",
		"dragonfly-seed-peer", "dragonfly-seed-peer.dragonfly-system.svc", "dragonfly-seed-peer.dragonfly-system.svc.cluster.local",
		"dragonfly-proxy", "dragonfly-proxy.dragonfly-system.svc", "dragonfly-proxy.dragonfly-system.svc.cluster.local",
		"dragonfly-dfdaemon", "dragonfly-dfdaemon.dragonfly-system.svc", "dragonfly-dfdaemon.dragonfly-system.svc.cluster.local",
	}

	// DefaultCertValidityPeriod is default validity period of certificate.
	DefaultCertValidityPeriod = 180 * 24 * time.Hour
)
View Source 
var (
	// DefaultAnnouncerSchedulerInterval is default interface of announcing scheduler.
	DefaultAnnouncerSchedulerInterval = 30 * time.Second
)
View Source 
var DefaultSupernodesValue = &SupernodesValue{
	Nodes: []string{
		fmt.Sprintf("%s:%d", DefaultSchedulerIP, DefaultSchedulerPort),
	},
}
View Source 
var (
	ErrUnimplemented = errors.New("operation is not implemented")
)

Functions

func ConvertCacheDelete added in v2.0.3 

func ConvertCacheDelete(cfg *CacheOption, args []string) error

func ConvertCacheExport added in v2.0.3 

func ConvertCacheExport(cfg *CacheOption, args []string) error

func ConvertCacheStat added in v2.0.3 

func ConvertCacheStat(cfg *CacheOption, args []string) error

func GetCertificate added in v2.0.6 

func GetCertificate(certifyClient *certify.Certify) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error)

func MkdirAll 

func MkdirAll(dir string, perm os.FileMode, uid, gid int) error

MkdirAll make directories recursive, and changes uid, gid to the latest directory. For example: the path /data/x exists, uid=1, gid=1 when call MkdirAll("/data/x/y/z", 0755, 2, 2) MkdirAll creates /data/x/y and change owner to 2:2, creates /data/x/y/z and change owner to 2:2

func ValidateCacheDelete added in v2.0.3 

func ValidateCacheDelete(cfg *CacheOption) error

func ValidateCacheExport added in v2.0.3 

func ValidateCacheExport(cfg *CacheOption) error

Types

type AnnouncerOption added in v2.0.8 

type AnnouncerOption struct {
	// SchedulerInterval is the interval of announcing scheduler.
	SchedulerInterval time.Duration `mapstructure:"schedulerInterval" yaml:"schedulerInterval"`
}

type BasicAuth 

type BasicAuth struct {
	Username string `json:"username" yaml:"username"`
	Password string `json:"password" yaml:"password"`
}

type CacheOption added in v2.0.3 

type CacheOption struct {
	base.Options `yaml:",inline" mapstructure:",squash"`

	// Cid content/cache ID
	Cid string `yaml:"cid,omitempty" mapstructure:"cid,omitempty"`

	// Tag identify task
	Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"`

	// Timeout operation timeout(second).
	Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"`

	// LogDir is log directory of dfcache.
	LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"`

	// Maximum size in megabytes of log files before rotation (default: 1024)
	LogMaxSize int `yaml:"logMaxSize" mapstructure:"logMaxSize"`

	// Maximum number of days to retain old log files (default: 7)
	LogMaxAge int `yaml:"logMaxAge" mapstructure:"logMaxAge"`

	// Maximum number of old log files to keep (default: 20)
	LogMaxBackups int `yaml:"logMaxBackups" mapstructure:"logMaxBackups"`

	// WorkHome is working directory of dfcache.
	WorkHome string `yaml:"workHome,omitempty" mapstructure:"workHome,omitempty"`

	// DaemonSock is socket path of dfdaemon to connect.
	DaemonSock string `yaml:"daemonSocket,omitempty" mapstructure:"daemon-sock,omitempty"`

	// Output full output path for export task
	Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"`

	// Path full input path for import task
	// TODO: change to Input
	Path string `yaml:"path,omitempty" mapstructure:"path,omitempty"`

	// RateLimit limits export task
	RateLimit rate.Limit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"`

	// LocalOnly indicates check local cache only
	LocalOnly bool `yaml:"localOnly,omitempty" mapstructure:"localOnly,omitempty"`
}

CacheOption holds all the runtime config information.

func NewDfcacheConfig added in v2.0.3 

func NewDfcacheConfig() *CacheOption

func (*CacheOption) Convert added in v2.0.3 

func (cfg *CacheOption) Convert(cmd string, args []string) error

func (*CacheOption) String added in v2.0.3 

func (cfg *CacheOption) String() string

func (*CacheOption) Validate added in v2.0.3 

func (cfg *CacheOption) Validate(cmd string) error

type CertPool 

type CertPool struct {
	Files []string
	*x509.CertPool
}

CertPool is a wrapper around x509.CertPool, which can be unmarshalled and constructed from a list of filenames.

func (*CertPool) MarshalJSON 

func (cp *CertPool) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaller to print the cert pool.

func (*CertPool) MarshalYAML 

func (cp *CertPool) MarshalYAML() (any, error)

MarshalYAML implements yaml.Marshaller to print the cert pool.

func (*CertPool) UnmarshalJSON 

func (cp *CertPool) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler.

func (*CertPool) UnmarshalYAML 

func (cp *CertPool) UnmarshalYAML(unmarshal func(any) error) error

UnmarshalYAML implements yaml.Unmarshaler.

type ClientOption 

type ClientOption struct {
	base.Options `yaml:",inline" mapstructure:",squash"`
	// URL download URL.
	URL string `yaml:"url,omitempty" mapstructure:"url,omitempty"`

	// Output full output path.
	Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"`

	// Timeout download timeout(second).
	Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"`

	BenchmarkRate unit.Bytes `yaml:"benchmarkRate,omitempty" mapstructure:"benchmarkRate,omitempty"`

	// Md5 expected file md5.
	// Deprecated: Md5 is deprecated, use DigestMethod with DigestValue instead
	Md5    string `yaml:"md5,omitempty" mapstructure:"md5,omitempty"`
	Digest string `yaml:"digest,omitempty" mapstructure:"digest,omitempty"`
	// DigestMethod indicates digest method, like md5, sha256
	DigestMethod string `yaml:"digestMethod,omitempty" mapstructure:"digestMethod,omitempty"`

	// DigestValue indicates digest value
	DigestValue string `yaml:"digestValue,omitempty" mapstructure:"digestValue,omitempty"`

	// Tag identify download task, it is available merely when md5 param not exist.
	Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"`

	// Application name that executes dfget.
	Application string `yaml:"application,omitempty" mapstructure:"application,omitempty"`

	// DaemonSock is daemon download socket path.
	DaemonSock string `yaml:"daemonSock,omitempty" mapstructure:"daemon-sock,omitempty"`

	// Priority scheduler will schedule tasks according to priority
	Priority int32 `yaml:"priority,omitempty" mapstructure:"priority,omitempty"`

	// CA certificate to verify when supernode interact with the source.
	Cacerts []string `yaml:"cacert,omitempty" mapstructure:"cacert,omitempty"`

	// Filter filter some query params of url, use char '&' to separate different params.
	// eg: -f 'key&sign' will filter 'key' and 'sign' query param.
	// in this way, different urls correspond one same download task that can use p2p mode.
	Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"`

	// Header of http request.
	// eg: --header='Accept: *' --header='Host: abc'.
	Header []string `yaml:"header,omitempty" mapstructure:"header,omitempty"`

	// DisableBackSource indicates whether to not back source to download when p2p fails.
	DisableBackSource bool `yaml:"disableBackSource,omitempty" mapstructure:"disable-back-source,omitempty"`

	// Insecure indicates whether skip secure verify when supernode interact with the source.
	Insecure bool `yaml:"insecure,omitempty" mapstructure:"insecure,omitempty"`

	// ShowProgress shows progress bar, it's conflict with `--console`.
	ShowProgress bool `yaml:"show-progress,omitempty" mapstructure:"show-progress,omitempty"`

	// LogDir is log directory of dfget.
	LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"`

	// Maximum size in megabytes of log files before rotation (default: 1024)
	LogMaxSize int `yaml:"logMaxSize" mapstructure:"logMaxSize"`

	// Maximum number of days to retain old log files (default: 7)
	LogMaxAge int `yaml:"logMaxAge" mapstructure:"logMaxAge"`

	// Maximum number of old log files to keep (default: 20)
	LogMaxBackups int `yaml:"logMaxBackups" mapstructure:"logMaxBackups"`

	// DataDir is data directory of dfget.
	DataDir string `yaml:"dataDir,omitempty" mapstructure:"dataDir,omitempty"`

	// CacheDir is cache directory of dfget.
	CacheDir string `yaml:"cacheDir,omitempty" mapstructure:"cacheDir,omitempty"`

	// WorkHome is working directory of dfget.
	WorkHome string `yaml:"workHome,omitempty" mapstructure:"workHome,omitempty"`

	RateLimit util.RateLimit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"`

	// MoreDaemonOptions indicates more options passed to daemon by command line.
	MoreDaemonOptions string `yaml:"moreDaemonOptions,omitempty" mapstructure:"moreDaemonOptions,omitempty"`

	// Recursive indicates to download all resources in target url, the target source client must support list action
	Recursive bool `yaml:"recursive,omitempty" mapstructure:"recursive,omitempty"`

	// RecursiveList indicates to list all resources in target url, the target source client must support list action
	RecursiveList bool `yaml:"recursiveList,omitempty" mapstructure:"list,omitempty"`

	// RecursiveLevel indicates to the maximum number of subdirectories that dfget will recurse into
	RecursiveLevel uint `yaml:"recursiveLevel,omitempty" mapstructure:"level,omitempty"`

	RecursiveAcceptRegex string `yaml:"acceptRegex,omitempty" mapstructure:"accept-regex,omitempty"`

	RecursiveRejectRegex string `yaml:"rejectRegex,omitempty" mapstructure:"reject-regex,omitempty"`

	KeepOriginalOffset bool `yaml:"keepOriginalOffset,omitempty" mapstructure:"original-offset,omitempty"`

	// Range stands download range for url, like: 0-9, will download 10 bytes from 0 to 9 ([0:9])
	Range string `yaml:"range,omitempty" mapstructure:"range,omitempty"`
}

ClientOption holds all the runtime config information.

func NewDfgetConfig 

func NewDfgetConfig() *ClientOption

func (*ClientOption) Convert 

func (cfg *ClientOption) Convert(args []string) error

func (*ClientOption) String 

func (cfg *ClientOption) String() string

func (*ClientOption) Validate 

func (cfg *ClientOption) Validate() error

type ConcurrentOption added in v2.0.5 

type ConcurrentOption struct {
	// ThresholdSize indicates the threshold to download pieces concurrently
	ThresholdSize util.Size `mapstructure:"thresholdSize" yaml:"thresholdSize"`
	// ThresholdSpeed indicates the threshold download speed to download pieces concurrently
	ThresholdSpeed unit.Bytes `mapstructure:"thresholdSpeed" yaml:"thresholdSpeed"`
	// GoroutineCount indicates the concurrent goroutine count for every task
	GoroutineCount int `mapstructure:"goroutineCount" yaml:"goroutineCount"`
	// InitBackoff second for every piece failed, default: 0.5
	InitBackoff float64 `mapstructure:"initBackoff" yaml:"initBackoff"`
	// MaxBackoff second for every piece failed, default: 3
	MaxBackoff float64 `mapstructure:"maxBackoff" yaml:"maxBackoff"`
	// MaxAttempts for every piece failed,default: 3
	MaxAttempts int `mapstructure:"maxAttempts" yaml:"maxAttempts"`
}

type DaemonConfig 

type DaemonConfig = DaemonOption

type DaemonOption 

type DaemonOption struct {
	base.Options `yaml:",inline" mapstructure:",squash"`
	// AliveTime indicates alive duration for which daemon keeps no accessing by any uploading and download requests,
	// after this period daemon will automatically exit
	// when AliveTime == 0, will run infinitely
	AliveTime  util.Duration `mapstructure:"aliveTime" yaml:"aliveTime"`
	GCInterval util.Duration `mapstructure:"gcInterval" yaml:"gcInterval"`
	Metrics    string        `mapstructure:"metrics" yaml:"metrics"`
	// MaxThreads indicates to invoke debug.SetMaxThreads to set max threads that golang runtime can create,
	// it's useful for large-scale downloading resource with none native golang source code, like call cgo to invoke c++
	// one cgo call will create a standalone os tread, the default 10000 limit is to small.
	// Normally, we can also set seed concurrent to limit for large-scale downloading at same time.
	MaxThreads int `mapstructure:"maxThreads" yaml:"maxThreads"`

	WorkHome      string `mapstructure:"workHome" yaml:"workHome"`
	WorkHomeMode  uint32 `mapstructure:"workHomeMode" yaml:"workHomeMode"`
	CacheDir      string `mapstructure:"cacheDir" yaml:"cacheDir"`
	CacheDirMode  uint32 `mapstructure:"cacheDirMode" yaml:"cacheDirMode"`
	LogDir        string `mapstructure:"logDir" yaml:"logDir"`
	LogMaxSize    int    `yaml:"logMaxSize" mapstructure:"logMaxSize"`
	LogMaxAge     int    `yaml:"logMaxAge" mapstructure:"logMaxAge"`
	LogMaxBackups int    `yaml:"logMaxBackups" mapstructure:"logMaxBackups"`
	PluginDir     string `mapstructure:"pluginDir" yaml:"pluginDir"`
	DataDir       string `mapstructure:"dataDir" yaml:"dataDir"`
	DataDirMode   uint32 `mapstructure:"dataDirMode" yaml:"dataDirMode"`
	KeepStorage   bool   `mapstructure:"keepStorage" yaml:"keepStorage"`

	Scheduler     SchedulerOption     `mapstructure:"scheduler" yaml:"scheduler"`
	Host          HostOption          `mapstructure:"host" yaml:"host"`
	Download      DownloadOption      `mapstructure:"download" yaml:"download"`
	Proxy         *ProxyOption        `mapstructure:"proxy" yaml:"proxy"`
	Upload        UploadOption        `mapstructure:"upload" yaml:"upload"`
	ObjectStorage ObjectStorageOption `mapstructure:"objectStorage" yaml:"objectStorage"`
	Storage       StorageOption       `mapstructure:"storage" yaml:"storage"`
	Health        *HealthOption       `mapstructure:"health" yaml:"health"`
	Reload        ReloadOption        `mapstructure:"reload" yaml:"reload"`
	Network       *NetworkOption      `mapstructure:"network" yaml:"network"`
	Announcer     AnnouncerOption     `mapstructure:"announcer" yaml:"announcer"`
	PeerExchange  PeerExchangeOption  `mapstructure:"peerExchange" yaml:"peerExchange"`
}

func NewDaemonConfig 

func NewDaemonConfig() *DaemonOption

func (*DaemonOption) Convert 

func (p *DaemonOption) Convert() error

func (*DaemonOption) IsSupportPeerExchange added in v2.1.40 

func (p *DaemonOption) IsSupportPeerExchange() bool

func (*DaemonOption) Load 

func (p *DaemonOption) Load(path string) error

func (*DaemonOption) Validate 

func (p *DaemonOption) Validate() error

type DfcacheConfig added in v2.0.3 

type DfcacheConfig = CacheOption

type DfgetConfig 

type DfgetConfig = ClientOption

type DfstoreConfig added in v2.0.5 

type DfstoreConfig struct {
	// Address of the object storage service.
	Endpoint string `yaml:"endpoint,omitempty" mapstructure:"endpoint,omitempty"`

	// Filter is used to generate a unique Task ID by
	// filtering unnecessary query params in the URL,
	// it is separated by & character.
	Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"`

	// Mode is the mode in which the backend is written,
	// including WriteBack and AsyncWriteBack.
	Mode int `yaml:"mode,omitempty" mapstructure:"mode,omitempty"`

	// MaxReplicas is the maximum number of
	// replicas of an object cache in seed peers.
	MaxReplicas int `yaml:"maxReplicas,omitempty" mapstructure:"mode,maxReplicas"`
}

func NewDfstore added in v2.0.5 

func NewDfstore() *DfstoreConfig

New dfstore configuration.

func (*DfstoreConfig) Validate added in v2.0.5 

func (cfg *DfstoreConfig) Validate() error

type DownloadOption 

type DownloadOption struct {
	TotalRateLimit       util.RateLimit    `mapstructure:"totalRateLimit" yaml:"totalRateLimit"`
	PerPeerRateLimit     util.RateLimit    `mapstructure:"perPeerRateLimit" yaml:"perPeerRateLimit"`
	TrafficShaperType    string            `mapstructure:"trafficShaperType" yaml:"trafficShaperType"`
	PieceDownloadTimeout time.Duration     `mapstructure:"pieceDownloadTimeout" yaml:"pieceDownloadTimeout"`
	GRPCDialTimeout      time.Duration     `mapstructure:"grpcDialTimeout" yaml:"grpcDialTimeout"`
	DownloadGRPC         ListenOption      `mapstructure:"downloadGRPC" yaml:"downloadGRPC"`
	PeerGRPC             ListenOption      `mapstructure:"peerGRPC" yaml:"peerGRPC"`
	CalculateDigest      bool              `mapstructure:"calculateDigest" yaml:"calculateDigest"`
	Transport            *TransportOption  `mapstructure:"transportOption" yaml:"transportOption"`
	GetPiecesMaxRetry    int               `mapstructure:"getPiecesMaxRetry" yaml:"getPiecesMaxRetry"`
	Prefetch             bool              `mapstructure:"prefetch" yaml:"prefetch"`
	WatchdogTimeout      time.Duration     `mapstructure:"watchdogTimeout" yaml:"watchdogTimeout"`
	Concurrent           *ConcurrentOption `mapstructure:"concurrent" yaml:"concurrent"`
	SyncPieceViaHTTPS    bool              `mapstructure:"syncPieceViaHTTPS" yaml:"syncPieceViaHTTPS"`
	SplitRunningTasks    bool              `mapstructure:"splitRunningTasks" yaml:"splitRunningTasks"`
	// resource clients option
	ResourceClients ResourceClientsOption `mapstructure:"resourceClients" yaml:"resourceClients"`

	RecursiveConcurrent    RecursiveConcurrent `mapstructure:"recursiveConcurrent" yaml:"recursiveConcurrent"`
	SeedConcurrent         int64               `mapstructure:"seedConcurrent" yaml:"seedConcurrent"`
	CacheRecursiveMetadata time.Duration       `mapstructure:"cacheRecursiveMetadata" yaml:"cacheRecursiveMetadata"`
}

type DurationValue 

type DurationValue time.Duration

DurationValue supports time.Duration format like 30s, 1m30s, 1h and also treat integer as seconds

func NewDurationValue 

func NewDurationValue(p *time.Duration) *DurationValue

func (*DurationValue) Set 

func (d *DurationValue) Set(s string) error

func (*DurationValue) String 

func (d *DurationValue) String() string

func (*DurationValue) Type 

func (d *DurationValue) Type() string

type Dynconfig added in v2.0.1 

type Dynconfig interface {
	// Get the dynamic seed peers config.
	GetSeedPeers() ([]*managerv1.SeedPeer, error)

	// Get the dynamic schedulers resolve addrs.
	GetResolveSchedulerAddrs() ([]resolver.Address, error)

	// Get the dynamic schedulers config.
	GetSchedulers() ([]*managerv1.Scheduler, error)

	// Get the dynamic schedulers cluster id.
	GetSchedulerClusterID() uint64

	// Get the dynamic object storage config.
	GetObjectStorage() (*managerv1.ObjectStorage, error)

	// Get the dynamic config.
	Get() (*DynconfigData, error)

	// Refresh refreshes dynconfig in cache.
	Refresh() error

	// Register allows an instance to register itself to listen/observe events.
	Register(Observer)

	// Deregister allows an instance to remove itself from the collection of observers/listeners.
	Deregister(Observer)

	// Notify publishes new events to listeners.
	Notify() error

	// OnNotify allows an event to be published to the dynconfig.
	// Used for listening changes of the local configuration.
	OnNotify(*DaemonOption)

	// Serve the dynconfig listening service.
	Serve() error

	// Stop the dynconfig listening service.
	Stop() error
}

func NewDynconfig added in v2.0.1 

func NewDynconfig(sourceType SourceType, cfg *DaemonOption, options ...DynconfigOption) (Dynconfig, error)

New returns a new dynconfig interface.

type DynconfigData added in v2.0.1 

type DynconfigData struct {
	SeedPeers     []*managerv1.SeedPeer
	Schedulers    []*managerv1.Scheduler
	ObjectStorage *managerv1.ObjectStorage
}

type DynconfigOption added in v2.0.9 

type DynconfigOption func(d *dynconfig) error

DynconfigOption is a functional option for configuring the dynconfig.

func WithCacheDir added in v2.0.6 

func WithCacheDir(dir string) DynconfigOption

WithCacheDir set the cache dir.

func WithManagerClient added in v2.0.6 

func WithManagerClient(c managerclient.V1) DynconfigOption

WithManagerClient set the manager client.

func WithTransportCredentials added in v2.0.9 

func WithTransportCredentials(creds credentials.TransportCredentials) DynconfigOption

WithTransportCredentials returns a DialOption which configures a connection level security credentials (e.g., TLS/SSL).

type HealthOption added in v2.0.2 

type HealthOption struct {
	ListenOption `yaml:",inline" mapstructure:",squash"`
	Path         string `mapstructure:"path" yaml:"path"`
}

type HijackConfig 

type HijackConfig struct {
	Cert  types.PEMContent   `yaml:"cert" mapstructure:"cert"`
	Key   types.PEMContent   `yaml:"key" mapstructure:"key"`
	Hosts []*HijackHost      `yaml:"hosts" mapstructure:"hosts"`
	SNI   []*TCPListenOption `yaml:"sni" mapstructure:"sni"`
}

HijackConfig represents how dfdaemon hijacks http requests.

type HijackHost 

type HijackHost struct {
	Regx     *Regexp   `yaml:"regx" mapstructure:"regx"`
	Insecure bool      `yaml:"insecure" mapstructure:"insecure"`
	Certs    *CertPool `yaml:"certs" mapstructure:"certs"`
}

HijackHost is a hijack rule for the hosts that matches Regx.

type HostOption 

type HostOption struct {
	// IDC for scheduler
	IDC string `mapstructure:"idc" yaml:"idc"`
	// Location for scheduler
	Location string `mapstructure:"location" yaml:"location"`
	// Hostname is daemon host name
	Hostname string `mapstructure:"hostname" yaml:"hostname"`
	// The ip report to scheduler, normal same with listen ip
	AdvertiseIP net.IP `mapstructure:"advertiseIP" yaml:"advertiseIP"`
}

type KeepAliveOption added in v2.0.3 

type KeepAliveOption struct {
	// Keep alive interval.
	Interval time.Duration `yaml:"interval" mapstructure:"interval"`
}

type ListenOption 

type ListenOption struct {
	Security   SecurityOption    `mapstructure:"security" yaml:"security"`
	TCPListen  *TCPListenOption  `mapstructure:"tcpListen,omitempty" yaml:"tcpListen,omitempty"`
	UnixListen *UnixListenOption `mapstructure:"unixListen,omitempty" yaml:"unixListen,omitempty"`
}

type ManagerOption added in v2.0.1 

type ManagerOption struct {
	// Enable get configuration from manager.
	Enable bool `mapstructure:"enable" yaml:"enable"`
	// NetAddrs is manager addresses.
	NetAddrs []dfnet.NetAddr `mapstructure:"netAddrs" yaml:"netAddrs"`
	// RefreshInterval is the refresh interval.
	RefreshInterval time.Duration `mapstructure:"refreshInterval" yaml:"refreshInterval"`
	// SeedPeer configuration.
	SeedPeer SeedPeerOption `mapstructure:"seedPeer" yaml:"seedPeer"`
}

type NetAddrsValue 

type NetAddrsValue struct {
	// contains filtered or unexported fields
}

func NewNetAddrsValue 

func NewNetAddrsValue(n *[]dfnet.NetAddr) *NetAddrsValue

func (*NetAddrsValue) Set 

func (nv *NetAddrsValue) Set(value string) error

func (*NetAddrsValue) String 

func (nv *NetAddrsValue) String() string

func (*NetAddrsValue) Type 

func (nv *NetAddrsValue) Type() string

type NetworkOption added in v2.0.7 

type NetworkOption struct {
	// EnableIPv6 enables ipv6 for server.
	EnableIPv6 bool `mapstructure:"enableIPv6" yaml:"enableIPv6"`
}

type ObjectStorageOption added in v2.0.4 

type ObjectStorageOption struct {
	// Enable object storage.
	Enable bool `mapstructure:"enable" yaml:"enable"`
	// Filter is used to generate a unique Task ID by
	// filtering unnecessary query params in the URL,
	// it is separated by & character.
	Filter string `mapstructure:"filter" yaml:"filter"`
	// MaxReplicas is the maximum number of replicas of an object cache in seed peers.
	MaxReplicas int `mapstructure:"maxReplicas" yaml:"maxReplicas"`
	// ListenOption is object storage service listener.
	ListenOption `yaml:",inline" mapstructure:",squash"`
}

type Observer added in v2.0.1 

type Observer interface {
	// OnNotify allows an event to be published to interface implementations.
	OnNotify(*DynconfigData)
}

type PeerExchangeOption added in v2.1.40 

type PeerExchangeOption struct {
	// Enable peer exchange service.
	Enable bool `mapstructure:"enable" yaml:"enable"`
	// InitialInterval is the initial retry interval when start and join gossip.
	InitialInterval time.Duration `mapstructure:"initialInterval" yaml:"initialInterval"`
	// InitialBroadcastDelay is the initial broadcast delay when daemon start due to we can only broadcast peers after join gossip with all daemons, it should be less than TaskExpireTime
	InitialBroadcastDelay time.Duration `mapstructure:"initialBroadcastDelay" yaml:"initialBroadcastDelay"`
	// ReSyncInterval is the re-sync interval for check running gossip members.
	ReSyncInterval time.Duration `mapstructure:"reSyncInterval" yaml:"reSyncInterval"`
	// ReplicaThreshold is used for keeping replicas in all peers is not bigger than threshold to save storage
	ReplicaThreshold int `mapstructure:"replicaThreshold" yaml:"replicaThreshold"`
	// ReplicaCleanPercentage is percentage probability to clean local replica when reach threshold, available values: [0, 100]
	ReplicaCleanPercentage int32 `mapstructure:"replicaCleanPercentage" yaml:"replicaCleanPercentage"`
}

type ProxyOption 

type ProxyOption struct {
	// WARNING: when add more option, please update ProxyOption.unmarshal function
	ListenOption       `mapstructure:",squash" yaml:",inline"`
	BasicAuth          *BasicAuth        `mapstructure:"basicAuth" yaml:"basicAuth"`
	DefaultFilter      string            `mapstructure:"defaultFilter" yaml:"defaultFilter"`
	DefaultTag         string            `mapstructure:"defaultTag" yaml:"defaultTag"`
	DefaultApplication string            `mapstructure:"defaultApplication" yaml:"defaultApplication"`
	DefaultPriority    commonv1.Priority `mapstructure:"defaultPriority" yaml:"defaultPriority"`
	MaxConcurrency     int64             `mapstructure:"maxConcurrency" yaml:"maxConcurrency"`
	RegistryMirror     *RegistryMirror   `mapstructure:"registryMirror" yaml:"registryMirror"`
	WhiteList          []*WhiteList      `mapstructure:"whiteList" yaml:"whiteList"`
	ProxyRules         []*ProxyRule      `mapstructure:"proxies" yaml:"proxies"`
	HijackHTTPS        *HijackConfig     `mapstructure:"hijackHTTPS" yaml:"hijackHTTPS"`
	DumpHTTPContent    bool              `mapstructure:"dumpHTTPContent" yaml:"dumpHTTPContent"`
	// ExtraRegistryMirrors add more mirrors for different ports
	ExtraRegistryMirrors []*RegistryMirror `mapstructure:"extraRegistryMirrors" yaml:"extraRegistryMirrors"`
}

func (*ProxyOption) UnmarshalJSON 

func (p *ProxyOption) UnmarshalJSON(b []byte) error

func (*ProxyOption) UnmarshalYAML 

func (p *ProxyOption) UnmarshalYAML(node *yaml.Node) error

type ProxyRule added in v2.0.2 

type ProxyRule struct {
	Regx     *Regexp `yaml:"regx" mapstructure:"regx"`
	UseHTTPS bool    `yaml:"useHTTPS" mapstructure:"useHTTPS"`
	Direct   bool    `yaml:"direct" mapstructure:"direct"`

	// Redirect is the host to redirect to, if not empty
	Redirect string `yaml:"redirect" mapstructure:"redirect"`
}

ProxyRule describes a regular expression matching rule for how to proxy a request.

func NewProxyRule added in v2.0.2 

func NewProxyRule(regx string, useHTTPS bool, direct bool, redirect string) (*ProxyRule, error)

func (*ProxyRule) Match added in v2.0.2 

func (r *ProxyRule) Match(url string) bool

Match checks if the given url matches the rule.

type RateLimitValue 

type RateLimitValue struct {
	// contains filtered or unexported fields
}

func NewLimitRateValue 

func NewLimitRateValue(rate *util.RateLimit) *RateLimitValue

func (*RateLimitValue) Set 

func (r *RateLimitValue) Set(s string) error

func (*RateLimitValue) String 

func (r *RateLimitValue) String() string

func (*RateLimitValue) Type 

func (r *RateLimitValue) Type() string

type RecursiveConcurrent added in v2.0.7 

type RecursiveConcurrent struct {
	// GoroutineCount indicates the concurrent goroutine count for every recursive task
	GoroutineCount int `mapstructure:"goroutineCount" yaml:"goroutineCount"`
}

type Regexp 

type Regexp struct {
	*regexp.Regexp
}

Regexp is a simple wrapper around regexp. Regexp to make it unmarshallable from a string.

func NewRegexp 

func NewRegexp(exp string) (*Regexp, error)

NewRegexp returns a new Regexp instance compiled from the given string.

func (*Regexp) MarshalJSON 

func (r *Regexp) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaller to print the regexp.

func (*Regexp) MarshalYAML 

func (r *Regexp) MarshalYAML() (any, error)

MarshalYAML implements yaml.Marshaller to print the regexp.

func (*Regexp) UnmarshalJSON 

func (r *Regexp) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler.

func (*Regexp) UnmarshalYAML 

func (r *Regexp) UnmarshalYAML(unmarshal func(any) error) error

UnmarshalYAML implements yaml.Unmarshaler.

type RegistryMirror 

type RegistryMirror struct {
	// Remote url for the registry mirror, default is https://index.docker.io
	Remote *URL `yaml:"url" mapstructure:"url"`

	// DynamicRemote indicates using header "X-Dragonfly-Registry" for remote instead of Remote
	// if header "X-Dragonfly-Registry" does not exist, use Remote by default
	DynamicRemote bool `yaml:"dynamic" mapstructure:"dynamic"`

	// Optional certificates if the mirror uses self-signed certificates
	Certs *CertPool `yaml:"certs" mapstructure:"certs"`

	// Whether to ignore certificates errors for the registry
	Insecure bool `yaml:"insecure" mapstructure:"insecure"`

	// Request the remote registry directly.
	Direct bool `yaml:"direct" mapstructure:"direct"`

	// Whether to use proxies to decide when to use dragonfly
	UseProxies bool `yaml:"useProxies" mapstructure:"useProxies"`
}

RegistryMirror configures the mirror of the official docker registry

func (*RegistryMirror) TLSConfig 

func (r *RegistryMirror) TLSConfig() *tls.Config

TLSConfig returns the tls.Config used to communicate with the mirror.

type ReloadOption added in v2.0.2 

type ReloadOption struct {
	Interval util.Duration `mapstructure:"interval" yaml:"interval"`
}

type ResourceClientsOption added in v2.0.9 

type ResourceClientsOption map[string]any

type SchedulerOption 

type SchedulerOption struct {
	// Manager is to get the scheduler configuration remotely.
	Manager ManagerOption `mapstructure:"manager" yaml:"manager"`
	// NetAddrs is scheduler addresses.
	NetAddrs []dfnet.NetAddr `mapstructure:"netAddrs" yaml:"netAddrs"`
	// ScheduleTimeout is request timeout.
	ScheduleTimeout util.Duration `mapstructure:"scheduleTimeout" yaml:"scheduleTimeout"`
	// DisableAutoBackSource indicates not back source normally, only scheduler says back source.
	DisableAutoBackSource bool `mapstructure:"disableAutoBackSource" yaml:"disableAutoBackSource"`
}

type SecurityOption 

type SecurityOption struct {
	// Insecure indicate enable tls or not
	Insecure  bool             `mapstructure:"insecure" yaml:"insecure"`
	CACert    types.PEMContent `mapstructure:"caCert" yaml:"caCert"`
	Cert      types.PEMContent `mapstructure:"cert" yaml:"cert"`
	Key       types.PEMContent `mapstructure:"key" yaml:"key"`
	TLSVerify bool             `mapstructure:"tlsVerify" yaml:"tlsVerify"`
	TLSConfig *tls.Config      `mapstructure:"tlsConfig" yaml:"tlsConfig"`
}

type SeedPeerOption added in v2.0.3 

type SeedPeerOption struct {
	// Enable seed peer mode.
	Enable bool `mapstructure:"enable" yaml:"enable"`
	// Type is seed peer type.
	Type string `mapstructure:"type" yaml:"type"`
	// ClusterID is seed peer cluster id.
	ClusterID uint `mapstructure:"clusterID" yaml:"clusterID"`
	// KeepAlive configuration.
	KeepAlive KeepAliveOption `yaml:"keepAlive" mapstructure:"keepAlive"`
}

type SourceType added in v2.0.6 

type SourceType string

type StorageOption 

type StorageOption struct {
	// DataPath indicates directory which stores temporary files for p2p uploading
	DataPath string `mapstructure:"dataPath" yaml:"dataPath"`
	// TaskExpireTime indicates caching duration for which cached file keeps no accessed by any process,
	// after this period cache file will be gc
	TaskExpireTime util.Duration `mapstructure:"taskExpireTime" yaml:"taskExpireTime"`
	// DiskGCThreshold indicates the threshold to gc the oldest tasks
	DiskGCThreshold unit.Bytes `mapstructure:"diskGCThreshold" yaml:"diskGCThreshold"`
	// DiskGCThresholdPercent indicates the threshold to gc the oldest tasks according the disk usage
	// Eg, DiskGCThresholdPercent=80, when the disk usage is above 80%, start to gc the oldest tasks
	DiskGCThresholdPercent float64 `mapstructure:"diskGCThresholdPercent" yaml:"diskGCThresholdPercent"`
	// Multiplex indicates reusing underlying storage for same task id
	Multiplex     bool          `mapstructure:"multiplex" yaml:"multiplex"`
	StoreStrategy StoreStrategy `mapstructure:"strategy" yaml:"strategy"`
	// WriteBufferSize indicates the buffer size when read from source, same usage with io.Copy
	// for some resource plugins, bigger buffer size with better performance, on the other hand, bigger buffer size cost huge memory
	WriteBufferSize unit.Bytes `mapstructure:"writeBufferSize" yaml:"writeBufferSize"`
	// ReloadGoroutineCount indicates concurrent goroutine count when daemon load cache data
	ReloadGoroutineCount int `mapstructure:"reloadGoroutineCount" yaml:"reloadGoroutineCount"`
}

type StoreStrategy 

type StoreStrategy string

type SupernodesValue 

type SupernodesValue struct {
	Nodes []string
}

func (*SupernodesValue) Set 

func (sv *SupernodesValue) Set(value string) error

Set implements the pflag.Value interface.

func (*SupernodesValue) String 

func (sv *SupernodesValue) String() string

String implements the pflag.Value interface.

func (*SupernodesValue) Type 

func (sv *SupernodesValue) Type() string

Type implements the pflag.Value interface.

type TCPListenOption 

type TCPListenOption struct {
	// Listen stands listen interface, like: 0.0.0.0, 192.168.0.1
	Listen string `mapstructure:"listen" yaml:"listen"`

	// PortRange stands listen port
	// yaml example 1:
	//   port: 12345
	// yaml example 2:
	//   port:
	//     start: 12345
	//     end: 12346
	PortRange TCPListenPortRange `mapstructure:"port" yaml:"port"`

	// Namespace stands the linux net namespace, like /proc/1/ns/net
	// It's useful for running daemon in pod with ip allocated and listen in host
	Namespace string `mapstructure:"namespace" yaml:"namespace"`
}

type TCPListenPortRange 

type TCPListenPortRange struct {
	Start int
	End   int
}

func (*TCPListenPortRange) UnmarshalJSON 

func (t *TCPListenPortRange) UnmarshalJSON(b []byte) error

func (*TCPListenPortRange) UnmarshalYAML 

func (t *TCPListenPortRange) UnmarshalYAML(node *yaml.Node) error

type TLSConfig 

type TLSConfig struct {
	tls.Config
}

func (*TLSConfig) UnmarshalJSON 

func (t *TLSConfig) UnmarshalJSON(b []byte) error

func (*TLSConfig) UnmarshalYAML added in v2.0.6 

func (t *TLSConfig) UnmarshalYAML(node *yaml.Node) error

type TelemetryOption 

type TelemetryOption struct {
	Jaeger string `yaml:"jaeger" mapstructure:"jaeger"`
}

TelemetryOption is the option for telemetry

type TransportOption added in v2.0.1 

type TransportOption struct {
	DialTimeout           time.Duration `mapstructure:"dialTimeout" yaml:"dialTimeout"`
	KeepAlive             time.Duration `mapstructure:"keepAlive" yaml:"keepAlive"`
	MaxIdleConns          int           `mapstructure:"maxIdleConns" yaml:"maxIdleConns"`
	IdleConnTimeout       time.Duration `mapstructure:"idleConnTimeout" yaml:"idleConnTimeout"`
	ResponseHeaderTimeout time.Duration `mapstructure:"responseHeaderTimeout" yaml:"responseHeaderTimeout"`
	TLSHandshakeTimeout   time.Duration `mapstructure:"tlsHandshakeTimeout" yaml:"tlsHandshakeTimeout"`
	ExpectContinueTimeout time.Duration `mapstructure:"expectContinueTimeout" yaml:"expectContinueTimeout"`
}

type URL 

type URL struct {
	*url.URL
}

URL is simple wrapper around url.URL to make it unmarshallable from a string.

func (*URL) MarshalJSON 

func (u *URL) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaller to print the url.

func (*URL) MarshalYAML 

func (u *URL) MarshalYAML() (any, error)

MarshalYAML implements yaml.Marshaller to print the url.

func (*URL) UnmarshalJSON 

func (u *URL) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler.

func (*URL) UnmarshalYAML 

func (u *URL) UnmarshalYAML(unmarshal func(any) error) error

UnmarshalYAML implements yaml.Unmarshaler.

type UnixListenOption 

type UnixListenOption struct {
	Socket string `mapstructure:"socket" yaml:"socket"`
}

type UploadOption 

type UploadOption struct {
	ListenOption `yaml:",inline" mapstructure:",squash"`
	RateLimit    util.RateLimit `mapstructure:"rateLimit" yaml:"rateLimit"`
}

type WhiteList 

type WhiteList struct {
	Host  string   `yaml:"host" mapstructure:"host"`
	Regx  *Regexp  `yaml:"regx" mapstructure:"regx"`
	Ports []string `yaml:"ports" mapstructure:"ports"`
}

Source Files

View all Source files

Directories

Path Synopsis
Package mocks is a generated GoMock package.
 Package mocks is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.