Documentation ¶
Overview ¶
Package config holds all DaemonConfig of dfget.
Package config holds all options of peerhost.
Index ¶
- Constants
- Variables
- func MkdirAll(dir string, perm os.FileMode, uid, gid int) error
- type BasicAuth
- type CertPool
- type ClientOption
- type DaemonConfig
- type DaemonOption
- type DfgetConfig
- type DownloadOption
- type DurationValue
- type Dynconfig
- type DynconfigData
- type FileString
- type HijackConfig
- type HijackHost
- type HostOption
- type ListenOption
- type ManagerOption
- type NetAddrsValue
- type Observer
- type Proxy
- type ProxyOption
- type RateLimitValue
- type Regexp
- type RegistryMirror
- type SchedulerOption
- type SecurityOption
- type StorageOption
- type StoreStrategy
- type SupernodesValue
- type TCPListenOption
- type TCPListenPortRange
- type TLSConfig
- type TelemetryOption
- type URL
- type UnixListenOption
- type UploadOption
- type WhiteList
Constants ¶
const ( BackSourceReasonNone = 0 BackSourceReasonRegisterFail = 1 BackSourceReasonMd5NotMatch = 2 BackSourceReasonDownloadError = 3 BackSourceReasonNoSpace = 4 BackSourceReasonInitError = 5 BackSourceReasonWriteError = 6 BackSourceReasonHostSysError = 7 BackSourceReasonNodeEmpty = 8 BackSourceReasonSourceError = 10 BackSourceReasonUserSpecified = 100 ForceNotBackSourceAddition = 1000 )
the reason of backing to source
const ( PatternP2P = "p2p" PatternCDN = "cdn" PatternSource = "source" )
download pattern
const ( DefaultPerPeerDownloadLimit = 20 * unit.MB DefaultTotalDownloadLimit = 100 * unit.MB DefaultUploadLimit = 100 * unit.MB DefaultMinRate = 64 * unit.KB )
const ( DefaultTimestampFormat = "2006-01-02 15:04:05" SchemaHTTP = "http" ServerPortLowerLimit = 15000 ServerPortUpperLimit = 65000 DefaultTaskExpireTime = 3 * time.Minute DefaultGCInterval = 1 * time.Minute DefaultDaemonAliveTime = 5 * time.Minute DefaultScheduleTimeout = 5 * time.Minute DefaultDownloadTimeout = 5 * time.Minute DefaultSchedulerSchema = "http" DefaultSchedulerIP = "127.0.0.1" DefaultSchedulerPort = 8002 DefaultPieceChanSize = 16 )
others
const ( SimpleLocalTaskStoreStrategy = StoreStrategy("io.d7y.storage.v2.simple") AdvanceLocalTaskStoreStrategy = StoreStrategy("io.d7y.storage.v2.advance") )
const ( AttributePeerHost = attribute.Key("d7y.peer.host") AttributeTaskID = attribute.Key("d7y.peer.task.id") AttributeTaskContentLength = attribute.Key("d7y.peer.task.content_length") AttributePeerID = attribute.Key("d7y.peer.id") AttributeTargetPeerID = attribute.Key("d7y.peer.target.id") AttributeReusePeerID = attribute.Key("d7y.peer.reuse.id") AttributeTargetPeerAddr = attribute.Key("d7y.peer.target.addr") AttributeMainPeer = attribute.Key("d7y.peer.task.main_peer") AttributePeerPacketCode = attribute.Key("d7y.peer.packet.code") AttributePeerTaskSizeScope = attribute.Key("d7y.peer.size.scope") AttributePeerTaskSize = attribute.Key("d7y.peer.size") AttributePeerTaskSuccess = attribute.Key("d7y.peer.task.success") AttributePeerTaskCode = attribute.Key("d7y.peer.task.code") AttributePeerTaskMessage = attribute.Key("d7y.peer.task.message") AttributePeerTaskCost = attribute.Key("d7y.peer.task.cost") AttributePiece = attribute.Key("d7y.peer.piece") AttributePieceSize = attribute.Key("d7y.peer.piece.size") AttributePieceWorker = attribute.Key("d7y.peer.piece.worker") AttributePieceSuccess = attribute.Key("d7y.peer.piece.success") AttributeGetPieceStartNum = attribute.Key("d7y.peer.piece.start") AttributeGetPieceLimit = attribute.Key("d7y.peer.piece.limit") AttributeGetPieceCount = attribute.Key("d7y.peer.piece.count") AttributeGetPieceRetry = attribute.Key("d7y.peer.piece.retry") AttributeWritePieceSuccess = attribute.Key("d7y.peer.piece.write.success") SpanFilePeerTask = "file-peer-task" SpanStreamPeerTask = "stream-peer-task" SpanReusePeerTask = "reuse-peer-task" SpanRegisterTask = "register" SpanReportPeerResult = "report-peer-result" SpanReportPieceResult = "report-piece-result" SpanBackSource = "client-back-source" SpanFirstSchedule = "schedule-#1" SpanGetPieceTasks = "get-piece-tasks" SpanDownloadPiece = "download-piece-#%d" SpanProxy = "proxy" SpanWritePiece = "write-piece" SpanWriteBackPiece = "write-back-piece" SpanWaitPieceLimit = "wait-limit" SpanPushPieceResult = "push-result" SpanPeerGC = "peer-gc" )
const ( HeaderDragonflyFilter = "X-Dragonfly-Filter" HeaderDragonflyPeer = "X-Dragonfly-Peer" HeaderDragonflyTask = "X-Dragonfly-Task" HeaderDragonflyBiz = "X-Dragonfly-Biz" // HeaderDragonflyRegistry is used for dynamic registry mirrors HeaderDragonflyRegistry = "X-Dragonfly-Registry" )
Variables ¶
var DefaultSupernodesValue = &SupernodesValue{ Nodes: []string{ fmt.Sprintf("%s:%d", DefaultSchedulerIP, DefaultSchedulerPort), }, }
var (
PeerHostConfigPath = "/etc/dragonfly/dfget-daemon.yaml"
)
Functions ¶
Types ¶
type CertPool ¶
CertPool is a wrapper around x509.CertPool, which can be unmarshalled and constructed from a list of filenames.
func (*CertPool) MarshalJSON ¶
MarshalJSON implements json.Marshaller to print the cert pool.
func (*CertPool) MarshalYAML ¶
MarshalYAML implements yaml.Marshaller to print the cert pool.
func (*CertPool) UnmarshalJSON ¶
UnmarshalJSON implements json.Unmarshaler.
func (*CertPool) UnmarshalYAML ¶
UnmarshalYAML implements yaml.Unmarshaler.
type ClientOption ¶
type ClientOption struct { base.Options `yaml:",inline" mapstructure:",squash"` // URL download URL. URL string `yaml:"url,omitempty" mapstructure:"url,omitempty"` // Output full output path. Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"` // Timeout download timeout(second). Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"` BenchmarkRate unit.Bytes `yaml:"benchmarkRate,omitempty" mapstructure:"benchmarkRate,omitempty"` // Md5 expected file md5. // Deprecated: Md5 is deprecated, use DigestMethod with DigestValue instead Md5 string `yaml:"md5,omitempty" mapstructure:"md5,omitempty"` Digest string `yaml:"digest,omitempty" mapstructure:"digest,omitempty"` // DigestMethod indicates digest method, like md5, sha256 DigestMethod string `yaml:"digestMethod,omitempty" mapstructure:"digestMethod,omitempty"` // DigestValue indicates digest value DigestValue string `yaml:"digestValue,omitempty" mapstructure:"digestValue,omitempty"` // Tag identify download task, it is available merely when md5 param not exist. Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"` // CallSystem system name that executes dfget. CallSystem string `yaml:"callSystem,omitempty" mapstructure:"callSystem,omitempty"` // Pattern download pattern, must be 'p2p' or 'cdn' or 'source', // default:`p2p`. Pattern string `yaml:"pattern,omitempty" mapstructure:"pattern,omitempty"` // CA certificate to verify when supernode interact with the source. Cacerts []string `yaml:"cacert,omitempty" mapstructure:"cacert,omitempty"` // Filter filter some query params of url, use char '&' to separate different params. // eg: -f 'key&sign' will filter 'key' and 'sign' query param. // in this way, different urls correspond one same download task that can use p2p mode. Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"` // Header of http request. // eg: --header='Accept: *' --header='Host: abc'. Header []string `yaml:"header,omitempty" mapstructure:"header,omitempty"` // DisableBackSource indicates whether to not back source to download when p2p fails. DisableBackSource bool `yaml:"disableBackSource,omitempty" mapstructure:"disableBackSource,omitempty"` // Insecure indicates whether skip secure verify when supernode interact with the source. Insecure bool `yaml:"insecure,omitempty" mapstructure:"insecure,omitempty"` // ShowProgress shows progress bar, it's conflict with `--console`. ShowProgress bool `yaml:"show-progress,omitempty" mapstructure:"show-progress,omitempty"` RateLimit rate.Limit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"` // MoreDaemonOptions indicates more options passed to daemon by command line. MoreDaemonOptions string `yaml:"moreDaemonOptions,omitempty" mapstructure:"moreDaemonOptions,omitempty"` }
ClientOption holds all the runtime config information.
func NewDfgetConfig ¶
func NewDfgetConfig() *ClientOption
func (*ClientOption) Convert ¶
func (cfg *ClientOption) Convert(args []string) error
func (*ClientOption) String ¶
func (cfg *ClientOption) String() string
func (*ClientOption) Validate ¶
func (cfg *ClientOption) Validate() error
type DaemonConfig ¶
type DaemonConfig = DaemonOption
type DaemonOption ¶
type DaemonOption struct { base.Options `yaml:",inline" mapstructure:",squash"` // AliveTime indicates alive duration for which daemon keeps no accessing by any uploading and download requests, // after this period daemon will automatically exit // when AliveTime == 0, will run infinitely AliveTime clientutil.Duration `mapstructure:"aliveTime" yaml:"aliveTime"` GCInterval clientutil.Duration `mapstructure:"gcInterval" yaml:"gcInterval"` DataDir string `mapstructure:"dataDir" yaml:"dataDir"` WorkHome string `mapstructure:"workHome" yaml:"workHome"` KeepStorage bool `mapstructure:"keepStorage" yaml:"keepStorage"` Scheduler SchedulerOption `mapstructure:"scheduler" yaml:"scheduler"` Host HostOption `mapstructure:"host" yaml:"host"` Download DownloadOption `mapstructure:"download" yaml:"download"` Proxy *ProxyOption `mapstructure:"proxy" yaml:"proxy"` Upload UploadOption `mapstructure:"upload" yaml:"upload"` Storage StorageOption `mapstructure:"storage" yaml:"storage"` ConfigServer string `mapstructure:"configServer" yaml:"configServer"` }
func NewDaemonConfig ¶
func NewDaemonConfig() *DaemonOption
func (*DaemonOption) Convert ¶
func (p *DaemonOption) Convert() error
func (*DaemonOption) Load ¶
func (p *DaemonOption) Load(path string) error
func (*DaemonOption) Validate ¶
func (p *DaemonOption) Validate() error
type DfgetConfig ¶
type DfgetConfig = ClientOption
type DownloadOption ¶
type DownloadOption struct { TotalRateLimit clientutil.RateLimit `mapstructure:"totalRateLimit" yaml:"totalRateLimit"` PerPeerRateLimit clientutil.RateLimit `mapstructure:"perPeerRateLimit" yaml:"perPeerRateLimit"` PieceDownloadTimeout time.Duration `mapstructure:"pieceDownloadTimeout" yaml:"pieceDownloadTimeout"` DownloadGRPC ListenOption `mapstructure:"downloadGRPC" yaml:"downloadGRPC"` PeerGRPC ListenOption `mapstructure:"peerGRPC" yaml:"peerGRPC"` CalculateDigest bool `mapstructure:"calculateDigest" yaml:"calculateDigest"` }
type DurationValue ¶
DurationValue supports time.Duration format like 30s, 1m30s, 1h and also treat integer as seconds
func NewDurationValue ¶
func NewDurationValue(p *time.Duration) *DurationValue
func (*DurationValue) Set ¶
func (d *DurationValue) Set(s string) error
func (*DurationValue) String ¶
func (d *DurationValue) String() string
func (*DurationValue) Type ¶
func (d *DurationValue) Type() string
type Dynconfig ¶ added in v2.0.1
type Dynconfig interface { // Get the dynamic config from manager. GetSchedulers() ([]*manager.Scheduler, error) // Get the dynamic config from manager. Get() (*DynconfigData, error) // Register allows an instance to register itself to listen/observe events. Register(Observer) // Deregister allows an instance to remove itself from the collection of observers/listeners. Deregister(Observer) // Notify publishes new events to listeners. Notify() error // Serve the dynconfig listening service. Serve() error // Stop the dynconfig listening service. Stop() error }
func NewDynconfig ¶ added in v2.0.1
func NewDynconfig(rawManagerClient managerclient.Client, hostOption HostOption, expire time.Duration) (Dynconfig, error)
type DynconfigData ¶ added in v2.0.1
type FileString ¶
type FileString string
func (*FileString) UnmarshalJSON ¶
func (f *FileString) UnmarshalJSON(b []byte) error
func (*FileString) UnmarshalYAML ¶
func (f *FileString) UnmarshalYAML(node *yaml.Node) error
type HijackConfig ¶
type HijackConfig struct { Cert string `yaml:"cert" mapstructure:"cert"` Key string `yaml:"key" mapstructure:"key"` Hosts []*HijackHost `yaml:"hosts" mapstructure:"hosts"` SNI []*TCPListenOption `yaml:"sni" mapstructure:"sni"` }
HijackConfig represents how dfdaemon hijacks http requests.
type HijackHost ¶
type HijackHost struct { Regx *Regexp `yaml:"regx" mapstructure:"regx"` Insecure bool `yaml:"insecure" mapstructure:"insecure"` Certs *CertPool `yaml:"certs" mapstructure:"certs"` }
HijackHost is a hijack rule for the hosts that matches Regx.
type HostOption ¶
type HostOption struct { // SecurityDomain is the security domain SecurityDomain string `mapstructure:"securityDomain" yaml:"securityDomain"` // Location for scheduler Location string `mapstructure:"location" yaml:"location"` // IDC for scheduler IDC string `mapstructure:"idc" yaml:"idc"` // Peerhost net topology for scheduler NetTopology string `mapstructure:"netTopology" yaml:"netTopology"` // Hostname is daemon host name Hostname string `mapstructure:"hostname" yaml:"hostname"` // The listen ip for all tcp services of daemon ListenIP string `mapstructure:"listenIP" yaml:"listenIP"` // The ip report to scheduler, normal same with listen ip AdvertiseIP string `mapstructure:"advertiseIP" yaml:"advertiseIP"` }
type ListenOption ¶
type ListenOption struct { Security SecurityOption `mapstructure:"security" yaml:"security"` TCPListen *TCPListenOption `mapstructure:"tcpListen,omitempty" yaml:"tcpListen,omitempty"` UnixListen *UnixListenOption `mapstructure:"unixListen,omitempty" yaml:"unixListen,omitempty"` }
type ManagerOption ¶ added in v2.0.1
type ManagerOption struct { // Enable get configuration from manager Enable bool `mapstructure:"enable" yaml:"enable"` // NetAddrs is manager addresses. NetAddrs []dfnet.NetAddr `mapstructure:"netAddrs" yaml:"netAddrs"` // RefreshInterval is the refresh interval RefreshInterval time.Duration `mapstructure:"refreshInterval" yaml:"refreshInterval"` }
type NetAddrsValue ¶
type NetAddrsValue struct {
// contains filtered or unexported fields
}
func NewNetAddrsValue ¶
func NewNetAddrsValue(n *[]dfnet.NetAddr) *NetAddrsValue
func (*NetAddrsValue) Set ¶
func (nv *NetAddrsValue) Set(value string) error
func (*NetAddrsValue) String ¶
func (nv *NetAddrsValue) String() string
func (*NetAddrsValue) Type ¶
func (nv *NetAddrsValue) Type() string
type Observer ¶ added in v2.0.1
type Observer interface { // OnNotify allows an event to be "published" to interface implementations. OnNotify(*DynconfigData) }
type Proxy ¶
type Proxy struct { Regx *Regexp `yaml:"regx" mapstructure:"regx"` UseHTTPS bool `yaml:"useHTTPS" mapstructure:"useHTTPS"` Direct bool `yaml:"direct" mapstructure:"direct"` // Redirect is the host to redirect to, if not empty Redirect string `yaml:"redirect" mapstructure:"redirect"` }
Proxy describes a regular expression matching rule for how to proxy a request.
type ProxyOption ¶
type ProxyOption struct { // WARNING: when add more option, please update ProxyOption.unmarshal function ListenOption `mapstructure:",squash" yaml:",inline"` BasicAuth *BasicAuth `mapstructure:"basicAuth" yaml:"basicAuth"` DefaultFilter string `mapstructure:"defaultFilter" yaml:"defaultFilter"` MaxConcurrency int64 `mapstructure:"maxConcurrency" yaml:"maxConcurrency"` RegistryMirror *RegistryMirror `mapstructure:"registryMirror" yaml:"registryMirror"` WhiteList []*WhiteList `mapstructure:"whiteList" yaml:"whiteList"` Proxies []*Proxy `mapstructure:"proxies" yaml:"proxies"` HijackHTTPS *HijackConfig `mapstructure:"hijackHTTPS" yaml:"hijackHTTPS"` DumpHTTPContent bool `mapstructure:"dumpHTTPContent" yaml:"dumpHTTPContent"` }
func (*ProxyOption) UnmarshalJSON ¶
func (p *ProxyOption) UnmarshalJSON(b []byte) error
func (*ProxyOption) UnmarshalYAML ¶
func (p *ProxyOption) UnmarshalYAML(node *yaml.Node) error
type RateLimitValue ¶
type RateLimitValue struct {
// contains filtered or unexported fields
}
func NewLimitRateValue ¶
func NewLimitRateValue(rate *clientutil.RateLimit) *RateLimitValue
func (*RateLimitValue) Set ¶
func (r *RateLimitValue) Set(s string) error
func (*RateLimitValue) String ¶
func (r *RateLimitValue) String() string
func (*RateLimitValue) Type ¶
func (r *RateLimitValue) Type() string
type Regexp ¶
Regexp is a simple wrapper around regexp. Regexp to make it unmarshallable from a string.
func (*Regexp) MarshalJSON ¶
MarshalJSON implements json.Marshaller to print the regexp.
func (*Regexp) MarshalYAML ¶
MarshalYAML implements yaml.Marshaller to print the regexp.
func (*Regexp) UnmarshalJSON ¶
UnmarshalJSON implements json.Unmarshaler.
func (*Regexp) UnmarshalYAML ¶
UnmarshalYAML implements yaml.Unmarshaler.
type RegistryMirror ¶
type RegistryMirror struct { // Remote url for the registry mirror, default is https://index.docker.io Remote *URL `yaml:"url" mapstructure:"url"` // DynamicRemote indicates using header "X-Dragonfly-Registry" for remote instead of Remote // if header "X-Dragonfly-Registry" does not exist, use Remote by default DynamicRemote bool `yaml:"dynamic" mapstructure:"dynamic"` // Optional certificates if the mirror uses self-signed certificates Certs *CertPool `yaml:"certs" mapstructure:"certs"` // Whether to ignore certificates errors for the registry Insecure bool `yaml:"insecure" mapstructure:"insecure"` // Request the remote registry directly. Direct bool `yaml:"direct" mapstructure:"direct"` }
RegistryMirror configures the mirror of the official docker registry
func (*RegistryMirror) TLSConfig ¶
func (r *RegistryMirror) TLSConfig() *tls.Config
TLSConfig returns the tls.Config used to communicate with the mirror.
type SchedulerOption ¶
type SchedulerOption struct { // Manager is to get the scheduler configuration remotely Manager ManagerOption `mapstructure:"manager" yaml:"manager"` // NetAddrs is scheduler addresses. NetAddrs []dfnet.NetAddr `mapstructure:"netAddrs" yaml:"netAddrs"` // ScheduleTimeout is request timeout. ScheduleTimeout clientutil.Duration `mapstructure:"scheduleTimeout" yaml:"scheduleTimeout"` // DisableAutoBackSource indicates not back source normally, only scheduler says back source DisableAutoBackSource bool `mapstructure:"disableAutoBackSource" yaml:"disableAutoBackSource"` }
type SecurityOption ¶
type SecurityOption struct { // Insecure indicate enable tls or not Insecure bool `mapstructure:"insecure" yaml:"insecure"` CACert string `mapstructure:"caCert" yaml:"caCert"` Cert string `mapstructure:"cert" yaml:"cert"` Key string `mapstructure:"key" yaml:"key"` TLSConfig *tls.Config `mapstructure:"tlsConfig" yaml:"tlsConfig"` }
type StorageOption ¶
type StorageOption struct { // DataPath indicates directory which stores temporary files for p2p uploading DataPath string `mapstructure:"dataPath" yaml:"dataPath"` // TaskExpireTime indicates caching duration for which cached file keeps no accessed by any process, // after this period cache file will be gc TaskExpireTime clientutil.Duration `mapstructure:"taskExpireTime" yaml:"taskExpireTime"` // DiskGCThreshold indicates the threshold to gc the oldest tasks DiskGCThreshold unit.Bytes `mapstructure:"diskGCThreshold" yaml:"diskGCThreshold"` // Multiplex indicates reusing underlying storage for same task id Multiplex bool `mapstructure:"multiplex" yaml:"multiplex"` StoreStrategy StoreStrategy `mapstructure:"strategy" yaml:"strategy"` }
type StoreStrategy ¶
type StoreStrategy string
type SupernodesValue ¶
type SupernodesValue struct {
Nodes []string
}
func (*SupernodesValue) Set ¶
func (sv *SupernodesValue) Set(value string) error
Set implements the pflag.Value interface.
func (*SupernodesValue) String ¶
func (sv *SupernodesValue) String() string
String implements the pflag.Value interface.
func (*SupernodesValue) Type ¶
func (sv *SupernodesValue) Type() string
Type implements the pflag.Value interface.
type TCPListenOption ¶
type TCPListenOption struct { // Listen stands listen interface, like: 0.0.0.0, 192.168.0.1 Listen string `mapstructure:"listen" yaml:"listen"` // PortRange stands listen port // yaml example 1: // port: 12345 // yaml example 2: // port: // start: 12345 // end: 12346 PortRange TCPListenPortRange `mapstructure:"port" yaml:"port"` // Namespace stands the linux net namespace, like /proc/1/ns/net // It's useful for running daemon in pod with ip allocated and listen in host Namespace string `mapstructure:"namespace" yaml:"namespace"` }
type TCPListenPortRange ¶
func (*TCPListenPortRange) UnmarshalJSON ¶
func (t *TCPListenPortRange) UnmarshalJSON(b []byte) error
func (*TCPListenPortRange) UnmarshalYAML ¶
func (t *TCPListenPortRange) UnmarshalYAML(node *yaml.Node) error
type TLSConfig ¶
func (*TLSConfig) UnmarshalJSON ¶
type TelemetryOption ¶
type TelemetryOption struct {
Jaeger string `yaml:"jaeger" mapstructure:"jaeger"`
}
TelemetryOption is the option for telemetry
type URL ¶
URL is simple wrapper around url.URL to make it unmarshallable from a string.
func (*URL) MarshalJSON ¶
MarshalJSON implements json.Marshaller to print the url.
func (*URL) MarshalYAML ¶
MarshalYAML implements yaml.Marshaller to print the url.
func (*URL) UnmarshalJSON ¶
UnmarshalJSON implements json.Unmarshaler.
func (*URL) UnmarshalYAML ¶
UnmarshalYAML implements yaml.Unmarshaler.
type UnixListenOption ¶
type UnixListenOption struct {
Socket string `mapstructure:"socket" yaml:"socket"`
}
type UploadOption ¶
type UploadOption struct { ListenOption `yaml:",inline" mapstructure:",squash"` RateLimit clientutil.RateLimit `mapstructure:"rateLimit" yaml:"rateLimit"` }