dhtcrawler

Documentation

Index

• type Config
  • func NewDefaultConfig() Config
• type DiscoveredNodesParams
• type DiscoveredNodesResult
  • func NewDiscoveredNodes(params DiscoveredNodesParams) DiscoveredNodesResult
• type Params
• type Result
  • func New(params Params) Result

Types

type Config

type Config struct {
	// ScalingFactor is a rough proxy for resource usage of the crawler; concurrency and buffer size of the various
	// pipeline channels are multiplied by this value. Diminishing returns may result from exceeding the default value of 10.
	// Since the software has not been tested on a wide variety of hardware and network conditions your mileage may vary here...
	ScalingFactor                uint
	BootstrapNodes               []string
	ReseedBootstrapNodesInterval time.Duration
	// SaveFilesThreshold specifies a maximum number of files in a torrent before file information is discarded.
	// Some torrents contain thousands of files which can severely impact performance and uses a lot of disk space.
	SaveFilesThreshold uint
	// SavePieces when true, torrent pieces will be persisted to the database.
	// The pieces take up quite a lot of space, and aren't currently very useful, but they may be used by future features.
	SavePieces bool
	// RescrapeThreshold is the amount of time that must pass before a torrent is rescraped to count seeders and leechers.
	RescrapeThreshold time.Duration
}

func NewDefaultConfig

func NewDefaultConfig() Config

type DiscoveredNodesParams

type DiscoveredNodesParams struct {
	fx.In
	Config Config
}

type DiscoveredNodesResult

type DiscoveredNodesResult struct {
	fx.Out
	DiscoveredNodes concurrency.BatchingChannel[ktable.Node] `name:"dht_discovered_nodes"`
}

func NewDiscoveredNodes

func NewDiscoveredNodes(params DiscoveredNodesParams) DiscoveredNodesResult

NewDiscoveredNodes creates the channel for discovered nodes. It receives nodes discovered by the crawler, as well as nodes from incoming requests to the DHT server. It is provided as a separate service to avoid a circular dependency with the DHT server.

type Params

type Params struct {
	fx.In
	Config            Config
	KTable            ktable.Table
	Client            lazy.Lazy[client.Client]
	MetainfoRequester metainforequester.Requester
	BanningChecker    banning.Checker `name:"metainfo_banning_checker"`
	Search            lazy.Lazy[search.Search]
	Dao               lazy.Lazy[*dao.Query]
	BlockingManager   lazy.Lazy[blocking.Manager]
	DiscoveredNodes   concurrency.BatchingChannel[ktable.Node] `name:"dht_discovered_nodes"`
	Logger            *zap.SugaredLogger
}

type Result

type Result struct {
	fx.Out
	Worker         worker.Worker        `group:"workers"`
	PersistedTotal prometheus.Collector `group:"prometheus_collectors"`
}

func New

func New(params Params) Result