crawl

Documentation

Index

• Variables

Variables

var CrawlCmd = &cobra.Command{
	Use:   "crawl [nodes file]",
	Short: "Crawl a network on the devp2p layer and generate a nodes JSON file.",
	Long:  "If no nodes.json file exists, it will be created.",
	Args:  cobra.MinimumNArgs(1),
	PreRunE: func(cmd *cobra.Command, args []string) (err error) {
		inputCrawlParams.NodesFile = args[0]

		inputCrawlParams.timeout, err = time.ParseDuration(inputCrawlParams.Timeout)
		if err != nil {
			return err
		}

		inputCrawlParams.revalidationInterval, err = time.ParseDuration(inputCrawlParams.RevalidationInterval)
		if err != nil {
			return err
		}

		return nil
	},
	RunE: func(cmd *cobra.Command, args []string) error {

		if inputCrawlParams.DiscoveryDNS != "" {
			dnsclient := dnsdisc.NewClient(dnsdisc.Config{})
			tree, err := dnsclient.SyncTree(inputCrawlParams.DiscoveryDNS)
			if err != nil {
				return err
			}
			return p2p.WriteDNSTreeNodes(inputCrawlParams.NodesFile, tree)
		}

		nodes, err := p2p.ReadNodeSet(inputCrawlParams.NodesFile)
		if err != nil {
			log.Warn().Err(err).Msgf("Creating nodes file %v because it does not exist", inputCrawlParams.NodesFile)
		}

		var cfg discover.Config
		cfg.PrivateKey, _ = crypto.GenerateKey()
		cfg.Bootnodes, err = p2p.ParseBootnodes(inputCrawlParams.Bootnodes)
		if err != nil {
			return fmt.Errorf("unable to parse bootnodes: %w", err)
		}

		db, err := enode.OpenDB(inputCrawlParams.Database)
		if err != nil {
			return err
		}

		ln := enode.NewLocalNode(db, cfg.PrivateKey)
		socket, err := p2p.Listen(ln)
		if err != nil {
			return err
		}

		disc, err := discover.ListenV4(socket, ln, cfg)
		if err != nil {
			return err
		}
		defer disc.Close()

		c := newCrawler(nodes, disc, disc.RandomNodes())
		c.revalidateInterval = inputCrawlParams.revalidationInterval

		log.Info().Msg("Starting crawl")

		output := c.run(inputCrawlParams.timeout, inputCrawlParams.Threads)

		if inputCrawlParams.OnlyURLs {
			return p2p.WriteURLs(inputCrawlParams.NodesFile, output)
		}

		return p2p.WriteNodeSet(inputCrawlParams.NodesFile, output, false)

	},
}

crawlCmd represents the crawl command. This is responsible for crawling the devp2p layer and generating a nodes json file with peers.