cmd

Documentation

Index

• Constants
• Variables
• func ClearlinksMain(cmd *cobra.Command, _ []string) error
• func CrawlManagerMiddleware(manager *crawler.CrawlManager) echo.MiddlewareFunc
• func Execute()
• func InitializeManager(redisClient prowlredis.ClientInterface, appLogger logger.Logger) (*crawler.CrawlManager, error)

Constants

const (
	CertPathEnvKey string = "SSL_CERT_PATH"
	KeyPathEnvKey  string = "SSL_KEY_PATH"
)

Variables

var (
	Debug  bool
	Siteid string
)

var ClearlinksCmd = &cobra.Command{
	Use:   "clearlinks",
	Short: "Clear the Redis set for a given siteid",
	RunE:  ClearlinksMain,
}

var ErrCrawlManagerNotInitialized = errors.New("CrawlManager is not initialized")

var ErrSiteidRequired = errors.New("siteid is required")

var GetLinksCmd = &cobra.Command{
	Use:   "getlinks",
	Short: "Get the list of links for a given siteid",
	RunE: func(cmd *cobra.Command, _ []string) error {
		if Siteid == "" {
			return ErrSiteidRequired
		}

		manager, ok := cmd.Context().Value(common.CrawlManagerKey).(*crawler.CrawlManager)
		if !ok || manager == nil {
			return fmt.Errorf("CrawlManager is not initialized")
		}

		output, err := printLinks(cmd.Context(), manager, Siteid)
		if err != nil {
			log.Printf("Failed to print links: %v\n", err)
			return err
		}

		jsonOutput, err := consumer.MarshalOutput(output)
		if err != nil {
			return err
		}

		err = printJSON(jsonOutput)
		if err != nil {
			log.Printf("Failed to print JSON output: %v\n", err)
			return err
		}

		return nil
	},
}

var RootCmd = &cobra.Command{
	Use:   "page-prowler",
	Short: "A tool for finding matchlinks from websites",
	Long: `Page Prowler is a tool that finds matchlinks from websites where the URL matches provided terms. It provides functionalities for:

1. Crawling specific websites and extracting matchlinks that match the provided terms ('matchlinks' command)

	In addition to the command line interface, Page Prowler also provides an HTTP API for interacting with the tool.`,
	SilenceErrors: false,
	PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {

		ctx := context.Background()

		appLogger, err := initializeLogger()
		if err != nil {
			log.Println("Error initializing logger:", err)
			return err
		}

		redisHost := viper.GetString("REDIS_HOST")
		redisPort := viper.GetString("REDIS_PORT")
		redisAuth := viper.GetString("REDIS_AUTH")

		if redisHost == "" {
			log.Println("REDIS_HOST is not set but is required")
			return fmt.Errorf("REDIS_HOST is not set but is required")
		}

		if redisPort == "" {
			log.Println("REDIS_PORT is not set but is required")
			return fmt.Errorf("REDIS_PORT is not set but is required")
		}

		cfg := &prowlredis.Options{
			Addr:     fmt.Sprintf("%s:%s", redisHost, redisPort),
			Password: redisAuth,
			DB:       0,
		}

		redisClient, err := prowlredis.NewClient(ctx, cfg)
		if err != nil {
			log.Printf("Failed to initialize Redis client: %v", err)
			return fmt.Errorf("failed to initialize Redis client: %v", err)
		}

		manager, err := InitializeManager(redisClient, appLogger)
		if err != nil {
			log.Printf("Error initializing manager: %v", err)
			return err
		}

		ctx = context.WithValue(ctx, common.CrawlManagerKey, manager)

		cmd.SetContext(ctx)

		return nil
	},
}

Functions

func ClearlinksMain

func ClearlinksMain(cmd *cobra.Command, _ []string) error

func CrawlManagerMiddleware

func CrawlManagerMiddleware(manager *crawler.CrawlManager) echo.MiddlewareFunc

func Execute

func Execute()

Execute adds all child commands to the root command and sets flags appropriately. This is called by main.main(). It only needs to happen once to the RootCmd.

func InitializeManager

func InitializeManager(
	redisClient prowlredis.ClientInterface,
	appLogger logger.Logger,
) (*crawler.CrawlManager, error)