spiderconfig

Documentation

Index

• type Config
  • func Load() (*Config, error)
  • func (c *Config) GetJobsFrequency() time.Duration
  • func (c *Config) GetLoggerContextExtraFields() []string
  • func (c *Config) GetLoggerFormatter() string
  • func (c *Config) GetLoggerLevel() string
  • func (c *Config) GetLoggerOutput() string
  • func (c *Config) GetMongoDatabase() string
  • func (c *Config) GetMongoHost() string
  • func (c *Config) GetMongoLogin() string
  • func (c *Config) GetMongoPagesCollection() string
  • func (c *Config) GetMongoPassword() string
  • func (c *Config) GetMongoPort() int
  • func (c *Config) GetMongoRequestTimeout() time.Duration
  • func (c *Config) GetRequestRetries() int
  • func (c *Config) GetTasksConcurrencyLimit() int
  • func (c *Config) GetTasksPerSecondLimit() int
  • func (c *Config) GetTimeoutPerURL() time.Duration
  • func (c *Config) GetURLsFilepath() string
  • func (c *Config) GetUserAgents() []string
• type UserAgents
  • func (c *UserAgents) Decode(value string) error

Types

type Config

type Config struct {
	// LoggerLevel: info, debug, warning, error, fatal, panic.
	LoggerLevel string `envconfig:"LOGGER_LEVEL"  default:"debug"`
	// LoggerLevel: /dev/null/, stdout, stderr, or filename (logs will store in the {projectRoot}/var/log dir.).
	LoggerOutput string `envconfig:"LOGGER_OUTPUT" default:"stdout"`
	// LoggerFormatter: text, json.
	LoggerFormatter string `envconfig:"LOGGER_FORMAT" default:"json"`
	// LoggerContextExtraFields determines which fields must be extract from
	// context.Context and passed into log record (see more into ctxenum package).
	LoggerContextExtraFields []string `envconfig:"LOGGER_CONTEXT_EXTRA_FIELD" default:"jobId,taskId"`

	// MongoHost is a host from docker-compose (mongodb container name).
	MongoHost string `envconfig:"MONGO_HOST" default:"mongodb"`
	// MongoPort is an exposed port of mongodb.
	MongoPort int `envconfig:"MONGO_PORT" default:"27017"`
	// MongoLogin is a login for simple auth.
	MongoLogin string `envconfig:"MONGO_LOGIN" default:"spider"`
	// MongoPassword is a password for simple auth.
	MongoPassword string `envconfig:"MONGO_PASSWORD" default:"spider"`
	// MongoDatabase is a name of target mongodb.
	MongoDatabase string `envconfig:"MONGO_DATABASE" default:"spider"`
	// MongoRequestTimeout is a timeout per request to mongodb.
	MongoRequestTimeout time.Duration `envconfig:"MONGO_REQUEST_TIMEOUT" default:"5s"`

	// URLsFilepath is an additional path to file with URLs.
	URLsFilepath string `envconfig:"URLS_FILEPATH" default:"/public/data/urls_test.csv"`
	// JobFrequency is an interval between jobs running (interval between spider execution).
	JobFrequency time.Duration `envconfig:"JOB_FREQUENCY" default:"1h"`
	// TasksPerSecondLimit indicates how many tasks will be processed ber one second.
	TasksPerSecondLimit int `envconfig:"TASKS_PER_SECOND_LIMIT" default:"10"`
	// TasksConcurrencyLimit indicates how many tasks will be processed at the same time.
	TasksConcurrencyLimit int `envconfig:"TASKS_CONCURRENCY_LIMIT" default:"10"`
	// TimeoutPerURL is a timeout per request.
	TimeoutPerURL time.Duration `envconfig:"TIMEOUT_PER_URL" default:"1m"`
	UserAgents    UserAgents    `` /* 233-byte string literal not displayed */
	//RequestRetries is a value that is used to determine the number of repetitions of the request if an error occurs.
	RequestRetries int `envconfig:"REQUEST_RETRIES" default:"3"`
	// MongoPagesCollection is a name of page entities collection.
	MongoPagesCollection string `envconfig:"MONGO_PAGES_COLLECTION" default:"pages"`
}

func Load

func Load() (*Config, error)

func (*Config) GetJobsFrequency

func (c *Config) GetJobsFrequency() time.Duration

func (*Config) GetLoggerContextExtraFields

func (c *Config) GetLoggerContextExtraFields() []string

func (*Config) GetLoggerFormatter

func (c *Config) GetLoggerFormatter() string

func (*Config) GetLoggerLevel

func (c *Config) GetLoggerLevel() string

func (*Config) GetLoggerOutput

func (c *Config) GetLoggerOutput() string

func (*Config) GetMongoDatabase

func (c *Config) GetMongoDatabase() string

func (*Config) GetMongoHost

func (c *Config) GetMongoHost() string

func (*Config) GetMongoLogin

func (c *Config) GetMongoLogin() string

func (*Config) GetMongoPagesCollection

func (c *Config) GetMongoPagesCollection() string

func (*Config) GetMongoPassword

func (c *Config) GetMongoPassword() string

func (*Config) GetMongoPort

func (c *Config) GetMongoPort() int

func (*Config) GetMongoRequestTimeout

func (c *Config) GetMongoRequestTimeout() time.Duration

func (*Config) GetRequestRetries

func (c *Config) GetRequestRetries() int

func (*Config) GetTasksConcurrencyLimit

func (c *Config) GetTasksConcurrencyLimit() int

func (*Config) GetTasksPerSecondLimit

func (c *Config) GetTasksPerSecondLimit() int

func (*Config) GetTimeoutPerURL

func (c *Config) GetTimeoutPerURL() time.Duration

func (*Config) GetURLsFilepath

func (c *Config) GetURLsFilepath() string

func (*Config) GetUserAgents

func (c *Config) GetUserAgents() []string

type UserAgents

type UserAgents []string

func (*UserAgents) Decode

func (c *UserAgents) Decode(value string) error