config

package
v1.0.15 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 28, 2024 License: MIT Imports: 2 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type ArrayConfig 

type ArrayConfig struct {
	RootPath    string        `json:"root_path" yaml:"root_path"`
	Reverse     bool          `yaml:"reverse" json:"reverse"`
	ItemConfig  *ObjectConfig `json:"item_config" yaml:"item_config"`
	LengthLimit uint32        `json:"length_limit" yaml:"length_limit"`

	StaticConfig *StaticArrayConfig `json:"static_array"  yaml:"static_array"`
}

type BaseField 

type BaseField struct {
	Type FieldType `yaml:"type" json:"type"`
	Path string    `yaml:"path" json:"path"`

	HTMLAttribute string `json:"html_attribute" yaml:"html_attribute"`

	Generated *GeneratedFieldConfig `yaml:"generated" json:"generated"`

	FirstOf []*BaseField `json:"first_of" yaml:"first_of"`
}

type BrowserConnectorConfig 

type BrowserConnectorConfig struct {
	Chromium   *ChromiumConfig   `json:"chromium" yaml:"chromium"`
	Docker     *DockerConfig     `json:"docker" yaml:"docker"`
	Playwright *PlaywrightConfig `json:"playwright" yaml:"playwright"`
}

type CalculatedConfig added in v0.0.19 

type CalculatedConfig struct {
	Type       FieldType `yaml:"type" json:"type"`
	Expression string    `yaml:"expression" json:"expression"`
}

type ChromiumConfig 

type ChromiumConfig struct {
	Path    string   `yaml:"path" json:"path"`
	Timeout uint32   `yaml:"timeout" json:"timeout"`
	Wait    uint32   `yaml:"wait" json:"wait"`
	Flags   []string `yaml:"flags" json:"flags"`
}

type CliItem 

type CliItem struct {
	Item *Item `yaml:"item" json:"item"`

	Limits     *Limits `yaml:"limits" json:"limits"`
	References RefMap  `json:"references" yaml:"references"`
}

type Config 

type Config struct {
	Items []*Item `yaml:"items" json:"items"`

	Limits     *Limits `yaml:"limits" json:"limits"`
	References RefMap  `json:"references" yaml:"references"`

	HttpServer *HttpServerCfg `json:"http_server" yaml:"http_server"`
}

type Connector 

type Connector string

type ConnectorConfig 

type ConnectorConfig struct {
	ResponseType ParserType `json:"response_type" yaml:"response_type"`
	Url          string     `json:"url" yaml:"url"`
	Attempts     uint32     `json:"attempts" yaml:"attempts"`

	NullOnError bool `yaml:"null_on_error" json:"null_on_error"`

	StaticConfig          *StaticConnectorConfig      `json:"static_config" yaml:"static_config"`
	IntSequenceConfig     *IntSequenceConnectorConfig `json:"int_sequence_config" yaml:"int_sequence_config"`
	ServerConfig          *ServerConnectorConfig      `json:"server_config" yaml:"server_config"`
	BrowserConfig         *BrowserConnectorConfig     `yaml:"browser_config" json:"browser_config"`
	PluginConnectorConfig *PluginConnectorConfig      `json:"plugin_connector_config" yaml:"plugin_connector_config"`
	ReferenceConfig       *ReferenceConnectorConfig   `yaml:"reference_config" json:"reference_config"`
	FileConfig            *FileConnectorConfig        `json:"file_config" yaml:"file_config"`
}

type ConsoleConfig added in v0.0.13 

type ConsoleConfig struct {
	OnlyResult bool `json:"only_result" yaml:"only_result"`
}

type DockerConfig added in v0.0.2 

type DockerConfig struct {
	Image       string   `yaml:"image" json:"image"`
	EntryPoint  string   `json:"entry_point" yaml:"entry_point"`
	Timeout     uint32   `yaml:"timeout" json:"timeout"`
	Wait        uint32   `yaml:"wait" json:"wait"`
	Flags       []string `yaml:"flags" json:"flags"`
	Purge       bool     `json:"purge" yaml:"purge"`
	NoPull      bool     `yaml:"no_pull" json:"no_pull"`
	PullTimeout uint32   `yaml:"pull_timeout" json:"pull_timeout"`
}

type Field 

type Field struct {
	BaseField    *BaseField    `json:"base_field" yaml:"base_field"`
	ObjectConfig *ObjectConfig `json:"object_config" yaml:"object_config"`
	ArrayConfig  *ArrayConfig  `json:"array_config" yaml:"array_config"`

	FirstOf []*Field `json:"first_of" yaml:"first_of"`
}

type FieldType 

type FieldType string
const (
	Null       FieldType = "null"
	Bool       FieldType = "boolean"
	String     FieldType = "string"
	Int        FieldType = "int"
	Int64      FieldType = "int64"
	Float      FieldType = "float"
	Float64    FieldType = "float64"
	HtmlString FieldType = "html"
	RawString  FieldType = "raw_string"

	Array  FieldType = "array"
	Object FieldType = "object"
)

type FileConnectorConfig added in v0.0.39 

type FileConnectorConfig struct {
	Path          string `yaml:"path" json:"path"`
	UseFormatting bool   `yaml:"use_formatting" json:"use_formatting"`
}

type FileFieldConfig added in v0.0.23 

type FileFieldConfig struct {
	Config *ServerConnectorConfig `yaml:"config" json:"config"`

	Url      string `yaml:"url" json:"url"`
	FileName string `json:"file_name" yaml:"file_name"`
	Path     string `json:"path" yaml:"path"`
}

type FileStorageField added in v1.0.1 

type FileStorageField struct {
	Content string          `json:"content" yaml:"content"`
	Raw     json.RawMessage `json:"raw" yaml:"raw"`

	FileName string `json:"file_name" yaml:"file_name"`
	Path     string `json:"path" yaml:"path"`
	Append   bool   `json:"append" yaml:"append"`
}

type FormattedFieldConfig 

type FormattedFieldConfig struct {
	Template string `yaml:"template" json:"template"`
}

type GeneratedFieldConfig 

type GeneratedFieldConfig struct {
	UUID             *UUIDGeneratedFieldConfig   `yaml:"uuid" json:"uuid"`
	Static           *StaticGeneratedFieldConfig `yaml:"static" json:"static"`
	Formatted        *FormattedFieldConfig       `json:"formatted" yaml:"formatted"`
	Plugin           *PluginFieldConfig          `yaml:"plugin" json:"plugin"`
	Calculated       *CalculatedConfig           `yaml:"calculated" json:"calculated"`
	File             *FileFieldConfig            `yaml:"file" json:"file"`
	Model            *ModelField                 `yaml:"model" json:"model"`
	FileStorageField *FileStorageField           `json:"file_storage" yaml:"file_storage"`
}

type HTTPTrigger 

type HTTPTrigger struct {
}

type HostRequestLimiter 

type HostRequestLimiter map[string]int64

type HttpConfig added in v0.0.39 

type HttpConfig struct {
	Url     string            `yaml:"url" json:"url"`
	Method  string            `json:"method" yaml:"method"`
	Headers map[string]string `yaml:"headers" json:"headers"`
	Timeout uint32            `yaml:"timeout" json:"timeout"`
}

type HttpServerCfg added in v0.0.47 

type HttpServerCfg struct {
	Port int `yaml:"port" json:"port"`
}

type IntSequenceConnectorConfig added in v0.0.31 

type IntSequenceConnectorConfig struct {
	Start int `json:"start" yaml:"start"`
	End   int `json:"end" yaml:"end"`
	Step  int `json:"step" yaml:"step"`
}

type Item 

type Item struct {
	Name string `yaml:"name" json:"name"`

	// Type of parsing
	ConnectorConfig *ConnectorConfig `yaml:"connector_config" json:"connector_config"`

	// TriggerConfig
	TriggerConfig *TriggerConfig `yaml:"trigger_config" json:"trigger_config"`
	// Model of the response
	Model *Model `yaml:"model" json:"model"`
	// Where to report result
	NotifierConfig *NotifierConfig `json:"notifier_config" yaml:"notifier_config"`
}

type Limits 

type Limits struct {
	HostRequestLimiter HostRequestLimiter `yaml:"host_request_limiter" json:"host_request_limiter"`
	ChromiumInstance   uint32             `yaml:"chromium_instance" json:"chromium_instance"`
	DockerContainers   uint32             `yaml:"docker_containers" json:"docker_containers"`
	PlaywrightInstance uint32             `yaml:"playwright_instance" json:"playwright_instance"`
}

type Model 

type Model struct {
	ObjectConfig *ObjectConfig `yaml:"object_config" json:"object_config"`
	ArrayConfig  *ArrayConfig  `json:"array_config" yaml:"array_config"`
	BaseField    *BaseField    `json:"base_field" yaml:"base_field"`
	IsArray      bool          `json:"is_array" yaml:"is_array"`
}

type ModelField 

type ModelField struct {
	// Type of parsing
	ConnectorConfig *ConnectorConfig `yaml:"connector_config" json:"connector_config"`
	// Model of the response
	Model *Model `yaml:"model" json:"model"`

	Type       FieldType `yaml:"type" json:"type"`
	Path       string    `yaml:"path" json:"path"`
	Expression string    `yaml:"expression" json:"expression"`
}

type ModelType 

type ModelType string

type NotifierConfig 

type NotifierConfig struct {
	Expression      string `yaml:"expression" json:"expression"`
	Force           bool   `json:"force" yaml:"force"`
	SendArrayByItem bool   `yaml:"send_array_by_item" json:"send_array_by_item"`
	Template        string `yaml:"template" json:"template"`

	Console     *ConsoleConfig       `yaml:"console" json:"console"`
	TelegramBot *TelegramBotConfig   `yaml:"telegram_bot" json:"telegram_bot"`
	Http        *HttpConfig          `yaml:"http" json:"http"`
	Redis       *RedisNotifierConfig `json:"redis" yaml:"redis"`
	File        *FileStorageField    `json:"file" yaml:"file"`
}

type ObjectConfig 

type ObjectConfig struct {
	Field       *BaseField        `json:"field" yaml:"field"`
	Fields      map[string]*Field `json:"fields" yaml:"fields"`
	ArrayConfig *ArrayConfig      `json:"array_config" yaml:"array_config"`
}

type ParserType 

type ParserType string
const (
	HTML  ParserType = "HTML"
	Json  ParserType = "json"
	XML   ParserType = "XML"
	XPath ParserType = "xpath"
)

type PlaywrightBrowser added in v0.0.3 

type PlaywrightBrowser string
const (
	Chromium PlaywrightBrowser = "Chromium"
	FireFox  PlaywrightBrowser = "FireFox"
	WebKit   PlaywrightBrowser = "WebKit"
)

type PlaywrightConfig added in v0.0.3 

type PlaywrightConfig struct {
	Browser      PlaywrightBrowser          `json:"browser" yaml:"browser"`
	Install      bool                       `yaml:"install" json:"install"`
	Timeout      uint32                     `yaml:"timeout" json:"timeout"`
	Wait         uint32                     `yaml:"wait" json:"wait"`
	TypeOfWait   *playwright.WaitUntilState `json:"type_of_wait" yaml:"type_of_wait"`
	PreRunScript string                     `json:"pre_run_script" yaml:"pre_run_script"`
	Stealth      bool                       `json:"stealth" yaml:"stealth"`

	Proxy *ProxyConfig `json:"proxy" yaml:"proxy"`
}

type PluginConnectorConfig added in v0.0.20 

type PluginConnectorConfig struct {
	Name   string          `json:"name" yaml:"name"`
	Config json.RawMessage `json:"config" yaml:"config"`
}

type PluginFieldConfig added in v0.0.16 

type PluginFieldConfig struct {
	Name   string          `json:"name" yaml:"name"`
	Config json.RawMessage `json:"config" yaml:"config"`
}

type ProxyConfig added in v0.0.26 

type ProxyConfig struct {
	// Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example
	// `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128`
	// is considered an HTTP proxy.
	Server string `json:"server" yaml:"server"`
	// Optional username to use if HTTP proxy requires authentication.
	Username string `json:"username" yaml:"username"`
	// Optional password to use if HTTP proxy requires authentication.
	Password string `json:"password" yaml:"password"`
}

type RedisNotifierConfig added in v0.0.39 

type RedisNotifierConfig struct {
	Addr     string `json:"addr" yaml:"addr"`
	Password string `json:"password" yaml:"password"`
	DB       int    `json:"db" yaml:"db"`
	Channel  string `json:"channel" yaml:"channel"`
}

type RefMap added in v0.0.30 

type RefMap map[string]*Reference

type Reference added in v0.0.37 

type Reference struct {
	*ModelField

	Expire *uint32 `yaml:"expire" json:"expire"`
}

type ReferenceConnectorConfig added in v0.0.30 

type ReferenceConnectorConfig struct {
	Name string `yaml:"name" json:"name"`
}

type SchedulerTrigger 

type SchedulerTrigger struct {
	// Interval for update rerun process in second
	Interval int `yaml:"interval" json:"interval"`
}

type ServerConnectorConfig 

type ServerConnectorConfig struct {
	Method  string            `json:"method" yaml:"method"`
	Headers map[string]string `yaml:"headers" json:"headers"`
	Timeout uint32            `yaml:"timeout" json:"timeout"`
	Body    string            `yaml:"body" json:"body"`

	Proxy *ProxyConfig `yaml:"proxy" json:"proxy"`
}

type StaticArrayConfig added in v0.0.5 

type StaticArrayConfig struct {
	Items  map[uint32]*Field `yaml:"items" json:"items"`
	Length uint32            `yaml:"length" json:"length"`
}

type StaticConnectorConfig added in v0.0.10 

type StaticConnectorConfig struct {
	Value string          `json:"value" yaml:"value"`
	Raw   json.RawMessage `json:"raw" yaml:"raw"`
}

type StaticGeneratedFieldConfig 

type StaticGeneratedFieldConfig struct {
	Type  FieldType       `yaml:"type" json:"type"`
	Value string          `json:"value" yaml:"value"`
	Raw   json.RawMessage `json:"raw" yaml:"raw"`
}

type TelegramBotConfig added in v0.0.13 

type TelegramBotConfig struct {
	Token   string  `json:"token" yaml:"token"`
	UsersId []int64 `json:"users_id" yaml:"users_id"`
	Pretty  bool    `json:"pretty" yaml:"pretty"`
	OnlyMsg bool    `json:"only_msg" yaml:"only_msg"`
}

type TriggerConfig 

type TriggerConfig struct {
	SchedulerTrigger *SchedulerTrigger `yaml:"scheduler_trigger" json:"scheduler_trigger"`
	HTTPTrigger      *HTTPTrigger      `json:"http_trigger" yaml:"http_trigger"`
}

type UUIDGeneratedFieldConfig 

type UUIDGeneratedFieldConfig struct {
	Regexp string `yaml:"regexp" json:"regexp"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.