config

Documentation

Index

• Constants
• type ArrayConfig
• type BaseField
• type BrowserConnectorConfig
• type ChromiumConfig
• type CliItem
• type Config
• type Connector
• type ConnectorConfig
• type DockerConfig
• type Field
• type FieldType
• type FormattedFieldConfig
• type GeneratedFieldConfig
• type GeneratedFieldType
• type HTTPTrigger
• type HostRequestLimiter
• type Item
• type Limits
• type Model
• type ModelField
• type ModelType
• type NotifierConfig
• type ObjectConfig
• type ParserType
• type PlaywrightBrowser
• type PlaywrightConfig
• type SchedulerTrigger
• type ServerConnectorConfig
• type StaticGeneratedFieldConfig
• type TriggerConfig
• type UUIDGeneratedFieldConfig

Constants

const (
	Browser Connector = "browser"
	Server  Connector = "server"

	HTML  ParserType = "HTML"
	Json  ParserType = "json"
	XML   ParserType = "XML"
	XPath ParserType = "xpath"

	ObjectModel ModelType = "object"
	ArrayModel  ModelType = "array"
)

const (
	Null    FieldType = "null"
	Bool    FieldType = "boolean"
	String  FieldType = "string"
	Int     FieldType = "int"
	Int64   FieldType = "int64"
	Float   FieldType = "float"
	Float64 FieldType = "float64"

	Array  GeneratedFieldType = "array"
	Object GeneratedFieldType = "object"
)

Types

type ArrayConfig

type ArrayConfig struct {
	RootPath   string        `json:"root_path" yaml:"root_path"`
	ItemConfig *ObjectConfig `json:"item_config" yaml:"item_config"`
}

type BaseField

type BaseField struct {
	Type FieldType `yaml:"type" json:"type"`
	Path string    `yaml:"path" json:"path"`

	Generated *GeneratedFieldConfig `yaml:"generated" json:"generated"`
}

type BrowserConnectorConfig

type BrowserConnectorConfig struct {
	Url        string            `json:"url" yaml:"url"`
	Chromium   *ChromiumConfig   `json:"chromium" yaml:"chromium"`
	Docker     *DockerConfig     `json:"docker" yaml:"docker"`
	Playwright *PlaywrightConfig `json:"playwright" yaml:"playwright"`
}

type ChromiumConfig

type ChromiumConfig struct {
	Path    string   `yaml:"path" json:"path"`
	Timeout uint32   `yaml:"timeout" json:"timeout"`
	Wait    uint32   `yaml:"wait" json:"wait"`
	Flags   []string `yaml:"flags" json:"flags"`
}

type CliItem

type CliItem struct {
	Item   *Item   `yaml:"item" json:"item"`
	Limits *Limits `yaml:"limits" json:"limits"`
}

type Config

type Config struct {
	Items  []*Item `yaml:"items" json:"items"`
	Limits *Limits `yaml:"limits" json:"limits"`
}

type Connector

type Connector string

type ConnectorConfig

type ConnectorConfig struct {
	ResponseType  ParserType              `json:"response_type" yaml:"response_type"`
	ConnectorType Connector               `json:"connector_type" yaml:"connector_type"`
	ServerConfig  *ServerConnectorConfig  `json:"server_config" yaml:"server_config"`
	BrowserConfig *BrowserConnectorConfig `yaml:"browser_config" json:"browser_config"`
	Attempts      uint32                  `json:"attempts" yaml:"attempts"`
}

type DockerConfig

type DockerConfig struct {
	Image       string   `yaml:"image" json:"image"`
	EntryPoint  string   `json:"entry_point" yaml:"entry_point"`
	Timeout     uint32   `yaml:"timeout" json:"timeout"`
	Wait        uint32   `yaml:"wait" json:"wait"`
	Flags       []string `yaml:"flags" json:"flags"`
	Purge       bool     `json:"purge" yaml:"purge"`
	NoPull      bool     `yaml:"no_pull" json:"no_pull"`
	PullTimeout uint32   `yaml:"pull_timeout" json:"pull_timeout"`
}

type Field

type Field struct {
	BaseField    *BaseField    `json:"base_field" yaml:"base_field"`
	ObjectConfig *ObjectConfig `yaml:"object_config" yaml:"object_config"`
	ArrayConfig  *ArrayConfig  `json:"array_config" yaml:"array_config"`
}

type FieldType

type FieldType string

type FormattedFieldConfig

type FormattedFieldConfig struct {
	Template string `yaml:"template" json:"template"`
}

type GeneratedFieldConfig

type GeneratedFieldConfig struct {
	UUID      *UUIDGeneratedFieldConfig   `yaml:"uuid" json:"uuid"`
	Static    *StaticGeneratedFieldConfig `yaml:"static" json:"static"`
	Formatted *FormattedFieldConfig       `json:"formatted" yaml:"formatted"`
	Model     *ModelField                 `yaml:"model" json:"model"`
}

type GeneratedFieldType

type GeneratedFieldType FieldType

type HTTPTrigger

type HTTPTrigger struct {
}

type HostRequestLimiter

type HostRequestLimiter map[string]int64

type Item

type Item struct {
	Name string `yaml:"name" json:"name"`

	// Type of parsing
	ConnectorConfig *ConnectorConfig `yaml:"connector_config" json:"connector_config"`

	// TriggerConfig
	TriggerConfig *TriggerConfig `yaml:"trigger_config" json:"trigger_config"`
	// Model of the response
	Model *Model `yaml:"model" json:"model"`
	// Where to report result
	NotifierConfig *NotifierConfig `json:"notifier_config" yaml:"notifier_config"`
}

type Limits

type Limits struct {
	HostRequestLimiter HostRequestLimiter `yaml:"host_request_limiter" json:"host_request_limiter"`
	ChromiumInstance   uint32             `yaml:"chromium_instance" json:"chromium_instance"`
	DockerContainers   uint32             `yaml:"docker_containers" json:"docker_containers"`
	PlaywrightInstance uint32             `yaml:"playwright_instance" json:"playwright_instance"`
}

type Model

type Model struct {
	Type         ModelType     `yaml:"type" json:"type"`
	ObjectConfig *ObjectConfig `yaml:"object_config" json:"object_config"`
	ArrayConfig  *ArrayConfig  `json:"array_config" yaml:"array_config"`
}

type ModelField

type ModelField struct {
	// Type of parsing
	ConnectorConfig *ConnectorConfig `yaml:"connector_config" json:"connector_config"`
	// Model of the response
	Model *Model `yaml:"model" json:"model"`

	Type GeneratedFieldType `yaml:"type" json:"type"`
	Path string             `yaml:"path" json:"path"`
}

type ModelType

type ModelType string

type NotifierConfig

type NotifierConfig struct {
	Console bool `yaml:"console" json:"console"`
}

type ObjectConfig

type ObjectConfig struct {
	Field       *BaseField        `json:"field" yaml:"field"`
	Fields      map[string]*Field `json:"fields" yaml:"fields"`
	ArrayConfig *ArrayConfig      `json:"array_config" yaml:"array_config"`
}

type ParserType

type ParserType string

type PlaywrightBrowser

type PlaywrightBrowser string

const (
	Chromium PlaywrightBrowser = "Chromium"
	FireFox  PlaywrightBrowser = "FireFox"
	WebKit   PlaywrightBrowser = "WebKit"
)

type PlaywrightConfig

type PlaywrightConfig struct {
	Browser    PlaywrightBrowser          `json:"browser" yaml:"browser"`
	Install    bool                       `yaml:"install" json:"install"`
	Timeout    uint32                     `yaml:"timeout" json:"timeout"`
	Wait       uint32                     `yaml:"wait" json:"wait"`
	TypeOfWait *playwright.WaitUntilState `json:"type_of_wait" yaml:"type_of_wait"`
}

type SchedulerTrigger

type SchedulerTrigger struct {
	// Interval for update rerun process in second
	Interval int `yaml:"interval" json:"interval"`
}

type ServerConnectorConfig

type ServerConnectorConfig struct {
	Method  string            `json:"method" yaml:"method"`
	Headers map[string]string `yaml:"headers" json:"headers"`
	Url     string            `json:"url" yaml:"url"`
}

type StaticGeneratedFieldConfig

type StaticGeneratedFieldConfig struct {
	Type  FieldType `yaml:"type" json:"type"`
	Value string    `json:"value" yaml:"value"`
}

type TriggerConfig

type TriggerConfig struct {
	SchedulerTrigger *SchedulerTrigger `yaml:"scheduler_trigger" json:"scheduler_trigger"`
	HTTPTrigger      *HTTPTrigger      `json:"http_trigger" yaml:"http_trigger"`
}

type UUIDGeneratedFieldConfig

type UUIDGeneratedFieldConfig struct {
	Regexp string `yaml:"regexp" json:"regexp"`
}