nuggit

README

nuggit

Declarative pipelines for web scraping.

Architecture

Client Flow

Documentation

Index

• Constants
• type Action
  • func (a Action) GetAction() string
  • func (a Action) GetArg(arg string) (string, bool)
  • func (a Action) GetOrDefaultArg(arg string) string
  • func (a Action) GetSpec() any
  • func (a Action) Set(key, value string)
  • func (a Action) SetAction(action string)
  • func (a Action) SetDigest(digest string)
  • func (a Action) SetName(name string)
  • func (a Action) SetOrDefault(key, value string)
• type AggColumn
• type AggOp
• type Pipe
  • func (p Pipe) GetSpec() any
• type Point
  • func NewPointFromNumber(x int) Point
  • func (p Point) AsNullable() Point
  • func (t Point) AsNumber() int
  • func (p Point) String() string
• type Rule
  • func (c *Rule) GetAlwaysTrigger() bool
  • func (c *Rule) GetHostname() string
  • func (c *Rule) GetLabels() []string
  • func (c *Rule) GetURLPattern() string
• type Scalar
• type View
  • func (v *View) GetAlias() string
  • func (v *View) GetColumns() []ViewColumn
  • func (v *View) GetSpec() any
• type ViewColumn

Constants

const (
	AggOpCount     = "count"
	AggOpSum       = "sum"
	AggOpMin       = "min"
	AggOpMax       = "max"
	AggOpAvg       = "avg"
	AggOpStringAgg = "string_agg"
)

Types

type Action

type Action map[string]string

func (Action) GetAction

func (a Action) GetAction() string

func (Action) GetArg

func (a Action) GetArg(arg string) (string, bool)

func (Action) GetOrDefaultArg

func (a Action) GetOrDefaultArg(arg string) string

func (Action) GetSpec

func (a Action) GetSpec() any

func (Action) Set

func (a Action) Set(key, value string)

func (Action) SetAction

func (a Action) SetAction(action string)

func (Action) SetDigest

func (a Action) SetDigest(digest string)

func (Action) SetName

func (a Action) SetName(name string)

func (Action) SetOrDefault

func (a Action) SetOrDefault(key, value string)

type AggColumn

type AggColumn struct {
	ViewColumn `json:",omitempty"`
	Op         AggOp  `json:"op,omitempty"`
	Arg        string `json:"arg,omitempty"`
	Distinct   bool   `json:"distinct,omitempty"`
	Expr       string `json:"expr,omitempty"`
	Filter     string `json:"filter,omitempty"`
	OrderBy    string `json:"order_by,omitempty"`
}

type AggOp

type AggOp = string

type Pipe

type Pipe struct {
	Actions []Action `json:"actions,omitempty"`
	Point   Point    `json:"point,omitempty"`
}

func (Pipe) GetSpec

func (p Pipe) GetSpec() any

type Point

type Point struct {
	Nullable bool   `json:"nullable,omitempty"`
	Scalar   Scalar `json:"scalar,omitempty"`
}

func NewPointFromNumber

func NewPointFromNumber(x int) Point

func (Point) AsNullable

func (p Point) AsNullable() Point

func (Point) AsNumber

func (t Point) AsNumber() int

func (Point) String

func (p Point) String() string

type Rule

type Rule struct {
	Hostname      string   `json:"hostname,omitempty"`
	URLPattern    string   `json:"url_pattern,omitempty"`
	AlwaysTrigger bool     `json:"always_trigger,omitempty"`
	Disable       bool     `json:"disable,omitempty"`
	Labels        []string `json:"labels,omitempty"`
}

func (*Rule) GetAlwaysTrigger

func (c *Rule) GetAlwaysTrigger() bool

func (*Rule) GetHostname

func (c *Rule) GetHostname() string

func (*Rule) GetLabels

func (c *Rule) GetLabels() []string

func (*Rule) GetURLPattern

func (c *Rule) GetURLPattern() string

type Scalar

type Scalar = string

const (
	Bytes  Scalar = "bytes"
	String Scalar = "string"
	Bool   Scalar = "bool"
	Int    Scalar = "int"
	Float  Scalar = "float"
)

type View

type View struct {
	Alias      string       `json:"alias,omitempty"`
	Columns    []ViewColumn `json:"columns,omitempty"`
	AggColumns []AggColumn  `json:"agg_columns,omitempty"`
}

func (*View) GetAlias

func (v *View) GetAlias() string

func (*View) GetColumns

func (v *View) GetColumns() []ViewColumn

func (*View) GetSpec

func (v *View) GetSpec() any

type ViewColumn

type ViewColumn struct {
	Alias string `json:"alias,omitempty"`
	Pipe  string `json:"pipe,omitempty"`
	Point Point  `json:"point,omitempty"`
}