crawler

Documentation

Index

• Constants
• Variables
• type Crawler
  • func New(resourceURL string, visitor Visitor) *Crawler
  • func NewWithOptions(resourceUrl string, visitor Visitor, options *Options) *Crawler
  • func (c *Crawler) Crawl(ctx context.Context) error
• type Link
• type Options
• type RecursionType
• type Resource
  • func (r Resource) Extensions() []string
  • func (r Resource) Links() []Link
  • func (r Resource) Type() ResourceType
  • func (r Resource) Version() string
• type ResourceType
• type Visitor

Constants

const (
	Item       = ResourceType("item")
	Catalog    = ResourceType("catalog")
	Collection = ResourceType("collection")
)

Variables

var DefaultOptions = &Options{
	Recursion:   Children,
	Concurrency: 1,
}

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func New

func New(resourceURL string, visitor Visitor) *Crawler

func NewWithOptions

func NewWithOptions(resourceUrl string, visitor Visitor, options *Options) *Crawler

func (*Crawler) Crawl

func (c *Crawler) Crawl(ctx context.Context) error

type Link

type Link map[string]string

type Options

type Options struct {
	Recursion   RecursionType
	Concurrency int
}

type RecursionType

type RecursionType string

const (
	All      RecursionType = "all"
	None     RecursionType = "none"
	Children RecursionType = "children"
)

type Resource

type Resource map[string]interface{}

func (Resource) Extensions

func (r Resource) Extensions() []string

func (Resource) Links

func (r Resource) Links() []Link

func (Resource) Type

func (r Resource) Type() ResourceType

func (Resource) Version

func (r Resource) Version() string

type ResourceType

type ResourceType string

type Visitor

type Visitor func(string, Resource) error