crawler

package
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 31, 2022 License: Apache-2.0 Imports: 11 Imported by: 0

Documentation

Index

Constants

View Source
const (
	Item       = ResourceType("item")
	Catalog    = ResourceType("catalog")
	Collection = ResourceType("collection")
)

Variables

View Source
var DefaultOptions = &Options{
	Recursion:   Children,
	Concurrency: 1,
}

Functions

This section is empty.

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func New

func New(resourceURL string, visitor Visitor) *Crawler

func NewWithOptions

func NewWithOptions(resourceUrl string, visitor Visitor, options *Options) *Crawler

func (*Crawler) Crawl

func (c *Crawler) Crawl(ctx context.Context) error
type Link map[string]string

type Options

type Options struct {
	Recursion   RecursionType
	Concurrency int
}

type RecursionType

type RecursionType string
const (
	All      RecursionType = "all"
	None     RecursionType = "none"
	Children RecursionType = "children"
)

type Resource

type Resource map[string]interface{}

func (Resource) Extensions

func (r Resource) Extensions() []string
func (r Resource) Links() []Link

func (Resource) Type

func (r Resource) Type() ResourceType

func (Resource) Version

func (r Resource) Version() string

type ResourceType

type ResourceType string

type Visitor

type Visitor func(string, Resource) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL