sciencebase

Documentation

Index

• Variables
• func ArchiveCatalog(store datastore.Datastore, ipfsApiUrl string, col *core.Collection, ...) error
• func ArchiveChild(store datastore.Datastore, ipfsApiUrl string, collection *core.Collection, ...) error
• func NewAddCatalogTree() tasks.Taskable
• type AddCatalogTree
  • func (t *AddCatalogTree) Do(pch chan tasks.Progress)
  • func (t *AddCatalogTree) SetDatastore(store datastore.Datastore)
  • func (t *AddCatalogTree) Valid() error

Variables

var IpfsApiServerUrl = ""

Functions

func ArchiveCatalog

func ArchiveCatalog(store datastore.Datastore, ipfsApiUrl string, col *core.Collection, index *cdxj.Writer, rootUrl string, maxDepth, parallelism int) error

func ArchiveChild

func ArchiveChild(store datastore.Datastore, ipfsApiUrl string, collection *core.Collection, index *cdxj.Writer, child childItem, visit, visited chan childItem) error

func NewAddCatalogTree

func NewAddCatalogTree() tasks.Taskable

Types

type AddCatalogTree

type AddCatalogTree struct {
	// title for collection if no collection present
	CollectionTitle string
	// url that points to catalog
	Url string `json:"url"`
	// how many items deep to crawl at most, -1 == no max
	MaxDepth int
	// how many fetching goroutines to spin up. max 5
	Parallelism int
	// skip items that already have a hash value
	SkipArchived bool
	// how long to sleep between requests in seconds(inside of parallel routines)
	CrawDelay time.Duration
	// contains filtered or unexported fields
}

AddCatalogTree injests a collection to IPFS, it iterates through each setting hashes on collection urls and, eventually, generates a cdxj index of the archive

func (*AddCatalogTree) Do

func (t *AddCatalogTree) Do(pch chan tasks.Progress)

func (*AddCatalogTree) SetDatastore

func (t *AddCatalogTree) SetDatastore(store datastore.Datastore)

AddCatalogTree task needs to talk to an underlying database it's expected that the task executor will call this method before calling Do

func (*AddCatalogTree) Valid

func (t *AddCatalogTree) Valid() error