etl

Documentation

Overview

package etl

Index

• func CorporationFromAuthority(rec marc.Record) (sirkulator.Resource, error)
• func PersonFromAuthority(rec marc.Record) (sirkulator.Resource, error)
• type FileFetch
• type HarvestNBLinksJob
  • func (h *HarvestNBLinksJob) Name() string
  • func (h *HarvestNBLinksJob) Run(ctx context.Context, w io.Writer) error
• type HarvestSNLDescriptions
  • func (h *HarvestSNLDescriptions) Name() string
  • func (h *HarvestSNLDescriptions) Run(ctx context.Context, w io.Writer) error
• type HarvestSNLLinksJob
  • func (h *HarvestSNLLinksJob) Name() string
  • func (h *HarvestSNLLinksJob) Run(ctx context.Context, w io.Writer) error
• type HarvestWikipediaLinks
  • func (h *HarvestWikipediaLinks) Name() string
  • func (h *HarvestWikipediaLinks) Run(ctx context.Context, w io.Writer) error
• type HarvestWikipediaSummaries
  • func (h *HarvestWikipediaSummaries) Name() string
  • func (h *HarvestWikipediaSummaries) Run(ctx context.Context, w io.Writer) error
• type ImportEntry
• type Ingestion
• type Ingestor
  • func NewIngestor(db *sqlitex.Pool, idx *search.Index) *Ingestor
  • func NewPreviewIngestor(db *sqlitex.Pool) *Ingestor
  • func (ig *Ingestor) Ingest(ctx context.Context, data Ingestion, persist bool) ([]sirkulator.SimpleResource, error)
  • func (ig *Ingestor) IngestISBN(ctx context.Context, id string, persist bool) ImportEntry
• type UpdateSNLDescriptions
  • func (u *UpdateSNLDescriptions) Name() string
  • func (u *UpdateSNLDescriptions) Run(ctx context.Context, w io.Writer) error

Functions

func CorporationFromAuthority

func CorporationFromAuthority(rec marc.Record) (sirkulator.Resource, error)

func PersonFromAuthority

func PersonFromAuthority(rec marc.Record) (sirkulator.Resource, error)

Types

type FileFetch

type FileFetch struct {
	ResourceID string
	URL        string
}

type HarvestNBLinksJob

type HarvestNBLinksJob struct {
	DB *sqlitex.Pool
}

func (*HarvestNBLinksJob) Name

func (h *HarvestNBLinksJob) Name() string

func (*HarvestNBLinksJob) Run

func (h *HarvestNBLinksJob) Run(ctx context.Context, w io.Writer) error

type HarvestSNLDescriptions

type HarvestSNLDescriptions struct {
	DB *sqlitex.Pool
}

func (*HarvestSNLDescriptions) Name

func (h *HarvestSNLDescriptions) Name() string

func (*HarvestSNLDescriptions) Run

func (h *HarvestSNLDescriptions) Run(ctx context.Context, w io.Writer) error

type HarvestSNLLinksJob

type HarvestSNLLinksJob struct {
	DB *sqlitex.Pool
}

func (*HarvestSNLLinksJob) Name

func (h *HarvestSNLLinksJob) Name() string

func (*HarvestSNLLinksJob) Run

func (h *HarvestSNLLinksJob) Run(ctx context.Context, w io.Writer) error

type HarvestWikipediaLinks

type HarvestWikipediaLinks struct {
	DB *sqlitex.Pool
}

func (*HarvestWikipediaLinks) Name

func (h *HarvestWikipediaLinks) Name() string

func (*HarvestWikipediaLinks) Run

func (h *HarvestWikipediaLinks) Run(ctx context.Context, w io.Writer) error

type HarvestWikipediaSummaries

type HarvestWikipediaSummaries struct {
	DB *sqlitex.Pool
}

func (*HarvestWikipediaSummaries) Name

func (h *HarvestWikipediaSummaries) Name() string

func (*HarvestWikipediaSummaries) Run

func (h *HarvestWikipediaSummaries) Run(ctx context.Context, w io.Writer) error

type ImportEntry

type ImportEntry struct {
	Source    string // bibsys, openlibrary, discogs etc
	Exists    bool   // allready in local DB
	Error     string // not found, unable to map to local metadata, insuficcient data etc
	Resources []sirkulator.SimpleResource
}

type Ingestion

type Ingestion struct {
	Resources []sirkulator.Resource
	Relations []sirkulator.Relation
	Covers    []FileFetch
}

type Ingestor

type Ingestor struct {

	// Options
	UseRemote     bool // if true, use external sources in additin to local
	ImageDownload bool // if true, download images found in imported records
	ImageAsync    bool // if true, download images after IngestISBN has returned
	ImageWidth    int  // scale to this with, calculating width to preserve aspect ratio
	// contains filtered or unexported fields
}

func NewIngestor

func NewIngestor(db *sqlitex.Pool, idx *search.Index) *Ingestor

func NewPreviewIngestor

func NewPreviewIngestor(db *sqlitex.Pool) *Ingestor

func (*Ingestor) Ingest

func (ig *Ingestor) Ingest(ctx context.Context, data Ingestion, persist bool) ([]sirkulator.SimpleResource, error)

Ingest will merge the ingestion with locally matching resources before storing the data to db and trigger indexing of documents. if persist=false, nothing is persisted, and the returnet results represents a preview of which resources would have been stored if persist=true.

func (*Ingestor) IngestISBN

func (ig *Ingestor) IngestISBN(ctx context.Context, id string, persist bool) ImportEntry

IngestISBN will try to ingest a publication and related resources given an ISBN-number, optionally persisting it to DB. TODO refactor to remove duplication

type UpdateSNLDescriptions

type UpdateSNLDescriptions struct {
	DB *sqlitex.Pool
}

func (*UpdateSNLDescriptions) Name

func (u *UpdateSNLDescriptions) Name() string

func (*UpdateSNLDescriptions) Run

func (u *UpdateSNLDescriptions) Run(ctx context.Context, w io.Writer) error