ingestors

package
v0.0.0-...-b114151 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 27, 2024 License: AGPL-3.0 Imports: 21 Imported by: 0

Documentation

Index

Constants

View Source 
const NPMRegistryDatabase = "registry"
View Source 
const NPMRegistryHostname = "https://replicate.npmjs.com"
View Source 
const RetryDelaySeconds = 5
View Source 
const UserAgent = "LibrariesIngestify/1.0 (support@khulnasoft.com)"

Variables

This section is empty.

Functions

func NewCocoaPods 

func NewCocoaPods() *cocoapods

func NewHackage 

func NewHackage() *hackage

func NewHex 

func NewHex() *hex

func NewPub 

func NewPub() *pub

Types

type CPAN 

type CPAN struct {
	LatestRun time.Time
}

func NewCPAN 

func NewCPAN() *CPAN

func (*CPAN) Ingest 

func (ingestor *CPAN) Ingest() []data.PackageVersion

func (*CPAN) Name 

func (ingestor *CPAN) Name() string

func (*CPAN) Schedule 

func (ingestor *CPAN) Schedule() string

type Cargo 

type Cargo struct {
	LatestRun time.Time
}

func NewCargo 

func NewCargo() *Cargo

func (*Cargo) Ingest 

func (ingestor *Cargo) Ingest() []data.PackageVersion

func (*Cargo) Schedule 

func (ingestor *Cargo) Schedule() string

type CondaIngestor 

type CondaIngestor struct {
	LatestRun  time.Time
	Repository CondaRepository
}

func NewConda 

func NewConda(repository CondaRepository) *CondaIngestor

func (*CondaIngestor) GetParser 

func (ingestor *CondaIngestor) GetParser() *CondaParser

func (*CondaIngestor) Ingest 

func (ingestor *CondaIngestor) Ingest() []data.PackageVersion

func (*CondaIngestor) Name 

func (ingestor *CondaIngestor) Name() string

func (*CondaIngestor) Schedule 

func (ingestor *CondaIngestor) Schedule() string

type CondaParser 

type CondaParser struct {
	URL      string
	Platform string
}

func NewCondaParser 

func NewCondaParser(url string, platform string) *CondaParser

func (*CondaParser) GetPackages 

func (parser *CondaParser) GetPackages(lastRun time.Time) ([]data.PackageVersion, error)

type CondaRepository 

type CondaRepository string
const (
	CondaForge CondaRepository = "conda_forge"
	CondaMain  CondaRepository = "conda_main"
)

type Drupal 

type Drupal struct {
	LatestRun time.Time
}

func NewDrupal 

func NewDrupal() *Drupal

func (*Drupal) Ingest 

func (ingestor *Drupal) Ingest() []data.PackageVersion

func (*Drupal) Name 

func (ingestor *Drupal) Name() string

func (*Drupal) Schedule 

func (ingestor *Drupal) Schedule() string

type Elm 

type Elm struct {
	LatestRun time.Time
}

func NewElm 

func NewElm() *Elm

func (*Elm) Ingest 

func (ingestor *Elm) Ingest() []data.PackageVersion

func (*Elm) Schedule 

func (ingestor *Elm) Schedule() string

type Go 

type Go struct {
	LatestRun time.Time
}

func NewGo 

func NewGo() *Go

func (*Go) Ingest 

func (ingestor *Go) Ingest() []data.PackageVersion

func (*Go) Schedule 

func (ingestor *Go) Schedule() string

type Ingestor 

type Ingestor interface {
	Schedule() string
	Ingest() []data.PackageVersion
}

Regular ingestors provide an API we can poll for changes. This polling is done on a regular schedule.

type MavenIngestor 

type MavenIngestor struct {
	LatestRun  time.Time
	Repository MavenRepository
}

func NewMaven 

func NewMaven(repository MavenRepository) *MavenIngestor

func (*MavenIngestor) GetParser 

func (ingestor *MavenIngestor) GetParser() *MavenParser

func (*MavenIngestor) Ingest 

func (ingestor *MavenIngestor) Ingest() []data.PackageVersion

func (*MavenIngestor) Name 

func (ingestor *MavenIngestor) Name() string

func (*MavenIngestor) Schedule 

func (ingestor *MavenIngestor) Schedule() string

func (*MavenIngestor) TTL 

func (ingestor *MavenIngestor) TTL() time.Duration

type MavenParser 

type MavenParser struct {
	URL      string
	Platform string
}

func NewMavenParser 

func NewMavenParser(url string, platform string) *MavenParser

func (*MavenParser) GetPackages 

func (parser *MavenParser) GetPackages() ([]data.PackageVersion, error)

type MavenRepository 

type MavenRepository string
const (
	MavenCentral MavenRepository = "maven_mavencentral"
	GoogleMaven  MavenRepository = "maven_google"
)

type NPM 

type NPM struct {
	// contains filtered or unexported fields
}

func NewNPM 

func NewNPM() *NPM

func (*NPM) Ingest 

func (ingestor *NPM) Ingest(results chan data.PackageVersion)

*

  • NPM package updates are ingested from a continuous reading of a remote
  • CouchDB database at replicate.npmjs.com. CouchDB databases provide a Changes
  • feed that provide the changes since the last set of changes were published:
  • https://docs.couchbase.com/sync-gateway/current/changes-feed.html *
  • We connect to the CouchDB database and continually read for the next set
  • of changes to the database. Once we receive changes, we take the found releases
  • and add them to the processing queue. If no changes are available, we
  • reconnect to the database in a number of seconds and try again. *
  • Since this is based on detecting changes to the NPM database, it may be
  • possible for a client to miss out on changes for some reason. Libraries only
  • processes individual NPM versions delivered by ingestify, rather than reprocessing
  • the whole package, so in that case, Libraries may not know about that version
  • onless something triggers a full package resync on Libraries.

func (*NPM) Name 

func (ingestor *NPM) Name() string

type NPMChangeDoc 

type NPMChangeDoc struct {
	ID       string `json:"_id"`
	Rev      string `json:"_rev,omitempty"`
	Name     string `json:"name"`
	DistTags struct {
		Latest string `json:"latest"`
	} `json:"dist-tags"`
	Time map[string]string `json:"time"`
}

See https://github.com/npm/registry-follower-tutorial#moar-data-please

type Namer 

type Namer interface {
	Name() string
}

type Nuget 

type Nuget struct {
	LatestRun time.Time
}

func NewNuget 

func NewNuget() *Nuget

func (*Nuget) Ingest 

func (ingestor *Nuget) Ingest() []data.PackageVersion

func (*Nuget) Schedule 

func (ingestor *Nuget) Schedule() string

type Packagist 

type Packagist struct {
	LatestRun time.Time
}

func NewPackagist 

func NewPackagist() *Packagist

func (*Packagist) Ingest 

func (ingestor *Packagist) Ingest() []data.PackageVersion

func (*Packagist) Name 

func (ingestor *Packagist) Name() string

func (*Packagist) Schedule 

func (ingestor *Packagist) Schedule() string

type PyPiRss 

type PyPiRss struct {
	LatestRun time.Time
}

func NewPyPiRss 

func NewPyPiRss() *PyPiRss

func (*PyPiRss) Ingest 

func (ingestor *PyPiRss) Ingest() []data.PackageVersion

func (*PyPiRss) Name 

func (ingestor *PyPiRss) Name() string

func (*PyPiRss) Schedule 

func (ingestor *PyPiRss) Schedule() string

type PyPiXmlRpc 

type PyPiXmlRpc struct {
	LatestRun time.Time
}

func NewPyPiXmlRpc 

func NewPyPiXmlRpc() *PyPiXmlRpc

func (*PyPiXmlRpc) Ingest 

func (ingestor *PyPiXmlRpc) Ingest() []data.PackageVersion

Retrieve a list of [name, version, timestamp, action] since the given since. All since timestamps are UTC values. The argument is a UTC integer seconds since the epoch (e.g., the timestamp method to a datetime.datetime object). calls "changelog(since, with_ids=False)" RPC

func (*PyPiXmlRpc) Name 

func (ingestor *PyPiXmlRpc) Name() string

func (*PyPiXmlRpc) Schedule 

func (ingestor *PyPiXmlRpc) Schedule() string

type PyPiXmlRpcResponse 

type PyPiXmlRpcResponse struct {
	Name      string
	Version   string
	Timestamp int64
	Action    string
	Serial    int64
}

Structured storage for the tuple returned by the xmlrpc client

func (*PyPiXmlRpcResponse) GetPackageVersion 

func (response *PyPiXmlRpcResponse) GetPackageVersion() data.PackageVersion

Get the PackageVersion struct for this response

func (*PyPiXmlRpcResponse) IsIngestionAction 

func (response *PyPiXmlRpcResponse) IsIngestionAction() bool

Return trhe if this response is an ingestable action

type RubyGems 

type RubyGems struct {
	LatestRun time.Time
}

func NewRubyGems 

func NewRubyGems() *RubyGems

func (*RubyGems) Ingest 

func (ingestor *RubyGems) Ingest() []data.PackageVersion

func (*RubyGems) Schedule 

func (ingestor *RubyGems) Schedule() string

type StreamingIngestor 

type StreamingIngestor interface {
	Ingest(chan data.PackageVersion)
}

Streaming Ingestors continually pull new release information from a persistent source. NPM is an example of this, as it provides a CouchDB API endpoint from which we can continually pull new package data.

type TTLer 

type TTLer interface {
	TTL() time.Duration
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.