scrap

Documentation

Index

• type Repository
• type RepositorySnapshot
• type RepositorySnapshots
• type Scraper
  • func NewScraper(tasks int, l logrus.FieldLogger, db *sqlx.DB, discoverEvery time.Duration, ...) *Scraper
  • func (i *Scraper) Discover()
  • func (i *Scraper) FindSnapshots(slug string) (RepositorySnapshots, error)
  • func (i *Scraper) ListRepositorySlugs(ctx context.Context) ([]string, error)
  • func (i *Scraper) Scrap()
  • func (i *Scraper) Stats(c context.Context) *Stats
• type Stats

Types

type Repository

type Repository struct {
	ID             int       `json:"-" db:"id"`
	Slug           string    `json:"slug" db:"slug"`
	LastScrappedAt time.Time `json:"last_scrapped_at" db:"last_scrapped_at"`
	DiscoveredAt   time.Time `json:"discovered_at" db:"discovered_at"`
	Source         string    `json:"source" db:"source"`
	ErrorCode      int       `json:"-" db:"error_code"`
	ErrorAt        time.Time `json:"-" db:"error_at"`
}

type RepositorySnapshot

type RepositorySnapshot struct {
	ID           int       `json:"-" db:"id"`
	RepositoryID int       `json:"-" db:"repository_id"`
	Stars        int64     `json:"star_count" db:"stars"`
	Pulls        int64     `json:"pull_count" db:"pulls"`
	Timestamp    time.Time `json:"timestamp" db:"fetched_at"`
}

type RepositorySnapshots

type RepositorySnapshots []*RepositorySnapshot

type Scraper

type Scraper struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper(
	tasks int,
	l logrus.FieldLogger,
	db *sqlx.DB,
	discoverEvery time.Duration,
	delay time.Duration,
	scrapRefreshQueue time.Duration,
	pageSize int,
	daysRefresh int,
) *Scraper

func (*Scraper) Discover

func (i *Scraper) Discover()

func (*Scraper) FindSnapshots

func (i *Scraper) FindSnapshots(slug string) (RepositorySnapshots, error)

func (*Scraper) ListRepositorySlugs

func (i *Scraper) ListRepositorySlugs(ctx context.Context) ([]string, error)

func (*Scraper) Scrap

func (i *Scraper) Scrap()

func (*Scraper) Stats

func (i *Scraper) Stats(c context.Context) *Stats

type Stats

type Stats struct {
	TotalRepositories         int64 `json:"discovered_repositories_total"`
	RepositoriesWithoutErrors int64 `json:"discovered_repositories_without_errors"`
	RepositoriesWithErrors    int64 `json:"discovered_repositories_with_errors"`

	TotalSnapshotsCompleted int64 `json:"snapshots_completed_total"`
	SnapShotQueue           int64 `json:"snapshot_queue"`

	SnapshotsCompleted      uint64 `json:"proc_snapshots_completed"`
	DiscoveriesCompleted    uint64 `json:"proc_discoveries_completed"`
	SnapshotQueueLength     int    `json:"proc_snapshot_queue_length"`
	SnapshotRefreshInterval string `json:"snapshot_refresh_interval"`
}