scrapers

Documentation

Index

• type DatasetScraper
• type FieldScraper
• type PartitionScraper
• type ProjectScraper
• type Row
• type Scraper
  • func (s *Scraper) Scrap()
• type TableScraper

Types

type DatasetScraper

type DatasetScraper struct {
	Context   context.Context
	DB        *gorm.DB
	Client    *bigquery.Client
	Config    *configs.Config
	Dataset   models.Dataset
	Logger    *log.Logger
	WaitGroup *sync.WaitGroup
}

DatasetScraper represents a dataset scraper instance

type FieldScraper

type FieldScraper struct {
	DB           *gorm.DB
	APIMetadata  *bigquery.TableMetadata
	Logger       *log.Logger
	ColumnNumber int
}

FieldScraper represents a field scraper instance

type PartitionScraper

type PartitionScraper struct {
	Context     context.Context
	DB          *gorm.DB
	Client      *bigquery.Client
	Config      *configs.Config
	APIMetadata *bigquery.TableMetadata
	Logger      *log.Logger
}

PartitionScraper represents a partition scraper instance

type ProjectScraper

type ProjectScraper struct {
	Context context.Context
	DB      *gorm.DB
	Client  *bigquery.Client
	Config  *configs.Config
	Project models.Project
	Logger  *log.Logger
}

ProjectScraper represents a project scraper instance

type Row

type Row struct {
	Partitiontime time.Time
	Count         int64
}

Row is a struct to be merged with BigQuery results

type Scraper

type Scraper struct {
	Context context.Context
	DB      *gorm.DB
	Client  *bigquery.Client
	Config  *configs.Config
	Logger  *log.Logger
}

Scraper represents a scraper instance

func (*Scraper) Scrap

func (s *Scraper) Scrap()

Scrap get data from BigQuery API

type TableScraper

type TableScraper struct {
	Context   context.Context
	DB        *gorm.DB
	Client    *bigquery.Client
	Config    *configs.Config
	Table     models.Table
	Logger    *log.Logger
	WaitGroup *sync.WaitGroup
}

TableScraper represents a table scraper instance