content

Documentation

Index

• type DBWorker
  • func (w *DBWorker) Start(wgParent *sync.WaitGroup)
• type DBrw
  • func GetDBrw() (*DBrw, error)
  • func (db *DBrw) AddHash(hash string, urlID int64)
  • func (db *DBrw) AddHost(host *proxy.Host, baseURL string) (int64, error)
  • func (db *DBrw) FindOrigin(hash string) sql.NullInt64
  • func (db *DBrw) GetHosts() (map[int64]*proxy.Host, error)
  • func (db *DBrw) GetNewURLs(hostID int64, cnt int) ([]URL, error)
  • func (db *DBrw) Transaction(fn func(*DBrw) error) error
• type Link
• type Logger
  • func (logger Logger) Print(values ...interface{})
• type URL

Types

type DBWorker

type DBWorker struct {
	DB   *DBrw
	ChDB <-chan *proxy.PageData
}

DBWorker - worker for save data to db

func (*DBWorker) Start

func (w *DBWorker) Start(wgParent *sync.WaitGroup)

Start - run db write worker

type DBrw

type DBrw struct {
	*gorm.DB
	// contains filtered or unexported fields
}

DBrw - content database (enable read/write operations)

func GetDBrw

func GetDBrw() (*DBrw, error)

GetDBrw - create/open content.db

func (*DBrw) AddHash

func (db *DBrw) AddHash(hash string, urlID int64)

AddHash - add new hash to hash storage

func (*DBrw) AddHost

func (db *DBrw) AddHost(host *proxy.Host, baseURL string) (int64, error)

AddHost - add new host

func (*DBrw) FindOrigin

func (db *DBrw) FindOrigin(hash string) sql.NullInt64

FindOrigin - find origin url id in table 'URL'

func (*DBrw) GetHosts

func (db *DBrw) GetHosts() (map[int64]*proxy.Host, error)

GetHosts - get rows from table 'Host'

func (*DBrw) GetNewURLs

func (db *DBrw) GetNewURLs(hostID int64, cnt int) ([]URL, error)

GetNewURLs - get URLs for downloads for host

func (*DBrw) Transaction

func (db *DBrw) Transaction(fn func(*DBrw) error) error

Transaction - execute fn in transaction

type Link

type Link struct {
	Master int64 `gorm:"type:integer REFERENCES url(id);index;not null"`
	Slave  int64 `gorm:"type:integer REFERENCES url(id);index;not null"`
}

Link - links between pages

type Logger

type Logger struct {
	gorm.LogWriter
}

Logger implemet gorm logger interface

func (Logger) Print

func (logger Logger) Print(values ...interface{})

Print -

type URL

type URL struct {
	ID     int64         `gorm:"primary_key;not null"`
	URL    string        `gorm:"size:2048;not null;unique_index"`
	HostID sql.NullInt64 `gorm:"type:integer REFERENCES host(id);index"`
	Loaded bool          `gorm:"not null;index"`
}

URL - struct for save all URLs in db