database

Documentation

Index

• Constants
• type Compressed
  • func (c *Compressed) Compress() []byte
  • func (c *Compressed) Scan(value interface{}) error
  • func (c Compressed) Value() (driver.Value, error)
• type Content
• type Host
• type Meta
• type State

Constants

const (
	//StateSuccess - page without error
	StateSuccess State = 0
	//StateDisabledByRobotsTxt - URL disabled in robots.txt
	StateDisabledByRobotsTxt = 1
	//StateConnectError - can not connect to server
	StateConnectError = 2
	//StateErrorStatusCode - status code != 200
	StateErrorStatusCode = 3
	//StateUnsupportedFormat - MIME type != text/html
	StateUnsupportedFormat = 4
	//StateAnswerError - can not read body
	StateAnswerError = 5
	//StateParseError - can not parse body or body not html (body not save)
	StateParseError = 6
	//StateEncodingError - can not find or apply encoding (body not save)
	StateEncodingError = 7
	//StateDublicate - dublicate see "Origin" field for origin URL
	StateDublicate = 8
	//StateExternal - after redirect - host is external (body not save)
	StateExternal = 9
	//StateNoFollow - found meta tag nofollow (body not save)
	StateNoFollow = 10
)

const (
	// ErrScanArgument - Scan argument is not array of bytes
	ErrScanArgument = "Compressed.Scan: argument is not array of bytes"
)

Types

type Compressed

type Compressed struct {
	Data []byte
}

Compressed - field compressed by zlib

func (*Compressed) Compress

func (c *Compressed) Compress() []byte

Compress - compress c.Data value

func (*Compressed) Scan

func (c *Compressed) Scan(value interface{}) error

Scan - load data from DB to value

func (Compressed) Value

func (c Compressed) Value() (driver.Value, error)

Value - prepare value for save to DB

type Content

type Content struct {
	URL   int64      `gorm:"type:integer REFERENCES url(id);unique_index;not null"`
	Hash  string     `gorm:"size:64;not null"`
	Body  Compressed `gorm:"not null"`
	Title string     `gorm:"size:100;not null"`
}

Content - store page content Hash - hash of uncompressed content

type Host

type Host struct {
	ID               int64  `gorm:"primary_key;not null"`
	Name             string `gorm:"size:255;unique_index;not null"`
	RobotsStatusCode int    `gorm:"not null"`
	RobotsData       []byte
}

Host - host information

type Meta

type Meta struct {
	URL         int64         `gorm:"type:integer REFERENCES url(id);unique_index;not null"`
	State       State         `gorm:"not null"`
	Origin      sql.NullInt64 `gorm:"type:integer REFERENCES url(id)"`
	RedirectCnt int
	StatusCode  sql.NullInt64
}

Meta - meta information about processed URL Origin - link to origin document (for State == CtStateDublicate)

type State

type State uint8

State - current state of page