model

Documentation

Index

• func InitDB(dst string) (*sql.DB, error)
• func InsertArchives(db *sql.DB, archives []*Archive) error
• func InsertStream(db *sql.DB, archiveID, byteBegin, byteEnd int64, pageIDs []int64, ...) error
• func MarkArchiveProcessed(db *sql.DB, id int64) error
• func SearchPageName(db *sql.DB, name string) ([]int64, error)
• func SelectPage(db *sql.DB, pageID int64) (int64, error)
• func SelectPages(db *sql.DB, pageIDs []int64) ([]int64, error)
• type Archive
  • func SelectArchives(db *sql.DB) ([]*Archive, error)
• type Stream
  • func SelectArchiveStreams(db *sql.DB, archivePath string) ([]*Stream, error)
  • func SelectStream(db *sql.DB, streamID int64) (*Stream, error)
  • func SelectStreams(db *sql.DB, streamIDs []int64) ([]*Stream, error)

Functions

func InitDB

func InitDB(dst string) (*sql.DB, error)

func InsertArchives

func InsertArchives(db *sql.DB, archives []*Archive) error

func InsertStream

func InsertStream(db *sql.DB, archiveID, byteBegin, byteEnd int64, pageIDs []int64, pageNames []string) error

func MarkArchiveProcessed

func MarkArchiveProcessed(db *sql.DB, id int64) error

func SearchPageName

func SearchPageName(db *sql.DB, name string) ([]int64, error)

func SelectPage

func SelectPage(db *sql.DB, pageID int64) (int64, error)

func SelectPages

func SelectPages(db *sql.DB, pageIDs []int64) ([]int64, error)

Types

type Archive

type Archive struct {
	FilePath  string
	IndexPath string
	ID        int64
	FileSize  int64
}

func SelectArchives

func SelectArchives(db *sql.DB) ([]*Archive, error)

type Stream

type Stream struct {
	Path      string
	ID        int64
	ByteBegin int64
	ByteEnd   int64
}

Bz2 files are split into streams. Each stream contains 100 articles. A stream is the smallest block we can extract.

func SelectArchiveStreams

func SelectArchiveStreams(db *sql.DB, archivePath string) ([]*Stream, error)

func SelectStream

func SelectStream(db *sql.DB, streamID int64) (*Stream, error)

func SelectStreams

func SelectStreams(db *sql.DB, streamIDs []int64) ([]*Stream, error)