sirene

Documentation

Index

• func Do(sfs RemoteFiles, workers int, dPath string) error
• func FileTypeName(ft FileType) string
• func Grab() (*api.Dataset, error)
• func NewDateSirene(raw string) (dateSirene, error)
• type CSVImport
  • func (c *CSVImport) Copy(db *pgx.ConnPool) error
  • func (c *CSVImport) Update(db *pgx.ConnPool) error
• type FileType
• type RemoteFile
  • func NewFromResource(r api.Resource, dPath string) (*RemoteFile, error)
  • func (rf *RemoteFile) CalculateChecksum() (string, error)
  • func (rf *RemoteFile) ChecksumMatch() (bool, error)
  • func (rf *RemoteFile) Download(dPath string) error
  • func (rf *RemoteFile) Unzip(dPath string) error
• type RemoteFiles
  • func GrabLatestFull(dPath string) (RemoteFiles, error)
  • func (r RemoteFiles) String() string

Functions

func Do

func Do(sfs RemoteFiles, workers int, dPath string) error

Do downloads and processes the sirene files

func FileTypeName

func FileTypeName(ft FileType) string

FileTypeName returns the string representation of a FileType

func Grab

func Grab() (*api.Dataset, error)

Grab can be used to grab the full dataset object

func NewDateSirene

func NewDateSirene(raw string) (dateSirene, error)

Types

type CSVImport

type CSVImport struct {
	Path         string
	File         *os.File
	Kind         FileType
	ZipName      string
	ProgressChan chan *progress.Progress
}

CSVImport is a struct helper to import a CSV file to the database It implementes the pgx.Copy interface

func (*CSVImport) Copy

func (c *CSVImport) Copy(db *pgx.ConnPool) error

Copy actually copies the content of the CSV file to the database

func (*CSVImport) Update

func (c *CSVImport) Update(db *pgx.ConnPool) error

Update update stock table from daily update file

type FileType

type FileType int

FileType is the main representation for the filetype

const (
	OtherType FileType = iota
	StockType
	DailyType
	MonthlyType
)

Defines the different filetype constants

type RemoteFile

type RemoteFile struct {
	Checksum       api.Checksum
	URL            string
	FileName       string
	Path           string
	Type           FileType
	YearDay        int
	OnDisk         bool
	ExtractedFiles []string
	ProgressChan   chan *progress.Progress
}

RemoteFile is a struct that adds and remove some fields from a Resource struct and actually keep only useful fields

func NewFromResource

func NewFromResource(r api.Resource, dPath string) (*RemoteFile, error)

NewFromResource takes an Resource and transforms it to a gouv_sirene.RemoteFile

func (*RemoteFile) CalculateChecksum

func (rf *RemoteFile) CalculateChecksum() (string, error)

CalculateChecksum generates the checksum of the file using the hasher type as defined in the Checksum.Type field

func (*RemoteFile) ChecksumMatch

func (rf *RemoteFile) ChecksumMatch() (bool, error)

ChecksumMatch calculates the checksum of the file on disk and checks if it matches the checksum defined in the Checksum.Value field

func (*RemoteFile) Download

func (rf *RemoteFile) Download(dPath string) error

Download will download the file

func (*RemoteFile) Unzip

func (rf *RemoteFile) Unzip(dPath string) error

Unzip will un-compress a zip archive moving all files and folders to an output directory

type RemoteFiles

type RemoteFiles []*RemoteFile

RemoteFiles is a slice of pointers to RemoteFile

func GrabLatestFull

func GrabLatestFull(dPath string) (RemoteFiles, error)

GrabLatestFull retrieves all the files that needs to be downloaded and applied to the database in inverse order (stock first, then each daily file)

func (RemoteFiles) String

func (r RemoteFiles) String() string