urlstore

package
v0.0.0-...-f43e39b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 23, 2020 License: MIT Imports: 20 Imported by: 0

Documentation

Overview

Package urlstore contains the engine of distill

Package urlstore provides the main functionalities for distill

Index

Constants

This section is empty.

Variables

View Source 
var (
	// ColferSizeMax is the upper limit for serial byte sizes.
	ColferSizeMax = 16 * 1024 * 1024
)

Colfer configuration attributes

View Source 
var ErrInvalidBackupRecord = fmt.Errorf("Invalid backup record")

ErrInvalidBackupRecord when a csv record from backup is different from expected

View Source 
var ErrURLExhausted = fmt.Errorf("url exhausted")

ErrURLExhausted when url is expired

View Source 
var ErrURLExpired = fmt.Errorf("url expired")

ErrURLExpired when url is expired

Functions

func Backup 

func Backup(outFile string) (err error)

Backup the database as csv

func CloseSession 

func CloseSession()

CloseSession closes the underling storage

func Defaults 

func Defaults()

Defaults set the defaults for the configuration

func Delete 

func Delete(id string) (err error)

Delete deletes an url

func DeleteURL 

func DeleteURL(id string) (err error)

DeleteURL delete a url mapping

func GenerateDefaultConfig 

func GenerateDefaultConfig(outFile, version string)

GenerateDefaultConfig generate a default configuration file an writes it in the outFile

func GetURLRedirect 

func GetURLRedirect(id string) (redirectURL string, err error)

GetURLRedirect retrieve the redicrect url associated to an id it also fire an event of tipe opcodeGet

func ImportCSV 

func ImportCSV(inFile string) (rows int, err error)

ImportCSV import urls from a csv file

func Insert 

func Insert(u *URLInfo) (err error)

Insert an url into the url store

func LoadStats 

func LoadStats() (err error)

LoadStats write the URL's statistics

func NewSession 

func NewSession()

NewSession opens the underling storage

func ResetStats 

func ResetStats() (err error)

ResetStats reset global statistcs

func Restore 

func Restore(inFile string) (count int, err error)

Restore the database from a backup file

func SaveStats 

func SaveStats() (err error)

SaveStats write the URL's statistics

func UpdateStats 

func UpdateStats(s Statistics)

UpdateStats uppdate urls statistics

func Upsert 

func Upsert(u *URLInfo) (err error)

Upsert an url into the the urlstore

func UpsertURL 

func UpsertURL(url *URLReq, forceAlphabet, forceLength bool, boundAt time.Time) (id string, err error)

UpsertURL insert or udpdate a url mapping

func UpsertURLSimple 

func UpsertURLSimple(url *URLReq) (id string, err error)

UpsertURLSimple insert or updae an url shortcut for UpsertURL(true, true, time.Now())

Types

type BinSerializable 

type BinSerializable interface {
	MarshalBinary() (data []byte, err error)
	UnmarshalBinary(data []byte) error
}

BinSerializable interface for binary serializable structs

type CSVSerializable 

type CSVSerializable interface {
	MarshalRecord() (data []string, err error)
	UnmarshalRecord(data []string) error
}

CSVSerializable interface for binary serializable structs

type ColferError 

type ColferError int

ColferError signals a data mismatch as as a byte index.

func (ColferError) Error 

func (i ColferError) Error() string

Error honors the error interface.

type ColferMax 

type ColferMax string

ColferMax signals an upper limit breach.

func (ColferMax) Error 

func (m ColferMax) Error() string

Error honors the error interface.

type ColferTail 

type ColferTail int

ColferTail signals data continuation as a byte index.

func (ColferTail) Error 

func (i ColferTail) Error() string

Error honors the error interface.

type ConfigSchema 

type ConfigSchema struct {
	Server  ServerConfig  `yaml:"server" mapstructure:"server"`
	ShortID ShortIDConfig `yaml:"short_id" mapstructure:"short_id"`
	Tuning  TuningConfig  `yaml:"tuning" mapstructure:"tuning"`
}

ConfigSchema define the configuration object 

var Config ConfigSchema

Config system configuration

func (*ConfigSchema) Defaults 

func (c *ConfigSchema) Defaults()

Defaults generate configuration defaults

func (*ConfigSchema) Validate 

func (c *ConfigSchema) Validate()

Validate configuration

type ServerConfig 

type ServerConfig struct {
	APIKey string `yaml:"api_key" mapstructure:"api_key"`
	Host   string `yaml:"host" mapstructure:"host"`
	Port   int    `yaml:"port" mapstructure:"port"`
	DbPath string `yaml:"db_path" mapstructure:"db_path"`
}

ServerConfig configuration for the server

type ShortID 

type ShortID struct {
	ID string `json:"id"`
}

ShortID used in reply and channel comunication

func (*ShortID) Bind 

func (u *ShortID) Bind(r *http.Request) error

Bind will run after the unmarshalling is complete

type ShortIDConfig 

type ShortIDConfig struct {
	Alphabet             string    `yaml:"alphabet" mapstructure:"alphabet"`
	Length               int       `yaml:"length" mapstructure:"length"`
	MaxRequests          uint64    `yaml:"max_requests" mapstructure:"max_requests"`
	TTL                  uint64    `yaml:"ttl" mapstructure:"ttl"`
	ExpireOn             time.Time `yaml:"expire_on" mapstructure:"expire_on"`
	RootRedirectURL      string    `yaml:"root_redirect_url" mapstructure:"root_redirect_url"`
	ExpiredRedirectURL   string    `yaml:"expired_redirect_url" mapstructure:"expired_redirect_url"`
	ExhaustedRedirectURL string    `yaml:"exhausted_redirect_url" mapstructure:"exhausted_redirect_url"`
}

ShortIDConfig configuration for the short id

type Statistics 

type Statistics struct {
	Urls        uint64    `json:"urls"`
	Gets        uint64    `json:"gets"`
	GetsExpired uint64    `json:"gets_expired"`
	Upserts     uint64    `json:"upserts"`
	Deletes     uint64    `json:"deletes"`
	LastRequest time.Time `json:"last_request"`
}

Statistics contains the global statistics

func GetStats 

func GetStats() (s *Statistics)

GetStats get the statistics

func (*Statistics) String 

func (s *Statistics) String() string

type TuningConfig 

type TuningConfig struct {
	StatsEventsWorkerNum   int     `yaml:"stats_events_worker_num" mapstructure:"stats_events_worker_num"`
	StatsCacheSize         int     `yaml:"stats_cache_size" mapstructure:"stats_cache_size"`
	DbPurgeWritesCount     uint64  `yaml:"db_purge_writes_count" mapstructure:"db_purge_writes_count"`
	DbGCDeletesCount       uint64  `yaml:"db_gc_deletes_count" mapstructure:"db_gc_deletes_count"`
	DbGCDiscardRation      float64 `yaml:"db_gc_discard_ration" mapstructure:"db_gc_discard_ration"`
	URLCacheSize           int     `yaml:"url_cache_size" mapstructure:"url_cache_size"`
	BckCSVIterPrefetchSize int     `yaml:"export_iterator_prefetch_size" mapstructure:"export_iterator_prefetch_size"`
	APIKeyHeaderName       string  `yaml:"api_key_header_name" mapstructure:"api_key_header_name"`
}

TuningConfig fine tuning configuration

type URLInfo 

type URLInfo struct {
	ID string

	URL string

	Counter uint64

	BountAt time.Time

	MaxRequests uint64

	ExhaustedURL string

	TTL uint64

	ExpireOn time.Time

	ExpiredURL string
}

URLInfo contains the data associated to an id

func Get 

func Get(id string) (u *URLInfo, err error)

Get an url from the datastore

func GetURLInfo 

func GetURLInfo(id string) (urlInfo *URLInfo, err error)

GetURLInfo retrieve the url info associated to an id

func Peek 

func Peek(id string) (u *URLInfo, err error)

Peek retrive a url without incrementing the counter

func (URLInfo) ExpirationDate 

func (u URLInfo) ExpirationDate() time.Time

ExpirationDate return the expiration date of the URLInfo

func (*URLInfo) MarshalBinary 

func (o *URLInfo) MarshalBinary() (data []byte, err error)

MarshalBinary encodes o as Colfer conform encoding.BinaryMarshaler. The error return option is urlstore.ColferMax.

func (*URLInfo) MarshalLen 

func (o *URLInfo) MarshalLen() (int, error)

MarshalLen returns the Colfer serial byte size. The error return option is urlstore.ColferMax.

func (*URLInfo) MarshalRecord 

func (u *URLInfo) MarshalRecord() []string

MarshalRecord marshal a urlinfo to a string array (for csv)

func (*URLInfo) MarshalTo 

func (o *URLInfo) MarshalTo(buf []byte) int

MarshalTo encodes o as Colfer into buf and returns the number of bytes written. If the buffer is too small, MarshalTo will panic.

func (URLInfo) String 

func (u URLInfo) String() string

String version of urlinfo

func (*URLInfo) Unmarshal 

func (o *URLInfo) Unmarshal(data []byte) (int, error)

Unmarshal decodes data as Colfer and returns the number of bytes read. The error return options are io.EOF, urlstore.ColferError and urlstore.ColferMax.

func (*URLInfo) UnmarshalBinary 

func (o *URLInfo) UnmarshalBinary(data []byte) error

UnmarshalBinary decodes data as Colfer conform encoding.BinaryUnmarshaler. The error return options are io.EOF, urlstore.ColferError, urlstore.ColferTail and urlstore.ColferMax.

func (*URLInfo) UnmarshalRecord 

func (u *URLInfo) UnmarshalRecord(pieces []string) (err error)

UnmarshalRecord unmarshal a string array into a urlinfo

type URLIterator 

type URLIterator struct {
	Transaction *badger.Txn
	Iterator    *badger.Iterator
	KeyPrefix   []byte
}

URLIterator an iterator over URLs

func NewURLIterator 

func NewURLIterator() *URLIterator

NewURLIterator return an url iterator over the database

func (*URLIterator) Close 

func (i *URLIterator) Close()

Close the iterator

func (*URLIterator) HasNext 

func (i *URLIterator) HasNext() bool

HasNext tells if there are still elements in the list

func (*URLIterator) NextURL 

func (i *URLIterator) NextURL() (u *URLInfo, err error)

NextURL get the next URL from the iterator

type URLOp 

type URLOp struct {
	ID string
	// contains filtered or unexported fields
}

URLOp to track events on urls

type URLReq 

type URLReq struct {
	ID           string    `json:"id"`
	URL          string    `json:"url"`
	MaxRequests  uint64    `json:"max_requests,omitempty"`
	ExhaustedURL string    `json:"url_exhausted"`
	TTL          uint64    `json:"ttl,omitempty"`
	ExpireOn     time.Time `json:"expire_on,omitempty"`
	ExpiredURL   string    `json:"url_expired"`
}

URLReq request from a client to register an url

func (*URLReq) Bind 

func (u *URLReq) Bind(r *http.Request) error

Bind will run after the unmarshalling is complete

func (*URLReq) UnmarshalRecord 

func (u *URLReq) UnmarshalRecord(pieces []string) (err error)

UnmarshalRecord unmarshal a string array (csv record) to URLReq pointer

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.