models

Documentation

Index

• Constants
• Variables
• func GetWebsiteFromURL(u string) (string, error)
• type Archival
• type ArchiveRequestRepo
  • func NewArchiveRequest(db *sqlx.DB) *ArchiveRequestRepo
  • func (m *ArchiveRequestRepo) AddInferenceCategory(entry *proto.InferenceEntry) error
  • func (m *ArchiveRequestRepo) ApproveVideo(videoID string) error
  • func (m *ArchiveRequestRepo) DeleteArchivalRequest(userID, downloadID uint64) error
  • func (m *ArchiveRequestRepo) GetAllUnapprovedVideos() (*proto.UnapprovedList, error)
  • func (m *ArchiveRequestRepo) GetArchivalEvents(downloadID int64, showAll bool) ([]Event, error)
  • func (m *ArchiveRequestRepo) GetContentArchivalRequests(userID int64) ([]Archival, error)
  • func (m *ArchiveRequestRepo) GetDownloadsInProgress() ([]Video, error)
  • func (m *ArchiveRequestRepo) GetInferenceCategories() ([]InferenceCategory, error)
  • func (m *ArchiveRequestRepo) GetUnclassifiedVideoURLs() ([]VideoURL, error)
  • func (m *ArchiveRequestRepo) GetUnsyncedCategoryDLRequests() ([]CategoryDLRequest, error)
  • func (m *ArchiveRequestRepo) New(url string, userID int64) error
  • func (m *ArchiveRequestRepo) RetryArchivalRequest(userID, downloadID uint64) error
  • func (m *ArchiveRequestRepo) UnapproveVideo(videoID string) error
  • func (m *ArchiveRequestRepo) UpdateClassification(category string, id int) error
  • func (m *ArchiveRequestRepo) WipeDownloadsInProgress() error
• type CategoryDLRequest
  • func (v *CategoryDLRequest) AddVideo(videoID, url string) (bool, error)
  • func (v *CategoryDLRequest) IsBackingOff() (bool, error)
  • func (v *CategoryDLRequest) RefreshLock() error
  • func (v *CategoryDLRequest) ReportSyncHit() error
  • func (v *CategoryDLRequest) ReportSyncMiss() error
• type Event
• type InferenceCategory
• type ProgressNotification
• type Video
• type VideoDLRequest
  • func (v *VideoDLRequest) PublishVideoInprogress(dlStatus int, action string) error
  • func (v *VideoDLRequest) RecordEvent(inpEvent event, additionalErrorMsg string) error
  • func (v *VideoDLRequest) SetDownloadFailed() error
  • func (v *VideoDLRequest) SetDownloadInProgress() error
  • func (v *VideoDLRequest) SetDownloadQueued() error
  • func (v *VideoDLRequest) SetDownloadSucceeded() error
• type VideoProgress
• type VideoURL

Constants

const (
	MINIMUM_BACKOFF_TIME   = time.Hour * 24
	MAXIMUM_BACKOFF_FACTOR = 8 // 8 days
)

const (
	Scheduled  event = "Video %s from %s has been scheduled for download"
	Error      event = "Video %s from %s could not be downloaded, failed with an error. "
	Downloaded event = "Video %s from %s has been downloaded successfully, and uploaded to videoservice"
)

Variables

var NeverDownloaded = errors.New("no video for category")

Functions

func GetWebsiteFromURL

func GetWebsiteFromURL(u string) (string, error)

Types

type Archival

type Archival struct {
	Url            string
	DownloadID     uint64
	Denominator    uint64
	Numerator      uint64
	Undownloadable uint64
	LastSynced     string
	BackoffFactor  uint32
}

type ArchiveRequestRepo

type ArchiveRequestRepo struct {
	Db *sqlx.DB
}

ArchiveRequest is the model for the creation of new archive requests It's very similar to video_dl_request, but video_dl_request has different utility.

func NewArchiveRequest

func NewArchiveRequest(db *sqlx.DB) *ArchiveRequestRepo

func (*ArchiveRequestRepo) AddInferenceCategory

func (m *ArchiveRequestRepo) AddInferenceCategory(entry *proto.InferenceEntry) error

func (*ArchiveRequestRepo) ApproveVideo

func (m *ArchiveRequestRepo) ApproveVideo(videoID string) error

func (*ArchiveRequestRepo) DeleteArchivalRequest

func (m *ArchiveRequestRepo) DeleteArchivalRequest(userID, downloadID uint64) error

func (*ArchiveRequestRepo) GetAllUnapprovedVideos

func (m *ArchiveRequestRepo) GetAllUnapprovedVideos() (*proto.UnapprovedList, error)

func (*ArchiveRequestRepo) GetArchivalEvents

func (m *ArchiveRequestRepo) GetArchivalEvents(downloadID int64, showAll bool) ([]Event, error)

func (*ArchiveRequestRepo) GetContentArchivalRequests

func (m *ArchiveRequestRepo) GetContentArchivalRequests(userID int64) ([]Archival, error)

func (*ArchiveRequestRepo) GetDownloadsInProgress

func (m *ArchiveRequestRepo) GetDownloadsInProgress() ([]Video, error)

func (*ArchiveRequestRepo) GetInferenceCategories

func (m *ArchiveRequestRepo) GetInferenceCategories() ([]InferenceCategory, error)

func (*ArchiveRequestRepo) GetUnclassifiedVideoURLs

func (m *ArchiveRequestRepo) GetUnclassifiedVideoURLs() ([]VideoURL, error)

func (*ArchiveRequestRepo) GetUnsyncedCategoryDLRequests

func (m *ArchiveRequestRepo) GetUnsyncedCategoryDLRequests() ([]CategoryDLRequest, error)

func (*ArchiveRequestRepo) New

func (m *ArchiveRequestRepo) New(url string, userID int64) error

func (*ArchiveRequestRepo) RetryArchivalRequest

func (m *ArchiveRequestRepo) RetryArchivalRequest(userID, downloadID uint64) error

func (*ArchiveRequestRepo) UnapproveVideo

func (m *ArchiveRequestRepo) UnapproveVideo(videoID string) error

func (*ArchiveRequestRepo) UpdateClassification

func (m *ArchiveRequestRepo) UpdateClassification(category string, id int) error

func (*ArchiveRequestRepo) WipeDownloadsInProgress

func (m *ArchiveRequestRepo) WipeDownloadsInProgress() error

type CategoryDLRequest

type CategoryDLRequest struct {
	Url     string
	Website string
	Id      string
	Db      *sqlx.DB
}

func (*CategoryDLRequest) AddVideo

func (v *CategoryDLRequest) AddVideo(videoID, url string) (bool, error)

Idempotent, ensures that videos are added and correct associations are created returns bool indicating whether something was added

func (*CategoryDLRequest) IsBackingOff

func (v *CategoryDLRequest) IsBackingOff() (bool, error)

IsBackingOff indicates whether the archive request is backing off from full content syncs Context: videos can be added to a category of content at any time; some categories are updated frequently, and some tend to be stagnant. We should vary the rate at which we fully sync content from a given category based on how frequently it's updated. Exponential backoff is used as the backoff strategy.

func (*CategoryDLRequest) RefreshLock

func (v *CategoryDLRequest) RefreshLock() error

RefreshLock refreshes the lock for this download request, preventing it from being acquired by another scheduler.

func (*CategoryDLRequest) ReportSyncHit

func (v *CategoryDLRequest) ReportSyncHit() error

func (*CategoryDLRequest) ReportSyncMiss

func (v *CategoryDLRequest) ReportSyncMiss() error

type Event

type Event struct {
	ParentURL      string
	VideoURL       string
	Message        string
	EventTimestamp string
}

type InferenceCategory

type InferenceCategory struct {
	ID       int    `db:"id"`
	Tag      string `db:"tag"`
	Category string `db:"category"` // postgres lowercased it, lol
}

type ProgressNotification

type ProgressNotification struct {
	Type  string
	Video VideoProgress
}

type Video

type Video struct {
	ID       string `db:"video_id"`
	Website  string `db:"website"`
	DlStatus int    `db:"dlstatus"` // postgres lowercased it, lol
}

type VideoDLRequest

type VideoDLRequest struct {
	Rabbitmq    *stomp.Conn
	Db          *sqlx.DB
	VideoID     string // Foreign ID
	ID          int    // Domestic ID
	DownloaddID int
	URL         string
	ParentURL   string
}

func (*VideoDLRequest) PublishVideoInprogress

func (v *VideoDLRequest) PublishVideoInprogress(dlStatus int, action string) error

func (*VideoDLRequest) RecordEvent

func (v *VideoDLRequest) RecordEvent(inpEvent event, additionalErrorMsg string) error

func (*VideoDLRequest) SetDownloadFailed

func (v *VideoDLRequest) SetDownloadFailed() error

func (*VideoDLRequest) SetDownloadInProgress

func (v *VideoDLRequest) SetDownloadInProgress() error

func (*VideoDLRequest) SetDownloadQueued

func (v *VideoDLRequest) SetDownloadQueued() error

func (*VideoDLRequest) SetDownloadSucceeded

func (v *VideoDLRequest) SetDownloadSucceeded() error

type VideoProgress

type VideoProgress struct {
	VideoID  string
	Website  string
	DlStatus string
}

type VideoURL

type VideoURL struct {
	URL string `db:"url"`
	ID  int    `db:"id"`
}