youtube

README

YouTube Feeds

The feed plugin is complicated because we use the user's uploads playlist, and that is sorted by actual upload date instead of publish date.

So say you have this channel's videos:

1. vid2 - uploaded 10pm - published 10pm
2. vid1 - uploaded 6pm - published 6pm

Then a video was published (but uploaded a long time ago):

1. vid2 - uploaded 10pm - published 10pm
2. vid1 - uploaded 6pm - published 6pm
3. vid3 - uploaded 5pm - published 11pm

vid3 was published after the latest video, but still appears at the bottom. This causes issues, as we now have no idea when to stop looking. Currently YAGPDB handles this fine as long as it's not uploaded longer than 50 videos ago, in which case it may or may not catch it.

In the future, I'll do a hybrid mode with search. Those super late published videos however will show up in Discord super late. I cannot use search for 100% either because it costs 100 times for api quota to use, meaning it could be up to hours behind.

Storage layout:

Postgres tables: youtube_guild_subs - postgres - guild_id - channel_id - youtube_channel

youtube_playlist_ids - channel_name PRIMARY - playlist_id

Redis keys:

youtube_subbed_channels - sorted set

key is the channel name score is unix time in seconds when it was last checked

youtube_registered_websub_channels - sorted set

key is the channel name score is unix time in seconds when it expires

At the start of a poll, it uses zrange/zrevrange to grab an amount of entries to process, and if they do get processed, it updates the score to the current unix time.

youtube_last_video_time:{channel} - string

Holds the time of the last video in the processed channel. All videos before this will be ignored.

youtube_last_video_id:{channel} - string

Holds the last video id for a channel, it will stop processing videos when it hits this video.

youtube_push_registrations - sorted set

Key is the channel id, value is the time it expires

youtube_currently_adding:{channelid} - set, set when this channel is being added

Documentation

Index

• Constants
• Variables
• func BaseEditHandler(inner web.ControllerHandlerFunc) web.ControllerHandlerFunc
• func KeyLastVidID(channel string) string
• func KeyLastVidTime(channel string) string
• func MaxFeedsEnabledForContext(ctx context.Context) int
• func RegisterPlugin()
• type ContextKey
• type Link
• type LinkEntry
• type Plugin
  • func (p *Plugin) AddFeed(guildID, discordChannelID int64, ytChannel *youtube.Channel, ...) (*models.YoutubeChannelSubscription, error)
  • func (p *Plugin) CheckVideo(parsedVideo XMLFeed) error
  • func (p *Plugin) DisableChannelFeeds(channelID int64) error
  • func (p *Plugin) DisableFeed(elem *mqueue.QueuedElement, err error)
  • func (p *Plugin) DisableGuildFeeds(guildID int64) error
  • func (p *Plugin) HandleEdit(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)
  • func (p *Plugin) HandleFeedUpdate(w http.ResponseWriter, r *http.Request)
  • func (p *Plugin) HandleNew(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)
  • func (p *Plugin) HandleRemove(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)
  • func (p *Plugin) HandleYoutube(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)
  • func (p *Plugin) HandleYoutubeAnnouncement(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)
  • func (p *Plugin) InitWeb()
  • func (p *Plugin) LoadServerHomeWidget(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)
  • func (p *Plugin) MaybeAddChannelWatch(lock bool, channel string) error
  • func (p *Plugin) MaybeRemoveChannelWatch(channel string)
  • func (p *Plugin) OnRemovedPremiumGuild(guildID int64) error
  • func (p *Plugin) PluginInfo() *common.PluginInfo
  • func (p *Plugin) SetupClient() error
  • func (p *Plugin) StartFeed()
  • func (p *Plugin) Status() (string, string)
  • func (p *Plugin) StopFeed(wg *sync.WaitGroup)
  • func (p *Plugin) ValidateSubscription(w http.ResponseWriter, r *http.Request, query url.Values)
  • func (p *Plugin) WebSubSubscribe(ytChannelID string) error
  • func (p *Plugin) WebSubUnsubscribe(ytChannelID string) error
• type XMLFeed
• type YoutubeAnnouncementForm
• type YoutubeFeedForm

Constants

const (
	RedisChannelsLockKey       = "youtube_subbed_channel_lock"
	RedisKeyPublishedVideoList = "youtube_published_videos"
	RedisKeyWebSubChannels     = "youtube_registered_websub_channels"
	GoogleWebsubHub            = "https://pubsubhubbub.appspot.com/subscribe"
)

const (
	GuildMaxFeeds               = 300
	GuildMaxEnabledFeeds        = 10
	GuildMaxEnabledFeedsPremium = 250
)

const (
	WebSubCheckInterval = time.Second * 10
)

Variables

var (
	ErrNoChannel              = errors.New("no channel with that id found")
	ErrMaxCustomMessageLength = errors.New("max length of custom message can be 500 chars")
)

var DBSchemas = []string{`
CREATE TABLE IF NOT EXISTS youtube_channel_subscriptions (
	id SERIAL PRIMARY KEY,
	created_at TIMESTAMP WITH TIME ZONE NOT NULL,
	updated_at TIMESTAMP WITH TIME ZONE NOT NULL,

	guild_id TEXT NOT NULL,
	channel_id TEXT NOT NULL,
	youtube_channel_id TEXT NOT NULL,
	youtube_channel_name TEXT NOT NULL,

	mention_everyone BOOLEAN NOT NULL,
	mention_roles BIGINT[],
	publish_livestream BOOLEAN NOT NULL DEFAULT TRUE,
	publish_shorts BOOLEAN NOT NULL DEFAULT TRUE,
	enabled BOOLEAN NOT NULL DEFAULT TRUE
);
`, `

-- Old tables managed with gorm are missing NOT NULL constraints on some columns
-- that are never null in existing records; add them as needed.

ALTER TABLE youtube_channel_subscriptions ALTER COLUMN created_at SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN updated_at SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN guild_id SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN channel_id SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN youtube_channel_id SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN youtube_channel_name SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN mention_everyone SET NOT NULL;

-- Can't add a NOT NULL constraint to mention_roles because too many records in
-- production have it set to null.
`, `

-- The migration for the publish_livestream, publish_shorts, and enabled columns is
-- more involved. These columns were added later and so it is possible that they are
-- null in some older records. Therefore, we first replace these missing values with
-- defaults before adding the NOT NULL constraint.

DO $$
BEGIN

-- only run if we haven't added the NOT NULL constraint yet
IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name='youtube_channel_subscriptions' AND column_name='publish_livestream' AND is_nullable='yes') THEN
	UPDATE youtube_channel_subscriptions SET publish_livestream = TRUE WHERE publish_livestream IS NULL;
	UPDATE youtube_channel_subscriptions SET publish_shorts = TRUE WHERE publish_shorts IS NULL;
	UPDATE youtube_channel_subscriptions SET enabled = TRUE WHERE enabled IS NULL;

	ALTER TABLE youtube_channel_subscriptions ALTER COLUMN publish_livestream SET NOT NULL;
	ALTER TABLE youtube_channel_subscriptions ALTER COLUMN publish_shorts SET NOT NULL;
	ALTER TABLE youtube_channel_subscriptions ALTER COLUMN enabled SET NOT NULL;
END IF;
END $$;
`, `

CREATE TABLE IF NOT EXISTS youtube_announcements (
	guild_id BIGINT PRIMARY KEY,
	message TEXT NOT NULL,
	enabled BOOLEAN NOT NULL DEFAULT FALSE
);
`, `

ALTER TABLE youtube_announcements ALTER COLUMN guild_id SET NOT NULL;
`, `
ALTER TABLE youtube_announcements ALTER COLUMN message SET NOT NULL;
`, `
ALTER TABLE youtube_announcements ALTER COLUMN enabled SET NOT NULL;
`}

var (
	ErrIDNotFound = errors.New("ID not found")
)

var PageHTML string

Functions

func BaseEditHandler

func BaseEditHandler(inner web.ControllerHandlerFunc) web.ControllerHandlerFunc

func KeyLastVidID

func KeyLastVidID(channel string) string

func KeyLastVidTime

func KeyLastVidTime(channel string) string

func MaxFeedsEnabledForContext

func MaxFeedsEnabledForContext(ctx context.Context) int

func RegisterPlugin

func RegisterPlugin()

Types

type ContextKey

type ContextKey int

const (
	ContextKeySub ContextKey = iota
)

type Link

type Link struct {
	Href string `xml:"href,attr"`
	Rel  string `xml:"rel,attr"`
}

type LinkEntry

type LinkEntry struct {
	Href string `xml:"href,attr"`
	Rel  string `xml:"rel,attr"`
}

type Plugin

type Plugin struct {
	YTService *youtube.Service
	Stop      chan *sync.WaitGroup
}

func (*Plugin) AddFeed

func (p *Plugin) AddFeed(guildID, discordChannelID int64, ytChannel *youtube.Channel, mentionEveryone bool, publishLivestream bool, publishShorts bool, mentionRoles []int64) (*models.YoutubeChannelSubscription, error)

func (*Plugin) CheckVideo

func (p *Plugin) CheckVideo(parsedVideo XMLFeed) error

func (*Plugin) DisableChannelFeeds

func (p *Plugin) DisableChannelFeeds(channelID int64) error

func (*Plugin) DisableFeed

func (p *Plugin) DisableFeed(elem *mqueue.QueuedElement, err error)

Remove feeds if they don't point to a proper channel

func (*Plugin) DisableGuildFeeds

func (p *Plugin) DisableGuildFeeds(guildID int64) error

func (*Plugin) HandleEdit

func (p *Plugin) HandleEdit(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)

func (*Plugin) HandleFeedUpdate

func (p *Plugin) HandleFeedUpdate(w http.ResponseWriter, r *http.Request)

func (*Plugin) HandleNew

func (p *Plugin) HandleNew(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)

func (*Plugin) HandleRemove

func (p *Plugin) HandleRemove(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)

func (*Plugin) HandleYoutube

func (p *Plugin) HandleYoutube(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)

func (*Plugin) HandleYoutubeAnnouncement

func (p *Plugin) HandleYoutubeAnnouncement(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)

func (*Plugin) InitWeb

func (p *Plugin) InitWeb()

func (*Plugin) LoadServerHomeWidget

func (p *Plugin) LoadServerHomeWidget(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)

func (*Plugin) MaybeAddChannelWatch

func (p *Plugin) MaybeAddChannelWatch(lock bool, channel string) error

maybeAddChannelWatch adds a channel watch to redis, if there wasn't one before

func (*Plugin) MaybeRemoveChannelWatch

func (p *Plugin) MaybeRemoveChannelWatch(channel string)

maybeRemoveChannelWatch checks the channel for subs, if it has none then it removes it from the watchlist in redis.

func (*Plugin) OnRemovedPremiumGuild

func (p *Plugin) OnRemovedPremiumGuild(guildID int64) error

func (*Plugin) PluginInfo

func (p *Plugin) PluginInfo() *common.PluginInfo

func (*Plugin) SetupClient

func (p *Plugin) SetupClient() error

func (*Plugin) StartFeed

func (p *Plugin) StartFeed()

func (*Plugin) Status

func (p *Plugin) Status() (string, string)

func (*Plugin) StopFeed

func (p *Plugin) StopFeed(wg *sync.WaitGroup)

func (*Plugin) ValidateSubscription

func (p *Plugin) ValidateSubscription(w http.ResponseWriter, r *http.Request, query url.Values)

func (*Plugin) WebSubSubscribe

func (p *Plugin) WebSubSubscribe(ytChannelID string) error

func (*Plugin) WebSubUnsubscribe

func (p *Plugin) WebSubUnsubscribe(ytChannelID string) error

type XMLFeed

type XMLFeed struct {
	Xmlns        string `xml:"xmlns,attr"`
	Link         []Link `xml:"link"`
	ChannelID    string `xml:"entry>channelId"`
	Published    string `xml:"entry>published"`
	VideoId      string `xml:"entry>videoId"`
	Yt           string `xml:"yt,attr"`
	LinkEntry    Link   `xml:"entry>link"`
	AuthorUri    string `xml:"entry>author>uri"`
	AuthorName   string `xml:"entry>author>name"`
	UpdatedEntry string `xml:"entry>updated"`
	Title        string `xml:"title"`
	TitleEntry   string `xml:"entry>title"`
	Id           string `xml:"entry>id"`
	Updated      string `xml:"updated"`
}

type YoutubeAnnouncementForm

type YoutubeAnnouncementForm struct {
	Message string `json:"message" valid:"template,5000"`
	Enabled bool
}

type YoutubeFeedForm

type YoutubeFeedForm struct {
	YoutubeUrl        string
	DiscordChannel    int64 `valid:"channel,false"`
	ID                uint
	MentionEveryone   bool
	MentionRoles      []int64
	PublishShorts     bool
	PublishLivestream bool
	Enabled           bool
	CustomMessage     string
}