youtube

package
v2.49.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 18, 2024 License: MIT Imports: 36 Imported by: 0

README

YouTube Feeds

The feed plugin is complicated because we use the user's uploads playlist, and that is sorted by actual upload date instead of publish date.

So say you have this channel's videos:

  1. vid2 - uploaded 10pm - published 10pm
  2. vid1 - uploaded 6pm - published 6pm

Then a video was published (but uploaded a long time ago):

  1. vid2 - uploaded 10pm - published 10pm
  2. vid1 - uploaded 6pm - published 6pm
  3. vid3 - uploaded 5pm - published 11pm

vid3 was published after the latest video, but still appears at the bottom. This causes issues, as we now have no idea when to stop looking. Currently YAGPDB handles this fine as long as it's not uploaded longer than 50 videos ago, in which case it may or may not catch it.

In the future, I'll do a hybrid mode with search. Those super late published videos however will show up in Discord super late. I cannot use search for 100% either because it costs 100 times for api quota to use, meaning it could be up to hours behind.

Storage layout:

Postgres tables: youtube_guild_subs - postgres - guild_id - channel_id - youtube_channel

youtube_playlist_ids - channel_name PRIMARY - playlist_id

Redis keys:

youtube_subbed_channels - sorted set

key is the channel name score is unix time in seconds when it was last checked

youtube_registered_websub_channels - sorted set

key is the channel name score is unix time in seconds when it expires

At the start of a poll, it uses zrange/zrevrange to grab an amount of entries to process, and if they do get processed, it updates the score to the current unix time.

youtube_last_video_time:{channel} - string

Holds the time of the last video in the processed channel. All videos before this will be ignored.

youtube_last_video_id:{channel} - string

Holds the last video id for a channel, it will stop processing videos when it hits this video.

youtube_push_registrations - sorted set

Key is the channel id, value is the time it expires

youtube_currently_adding:{channelid} - set, set when this channel is being added

Documentation

Index

Constants

View Source 
const (
	RedisChannelsLockKey       = "youtube_subbed_channel_lock"
	RedisKeyPublishedVideoList = "youtube_published_videos"
	RedisKeyWebSubChannels     = "youtube_registered_websub_channels"
	GoogleWebsubHub            = "https://pubsubhubbub.appspot.com/subscribe"
)
View Source 
const (
	GuildMaxFeeds               = 300
	GuildMaxEnabledFeeds        = 10
	GuildMaxEnabledFeedsPremium = 250
)
View Source 
const (
	WebSubCheckInterval = time.Second * 10
)

Variables

View Source 
var (
	ErrNoChannel              = errors.New("no channel with that id found")
	ErrMaxCustomMessageLength = errors.New("max length of custom message can be 500 chars")
)
View Source 
var DBSchemas = []string{`
CREATE TABLE IF NOT EXISTS youtube_channel_subscriptions (
	id SERIAL PRIMARY KEY,
	created_at TIMESTAMP WITH TIME ZONE NOT NULL,
	updated_at TIMESTAMP WITH TIME ZONE NOT NULL,

	guild_id TEXT NOT NULL,
	channel_id TEXT NOT NULL,
	youtube_channel_id TEXT NOT NULL,
	youtube_channel_name TEXT NOT NULL,

	mention_everyone BOOLEAN NOT NULL,
	mention_roles BIGINT[],
	publish_livestream BOOLEAN NOT NULL DEFAULT TRUE,
	publish_shorts BOOLEAN NOT NULL DEFAULT TRUE,
	enabled BOOLEAN NOT NULL DEFAULT TRUE
);
`, `

-- Old tables managed with gorm are missing NOT NULL constraints on some columns
-- that are never null in existing records; add them as needed.

ALTER TABLE youtube_channel_subscriptions ALTER COLUMN created_at SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN updated_at SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN guild_id SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN channel_id SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN youtube_channel_id SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN youtube_channel_name SET NOT NULL;
`, `
ALTER TABLE youtube_channel_subscriptions ALTER COLUMN mention_everyone SET NOT NULL;

-- Can't add a NOT NULL constraint to mention_roles because too many records in
-- production have it set to null.
`, `

-- The migration for the publish_livestream, publish_shorts, and enabled columns is
-- more involved. These columns were added later and so it is possible that they are
-- null in some older records. Therefore, we first replace these missing values with
-- defaults before adding the NOT NULL constraint.

DO $$
BEGIN

-- only run if we haven't added the NOT NULL constraint yet
IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name='youtube_channel_subscriptions' AND column_name='publish_livestream' AND is_nullable='yes') THEN
	UPDATE youtube_channel_subscriptions SET publish_livestream = TRUE WHERE publish_livestream IS NULL;
	UPDATE youtube_channel_subscriptions SET publish_shorts = TRUE WHERE publish_shorts IS NULL;
	UPDATE youtube_channel_subscriptions SET enabled = TRUE WHERE enabled IS NULL;

	ALTER TABLE youtube_channel_subscriptions ALTER COLUMN publish_livestream SET NOT NULL;
	ALTER TABLE youtube_channel_subscriptions ALTER COLUMN publish_shorts SET NOT NULL;
	ALTER TABLE youtube_channel_subscriptions ALTER COLUMN enabled SET NOT NULL;
END IF;
END $$;
`, `

CREATE TABLE IF NOT EXISTS youtube_announcements (
	guild_id BIGINT PRIMARY KEY,
	message TEXT NOT NULL,
	enabled BOOLEAN NOT NULL DEFAULT FALSE
);
`, `

ALTER TABLE youtube_announcements ALTER COLUMN guild_id SET NOT NULL;
`, `
ALTER TABLE youtube_announcements ALTER COLUMN message SET NOT NULL;
`, `
ALTER TABLE youtube_announcements ALTER COLUMN enabled SET NOT NULL;
`}
View Source 
var (
	ErrIDNotFound = errors.New("ID not found")
)
View Source 
var PageHTML string

Functions

func BaseEditHandler 

func BaseEditHandler(inner web.ControllerHandlerFunc) web.ControllerHandlerFunc

func KeyLastVidID 

func KeyLastVidID(channel string) string

func KeyLastVidTime 

func KeyLastVidTime(channel string) string

func MaxFeedsEnabledForContext added in v2.45.0 

func MaxFeedsEnabledForContext(ctx context.Context) int

func RegisterPlugin 

func RegisterPlugin()

Types

type ContextKey 

type ContextKey int
const (
	ContextKeySub ContextKey = iota
)
type Link struct {
	Href string `xml:"href,attr"`
	Rel  string `xml:"rel,attr"`
}

type LinkEntry 

type LinkEntry struct {
	Href string `xml:"href,attr"`
	Rel  string `xml:"rel,attr"`
}

type Plugin 

type Plugin struct {
	YTService *youtube.Service
	Stop      chan *sync.WaitGroup
}

func (*Plugin) AddFeed 

func (p *Plugin) AddFeed(guildID, discordChannelID int64, ytChannel *youtube.Channel, mentionEveryone bool, publishLivestream bool, publishShorts bool, mentionRoles []int64) (*models.YoutubeChannelSubscription, error)

func (*Plugin) CheckVideo 

func (p *Plugin) CheckVideo(parsedVideo XMLFeed) error

func (*Plugin) DisableChannelFeeds added in v2.16.0 

func (p *Plugin) DisableChannelFeeds(channelID int64) error

func (*Plugin) DisableFeed 

func (p *Plugin) DisableFeed(elem *mqueue.QueuedElement, err error)

Remove feeds if they don't point to a proper channel

func (*Plugin) DisableGuildFeeds added in v2.16.0 

func (p *Plugin) DisableGuildFeeds(guildID int64) error

func (*Plugin) HandleEdit 

func (p *Plugin) HandleEdit(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)

func (*Plugin) HandleFeedUpdate 

func (p *Plugin) HandleFeedUpdate(w http.ResponseWriter, r *http.Request)

func (*Plugin) HandleNew 

func (p *Plugin) HandleNew(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)

func (*Plugin) HandleRemove 

func (p *Plugin) HandleRemove(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)

func (*Plugin) HandleYoutube 

func (p *Plugin) HandleYoutube(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)

func (*Plugin) HandleYoutubeAnnouncement added in v2.15.0 

func (p *Plugin) HandleYoutubeAnnouncement(w http.ResponseWriter, r *http.Request) (templateData web.TemplateData, err error)

func (*Plugin) InitWeb 

func (p *Plugin) InitWeb()

func (*Plugin) LoadServerHomeWidget 

func (p *Plugin) LoadServerHomeWidget(w http.ResponseWriter, r *http.Request) (web.TemplateData, error)

func (*Plugin) MaybeAddChannelWatch 

func (p *Plugin) MaybeAddChannelWatch(lock bool, channel string) error

maybeAddChannelWatch adds a channel watch to redis, if there wasn't one before

func (*Plugin) MaybeRemoveChannelWatch 

func (p *Plugin) MaybeRemoveChannelWatch(channel string)

maybeRemoveChannelWatch checks the channel for subs, if it has none then it removes it from the watchlist in redis.

func (*Plugin) OnRemovedPremiumGuild added in v2.17.0 

func (p *Plugin) OnRemovedPremiumGuild(guildID int64) error

func (*Plugin) PluginInfo 

func (p *Plugin) PluginInfo() *common.PluginInfo

func (*Plugin) SetupClient 

func (p *Plugin) SetupClient() error

func (*Plugin) StartFeed 

func (p *Plugin) StartFeed()

func (*Plugin) Status 

func (p *Plugin) Status() (string, string)

func (*Plugin) StopFeed 

func (p *Plugin) StopFeed(wg *sync.WaitGroup)

func (*Plugin) ValidateSubscription 

func (p *Plugin) ValidateSubscription(w http.ResponseWriter, r *http.Request, query url.Values)

func (*Plugin) WebSubSubscribe 

func (p *Plugin) WebSubSubscribe(ytChannelID string) error

func (*Plugin) WebSubUnsubscribe 

func (p *Plugin) WebSubUnsubscribe(ytChannelID string) error

type XMLFeed 

type XMLFeed struct {
	Xmlns        string `xml:"xmlns,attr"`
	Link         []Link `xml:"link"`
	ChannelID    string `xml:"entry>channelId"`
	Published    string `xml:"entry>published"`
	VideoId      string `xml:"entry>videoId"`
	Yt           string `xml:"yt,attr"`
	LinkEntry    Link   `xml:"entry>link"`
	AuthorUri    string `xml:"entry>author>uri"`
	AuthorName   string `xml:"entry>author>name"`
	UpdatedEntry string `xml:"entry>updated"`
	Title        string `xml:"title"`
	TitleEntry   string `xml:"entry>title"`
	Id           string `xml:"entry>id"`
	Updated      string `xml:"updated"`
}

type YoutubeAnnouncementForm added in v2.15.0 

type YoutubeAnnouncementForm struct {
	Message string `json:"message" valid:"template,5000"`
	Enabled bool
}

type YoutubeFeedForm added in v2.15.0 

type YoutubeFeedForm struct {
	YoutubeUrl        string
	DiscordChannel    int64 `valid:"channel,false"`
	ID                uint
	MentionEveryone   bool
	MentionRoles      []int64
	PublishShorts     bool
	PublishLivestream bool
	Enabled           bool
	CustomMessage     string
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.