rhema

README

rhema

rhema is the Greek word for "utterance" or "thing said". This project is all about taking the content of a URI and turning it into compressed audio for faster (for me anyway) consumption.

Building

Developing on this project is as easy as:

# Set some env vars
export BUCKET="my-bucket"
export LOCAL_DEV_PATH="/my/local/dev/path"
export LOCAL_CONTENT_PATH="/local/content/path"
export ELASTICSEARCH_URL="http://localhost:9001"
export GOOGLE_APPLICATION_CREDENTIALS="/local/gcp/json/creds"
# make the builder image
make builder-image

# execute the interactive development shell
make exec-interactive

Usage

This repo contains several related sub-projects that are all avaliable as public Docker images.

Scrape

Use the scrape image to test scraping a url of its content

docker run \
--rm \
kai5263499/rhema-scrape "${URI}"

process-url

Use the process-url image to process a URL into an mp3 stored in the ${LOCAL_PATH} directory

docker run \
--rm \
-e BUCKET="${BUCKET}" \
-v ${LOCAL_PATH}:/data \
--tmpfs /tmp \
kai5263499/rhema-process-url "${URI}"

bot

Use the bot image to connect to have a bot listen for URLS posted on a slack channel and convert those to audio, stored in a ${LOCAL_PATH} directory

docker run \
-it --rm \
-e BUCKET="${BUCKET}" \
-e SLACK_TOKEN="${SLACK_TOKEN}" \
-e LOG_LEVEL="${LOG_LEVEL}" \
-v ${LOCAL_CONTENT_PATH}:/data \
-v ${LOCAL_TMP_PATH}:/tmp \
kai5263499/rhema-bot

Misc

There are a few other optional parameters that can be set.

• WORDS_PER_MINUTE - How fast espeak should make its resulting speech
• ATEMPO - How much to make audio and video content. This is a floating point decimal where the default is 2.0 or 2x the origional speed

Possible espeak-ng voice options for the optional ESPEAK_VOICE parameter include the following from /usr/lib/x86_64-linux-gnu/espeak-ng-data/voices/!v/:

 Andy          Gene    Mario         Tweaky   f1   f5      john         klatt3   m2   m6       norbert   steph    whisper
 Annie         Gene2   Michael       aunty    f2   iven    kaukovalta   klatt4   m3   m7       quincy    steph2   whisperf
 AnxiousAndy   Jacky  'Mr serious'   boris    f3   iven2   klatt        linda    m4   max      rob       steph3   zac
 Denis         Lee     Storm         croak    f4   iven3   klatt2       m1       m5   michel   robert    travis

Deploy to Kubernetes

Helm3 charts are provided to deploy this service into a Kubernetes cluster

# Install the mechanism for generating let's encrypt certificates for SSL transport
kubectl create namespace cert-manager
helm install cert-manager jetstack/cert-manager --namespace cert-manager --version v1.2.0 --set installCRDs=true

# Install nginx ingress ingress with a public IP
helm install nginx-ingress ingress-nginx/ingress-nginx --set controller.publishService.enabled=true

# Manual step (sorry) - Set your domain's A record to the public IP exposed above

# Finally, install the Rhema suite of services
helm install rhema ./helm

Documentation

Index

• Constants
• func DownloadUriToFile(uri string, uriFullFilename string) error
• func GetFilePath(req *pb.Request) (string, error)
• func NewContentStorage(cfg *domain.Config) (domain.Storage, error)
• type Api
  • func NewApi(shutdownContext context.Context, shutdownCancelFunc context.CancelFunc, ...) (*Api, error)
  • func (a *Api) ListAllRequests(ctx echo.Context) error
  • func (a *Api) Live(ctx echo.Context) error
  • func (a *Api) Ping(ctx echo.Context) error
  • func (a *Api) Ready(ctx echo.Context) error
  • func (a *Api) RetrieveResultContent(ctx echo.Context, pType string, requestId string) error
  • func (a *Api) RetrieveResultStatus(ctx echo.Context, requestId string) error
  • func (a *Api) Start()
  • func (a *Api) SubmitRequest(ctx echo.Context, params v1.SubmitRequestParams) error
• type RequestProcessor
  • func NewRequestProcessor(cfg *domain.Config, scrape domain.Converter, youtube domain.Converter, ...) *RequestProcessor
  • func (rp *RequestProcessor) Process(ci *pb.Request) (err error)
• type Scrape
  • func NewScrape(cfg *domain.Config) *Scrape
  • func (s *Scrape) Convert(ci *pb.Request) error
• type SpeedupAudio
  • func NewSpeedupAudio(cfg *domain.Config, exec func(command string, args ...string) *exec.Cmd) *SpeedupAudio
  • func (sa *SpeedupAudio) Convert(ci *pb.Request) error
• type Text2Mp3
  • func NewText2Mp3(cfg *domain.Config) *Text2Mp3
  • func (tm *Text2Mp3) Convert(ci *pb.Request) (err error)
• type YouTube
  • func NewYoutube(cfg *domain.Config, scrape domain.Converter, speedupAudio domain.Converter) *YouTube
  • func (yt *YouTube) Convert(ci *pb.Request) error

Constants

const (
	STATSD_REQUEST_ACCEPTED = "rhema.request.accepted"
)

Functions

func DownloadUriToFile

func DownloadUriToFile(uri string, uriFullFilename string) error

DownloadUriToFile downloads the contents of a uri to a local file

func GetFilePath

func GetFilePath(req *pb.Request) (string, error)

GetFilePath returns the filename for a given Request

func NewContentStorage

func NewContentStorage(
	cfg *domain.Config,
) (domain.Storage, error)

NewContentStorage returns an instance of ContentStorage

Types

type Api

type Api struct {
	// contains filtered or unexported fields
}

func NewApi

func NewApi(
	shutdownContext context.Context,
	shutdownCancelFunc context.CancelFunc,
	cfg *domain.Config,
	requestProcessor domain.Processor,
	contentStorage domain.Storage,
) (*Api, error)

func (*Api) ListAllRequests

func (a *Api) ListAllRequests(ctx echo.Context) error

(GET /list-requests)

func (*Api) Live

func (a *Api) Live(ctx echo.Context) error

func (*Api) Ping

func (a *Api) Ping(ctx echo.Context) error

func (*Api) Ready

func (a *Api) Ready(ctx echo.Context) error

func (*Api) RetrieveResultContent

func (a *Api) RetrieveResultContent(ctx echo.Context, pType string, requestId string) error

(GET /result/{type}/{request_id})

func (*Api) RetrieveResultStatus

func (a *Api) RetrieveResultStatus(ctx echo.Context, requestId string) error

(GET /status/{request_id})

func (*Api) Start

func (a *Api) Start()

func (*Api) SubmitRequest

func (a *Api) SubmitRequest(ctx echo.Context, params v1.SubmitRequestParams) error

type RequestProcessor

type RequestProcessor struct {
	// contains filtered or unexported fields
}

func NewRequestProcessor

func NewRequestProcessor(cfg *domain.Config, scrape domain.Converter, youtube domain.Converter, text2mp3 domain.Converter, speedupAudio domain.Converter, contentStorage domain.Storage) *RequestProcessor

func (*RequestProcessor) Process

func (rp *RequestProcessor) Process(ci *pb.Request) (err error)

type Scrape

type Scrape struct {
	// contains filtered or unexported fields
}

Scrape represents everything needed to scrape content from a URL

func NewScrape

func NewScrape(cfg *domain.Config) *Scrape

NewScrape returns a new Scrape instance

func (*Scrape) Convert

func (s *Scrape) Convert(ci *pb.Request) error

Convert takes a URL, reads the content, stores the relevant body of the website, and returns a new TEXT

type SpeedupAudio

type SpeedupAudio struct {
	// contains filtered or unexported fields
}

func NewSpeedupAudio

func NewSpeedupAudio(cfg *domain.Config, exec func(command string, args ...string) *exec.Cmd) *SpeedupAudio

func (*SpeedupAudio) Convert

func (sa *SpeedupAudio) Convert(ci *pb.Request) error

type Text2Mp3

type Text2Mp3 struct {
	// contains filtered or unexported fields
}

func NewText2Mp3

func NewText2Mp3(cfg *domain.Config) *Text2Mp3

func (*Text2Mp3) Convert

func (tm *Text2Mp3) Convert(ci *pb.Request) (err error)

type YouTube

type YouTube struct {
	// contains filtered or unexported fields
}

func NewYoutube

func NewYoutube(cfg *domain.Config, scrape domain.Converter, speedupAudio domain.Converter) *YouTube

func (*YouTube) Convert

func (yt *YouTube) Convert(ci *pb.Request) error