archiver

README

RapidPro Archiver

Simple service for archiving messages, runs and sessions to S3 from the RapidPro database.

Usage

Archives RapidPro flows, msgs and sessions to S3

Usage of archiver:
  -aws-access-key-id string
    	the access key id to use when authenticating S3 (default "missing_aws_access_key_id")
  -aws-secret-access-key string
    	the secret access key id to use when authenticating S3 (default "missing_aws_secret_access_key")
  -db string
    	the connection string for our database (default "postgres://localhost/temba?sslmode=disable")
  -debug-conf
    	print where config values are coming from
  -help
    	print usage information
  -log-level string
    	the log level, one of error, warn, info, debug (default "info")
  -s3-bucket string
    	the S3 bucket we will write archives to (default "dl-temba-archives")
  -s3-disable-ssl
    	whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network
  -s3-endpoint string
    	the S3 endpoint we will write archives to (default "https://s3.amazonaws.com")
  -s3-force-path-style
    	whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service
  -s3-region string
    	the S3 region we will write archives to (default "us-east-1")
  -sentry-dsn string
    	the sentry configuration to log errors to, if any

Environment variables:
                  ARCHIVER_AWS_ACCESS_KEY_ID - string
              ARCHIVER_AWS_SECRET_ACCESS_KEY - string
                                 ARCHIVER_DB - string
                          ARCHIVER_LOG_LEVEL - string
                          ARCHIVER_S3_BUCKET - string
                     ARCHIVER_S3_DISABLE_SSL - bool
                        ARCHIVER_S3_ENDPOINT - string
                ARCHIVER_S3_FORCE_PATH_STYLE - bool
                          ARCHIVER_S3_REGION - string
                         ARCHIVER_SENTRY_DSN - string

Documentation

Index

• Constants
• func CreateMsgArchive(ctx context.Context, db *sqlx.DB, task *ArchiveTask) error
• func UploadArchive(ctx context.Context, s3Client s3iface.S3API, bucket string, task *ArchiveTask) error
• func WriteArchiveToDB(ctx context.Context, db *sqlx.DB, task *ArchiveTask) error
• type ArchiveTask
  • func GetArchiveTasks(ctx context.Context, db *sqlx.DB, org DBOrg, archiveType ArchiveType) ([]ArchiveTask, error)
• type ArchiveType
• type Config
  • func NewConfig() *Config
• type DBArchive
• type DBOrg
  • func GetActiveOrgs(ctx context.Context, db *sqlx.DB) ([]DBOrg, error)

Constants

const (
	FlowRunType = ArchiveType("flowrun")
	MessageType = ArchiveType("message")
	SessionType = ArchiveType("session")
)

Functions

func CreateMsgArchive

func CreateMsgArchive(ctx context.Context, db *sqlx.DB, task *ArchiveTask) error

func UploadArchive

func UploadArchive(ctx context.Context, s3Client s3iface.S3API, bucket string, task *ArchiveTask) error

func WriteArchiveToDB

func WriteArchiveToDB(ctx context.Context, db *sqlx.DB, task *ArchiveTask) error

Types

type ArchiveTask

type ArchiveTask struct {
	Org         DBOrg
	ArchiveType ArchiveType
	StartDate   time.Time
	EndDate     time.Time

	ID int

	RecordCount int
	Filename    string
	FileSize    int64
	FileHash    string
	URL         string

	BuildStart time.Time
}

func GetArchiveTasks

func GetArchiveTasks(ctx context.Context, db *sqlx.DB, org DBOrg, archiveType ArchiveType) ([]ArchiveTask, error)

type ArchiveType

type ArchiveType string

type Config

type Config struct {
	DB        string `help:"the connection string for our database"`
	LogLevel  string `help:"the log level, one of error, warn, info, debug"`
	SentryDSN string `help:"the sentry configuration to log errors to, if any"`

	S3Endpoint       string `help:"the S3 endpoint we will write archives to"`
	S3Region         string `help:"the S3 region we will write archives to"`
	S3Bucket         string `help:"the S3 bucket we will write archives to"`
	S3DisableSSL     bool   `` /* 158-byte string literal not displayed */
	S3ForcePathStyle bool   `` /* 127-byte string literal not displayed */

	AWSAccessKeyID     string `help:"the access key id to use when authenticating S3"`
	AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"`
}

func NewConfig

func NewConfig() *Config

type DBArchive

type DBArchive struct {
	ID              int       `db:"id"`
	ArchiveType     string    `db:"archive_type"`
	OrgID           int       `db:"org_id"`
	CreatedOn       time.Time `db:"created_on"`
	ArchiveDuration int       `db:"archive_duration"`

	StartDate time.Time `db:"start_date"`
	EndDate   time.Time `db:"end_date"`

	RecordCount int `db:"record_count"`

	ArchiveSize int    `db:"archive_size"`
	ArchiveHash string `db:"archive_hash"`
	ArchiveURL  string `db:"archive_url"`

	IsPurged  bool `db:"is_purged"`
	BuildTime int  `db:"build_time"`
}

type DBOrg

type DBOrg struct {
	ID         int       `db:"id"`
	Name       string    `db:"name"`
	CreatedOn  time.Time `db:"created_on"`
	ActiveDays int
}

func GetActiveOrgs

func GetActiveOrgs(ctx context.Context, db *sqlx.DB) ([]DBOrg, error)