downloader

Documentation

Overview

Package downloader provides the sync and async file download functionality.

Index

• Variables
• func Filename(f *slack.File) string
• type Client
  • func New(client Downloader, fs fsadapter.FS, opts ...Option) *Client
  • func (c *Client) AsyncDownloader(ctx context.Context, dir string, fileDlQueue <-chan *slack.File) (chan struct{}, error)
  • func (c *Client) DownloadFile(dir string, f slack.File) (string, error)
  • func (c *Client) SaveFile(ctx context.Context, dir string, f *slack.File) (int64, error)
  • func (c *Client) Start(ctx context.Context)
  • func (c *Client) Stop()
• type Downloader
• type FileRequest
• type Option
  • func Limiter(l *rate.Limiter) Option
  • func Logger(l logger.Interface) Option
  • func Retries(n int) Option
  • func Workers(n int) Option

Examples

• New (Advanced)
• New (Basic)

Variables

var ErrNotStarted = errors.New("downloader not started")

Functions

func Filename

func Filename(f *slack.File) string

Filename returns name of the file

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client is the instance of the downloader.

func New

func New(client Downloader, fs fsadapter.FS, opts ...Option) *Client

New initialises new file downloader.

Example (Advanced)

package main

import (
	"context"
	"fmt"

	"github.com/rusq/slackdump/v2/downloader"
	"github.com/rusq/slackdump/v2/fsadapter"
	"github.com/slack-go/slack"
	"golang.org/x/time/rate"
)

func main() {
	client := slack.New("token")

	// initialise the filesystem (files.zip archive)
	fs, err := fsadapter.NewZipFile("files.zip")
	if err != nil {
		fmt.Println("failed to initialise the file system")
		return
	}

	dl := downloader.New(
		client,
		fs,
		downloader.Retries(100), // 100 retries when rate limited
		downloader.Limiter(rate.NewLimiter(20, 1)), // rate limit
		downloader.Workers(8),                      // number of download workers
	)

	f := &slack.File{}

	if n, err := dl.SaveFile(context.Background(), "some_dir", f); err != nil {
		fmt.Printf("failed to save the file: %s", err)
	} else {
		fmt.Printf("downloaded: %d bytes", n)
	}
}

Example (Basic)

package main

import (
	"context"
	"fmt"

	"github.com/rusq/slackdump/v2/downloader"
	"github.com/rusq/slackdump/v2/fsadapter"
	"github.com/slack-go/slack"
)

func main() {
	client := slack.New("token")
	fs := fsadapter.NewDirectory("files/")

	dl := downloader.New(
		client,
		fs,
	)

	f := &slack.File{}

	if n, err := dl.SaveFile(context.Background(), "some_dir", f); err != nil {
		fmt.Printf("failed to save the file: %s", err)
	} else {
		fmt.Printf("downloaded: %d bytes", n)
	}
}

func (*Client) AsyncDownloader

func (c *Client) AsyncDownloader(ctx context.Context, dir string, fileDlQueue <-chan *slack.File) (chan struct{}, error)

AsyncDownloader starts Client.worker goroutines to download files concurrently. It will download any file that is received on fileDlQueue channel. It returns the "done" channel and an error. "done" channel will be closed once all downloads are complete.

func (*Client) DownloadFile

func (c *Client) DownloadFile(dir string, f slack.File) (string, error)

DownloadFile requires a started downloader, otherwise it will return ErrNotStarted. Will place the file to the download queue, and save the file to the directory that was specified when Start was called. If the file buffer is full, will block until it becomes empty. It returns the filepath within the filesystem.

func (*Client) SaveFile

func (c *Client) SaveFile(ctx context.Context, dir string, f *slack.File) (int64, error)

SaveFile saves a single file to the specified directory synchrounously.

func (*Client) Start

func (c *Client) Start(ctx context.Context)

Start starts an async file downloader. If the downloader is already started, it does nothing.

func (*Client) Stop

func (c *Client) Stop()

Stop stops the downloader.

type Downloader

type Downloader interface {
	// GetFile retreives a given file from its private download URL
	GetFile(downloadURL string, writer io.Writer) error
}

Downloader is the file downloader interface. It exists primarily for mocking in tests.

type FileRequest

type FileRequest struct {
	Directory string
	File      *slack.File
}

type Option

type Option func(*Client)

Option is the function signature for the option functions.

func Limiter

func Limiter(l *rate.Limiter) Option

Limiter uses the initialised limiter instead of built in.

func Logger

func Logger(l logger.Interface) Option

Logger allows to use an external log library, that satisfies the logger.Interface.

func Retries

func Retries(n int) Option

Retries sets the number of attempts that will be taken for the file download.

func Workers

func Workers(n int) Option

Workers sets the number of workers for the download queue.