s3writer

README

Implements io.Writer and io.ReaderFrom to upload to S3

This module provides implementations of io.Writer and io.ReaderFrom for S3 uploading needs.

package main

import (
	"context"
	"log"
	"os"
	"os/signal"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/nguyengg/xy3/s3writer"
)

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Kill, os.Interrupt)
	defer stop()

	cfg, err := config.LoadDefaultConfig(ctx)
	if err != nil {
		log.Fatal(err)
	}

	client := s3.NewFromConfig(cfg)

	// s3writer.Writer implements io.Writer and io.ReaderFrom so I can start piping local file to upload.
	w, err := s3writer.New(ctx, client, &s3.PutObjectInput{
		Bucket: aws.String("my-bucket"),
		Key:    aws.String("my-key"),
	})
	if err != nil {
		log.Fatal(err)
	}

	// either way below will work.
	f, _ := os.Open("/path/to/file")
	_, err = f.WriteTo(w)
	//_, err = w.ReadFrom(f)
	_ = f.Close()
}

Documentation

Index

• Constants
• Variables
• type AbortAttempt
• type MultipartUploadError
  • func (e MultipartUploadError) Error() string
  • func (e MultipartUploadError) Unwrap() error
• type Options
• type Writer
  • func New(ctx context.Context, client WriterClient, input *s3.PutObjectInput, ...) (Writer, error)
• type WriterClient

Constants

const (
	// MaxObjectSize is the maximum size of an S3 object.
	//
	// See [Amazon S3 multipart upload limits].
	//
	// [Amazon S3 multipart upload limits]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
	MaxObjectSize = int64(5_497_558_138_880)

	// MaxPartCount is the maximum number of parts per upload.
	//
	// See [Amazon S3 multipart upload limits].
	//
	// [Amazon S3 multipart upload limits]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
	MaxPartCount = 10_000

	// MinPartSize is the minimum number of bytes per part upload.
	//
	// See [Amazon S3 multipart upload limits].
	//
	// [Amazon S3 multipart upload limits]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
	MinPartSize = int64(5_242_880)

	// MaxPartSize is the maximum number of bytes per part upload.
	//
	// See [Amazon S3 multipart upload limits].
	//
	// [Amazon S3 multipart upload limits]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
	MaxPartSize = int64(5_368_709_120)

	// DefaultConcurrency is the default value for Options.Concurrency.
	DefaultConcurrency = 3
)

Variables

var ErrClosed = errors.New("writer already closed")

ErrClosed is returned by all Writer write methods after Close returns.

Types

type AbortAttempt

type AbortAttempt int

const (
	AbortNotAttempted AbortAttempt = iota
	AbortSuccess
	AbortFailure
)

type MultipartUploadError

type MultipartUploadError struct {
	// Err is the wrapped error encountered during multipart upload.
	Err error
	// UploadID is the upload Id of the multipart upload.
	UploadID string
	// Abort indicates whether an abort attempt was made successfully.
	Abort AbortAttempt
	// AbortErr is the error encountered while attempting to abort the multipart upload.
	AbortErr error
}

MultipartUploadError is the error returns from any writes including Writer.Close if there was an error while using multipart upload.

func (MultipartUploadError) Error

func (e MultipartUploadError) Error() string

func (MultipartUploadError) Unwrap

func (e MultipartUploadError) Unwrap() error

type Options

type Options struct {
	// Concurrency controls the number of goroutines in the pool that supports parallel UploadPart.
	//
	// Default to DefaultConcurrency. Must be a positive integer. Set to 1 to disable the feature.
	//
	// Because a single goroutine pool is shared for all Writer.Write calls, it is acceptable to set this value to
	// a high number (`runtime.NumCPU()`) and use MaxBytesInSecond instead to add rate limiting.
	Concurrency int

	// MaxBytesInSecond limits the number of bytes that are uploaded in one second.
	//
	// The zero-value indicates no limit. Must be a positive integer otherwise.
	MaxBytesInSecond int64

	// PartSize is the size of each parallel GetObject.
	//
	// Default to MinPartSize. Must be a positive integer; cannot be lower than MinPartSize.
	//
	// Be mindful of the MaxPartCount limit. Since Writer does not know the size of the content being uploaded,
	// caller must set PartSize to something reasonably large enough so that the number of parts does not exceed
	// limit.
	PartSize int64

	// DisableAbortOnError controls whether AbortMultipartUpload is automatically called on error.
	DisableAbortOnError bool
}

Options customises the returned Writer of NewWriter.

type Writer

type Writer interface {
	// Write writes len(p) bytes from p to the write buffer.
	//
	// Every Write may not end up uploading to S3 immediately due to the use of a buffer.
	//
	// See io.Writer for more information on the return values. This method always returns n == len(p) even though
	// the number of bytes uploaded to S3 may be less than n to meet io.Writer requirements for avoiding short
	// writes.
	Write(p []byte) (n int, err error)

	// ReadFrom reads data from src until EOF or error and writes to S3.
	//
	// See io.ReaderFrom for more information on the return values. It returns n as the number of bytes read from
	// src, not the number of bytes uploaded to S3 which can be less than n.
	ReadFrom(src io.Reader) (n int64, err error)

	// Close drains the write buffer and complete the upload if multipart upload is used.
	//
	// After Close completes successfully, subsequent writes including Close will return ErrClosed.
	Close() error
}

Writer uses either a single PutObject or multipart upload to upload content to S3.

Writer is implemented as a buffered partWriter. Close must be called to drain the write buffer and complete the multipart upload if multipart upload was used; the return value will be an MultipartUploadError instance in this case. If the number of bytes to upload is less than MinPartSize, a single PutObject is used.

Similar to bufio.Writer, if an error occurs writing to Writer, no more data will be accepted and subsequent writes including Close will return the initial error.

func New

func New(ctx context.Context, client WriterClient, input *s3.PutObjectInput, optFns ...func(*Options)) (Writer, error)

New returns a Writer given the PutObject input parameters.

Unlike s3/manager, you do not pass the content being uploaded via s3.PutObjectInput.Body here. Instead, use the returned Writer as either an io.Writer or an io.ReaderFrom.

New will only return a non-nil error if there are invalid options.

type WriterClient

type WriterClient interface {
	PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error)
	UploadPart(context.Context, *s3.UploadPartInput, ...func(*s3.Options)) (*s3.UploadPartOutput, error)
	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput, ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput, ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
}

WriterClient abstracts the S3 APIs that are needed to implement Writer.