s3

README

s3 - Amazon Web Service s3 for Abstract File Storage

Usage

import (
	"context"
	"fmt"
	"github.com/viant/afs"
	_ "github.com/viant/afsc/s3"
	"io/ioutil"
	"log"
)

func main() {
	service := afs.New()
	ctx := context.Background()
	objects, err := service.List(ctx, "s3://myBucket/folder")
	if err != nil {
		log.Fatal(err)
	}
	for _, object := range objects {
		fmt.Printf("%v %v\n", object.Name(), object.URL())
		if object.IsDir() {
			continue
		}
		reader, err := service.Download(ctx, object)
		if err != nil {
			log.Fatal(err)
		}
		defer reader.Close()
		data, err := ioutil.ReadAll(reader)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("%s\n", data)
	}
	
	err = service.Copy(ctx, "s3://myBucket/folder", "/tmp")
	if err != nil {
		log.Fatal(err)
	}
}

Auth

• AwsConfigProvider

To use auth provider, provide a type that implements the following interface, you can also use s3.NewAuthConfig

type AwsConfigProvider interface {
	AwsConfig() (*aws.Config, error)
}

Example:

    
    authConfig, err := s3.NewAuthConfig(option.NewLocation("credetnialsfile"))
    if err != nil {
		log.Fatal(err)
	}

	service := afs.New()
	reader, err := service.DownloadWithURL(ctx, "s3://my-bucket/myfolder/asset.txt", authConfig)
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("data: %s\n", data)
	

Options

• Md5: when uploading content with this option, supplied option is used for Put ContentMD5, otherwise md5 is computed for supplied content.

• aws.Config: s3 client.

• Region: s3 client region.

• Custom key encryption

   customKey, err := option.NewAES256Key([]byte("secret-key-that-is-32-bytes-long"))
   	if err != nil {
   		log.Fatal(err)
   	}
   	ctx := context.Background()
   	service := afs.New()
   	err = service.Upload(ctx, "s3://mybucket/folder/secret1.txt", 0644, strings.NewReader("my secret text"), customKey)
   	if err != nil {
   		log.Fatal(err)
   	}
   	reader, err := service.DownloadWithURL(ctx, "s3://mybucket/folder/secret1.txt", customKey)
   	data, err := ioutil.ReadAll(reader)
   	if err != nil {
   		log.Fatal(err)
   	}
   	fmt.Printf("data: %s\n", data)

• option.Checksum{Skip:true}: checksum (md5) is not computed to stream data in chunks
• option.Stream: download reader reads data with specified stream PartSize

	_ = os.Setenv("AWS_SDK_LOAD_CONFIG", "true")
	ctx := context.Background()
	fs := afs.New()
	sourceURL := "s3://myBucket/path/myasset.gz"
	reader, err := fs.DownloadWithURL(ctx, sourceURL, option.NewStream(64*1024*1024, 0))
	if err != nil {
		log.Fatal(err)
	}
	jwtConfig, err := gs.NewJwtConfig()
	if err != nil {
		log.Fatal(err)
	}
	destURL := "gs://myBucket/path/myasset.gz"
	err = fs.Upload(ctx, destURL, 0644, reader, jwtConfig, &option.Checksum{Skip:true})
	if err != nil {
		log.Fatal(err)
		return
	}

Documentation

Overview

Package s3 implement abstract file storage API with Amazon Web Service s3

Index

• Constants
• func New(options ...storage.Option) storage.Manager
• func NewReadSeeker(ctx context.Context, input *s3.GetObjectInput, ...) io.ReadSeeker
• func NewStorager(ctx context.Context, baseURL string, options ...storage.Option) (*storager, error)
• func Provider(options ...storage.Option) (storage.Manager, error)
• type AuthConfig
  • func NewAuthConfig(options ...storage.Option) (*AuthConfig, error)
  • func (c *AuthConfig) AwsConfig() (*aws.Config, error)
• type AwsConfigProvider
• type CompletedParts
  • func (a CompletedParts) Len() int
  • func (a CompletedParts) Less(i, j int) bool
  • func (a CompletedParts) Swap(i, j int)
• type Writer
  • func NewWriter(initSize int) *Writer
  • func (w *Writer) Bytes() []byte
  • func (w *Writer) Reset()
  • func (w *Writer) WriteAt(p []byte, offset int64) (n int, err error)

Examples

• New
• NewAuthConfig

Constants

const Scheme = "s3"

Scheme s3 URL proto scheme

Functions

func New

func New(options ...storage.Option) storage.Manager

New creates s3 manager

Example

package main

import (
	"context"
	"fmt"
	"io"
	"log"

	"github.com/viant/afsc/s3"
)

func main() {
	service := s3.New()
	ctx := context.Background()
	reader, err := service.OpenURL(ctx, "s3://my-bucket/folder/asset")
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()
	data, err := io.ReadAll(reader)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("data: %s\n", data)
}

func NewReadSeeker

func NewReadSeeker(ctx context.Context, input *s3.GetObjectInput, downloader *s3manager.Downloader, partSize, size int) io.ReadSeeker

NewReadSeeker create a reader seeker

func NewStorager

func NewStorager(ctx context.Context, baseURL string, options ...storage.Option) (*storager, error)

func Provider

func Provider(options ...storage.Option) (storage.Manager, error)

Provider returns a google storage manager

Types

type AuthConfig

type AuthConfig struct {
	Key       string    `json:",omitempty"`
	Secret    string    `json:",omitempty"`
	Region    string    `json:",omitempty"`
	AccountID string    `json:"-"`
	Token     string    `json:"-"`
	Expiry    time.Time `json:"-"`
	RoleArn   string    `json:",omitempty"`
}

AuthConfig represents an auth config

func NewAuthConfig

func NewAuthConfig(options ...storage.Option) (*AuthConfig, error)

NewAuthConfig returns new auth config from location

Example

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"path"

	"github.com/viant/afs"
	"github.com/viant/afs/option"
	"github.com/viant/afsc/s3"
)

func main() {

	authConfig, err := s3.NewAuthConfig(option.NewLocation(path.Join(os.Getenv("HOME"), ".aws/credentials")))
	if err != nil {
		log.Fatal(err)
	}

	ctx := context.Background()
	// add default import _ "github.com/viant/afsc/s3"

	service := afs.New()
	reader, err := service.OpenURL(ctx, "s3://my-bucket/myfolder/asset.txt", authConfig)
	data, err := io.ReadAll(reader)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("data: %s\n", data)

}

func (*AuthConfig) AwsConfig

func (c *AuthConfig) AwsConfig() (*aws.Config, error)

AwsConfig returns aws config

type AwsConfigProvider

type AwsConfigProvider interface {
	AwsConfig() (*aws.Config, error)
}

AwsConfigProvider represents aws config provider

type CompletedParts

type CompletedParts []*s3.CompletedPart

func (CompletedParts) Len

func (a CompletedParts) Len() int

func (CompletedParts) Less

func (a CompletedParts) Less(i, j int) bool

func (CompletedParts) Swap

func (a CompletedParts) Swap(i, j int)

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer represents a bytes writer at

func NewWriter

func NewWriter(initSize int) *Writer

NewWriter returns a writer

func (*Writer) Bytes

func (w *Writer) Bytes() []byte

func (*Writer) Reset

func (w *Writer) Reset()

Reset resets writer

func (*Writer) WriteAt

func (w *Writer) WriteAt(p []byte, offset int64) (n int, err error)

WriteAt returns number of written bytes or error