azure

README

azure blog store

Cloudstorage abstraction to Azure Blob storage.

config

Login to your https://portal.azure.com account and click "Storage Accounts" in menu. Then Click the storage account you want.

• config.Project is required. use "Account" in azure portal. This is the "Name" of cloudstorageazuretesting https://cloudstorageazuretesting.blob.core.windows.net/
• azure_key from your storage account go to the menu "Access Keys"
• Bucket go to Containers in the azure storage and get this name.

Example in Go:

package main

import (
	"context"
	"fmt"
	"os"

	"github.com/araddon/gou"
	"google.golang.org/api/iterator"

	"github.com/lytics/cloudstorage"
	"github.com/lytics/cloudstorage/azure"
)

/*

# to use azure tests ensure you have exported

export AZURE_KEY="aaa"
export AZURE_PROJECT="bbb"
export AZURE_BUCKET="cloudstorageunittests"

*/

func main() {
	tmpDir, err := os.MkdirTemp("/tmp", "azure_example")
	if err != nil {
		fmt.Println("Could not create temp dir", err)
		os.Exit(1)
	}
	defer os.RemoveAll(tmpDir)

	conf := &cloudstorage.Config{
		Type:       azure.StoreType,
		AuthMethod: azure.AuthKey,
		Bucket:     os.Getenv("AZURE_BUCKET"),
		Project:    os.Getenv("AZURE_PROJECT"),
		TmpDir:     filepath.Join(tempDir, "localcache", "azure"),
		Settings:   make(gou.JsonHelper),
	}

	conf.Settings[azure.ConfKeyAuthKey] = os.Getenv("AZURE_KEY")

	// Should error with empty config
	store, err := cloudstorage.NewStore(conf)
	if err != nil {
		fmt.Println("Could not get azure store ", err)
		os.Exit(1)
	}

	folders, err := store.Folders(context.Background(), cloudstorage.NewQueryForFolders(""))
	if err != nil {
		fmt.Println("Could not get folders ", err)
		os.Exit(1)
	}
	for _, folder := range folders {
		fmt.Println("found folder: ", folder)
	}

	// Create a search query for all objects
	q := cloudstorage.NewQuery("")
	// Create an Iterator
	iter, err := store.Objects(context.Background(), q)
	if err != nil {
		fmt.Println("Could not get iter ", err)
		os.Exit(1)
	}

	for {
		o, err := iter.Next()
		if err == iterator.Done {
			fmt.Println("done, exiting iterator")
			break
		}
		fmt.Println("found object", o.Name())
	}
}

Documentation

Index

• Constants
• Variables
• func NewClient(conf *cloudstorage.Config) (*az.Client, *az.BlobStorageClient, error)
• type FS
  • func NewStore(c *az.Client, blobClient *az.BlobStorageClient, conf *cloudstorage.Config) (*FS, error)
  • func (f *FS) Client() interface{}
  • func (f *FS) Delete(ctx context.Context, name string) error
  • func (f *FS) Folders(ctx context.Context, q cloudstorage.Query) ([]string, error)
  • func (f *FS) Get(ctx context.Context, objectpath string) (cloudstorage.Object, error)
  • func (f *FS) List(ctx context.Context, q cloudstorage.Query) (*cloudstorage.ObjectsResponse, error)
  • func (f *FS) NewObject(objectname string) (cloudstorage.Object, error)
  • func (f *FS) NewReader(o string) (io.ReadCloser, error)
  • func (f *FS) NewReaderWithContext(ctx context.Context, objectname string) (io.ReadCloser, error)
  • func (f *FS) NewWriter(objectName string, metadata map[string]string) (io.WriteCloser, error)
  • func (f *FS) NewWriterWithContext(ctx context.Context, name string, metadata map[string]string, ...) (io.WriteCloser, error)
  • func (f *FS) Objects(ctx context.Context, q cloudstorage.Query) (cloudstorage.ObjectIterator, error)
  • func (f *FS) String() string
  • func (f *FS) Type() string

Constants

const (
	// StoreType = "azure" this is used to define the storage type to create
	// from cloudstorage.NewStore(config)
	StoreType = "azure"

	// ConfKeyAuthKey config key name of the azure api key for auth
	ConfKeyAuthKey = "azure_key"

	// AuthKey is for using azure api key
	AuthKey cloudstorage.AuthMethod = "azure_key"
)

Variables

var (
	// Retries number of times to retry upon failures.
	Retries = 3
	// PageSize is default page size
	PageSize = 2000

	// ErrNoAzureSession no valid session
	ErrNoAzureSession = fmt.Errorf("no valid azure session was created")
	// ErrNoAccessKey error for no azure_key
	ErrNoAccessKey = fmt.Errorf("no settings.azure_key")
	// ErrNoAuth error for no findable auth
	ErrNoAuth = fmt.Errorf("No auth provided")
)

Functions

func NewClient

func NewClient(conf *cloudstorage.Config) (*az.Client, *az.BlobStorageClient, error)

NewClient create new AWS s3 Client. Uses cloudstorage.Config to read necessary config settings such as bucket, region, auth.

Types

type FS

type FS struct {
	PageSize int
	ID       string
	// contains filtered or unexported fields
}

FS Simple wrapper for accessing azure blob files, it doesn't currently implement a Reader/Writer interface so not useful for stream reading of large files yet.

func NewStore

func NewStore(c *az.Client, blobClient *az.BlobStorageClient, conf *cloudstorage.Config) (*FS, error)

NewStore Create AWS S3 storage client of type cloudstorage.Store

func (*FS) Client

func (f *FS) Client() interface{}

Client gets access to the underlying google cloud storage client.

func (*FS) Delete

func (f *FS) Delete(ctx context.Context, name string) error

Delete requested object path string.

func (*FS) Folders

func (f *FS) Folders(ctx context.Context, q cloudstorage.Query) ([]string, error)

Folders get folders list.

func (*FS) Get

func (f *FS) Get(ctx context.Context, objectpath string) (cloudstorage.Object, error)

Get a single File Object

func (*FS) List

func (f *FS) List(ctx context.Context, q cloudstorage.Query) (*cloudstorage.ObjectsResponse, error)

List objects from this store.

func (*FS) NewObject

func (f *FS) NewObject(objectname string) (cloudstorage.Object, error)

NewObject of Type azure.

func (*FS) NewReader

func (f *FS) NewReader(o string) (io.ReadCloser, error)

// Copy from src to destination func (f *FS) Copy(ctx context.Context, src, des cloudstorage.Object) error {

	so, ok := src.(*object)
	if !ok {
		return fmt.Errorf("Copy source file expected s3 but got %T", src)
	}
	do, ok := des.(*object)
	if !ok {
		return fmt.Errorf("Copy destination expected s3 but got %T", des)
	}

	oh := so.b.Object(so.name)
	dh := do.b.Object(do.name)

	_, err := dh.CopierFrom(oh).Run(ctx)
	return err
}

// Move which is a Copy & Delete func (f *FS) Move(ctx context.Context, src, des cloudstorage.Object) error {

	so, ok := src.(*object)
	if !ok {
		return fmt.Errorf("Move source file expected s3 but got %T", src)
	}
	do, ok := des.(*object)
	if !ok {
		return fmt.Errorf("Move destination expected s3 but got %T", des)
	}

	oh := so.b.Object(so.name)
	dh := do.b.Object(des.name)

	if _, err := dh.CopierFrom(oh).Run(ctx); err != nil {
		return err
	}

	return oh.Delete(ctx)
}

NewReader create file reader.

func (*FS) NewReaderWithContext

func (f *FS) NewReaderWithContext(ctx context.Context, objectname string) (io.ReadCloser, error)

NewReaderWithContext create new File reader with context.

func (*FS) NewWriter

func (f *FS) NewWriter(objectName string, metadata map[string]string) (io.WriteCloser, error)

NewWriter create Object Writer.

func (*FS) NewWriterWithContext

func (f *FS) NewWriterWithContext(ctx context.Context, name string, metadata map[string]string, opts ...cloudstorage.Opts) (io.WriteCloser, error)

NewWriterWithContext create writer with provided context and metadata.

func (*FS) Objects

func (f *FS) Objects(ctx context.Context, q cloudstorage.Query) (cloudstorage.ObjectIterator, error)

Objects returns an iterator over the objects in the google bucket that match the Query q. If q is nil, no filtering is done.

func (*FS) String

func (f *FS) String() string

String function to provide azure://..../file path

func (*FS) Type

func (f *FS) Type() string

Type of store = "azure"