azure

README

Azure Blob Storage

This sink implements Azure Blob Storage protocol. It can be enabled by adding the following configuration in the tables section:

tables:
  eventlog:
    compact:                               # enable compaction
      interval: 60                         # compact every 60 seconds
      nameFunc: "s3://bucket/namefunc.lua" # file name function
      azure:                               # sink to use
        container: "container-id-1"        # the container ID
        prefix: ""                         # (optional) prefix to add
...

This sink can write to different Storage Accounts randomly each time, if multiple storage accounts are configured

tables:
  eventlog:
    compact:                               # enable compaction
      interval: 60                         # compact every 60 seconds
      nameFunc: "s3://bucket/namefunc.lua" # file name function
      azure:                               # sink to use
        container: "container-id-1"        # the container ID
        prefix: ""                         # (optional) prefix to add
        parallelism: 0                     # (optional) default to 5
        blockSize: 5242880                 # (optional) if not set, use default
        defaultBlobServiceURL: ""          # (optional) if not set, use default
        storageAccounts:                   # (optional) if not set, use single storage writer
          - "storage-account-id-0"
          - "storage-account-id-1"
...

Documentation

Index

• func GetAzureStorageCredentials(monitor monitor.Monitor) (azblob.Credential, error)
• type MultiAccountWriter
  • func NewMultiAccountWriter(monitor monitor.Monitor, ...) (*MultiAccountWriter, error)
  • func (m *MultiAccountWriter) Write(key key.Key, blocks []block.Block) error
  • func (m *MultiAccountWriter) WriteToContanier(key key.Key, buffer []byte, containerURL *azblob.ContainerURL) error
• type Writer
  • func New(container, prefix, filter, encoding string, monitor monitor.Monitor) (*Writer, error)
  • func (w *Writer) Write(key key.Key, blocks []block.Block) error

Functions

func GetAzureStorageCredentials

func GetAzureStorageCredentials(monitor monitor.Monitor) (azblob.Credential, error)

Types

type MultiAccountWriter

type MultiAccountWriter struct {
	*base.Writer
	// contains filtered or unexported fields
}

MultiAccountWriter represents a writer for Microsoft Azure with multiple storage accounts.

func NewMultiAccountWriter

func NewMultiAccountWriter(monitor monitor.Monitor, filter, encoding, blobServiceURL, container, prefix string, storageAccount []string, weights []uint, parallelism uint16, blockSize int64) (*MultiAccountWriter, error)

NewMultiAccountWriter creates a new MultiAccountWriter.

func (*MultiAccountWriter) Write

func (m *MultiAccountWriter) Write(key key.Key, blocks []block.Block) error

Write writes the data to a randomly selected storage account sink.

func (*MultiAccountWriter) WriteToContanier

func (m *MultiAccountWriter) WriteToContanier(key key.Key, buffer []byte, containerURL *azblob.ContainerURL) error

type Writer

type Writer struct {
	*base.Writer
	// contains filtered or unexported fields
}

Writer represents a writer for Microsoft Azure.

func New

func New(container, prefix, filter, encoding string, monitor monitor.Monitor) (*Writer, error)

New creates a new writer.

func (*Writer) Write

func (w *Writer) Write(key key.Key, blocks []block.Block) error

Write writes the data to the sink.