s3setlock

README

s3-setlock

Like the setlock command using Amazon S3.

This tool uses conditional requests as described in the Amazon S3 User Guide to implement locking mechanisms. This ensures that operations are performed only if certain conditions are met, providing a robust way to handle concurrent access to S3 objects.

Usage as command line

Usage: s3-setlock [ -nNxX ] [--endpoint <endpoint>] [--timeout <duration>] [--log-level <level> --log-format <json/text>] [--version] s3://<bucket_name>/<object_key> your_command
Flags:
  -n
        No delay. If fn is locked by another process, setlock gives up.
  -N
        (Default.) Delay. If fn is locked by another process, setlock waits until it can obtain a new lock.
  -x
        If fn cannot be update-item (or put-item) or locked, setlock exits zero.
  -X
        (Default.) If fn cannot be update-item (or put-item) or locked, setlock prints an error message and exits nonzero.
  --log-format string
        log format (text, json)
  --log-level string
        minimum log level (debug, info, warn, error)
  --region string
        aws region
  --timeout string
        set command timeout
  --version
        show version

for Example

$s3-setlock s3://mybucket/mykey sleep 60

Usage as library

example code is below.

package s3setlock_test

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
	"time"

	s3setlock "github.com/mashiike/s3-setlock"
)

func main() {
	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
	defer cancel()
	locker, err := s3setlock.New(
        "s3://mybucket/myobject.lock",
		s3setlock.WithContext(ctx),
		s3setlock.WithDelay(true),
		s3setlock.WithLeaseDuration(60*time.Second),
	)
	if err != nil {
		log.Fatal(err)
	}
	var wg sync.WaitGroup
	counter := 0
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for i := 0; i < 10; i++ {
				err := s3setlock.HandleBailout(func() error {
					locker.Lock()
					counter++
					locker.Unlock()
					return nil
				})
				if err != nil {
					log.Fatal(err)
				}
			}
		}()
	}
	wg.Wait()
	fmt.Println(counter)
	// Output:
	// 100
}

Note: If Lock or Unlock fails, for example because you can't put object to Amazon S3, it will panic.
If you don't want it to panic, use LockWithError() and UnlockWithError(). Alternatively, use the WithNoBailout option.

more information see go doc.

Install

binary packages

Releases.

Homebrew tap

$ brew install mashiike/tap/s3-setlock

License

see LICENSE file.

Documentation

Overview

Example

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	s3setlock "github.com/mashiike/s3-setlock"
)

func main() {
	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
	defer cancel()
	bucketName, client, err := newS3Client(ctx)
	if err != nil {
		log.Fatal(err)
	}
	locker, err := s3setlock.New(fmt.Sprintf("s3://%s/myobject.lock", bucketName),
		s3setlock.WithContext(ctx),
		s3setlock.WithClient(client),
		s3setlock.WithDelay(true),
		s3setlock.WithLeaseDuration(60*time.Second),
	)
	if err != nil {
		log.Fatal(err)
	}
	var wg sync.WaitGroup
	counter := 0
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for i := 0; i < 10; i++ {
				locker.Lock()
				counter++
				locker.Unlock()
			}
		}()
	}
	wg.Wait()
	fmt.Println(counter)
}

func newS3Client(ctx context.Context) (string, s3setlock.S3Client, error) {
	bucketName, ok := os.LookupEnv("TEST_S3_SETLOCK_BUCKET_NAME")
	if !ok || bucketName == "" {
		return "s3-setlock-test", &mockClient{}, nil
	}
	awsCfg, err := config.LoadDefaultConfig(ctx)
	if err != nil {
		return "", nil, err
	}
	return bucketName, s3.NewFromConfig(awsCfg), nil
}

Output:

100

Index

• Variables
• func AsBailout(result interface{}, target *error) bool
• func HandleBailout(fn func() error) (err error)
• func WithClient(client S3Client) func(opts *Options)
• func WithContext(ctx context.Context) func(opts *Options)
• func WithDelay(delay bool) func(opts *Options)
• func WithLeaseDuration(d time.Duration) func(opts *Options)
• func WithLogger(logger *slog.Logger) func(opts *Options)
• func WithNoBailout() func(opts *Options)
• func WithRegion(region string) func(opts *Options)
• type Locker
  • func New(urlStr string, optFns ...func(*Options)) (*Locker, error)
  • func (l *Locker) LastError() error
  • func (l *Locker) Lock()
  • func (l *Locker) LockGranted() bool
  • func (l *Locker) LockWithError(ctx context.Context) (bool, error)
  • func (l *Locker) Reset()
  • func (l *Locker) Unlock()
  • func (l *Locker) UnlockWithError(ctx context.Context) error
• type Options
• type S3Client

Examples

• Package

Variables

var (
	DefaultLeaseDuration = 60 * time.Second
)

Default values

var Version = "0.2.0"

Functions

func AsBailout

func AsBailout(result interface{}, target *error) bool

AsBailout returns bailout errors example:

defer func() {
	var err error
	if s3setlock.AsBailout(recover(),&err) {
		log.Fatal(err)
	}
}()

func HandleBailout

func HandleBailout(fn func() error) (err error)

HandleBailout executes the provided function and recovers from any bailout errors, returning the bailout error if one occurs.

func WithClient

func WithClient(client S3Client) func(opts *Options)

WithClient specifies the AWS SDK Client. If not specified, the default client is created.

func WithContext

func WithContext(ctx context.Context) func(opts *Options)

WithContext specifies the Context used by Lock() and Unlock().

func WithDelay

func WithDelay(delay bool) func(opts *Options)

WithDelay will delay the acquisition of the lock if it fails to acquire the lock. This is similar to the N option of setlock. The default is delay enalbed(true). Specify false if you want to exit immediately if Lock acquisition fails.

func WithLeaseDuration

func WithLeaseDuration(d time.Duration) func(opts *Options)

WithLeaseDuration affects the heartbeat interval and TTL after Lock acquisition. The default is 10 seconds

func WithLogger

func WithLogger(logger *slog.Logger) func(opts *Options)

WithLogger is a setting to enable the log output of S3 Locker. By default, Logger that does not output anywhere is specified.

func WithNoBailout

func WithNoBailout() func(opts *Options)

WithNoBailout changes the behavior so that it does not panic if an error occurs in the Lock () and Unlock () functions. Check the LastErr () function to see if an error has occurred when WithNoBailout is specified.

func WithRegion

func WithRegion(region string) func(opts *Options)

WithRegion specifies the AWS Region. Default AWS_DEFAULT_REGION env

Types

type Locker

type Locker struct {
	NoBailout bool
	// contains filtered or unexported fields
}

Locker is a locker that uses S3 as a lock storage

func New

func New(urlStr string, optFns ...func(*Options)) (*Locker, error)

New returns *Locker

func (*Locker) LastError

func (l *Locker) LastError() error

func (*Locker) Lock

func (l *Locker) Lock()

func (*Locker) LockGranted

func (l *Locker) LockGranted() bool

func (*Locker) LockWithError

func (l *Locker) LockWithError(ctx context.Context) (bool, error)

func (*Locker) Reset

func (l *Locker) Reset()

func (*Locker) Unlock

func (l *Locker) Unlock()

func (*Locker) UnlockWithError

func (l *Locker) UnlockWithError(ctx context.Context) error

type Options

type Options struct {
	NoBailout     bool
	Logger        *slog.Logger
	Delay         bool
	Client        S3Client
	Region        string
	LeaseDuration time.Duration
	// contains filtered or unexported fields
}

type S3Client

type S3Client interface {
	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
	HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
	DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
}