lino_s3

package module
v0.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 21, 2024 License: MIT Imports: 13 Imported by: 0

README

Lino S3

High level S3 wrapper for Go.

Usage

NewLinoS3(sess *session.Session) -> LinoS3

LinoS3
  -> UseInterceptors(...interceptors) -> LinoS3 (sub entities will inherit these interceptors)
  -> Bucket(bucketName) -> LinoS3Bucket

LinoS3Bucket
  -> UseInterceptors(...interceptors) -> LinoS3Bucket
  -> Object(key) -> LinoS3Object
  -> SubPath(subPath) -> LinoS3Path
  -> Piece(objectKey, start, end) -> LinoS3Piece
  -> PieceByKey(pieceKey) -> LinoS3Piece

LinoS3Path
  -> UseInterceptors(...interceptors) -> LinoS3Path
  -> Object(key) -> LinoS3Object
  -> SubPath(subPath) -> LinoS3Path

LinoS3Object
  -> UseInterceptors(...interceptors) -> LinoS3Path
  -> HasInterceptors() -> bool
  -> Piece(start, end) -> LinoS3Piece

  -> Get() -> *s3.GetObjectOutput
  -> Put(s3.PutObjectInput) -> *s3.PutObjectOutput
     // key and bucket in input will be overrided,so it is not necessary to set them
  -> Upload(s3manager.UploadInput) -> *s3manager.UploadOutput
  -> Delete() => *s3.DeleteObjectOutput

  -> ReadTo(io.Writer)
  -> ReadBuffer() -> buffer
  -> ReadString() -> string
  -> ReadJSON(&value) // encoding/json
  -> ReadCBOR(&value) // github.com/fxamacker/cbor/v2
  -> ReadCSV(&value)  // github.com/gocarina/gocsv

  -> WriteFrom(io.Reader, contentType? string)
  -> WriteBuffer(buffer, contentType? string)
  -> WriteString(string, contentType? string)
  -> WriteJSON(&value)
  -> WriteCBOR(&value)
  -> WriteCSV(&value)

  -> Key() -> string // pieceKey

LinoS3Piece
  -> Get() -> *s3.GetObjectOutput

  -> ReadTo(io.Writer)
  -> ReadBuffer() -> buffer
  -> ReadString() -> string
  -> ReadJSON(&value)
  -> ReadCBOR(&value)
  -> ReadCSV(&value)

  -> Key() -> string // pieceKey
interceptors
import (
  "github.com/linolabx/lino_s3"
  "github.com/linolabx/lino_s3/interceptors"
)

s3 := lino_s3.NewLinoS3(session).Bucket("bucket")
blocksDir := bucket.SubPath("blocks:v1").UseInterceptors(interceptors.Gzip)

block SomeStruct
blockObject := blocksDir.Object("somekey.gz").ReadCBOR(&block);

block.SomeField = "new value"
blockObject.WriteCBOR(&block)
utils
import "github.com/linolabx/lino_s3/utils"

utils.HashSplit("123456") // "e1/0a/dc"
utils.HashSplit("123456", 5) // "e1/0a/dc/39/49"

utils.HashPrefix("hello.txt") // "2e/54/14/hello.txt"
utils.HashPrefix("hello.txt", 3) // "2e/54/14/hello.txt"
structures

LinoS3Map

import (
  "github.com/linolabx/lino_s3"
  "github.com/linolabx/lino_s3/structures/lino_s3_map"
)

bucket := s3.Bucket("bucket")
obj := bucket.Object("my-map")

someMap := lino_s3_map.NewMap(obj)
someMap.Set("foo1", []byte{"bar1"})
someMap.Set("foo2", []byte{"bar2"})
// save index of items in the first bytes, and then items
someMap.Save()

// read bytes from memory
someMap.Get("foo1") // []byte{"bar1"}

// directly read required bytes from s3
pieceKey, _ := someMap.PieceKey("foo1")
bucket.PieceByKey(pieceKey).ReadBuffer() // []byte{"bar1"}

// load map index, and then read required bytes from s3
secondMap := lino_s3_map.LoadMap(obj)
secondMap.Get("foo1") // []byte{"bar1"}

// load entire map, and then read required bytes from memory
thirdMap := lino_s3_map.NewMap(obj)
thirdMap.Get("foo1") // []byte{"bar1"}

Develop

start minio, and default access key and secret key is minioadmin:minioadmin

docker run --name lino-s3-test --rm -p 9000:9000 -p 9001:9001 quay.io/minio/minio server /data --console-address ":9001"

docker exec -it lino-s3-test mc alias set minio http://localhost:9000 minioadmin minioadmin
docker exec -it lino-s3-test mc mb minio/lino-stor

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Interceptor 

type Interceptor struct {
	PreHead    transformer[*s3.HeadObjectInput]
	PostHead   transformer[*s3.HeadObjectOutput]
	PreGet     transformer[*s3.GetObjectInput]
	PostGet    transformer[*s3.GetObjectOutput]
	PrePut     transformer[*s3.PutObjectInput]
	PostPut    transformer[*s3.PutObjectOutput]
	PreDelete  transformer[*s3.DeleteObjectInput]
	PostDelete transformer[*s3.DeleteObjectOutput]
	PreUpload  transformer[*s3manager.UploadInput]
	PostUpload transformer[*s3manager.UploadOutput]
}

type LinoS3 

type LinoS3 struct {
	// contains filtered or unexported fields
}

func NewLinoS3 

func NewLinoS3(sess *session.Session) *LinoS3

func (*LinoS3) Bucket 

func (s *LinoS3) Bucket(bucketname string) *LinoS3Bucket

func (*LinoS3) UseInterceptors 

func (s *LinoS3) UseInterceptors(interceptors ...*Interceptor) *LinoS3

type LinoS3Bucket 

type LinoS3Bucket struct {
	// contains filtered or unexported fields
}

func (*LinoS3Bucket) Object 

func (s *LinoS3Bucket) Object(key string) *LinoS3Object

func (*LinoS3Bucket) Piece added in v0.0.2 

func (s *LinoS3Bucket) Piece(key string, start int64, end int64) *LinoS3Piece

func (*LinoS3Bucket) PieceByKey added in v0.0.2 

func (s *LinoS3Bucket) PieceByKey(pieceKey string) (*LinoS3Piece, error)

func (*LinoS3Bucket) SubPath 

func (s *LinoS3Bucket) SubPath(path string) *LinoS3Path

func (*LinoS3Bucket) UseInterceptors 

func (s *LinoS3Bucket) UseInterceptors(interceptors ...*Interceptor) *LinoS3Bucket

type LinoS3Object 

type LinoS3Object struct {
	// contains filtered or unexported fields
}

func (*LinoS3Object) Delete 

func (s *LinoS3Object) Delete() (*s3.DeleteObjectOutput, error)

func (*LinoS3Object) Get 

func (s *LinoS3Object) Get() (*s3.GetObjectOutput, error)

func (*LinoS3Object) HasInterceptors added in v0.0.2 

func (s *LinoS3Object) HasInterceptors() bool

func (*LinoS3Object) Head 

func (s *LinoS3Object) Head() (*s3.HeadObjectOutput, error)

func (*LinoS3Object) Key added in v0.0.2 

func (s *LinoS3Object) Key() string

func (*LinoS3Object) Piece added in v0.0.2 

func (s *LinoS3Object) Piece(start int64, end int64) *LinoS3Piece

func (*LinoS3Object) Put 

func (s *LinoS3Object) Put(input s3.PutObjectInput) (*s3.PutObjectOutput, error)

func (*LinoS3Object) ReadBuffer 

func (s *LinoS3Object) ReadBuffer() ([]byte, error)

func (*LinoS3Object) ReadCBOR 

func (s *LinoS3Object) ReadCBOR(v interface{}) error

func (*LinoS3Object) ReadCSV 

func (s *LinoS3Object) ReadCSV(v interface{}) error

func (*LinoS3Object) ReadJSON 

func (s *LinoS3Object) ReadJSON(v interface{}) error

func (*LinoS3Object) ReadString 

func (s *LinoS3Object) ReadString() (string, error)

func (*LinoS3Object) ReadTo 

func (s *LinoS3Object) ReadTo(writer io.WriteCloser) error

func (*LinoS3Object) Upload 

func (s *LinoS3Object) Upload(input s3manager.UploadInput) (*s3manager.UploadOutput, error)

func (*LinoS3Object) UseInterceptors 

func (s *LinoS3Object) UseInterceptors(interceptors ...*Interceptor) *LinoS3Object

func (*LinoS3Object) WriteBuffer 

func (s *LinoS3Object) WriteBuffer(data []byte, contentType ...string) error

func (*LinoS3Object) WriteCBOR 

func (s *LinoS3Object) WriteCBOR(v interface{}) error

func (*LinoS3Object) WriteCSV 

func (s *LinoS3Object) WriteCSV(v interface{}) error

func (*LinoS3Object) WriteFrom 

func (s *LinoS3Object) WriteFrom(reader io.ReadCloser, contentType ...string) error

func (*LinoS3Object) WriteJSON 

func (s *LinoS3Object) WriteJSON(v interface{}) error

func (*LinoS3Object) WriteString 

func (s *LinoS3Object) WriteString(_string string, contentType ...string) error

type LinoS3Path 

type LinoS3Path struct {
	// contains filtered or unexported fields
}

func (*LinoS3Path) Object 

func (s *LinoS3Path) Object(key string) *LinoS3Object

func (*LinoS3Path) SubPath 

func (s *LinoS3Path) SubPath(path string) *LinoS3Path

func (*LinoS3Path) UseInterceptors 

func (s *LinoS3Path) UseInterceptors(interceptors ...*Interceptor) *LinoS3Path

type LinoS3Piece added in v0.0.2 

type LinoS3Piece struct {
	// contains filtered or unexported fields
}

func (*LinoS3Piece) Get added in v0.0.2 

func (s *LinoS3Piece) Get() (*s3.GetObjectOutput, error)

func (*LinoS3Piece) Key added in v0.0.2 

func (s *LinoS3Piece) Key() string

func (*LinoS3Piece) ReadBuffer added in v0.0.2 

func (s *LinoS3Piece) ReadBuffer() ([]byte, error)

func (*LinoS3Piece) ReadCBOR added in v0.0.2 

func (s *LinoS3Piece) ReadCBOR(v interface{}) error

func (*LinoS3Piece) ReadJSON added in v0.0.2 

func (s *LinoS3Piece) ReadJSON(v interface{}) error

func (*LinoS3Piece) ReadString added in v0.0.2 

func (s *LinoS3Piece) ReadString() (string, error)

func (*LinoS3Piece) ReadTo added in v0.0.2 

func (s *LinoS3Piece) ReadTo(writer io.WriteCloser) error

Source Files

View all Source files

Directories

Path Synopsis
test
structures
lino_s3_map

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.