minio

Documentation

Index

• Variables
• func ListFiles(pth string, opt Option) ([]stat.Stats, error)
• func Stat(pth string, Opt Option) (stat.Stats, error)
• type Option
• type Reader
  • func NewReader(pth string, opt Option) (*Reader, error)
  • func (r *Reader) Close() (err error)
  • func (r *Reader) Read(p []byte) (n int, err error)
  • func (r *Reader) ReadLine() (ln []byte, err error)
  • func (r *Reader) Stats() stat.Stats
• type Writer
  • func NewWriter(pth string, mOpt Option, opt *buf.Options) (*Writer, error)
  • func (w *Writer) Abort() (err error)
  • func (w *Writer) Close() error
  • func (w *Writer) Stats() stat.Stats
  • func (w *Writer) Write(p []byte) (n int, err error)
  • func (w *Writer) WriteLine(ln []byte) (err error)

Examples

• NewReader
• NewWriter
• Reader.Close
• Reader.Read
• Reader.ReadLine
• Reader.Stats
• Writer.Abort
• Writer.Close
• Writer.Stats
• Writer.Write
• Writer.WriteLine

Variables

var (
	// domain of s3 compatible api
	S3Host = "s3.amazonaws.com"
	GSHost = "storage.googleapis.com"
)

Functions

func ListFiles

func ListFiles(pth string, opt Option) ([]stat.Stats, error)

ListFiles will list all file objects in the provided pth directory. pth is assumed to be a directory and so a trailing "/" is appended if one does not already exist.

func Stat

func Stat(pth string, Opt Option) (stat.Stats, error)

Stat a directory or file for additional information

Types

type Option

type Option struct {
	//	*buf.Options
	Host      string
	AccessKey string
	SecretKey string
	Secure    bool
}

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader will read in streamed bytes from the s3 object.NewS3Client

func NewReader

func NewReader(pth string, opt Option) (*Reader, error)

Example

pth := fmt.Sprintf("mcs://%v/read/test.txt", testBucket)
r, err := NewReader(pth, testOption)
if r == nil {
	return
}

fmt.Println(err)        // output: <nil>
fmt.Println(r.sts.Path) // output: mcs://task-tools-test/read/test.txt
fmt.Println(r.sts.Size) // output: 20

Output:

<nil>
mcs://task-tools-test/read/test.txt
20

func (*Reader) Close

func (r *Reader) Close() (err error)

Example

pth := fmt.Sprintf("mcs://%v/read/test.txt", testBucket)
r, _ := NewReader(pth, testOption)
if r == nil {
	return
}

r.ReadLine()
r.ReadLine()
r.ReadLine()
err := r.Close()
sts := r.Stats()

fmt.Println(err)          // output: <nil>
fmt.Println(sts.ByteCnt)  // output: 20
fmt.Println(sts.LineCnt)  // output: 2
fmt.Println(sts.Checksum) // output: 54f30d75cf7374c7e524a4530dbc93c2

Output:

<nil>
20
2
54f30d75cf7374c7e524a4530dbc93c2

func (*Reader) Read

func (r *Reader) Read(p []byte) (n int, err error)

Example

pth := fmt.Sprintf("mcs://%v/read/test.txt", testBucket)
r, err := NewReader(pth, testOption)
if r == nil {
	return
}

b := make([]byte, 20)
n, err := r.Read(b)

fmt.Println(n)             // output: 20
fmt.Println(err)           // output: <nil>
fmt.Print(string(b))       // output: test line, test line
fmt.Println(r.sts.ByteCnt) // output: 20
fmt.Println(r.sts.LineCnt) // output: 0

Output:

20
<nil>
test line
test line
20
0

func (*Reader) ReadLine

func (r *Reader) ReadLine() (ln []byte, err error)

Example

pth := fmt.Sprintf("mcs://%v/read/test.txt", testBucket)
r, _ := NewReader(pth, testOption)
if r == nil {
	return
}

ln1, err1 := r.ReadLine()
ln2, err2 := r.ReadLine()

fmt.Println(string(ln1))   // output: test line
fmt.Println(err1)          // output: <nil>
fmt.Println(string(ln2))   // output: test line
fmt.Println(err2)          // output: <nil>
fmt.Println(r.sts.ByteCnt) // output: 20
fmt.Println(r.sts.LineCnt) // output: 2

Output:

test line
<nil>
test line
<nil>
20
2

func (*Reader) Stats

func (r *Reader) Stats() stat.Stats

Example

pth := fmt.Sprintf("mcs://%v/read/test.txt", testBucket)
r, _ := NewReader(pth, testOption)
if r == nil {
	return
}

r.ReadLine()
sts := r.Stats()
fmt.Println(sts.ByteCnt) // output: 10
fmt.Println(sts.LineCnt) // output: 1

Output:

10
1

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer will write to local buffer first and will copy all the written contents to the S3 destination after calling Close(). Close() must be called in order for the written contents to be written to S3.

Calling Abort() before Close() will cleanup the buffer. Calling Close() after Abort() will not result in any writing to S3.

Calling Abort() after Close() will do nothing.

func NewWriter

func NewWriter(pth string, mOpt Option, opt *buf.Options) (*Writer, error)

Example

pth := fmt.Sprintf("mcs://%v/write/test.txt", testBucket)
w, err := NewWriter(pth, testOption, nil)
if w == nil {
	return
}

fmt.Println(err)             // output: <nil>
fmt.Println(w.sts.Path)      // output: mcs://task-tools-test/write/test.txt
fmt.Println(w.client != nil) // output: true
fmt.Println(w.bfr != nil)    // output: true
fmt.Println(w.bucket)        // output: task-tools-test
fmt.Println(w.objPth)        // output: write/test.txt
fmt.Println(w.tmpPth == "")  // output: true

Output:

<nil>
mcs://task-tools-test/write/test.txt
true
true
task-tools-test
write/test.txt
true

func (*Writer) Abort

func (w *Writer) Abort() (err error)

Abort will: - clear and close buffer

Calling Close after Abort will do nothing. Writing after calling Abort has undefined behavior.

Example

pth := fmt.Sprintf("mcs://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testOption, nil)
if w == nil {
	return
}

w.WriteLine([]byte("test line"))

fmt.Println(w.done) // output: false
err := w.Abort()

fmt.Println(err)    // output: <nil>
fmt.Println(w.done) // output: true

Output:

false
<nil>
true

func (*Writer) Close

func (w *Writer) Close() error

Close will: - calculate final checksum - copy (mv) buffer to pth file - clear and close buffer - report any errors

If an error is returned it should be assumed that S3 object writing failed.

Calling Abort after Close will do nothing. Writing after calling Close has undefined behavior.

Example

pth := fmt.Sprintf("mcs://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testOption, nil)
if w == nil {
	return
}

w.WriteLine([]byte("test line"))
w.WriteLine([]byte("test line"))
err := w.Close()

fmt.Println(err)                     // output: <nil>
fmt.Println(w.done)                  // output: true
fmt.Println(w.objSts.Checksum != "") // output: true

// cleanup
rmTestFile(pth)

Output:

<nil>
true
true

func (*Writer) Stats

func (w *Writer) Stats() stat.Stats

Example

pth := fmt.Sprintf("mcs://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testOption, nil)
if w == nil {
	return
}

w.WriteLine([]byte("test line"))
w.WriteLine([]byte("test line"))
sts := w.Stats()

fmt.Println(sts.Path)          // output: mcs://task-tools-test/write/test.txt
fmt.Println(sts.ByteCnt)       // output: 20
fmt.Println(sts.LineCnt)       // output: 2
fmt.Println(sts.Size)          // output: 0
fmt.Println(sts.Checksum)      // output:
fmt.Println(sts.Created == "") // output: true

Output:

mcs://task-tools-test/write/test.txt
20
2
0

true

func (*Writer) Write

func (w *Writer) Write(p []byte) (n int, err error)

Example

pth := fmt.Sprintf("mcs://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testOption, nil)
if w == nil {
	return
}

n, err := w.Write([]byte("test line"))

fmt.Println(n)   // output: 9
fmt.Println(err) // output: <nil>

Output:

9
<nil>

func (*Writer) WriteLine

func (w *Writer) WriteLine(ln []byte) (err error)

Example

pth := fmt.Sprintf("mcs://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testOption, nil)
if w == nil {
	return
}

err := w.WriteLine([]byte("test line"))

fmt.Println(err) // output: <nil>

Output:

<nil>