ioz

Documentation

Overview

Package ioz contains supplemental io functionality.

Index

• Constants
• Variables
• func Close(ctx context.Context, c io.Closer)
• func CopyAsync(w io.Writer, r io.Reader, callback func(written int64, err error))
• func CopyFile(dst, src string, mkdir bool) error
• func DelayReader(r io.Reader, delay time.Duration, jitter bool) io.Reader
• func DirExists(dir string) bool
• func DirSize(path string) (int64, error)
• func DrainClose(rc io.ReadCloser) (n int, err error)
• func FPrintFile(w io.Writer, name string) error
• func FileAccessible(path string) bool
• func FileInfoEqual(a, b os.FileInfo) bool
• func Filesize(fp string) (int64, error)
• func IsPathToRegularFile(path string) bool
• func LimitRandReader(limit int64) io.Reader
• func MarshalYAML(v any) ([]byte, error)
• func NewErrorAfterBytesReader(p []byte, err error) io.Reader
• func NewErrorAfterRandNReader(n int, err error) io.Reader
• func NotifyOnEOFReader(r io.Reader, fn func(error) error) io.Reader
• func NotifyOnErrorReader(r io.Reader, fn func(error) error) io.Reader
• func NotifyOnceWriter(w io.Writer, fn func()) io.Writer
• func NotifyWriter(w io.Writer, fn func(n int)) io.Writer
• func PrintFile(name string) error
• func PrintTree(w io.Writer, loc string, showSize, colorize bool) error
• func PruneEmptyDirTree(ctx context.Context, dir string) (count int, err error)
• func ReadCloserNotifier(rc io.ReadCloser, fn func(closeErr error)) io.ReadCloser
• func ReadDir(dir string, includeDirPath, markDirs, includeDot bool) (paths []string, err error)
• func ReadFileToString(name string) (string, error)
• func RenameDir(oldDir, newpath string) error
• func RequireDir(dir string) error
• func UnmarshallYAML(data []byte, v any) error
• func WriteCloser(w io.Writer) io.WriteCloser
• func WriteFileAtomic(fp string, data []byte, mode os.FileMode) error
• func WriteToFile(ctx context.Context, fp string, r io.Reader) (written int64, err error)
• type Buffer
• type Buffers
  • func NewBuffers(dir string, memBufSize int) (*Buffers, error)
  • func (bs *Buffers) Close() error
  • func (bs *Buffers) NewMem2Disk() Buffer
• type EmptyReader
  • func (e EmptyReader) Read([]byte) (n int, err error)
• type ErrReader
  • func (e ErrReader) Read([]byte) (n int, err error)
• type WriteErrorCloser
  • func NewFuncWriteErrorCloser(w io.WriteCloser, fn func(error)) WriteErrorCloser
• type WrittenWriter
  • func (w *WrittenWriter) Write(p []byte) (n int, err error)

Constants

const RWPerms = os.FileMode(0o600)

RWPerms is the default file mode used for creating files.

Variables

var NewDefaultBuffer = func() Buffer {
	return &bytesBuffer{buf: &bytes.Buffer{}}
}

NewDefaultBuffer returns a Buffer backed by a bytes.Buffer.

Functions

func Close

func Close(ctx context.Context, c io.Closer)

Close is a convenience function to close c, logging a warning if c.Close returns an error. This is useful in defer, e.g.

defer ioz.Close(ctx, c)

func CopyAsync

func CopyAsync(w io.Writer, r io.Reader, callback func(written int64, err error))

CopyAsync asynchronously copies from r to w, invoking non-nil callback when done.

func CopyFile

func CopyFile(dst, src string, mkdir bool) error

CopyFile copies the contents from src to dst atomically. If dst does not exist, CopyFile creates it with src's perms. If the copy fails, CopyFile aborts and dst is preserved. If mkdir is true, the directory for dst is created if it doesn't exist.

func DelayReader

func DelayReader(r io.Reader, delay time.Duration, jitter bool) io.Reader

DelayReader returns an io.Reader that delays on each read from r. This is primarily intended for testing. If jitter is true, a randomized jitter factor is added to the delay. If r implements io.Closer, the returned reader will also implement io.Closer; if r doesn't implement io.Closer, the returned reader will not implement io.Closer. If r is nil, nil is returned.

func DirExists

func DirExists(dir string) bool

DirExists returns true if dir exists and is a directory.

func DirSize

func DirSize(path string) (int64, error)

DirSize returns total size of all regular files in path.

func DrainClose

func DrainClose(rc io.ReadCloser) (n int, err error)

DrainClose drains rc, returning the number of bytes read, and any error. The reader is always closed, even if the drain operation returned an error. If both the drain and the close operations return non-nil errors, the drain error is returned.

func FPrintFile

func FPrintFile(w io.Writer, name string) error

FPrintFile reads file from name and writes it to w.

func FileAccessible

func FileAccessible(path string) bool

FileAccessible returns true if path is a file that can be read.

func FileInfoEqual

func FileInfoEqual(a, b os.FileInfo) bool

FileInfoEqual returns true if a and b are equal. The FileInfo.Sys() field is ignored.

func Filesize

func Filesize(fp string) (int64, error)

Filesize returns the size of the file at fp. An error is returned if fp doesn't exist or is a directory.

func IsPathToRegularFile

func IsPathToRegularFile(path string) bool

IsPathToRegularFile return true if path is a regular file or a symlink that resolves to a regular file. False is returned on any error.

func LimitRandReader

func LimitRandReader(limit int64) io.Reader

LimitRandReader returns an io.Reader that reads up to limit bytes from crypto/rand.Reader.

func MarshalYAML

func MarshalYAML(v any) ([]byte, error)

MarshalYAML is our standard mechanism for encoding YAML.

func NewErrorAfterBytesReader

func NewErrorAfterBytesReader(p []byte, err error) io.Reader

NewErrorAfterBytesReader returns an io.Reader that returns err after p has been fully read. If err is nil, the reader will return io.EOF instead of err.

func NewErrorAfterRandNReader

func NewErrorAfterRandNReader(n int, err error) io.Reader

NewErrorAfterRandNReader returns an io.Reader that returns err after reading n random bytes from crypto/rand.Reader.

func NotifyOnEOFReader

func NotifyOnEOFReader(r io.Reader, fn func(error) error) io.Reader

NotifyOnEOFReader returns an io.Reader that invokes fn when r.Read returns io.EOF. The error that fn returns is what's returned to the r caller: fn can transform the error or return it unchanged. If r or fn is nil, r is returned.

If r is an io.ReadCloser, the returned reader will also implement io.ReadCloser.

See also: NotifyOnErrorReader, which is a generalization of NotifyOnEOFReader.

func NotifyOnErrorReader

func NotifyOnErrorReader(r io.Reader, fn func(error) error) io.Reader

NotifyOnErrorReader returns an io.Reader that invokes fn when r.Read returns an error. The error that fn returns is what's returned to the r caller: fn can transform the error or return it unchanged. If r or fn is nil, r is returned.

See also: NotifyOnEOFReader, which is a specialization of NotifyOnErrorReader.

func NotifyOnceWriter

func NotifyOnceWriter(w io.Writer, fn func()) io.Writer

NotifyOnceWriter returns an io.Writer that invokes fn before the first invocation of Write. If w or fn is nil, w is returned.

See also: NotifyWriter, which is a generalization of NotifyOnceWriter.

func NotifyWriter

func NotifyWriter(w io.Writer, fn func(n int)) io.Writer

NotifyWriter returns an io.Writer that invokes fn(n) before every invocation of Write, where the n arg to fn is the length of the byte slice to be written (which may be zero). If w or fn is nil, w is returned.

See also: NotifyOnceWriter.

func PrintFile

func PrintFile(name string) error

PrintFile reads file from name and writes it to stdout.

func PrintTree

func PrintTree(w io.Writer, loc string, showSize, colorize bool) error

PrintTree prints the file tree structure at loc to w. This function uses the github.com/a8m/tree library, which is a Go implementation of the venerable "tree" command.

func PruneEmptyDirTree

func PruneEmptyDirTree(ctx context.Context, dir string) (count int, err error)

PruneEmptyDirTree prunes empty dirs, and dirs that contain only other empty dirs, from the directory tree rooted at dir. If a dir contains at least one non-dir entry, that dir is spared. Arg dir must be an absolute path.

func ReadCloserNotifier

func ReadCloserNotifier(rc io.ReadCloser, fn func(closeErr error)) io.ReadCloser

ReadCloserNotifier returns a new io.ReadCloser that invokes fn after Close is called, passing along any error from Close. If rc or fn is nil, rc is returned. Note that any subsequent calls to Close are no-op, and return the same error (if any) as the first invocation of Close.

func ReadDir

func ReadDir(dir string, includeDirPath, markDirs, includeDot bool) (paths []string, err error)

ReadDir lists the contents of dir, returning the relative paths of the files. If markDirs is true, directories are listed with a "/" suffix (including symlinked dirs). If includeDirPath is true, the listing is of the form "dir/name". If includeDot is true, files beginning with period (dot files) are included. The function attempts to continue in the present of errors: the returned paths may contain values even in the presence of a returned error (which may be a multierr).

func ReadFileToString

func ReadFileToString(name string) (string, error)

ReadFileToString reads the file at name and returns its contents as a string.

func RenameDir

func RenameDir(oldDir, newpath string) error

RenameDir is like os.Rename, but it works even if newpath already exists and is a directory (which os.Rename fails on).

func RequireDir

func RequireDir(dir string) error

RequireDir ensures that dir exists and is a directory, creating it if necessary.

func UnmarshallYAML

func UnmarshallYAML(data []byte, v any) error

UnmarshallYAML is our standard mechanism for decoding YAML.

func WriteCloser

func WriteCloser(w io.Writer) io.WriteCloser

WriteCloser returns w as an io.WriteCloser. If w implements io.WriteCloser, w is returned. Otherwise, w is wrapped in a no-op decorator that implements io.WriteCloser.

WriteCloser is the missing sibling of io.NopCloser, which isn't implemented in stdlib. See: https://github.com/golang/go/issues/22823.

func WriteFileAtomic

func WriteFileAtomic(fp string, data []byte, mode os.FileMode) error

WriteFileAtomic writes data to fp atomically, but not on Windows.

func WriteToFile

func WriteToFile(ctx context.Context, fp string, r io.Reader) (written int64, err error)

WriteToFile writes the contents of r to fp. If fp doesn't exist, the file is created (including any parent dirs). If fp exists, it is truncated. The write operation is context-aware.

Types

type Buffer

type Buffer interface {
	io.Reader
	io.Writer

	// Len returns the number of bytes of the unread portion of the buffer;
	Len() int64

	// Cap returns the capacity of the buffer.
	Cap() int64

	// Reset resets the buffer to be empty.
	Reset()

	// Close MUST be invoked when done with the buffer, or resources may leak.
	Close() error
}

Buffer extracts the methods of bytes.Buffer to allow for alternative buffering strategies, such as file-backed buffers for large files. It also adds a [Buffer.Close] method that the caller MUST invoke when done.

type Buffers

type Buffers struct {
	// contains filtered or unexported fields
}

Buffers is a factory for creating buffers that overflow to disk. This is useful when dealing with large data that may not fit in memory.

func NewBuffers

func NewBuffers(dir string, memBufSize int) (*Buffers, error)

NewBuffers returns a new Buffers instance. The caller must invoke Buffers.Close when done. Arg dir is the directory in which file-backed buffers will be stored; the dir will be created if necessary, and is deleted by Buffers.Close. Arg memBufSize is the maximum size of in-memory buffers; if a buffer exceeds this size, it will be backed by a file.

func (*Buffers) Close

func (bs *Buffers) Close() error

Close removes the directory used for file-backed buffers.

func (*Buffers) NewMem2Disk

func (bs *Buffers) NewMem2Disk() Buffer

NewMem2Disk returns a Buffer whose head is in-memory, but overflows to disk when it reaches a threshold. The caller MUST invoke [Buffer.Close] when done, or resources may be leaked.

type EmptyReader

type EmptyReader struct{}

EmptyReader is an io.Reader whose Read methods always returns io.EOF.

func (EmptyReader) Read

func (e EmptyReader) Read([]byte) (n int, err error)

Read always returns (0, io.EOF).

type ErrReader

type ErrReader struct {
	Err error
}

ErrReader is an io.Reader that always returns an error.

func (ErrReader) Read

func (e ErrReader) Read([]byte) (n int, err error)

Read implements io.Reader: it always returns [ErrReader.Err].

type WriteErrorCloser

type WriteErrorCloser interface {
	io.WriteCloser

	// Error indicates that an upstream error has interrupted the
	// writing operation.
	Error(err error)
}

WriteErrorCloser supplements io.WriteCloser with an Error method, indicating to the io.WriteCloser that an upstream error has interrupted the writing operation. Note that clients should invoke only one of Close or Error.

func NewFuncWriteErrorCloser

func NewFuncWriteErrorCloser(w io.WriteCloser, fn func(error)) WriteErrorCloser

NewFuncWriteErrorCloser returns a new WriteErrorCloser that wraps w, and invokes non-nil fn when WriteErrorCloser.Error is called.

type WrittenWriter

type WrittenWriter struct {
	W       io.Writer
	Err     error
	Written int64
}

WrittenWriter is an io.Writer decorator that tracks the number of bytes written to the underlying writer, as well as any error returned. If an error occurs, it is stored in the Err field, and subsequent calls to Write are no-op.

func (*WrittenWriter) Write

func (w *WrittenWriter) Write(p []byte) (n int, err error)

Write writes p to the underlying writer, updating the number of bytes written. If an error occurs, it is stored in the Err field, and subsequent calls to Write are no-op.