wal

package
v2.0.0-beta.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 13, 2020 License: MIT Imports: 29 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefaultSegmentSize of 10MB is the size at which segment files will be rolled over.
	DefaultSegmentSize = 10 * 1024 * 1024

	// WALFileExtension is the file extension we expect for wal segments.
	WALFileExtension = "wal"

	// WALFilePrefix is the prefix on all wal segment files.
	WALFilePrefix = "_"
)

Variables

View Source 
var (
	// ErrWALClosed is returned when attempting to write to a closed WAL file.
	ErrWALClosed = fmt.Errorf("WAL closed")

	// ErrWALCorrupt is returned when reading a corrupt WAL entry.
	ErrWALCorrupt = fmt.Errorf("corrupted WAL entry")
)

Functions

func PrometheusCollectors 

func PrometheusCollectors() []prometheus.Collector

PrometheusCollectors returns all the metrics associated with the tsdb package.

func SegmentFileNames 

func SegmentFileNames(dir string) ([]string, error)

SegmentFileNames will return all files that are WAL segment files in sorted order by ascending ID.

Types

type DeleteBucketRangeWALEntry 

type DeleteBucketRangeWALEntry struct {
	OrgID     influxdb.ID
	BucketID  influxdb.ID
	Min, Max  int64
	Predicate []byte
}

DeleteBucketRangeWALEntry represents the deletion of data in a bucket.

func (*DeleteBucketRangeWALEntry) Encode 

func (w *DeleteBucketRangeWALEntry) Encode(b []byte) ([]byte, error)

Encode converts the entry into a byte stream using b if it is large enough. If b is too small, a newly allocated slice is returned.

func (*DeleteBucketRangeWALEntry) MarshalBinary 

func (w *DeleteBucketRangeWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*DeleteBucketRangeWALEntry) MarshalSize 

func (w *DeleteBucketRangeWALEntry) MarshalSize() int

MarshalSize returns the number of bytes the entry takes when marshaled.

func (*DeleteBucketRangeWALEntry) Type 

func (w *DeleteBucketRangeWALEntry) Type() WalEntryType

Type returns DeleteBucketRangeWALEntryType.

func (*DeleteBucketRangeWALEntry) UnmarshalBinary 

func (w *DeleteBucketRangeWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

type Dump 

type Dump struct {
	// Standard input/output
	Stderr io.Writer
	Stdout io.Writer

	// A list of files to dump
	FileGlobs []string

	// Whether or not to check for duplicate/out of order entries
	FindDuplicates bool
}

func (*Dump) Run 

func (w *Dump) Run(print bool) ([]*DumpReport, error)

Run executes the dumpwal command, generating a list of DumpReports for each requested file. The `print` flag indicates whether or not the command should log output during execution. If the command is run as a cli, Run(true) should be used, and if the tool is run programmatically, output should likely be suppressed with Run(false).

type DumpReport 

type DumpReport struct {
	// The file this report corresponds to
	File string
	// Any keys found to be duplicated/out of order
	DuplicateKeys []string
	// A list of all the write wal entries from this file
	Writes []*WriteWALEntry
	// A list of all the delete wal entries from this file
	Deletes []*DeleteBucketRangeWALEntry
}

type VerificationSummary 

type VerificationSummary struct {
	EntryCount   int
	FileCount    int
	CorruptFiles []string
}

type Verifier 

type Verifier struct {
	Stderr io.Writer
	Stdout io.Writer
	Dir    string
}

func (*Verifier) Run 

func (v *Verifier) Run(print bool) (*VerificationSummary, error)

type WAL 

type WAL struct {

	// SegmentSize is the file size at which a segment file will be rotated
	SegmentSize int
	// contains filtered or unexported fields
}

WAL represents the write-ahead log used for writing TSM files.

func NewWAL 

func NewWAL(path string) *WAL

NewWAL initializes a new WAL at the given directory.

func (*WAL) Close 

func (l *WAL) Close() error

Close will finish any flush that is currently in progress and close file handles.

func (*WAL) CloseSegment 

func (l *WAL) CloseSegment() error

CloseSegment closes the current segment if it is non-empty and opens a new one.

func (*WAL) ClosedSegments 

func (l *WAL) ClosedSegments() ([]string, error)

ClosedSegments returns a slice of the names of the closed segment files.

func (*WAL) DeleteBucketRange 

func (l *WAL) DeleteBucketRange(orgID, bucketID influxdb.ID, min, max int64, pred []byte) (int, error)

DeleteBucketRange deletes the data inside of the bucket between the two times, returning the segment ID for the operation.

func (*WAL) DiskSizeBytes 

func (l *WAL) DiskSizeBytes() int64

DiskSizeBytes returns the on-disk size of the WAL.

func (*WAL) LastWriteTime 

func (l *WAL) LastWriteTime() time.Time

LastWriteTime is the last time anything was written to the WAL.

func (*WAL) Open 

func (l *WAL) Open(ctx context.Context) error

Open opens and initializes the Log. Open can recover from previous unclosed shutdowns.

func (*WAL) Path 

func (l *WAL) Path() string

Path returns the directory the log was initialized with.

func (*WAL) Remove 

func (l *WAL) Remove(ctx context.Context, files []string) error

Remove deletes the given segment file paths from disk and cleans up any associated objects.

func (*WAL) SetDefaultMetricLabels 

func (l *WAL) SetDefaultMetricLabels(labels prometheus.Labels)

SetDefaultMetricLabels sets the default labels for metrics on the engine. It must be called before the Engine is opened.

func (*WAL) SetEnabled 

func (l *WAL) SetEnabled(enabled bool)

SetEnabled sets if the WAL is enabled and should be called before the WAL is opened.

func (*WAL) WithFsyncDelay 

func (l *WAL) WithFsyncDelay(delay time.Duration)

WithFsyncDelay sets the fsync delay and should be called before the WAL is opened.

func (*WAL) WithLogger 

func (l *WAL) WithLogger(log *zap.Logger)

WithLogger sets the WAL's logger.

func (*WAL) WriteMulti 

func (l *WAL) WriteMulti(ctx context.Context, values map[string][]value.Value) (int, error)

WriteMulti writes the given values to the WAL. It returns the WAL segment ID to which the points were written. If an error is returned the segment ID should be ignored. If the WAL is disabled, -1 and nil is returned.

type WALEntry 

type WALEntry interface {
	Type() WalEntryType
	Encode(dst []byte) ([]byte, error)
	MarshalBinary() ([]byte, error)
	UnmarshalBinary(b []byte) error
	MarshalSize() int
}

WALEntry is record stored in each WAL segment. Each entry has a type and an opaque, type dependent byte slice data attribute.

type WALReader 

type WALReader struct {
	// contains filtered or unexported fields
}

WALReader helps one read out the WAL into entries.

func NewWALReader 

func NewWALReader(files []string) *WALReader

NewWALReader constructs a WALReader over the given set of files.

func (*WALReader) Read 

func (r *WALReader) Read(cb func(WALEntry) error) error

Read calls the callback with every entry in the WAL files. If, during reading of a segment file, corruption is encountered, that segment file is truncated up to and including the last valid byte, and processing continues with the next segment file.

func (*WALReader) WithLogger 

func (r *WALReader) WithLogger(logger *zap.Logger)

WithLogger sets the logger for the WALReader.

type WALSegmentReader 

type WALSegmentReader struct {
	// contains filtered or unexported fields
}

WALSegmentReader reads WAL segments.

func NewWALSegmentReader 

func NewWALSegmentReader(r io.ReadCloser) *WALSegmentReader

NewWALSegmentReader returns a new WALSegmentReader reading from r.

func (*WALSegmentReader) Close 

func (r *WALSegmentReader) Close() error

Close closes the underlying io.Reader.

func (*WALSegmentReader) Count 

func (r *WALSegmentReader) Count() int64

Count returns the total number of bytes read successfully from the segment, as of the last call to Read(). The segment is guaranteed to be valid up to and including this number of bytes.

func (*WALSegmentReader) Error 

func (r *WALSegmentReader) Error() error

Error returns the last error encountered by the reader.

func (*WALSegmentReader) Next 

func (r *WALSegmentReader) Next() bool

Next indicates if there is a value to read.

func (*WALSegmentReader) Read 

func (r *WALSegmentReader) Read() (WALEntry, error)

Read returns the next entry in the reader.

func (*WALSegmentReader) Reset 

func (r *WALSegmentReader) Reset(rc io.ReadCloser)

type WALSegmentWriter 

type WALSegmentWriter struct {
	// contains filtered or unexported fields
}

WALSegmentWriter writes WAL segments.

func NewWALSegmentWriter 

func NewWALSegmentWriter(w io.WriteCloser) *WALSegmentWriter

NewWALSegmentWriter returns a new WALSegmentWriter writing to w.

func (*WALSegmentWriter) Flush 

func (w *WALSegmentWriter) Flush() error

func (*WALSegmentWriter) Write 

func (w *WALSegmentWriter) Write(entryType WalEntryType, compressed []byte) error

Write writes entryType and the buffer containing compressed entry data.

type WalEntryType 

type WalEntryType byte

WalEntryType is a byte written to a wal segment file that indicates what the following compressed block contains. 

const (
	// WriteWALEntryType indicates a write entry.
	WriteWALEntryType WalEntryType = 0x01

	// DeleteBucketRangeWALEntryType indicates a delete bucket range entry.
	DeleteBucketRangeWALEntryType WalEntryType = 0x04
)

type WriteWALEntry 

type WriteWALEntry struct {
	Values map[string][]value.Value
	// contains filtered or unexported fields
}

WriteWALEntry represents a write of points.

func (*WriteWALEntry) Encode 

func (w *WriteWALEntry) Encode(dst []byte) ([]byte, error)

Encode converts the WriteWALEntry into a byte stream using dst if it is large enough. If dst is too small, the slice will be grown to fit the encoded entry.

func (*WriteWALEntry) MarshalBinary 

func (w *WriteWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*WriteWALEntry) MarshalSize 

func (w *WriteWALEntry) MarshalSize() int

MarshalSize returns the number of bytes the entry takes when marshaled.

func (*WriteWALEntry) Type 

func (w *WriteWALEntry) Type() WalEntryType

Type returns WriteWALEntryType.

func (*WriteWALEntry) UnmarshalBinary 

func (w *WriteWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.