objstorageprovider

Documentation

Index

• Variables
• func NewFileWritable(file vfs.File) objstorage.Writable
• func NewRemoteWritable(obj io.WriteCloser) objstorage.Writable
• func Open(settings Settings) (objstorage.Provider, error)
• func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool
• func UsePreallocatedReadHandle(ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle) objstorage.ReadHandle
• type PreallocatedReadHandle
  • func (rh *PreallocatedReadHandle) Close() error
  • func (rh *PreallocatedReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error
  • func (rh *PreallocatedReadHandle) RecordCacheHit(_ context.Context, offset, size int64)
  • func (rh *PreallocatedReadHandle) SetupForCompaction()
• type ReadaheadConfig
• type ReadaheadMode
• type Settings
  • func DefaultSettings(fs vfs.FS, dirName string) Settings

Variables

var DefaultReadaheadConfig = ReadaheadConfig{
	Informed:    FadviseSequential,
	Speculative: FadviseSequential,
}

DefaultReadaheadConfig is the readahead config used when ReadaheadConfigFn is not specified.

Functions

func NewFileWritable

func NewFileWritable(file vfs.File) objstorage.Writable

NewFileWritable returns a Writable that uses a file as underlying storage.

func NewRemoteWritable

func NewRemoteWritable(obj io.WriteCloser) objstorage.Writable

NewRemoteWritable creates an objstorage.Writable out of an io.WriteCloser.

func Open

func Open(settings Settings) (objstorage.Provider, error)

Open creates the provider.

func TestingCheckMaxReadahead

func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool

TestingCheckMaxReadahead returns true if the ReadHandle has switched to OS-level read-ahead.

func UsePreallocatedReadHandle

func UsePreallocatedReadHandle(
	ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
) objstorage.ReadHandle

UsePreallocatedReadHandle is equivalent to calling readable.NewReadHandle() but uses the existing storage of a PreallocatedReadHandle when possible (currently this happens if we are reading from a local file). The returned handle still needs to be closed.

Types

type PreallocatedReadHandle

type PreallocatedReadHandle struct {
	// contains filtered or unexported fields
}

PreallocatedReadHandle is used to avoid an allocation in NewReadHandle; see UsePreallocatedReadHandle.

func (*PreallocatedReadHandle) Close

func (rh *PreallocatedReadHandle) Close() error

Close is part of the objstorage.ReadHandle interface.

func (*PreallocatedReadHandle) ReadAt

func (rh *PreallocatedReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error

ReadAt is part of the objstorage.ReadHandle interface.

func (*PreallocatedReadHandle) RecordCacheHit

func (rh *PreallocatedReadHandle) RecordCacheHit(_ context.Context, offset, size int64)

RecordCacheHit is part of the objstorage.ReadHandle interface.

func (*PreallocatedReadHandle) SetupForCompaction

func (rh *PreallocatedReadHandle) SetupForCompaction()

SetupForCompaction is part of the objstorage.ReadHandle interface.

type ReadaheadConfig

type ReadaheadConfig struct {
	// Informed is the type of read-ahead for operations that are known to read a
	// large consecutive chunk of a file.
	Informed ReadaheadMode

	// Speculative is the type of read-ahead used automatically, when consecutive
	// reads are detected.
	Speculative ReadaheadMode
}

ReadaheadConfig controls the use of read-ahead.

type ReadaheadMode

type ReadaheadMode uint8

ReadaheadMode indicates the type of read-ahead to use, either for informed read-ahead (e.g. compactions) or speculative read-ahead.

const (
	// NoReadahead disables readahead altogether.
	NoReadahead ReadaheadMode = iota

	// SysReadahead enables the use of SYS_READAHEAD call to prefetch data.
	// The prefetch window grows dynamically as consecutive writes are detected.
	SysReadahead

	// FadviseSequential enables to use of FADV_SEQUENTIAL. For informed
	// read-ahead, FADV_SEQUENTIAL is used from the beginning. For speculative
	// read-ahead SYS_READAHEAD is first used until the window reaches the maximum
	// size, then we siwtch to FADV_SEQUENTIAL.
	FadviseSequential
)

type Settings

type Settings struct {
	Logger base.Logger

	// Local filesystem configuration.
	FS        vfs.FS
	FSDirName string

	// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
	//
	// This is an optional optimization to avoid double listing on Open when the
	// higher layer already has a listing. When nil, we obtain the listing on
	// Open.
	FSDirInitialListing []string

	// Cleaner cleans obsolete files from the local filesystem.
	//
	// The default cleaner uses the DeleteCleaner.
	FSCleaner base.Cleaner

	// NoSyncOnClose decides whether the implementation will enforce a
	// close-time synchronization (e.g., fdatasync() or sync_file_range())
	// on files it writes to. Setting this to true removes the guarantee for a
	// sync on close. Some implementations can still issue a non-blocking sync.
	NoSyncOnClose bool

	// BytesPerSync enables periodic syncing of files in order to smooth out
	// writes to disk. This option does not provide any persistence guarantee, but
	// is used to avoid latency spikes if the OS automatically decides to write
	// out a large chunk of dirty filesystem buffers.
	BytesPerSync int

	// Local contains fields that are only relevant for files stored on the local
	// filesystem.
	Local struct {

		// ReadaheadConfigFn is a function used to retrieve the current readahead
		// mode. This function is run whenever a local object is open for reading.
		// If it is nil, DefaultReadaheadConfig is used.
		ReadaheadConfigFn func() ReadaheadConfig
	}

	// Fields here are set only if the provider is to support remote objects
	// (experimental).
	Remote struct {
		StorageFactory remote.StorageFactory

		// If CreateOnShared is non-zero, sstables are created on remote storage using
		// the CreateOnSharedLocator (when the PreferSharedStorage create option is
		// true).
		CreateOnShared        remote.CreateOnSharedStrategy
		CreateOnSharedLocator remote.Locator

		// CacheSizeBytes is the size of the on-disk block cache for objects
		// on remote storage. If it is 0, no cache is used.
		CacheSizeBytes int64

		// CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used.
		CacheBlockSize int

		// ShardingBlockSize is the size of a shard block. The cache is split into contiguous
		// ShardingBlockSize units. The units are distributed across multiple independent shards
		// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
		// policies operate at the level of shard, not whole cache. This is done to reduce lock
		// contention.
		//
		// If ShardingBlockSize is 0, the default of 1 MB is used.
		ShardingBlockSize int64

		// The number of independent shards the cache leverages. Each shard is the same size,
		// and a hash of filenum & offset map a read to a certain shard. If set to 0,
		// 2*runtime.GOMAXPROCS is used as the shard count.
		CacheShardCount int
	}
}

Settings that must be specified when creating the provider.

func DefaultSettings

func DefaultSettings(fs vfs.FS, dirName string) Settings

DefaultSettings initializes default settings (with no remote storage), suitable for tests and tools.