store

package
v0.0.0-...-309f818 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 2, 2021 License: Apache-2.0 Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CFD_DATA_CODEC       = "CompoundFileWriterData"
	CFD_VERSION_START    = 0
	CFD_VERSION_CHECKSUM = 1
	CFD_VERSION_CURRENT  = CFD_VERSION_CHECKSUM

	CFD_ENTRY_CODEC = "CompoundFileWriterEntries"

	COMPOUND_FILE_EXTENSION         = "cfs"
	COMPOUND_FILE_ENTRIES_EXTENSION = "cfe"
)
View Source 
const (
	CODEC_MAGIC_BYTE1 = byte(uint32(codec.CODEC_MAGIC) >> 24 & 0xFF)
	CODEC_MAGIC_BYTE2 = byte(uint32(codec.CODEC_MAGIC) >> 16 & 0xFF)
	CODEC_MAGIC_BYTE3 = byte(uint32(codec.CODEC_MAGIC) >> 8 & 0xFF)
	CODEC_MAGIC_BYTE4 = byte(codec.CODEC_MAGIC & 0xFF)
)
View Source 
const (
	IO_CONTEXT_TYPE_MERGE   = 1
	IO_CONTEXT_TYPE_READ    = 2
	IO_CONTEXT_TYPE_FLUSH   = 3
	IO_CONTEXT_TYPE_DEFAULT = 4
)
View Source 
const (
	BUFFER_SIZE       = 1024
	MERGE_BUFFER_SIZE = 4096
)
View Source 
const CHUNK_SIZE = 8192

The maximum chunk size is 8192 bytes, becaues Java RamdomAccessFile mallocs a native buffer outside of stack if the read buffer size is larger. GoLucene takes the same default value. TODO: test larger value here

View Source 
const DEFAULT_BUFFER_SIZE = 16384

The default buffer size in bytes.

View Source 
const LOCK_OBTAIN_WAIT_FOREVER = -1

Pass this value to obtain() to try forever to obtain the lock

View Source 
const LOCK_POOL_INTERVAL = 1000

How long obtain() waits, in milliseconds, in between attempts to acquire the lock.

View Source 
const MIN_BUFFER_SIZE = 8

Minimum buffer size allowed

View Source 
const NRT_VERBOSE = false

Variables

View Source 
var (
	IO_CONTEXT_DEFAULT  = NewIOContextFromType(IOContextType(IO_CONTEXT_TYPE_DEFAULT))
	IO_CONTEXT_READONCE = NewIOContextBool(true)
	IO_CONTEXT_READ     = NewIOContextBool(false)
)
View Source 
var SENTINEL = make(map[string]FileSlice)

Functions

func ChecksumEntireFile 

func ChecksumEntireFile(input IndexInput) (hash int64, err error)

Clones the provided input, reads all bytes from the file, and calls CheckFooter().

Note that this method may be slow, as it must process the entire file. If you just need to extract the checksum value, call retrieveChecksum().

func FSDirectoryListAll 

func FSDirectoryListAll(path string) (paths []string, err error)

func WithLock 

func WithLock(lock Lock, lockWaitTimeout int64, body func() interface{}) interface{}

Utility to execute code with exclusive access.

Types

type AtomicBool 

type AtomicBool struct {
	*sync.RWMutex
	// contains filtered or unexported fields
}

func NewAtomicBool 

func NewAtomicBool() *AtomicBool

func (*AtomicBool) CompareAndSet 

func (b *AtomicBool) CompareAndSet(from, to bool) bool

func (*AtomicBool) Get 

func (b *AtomicBool) Get() bool

type BaseDirectory 

type BaseDirectory struct {
	IsOpen bool
	// contains filtered or unexported fields
}

Base implementation for a concrete Directory.

func NewBaseDirectory 

func NewBaseDirectory(spi BaseDirectorySPI) *BaseDirectory

func (*BaseDirectory) ClearLock 

func (d *BaseDirectory) ClearLock(name string) error

func (*BaseDirectory) EnsureOpen 

func (d *BaseDirectory) EnsureOpen()

func (*BaseDirectory) LockFactory 

func (d *BaseDirectory) LockFactory() LockFactory

func (*BaseDirectory) MakeLock 

func (d *BaseDirectory) MakeLock(name string) Lock

func (*BaseDirectory) SetLockFactory 

func (d *BaseDirectory) SetLockFactory(lockFactory LockFactory)

type BaseDirectorySPI 

type BaseDirectorySPI interface {
	// DirectoryImplSPI
	LockID() string
}

type BufferedChecksum 

type BufferedChecksum struct {
	// contains filtered or unexported fields
}

Wraps another Checksum with an internal buffer to speed up checksum calculations.

func (*BufferedChecksum) BlockSize 

func (bc *BufferedChecksum) BlockSize() int

func (*BufferedChecksum) Reset 

func (bc *BufferedChecksum) Reset()

func (*BufferedChecksum) Size 

func (bc *BufferedChecksum) Size() int

func (*BufferedChecksum) Sum 

func (bc *BufferedChecksum) Sum(p []byte) []byte

func (*BufferedChecksum) Sum32 

func (bc *BufferedChecksum) Sum32() uint32

func (*BufferedChecksum) Write 

func (bc *BufferedChecksum) Write(p []byte) (int, error)

type BufferedChecksumIndexInput 

type BufferedChecksumIndexInput struct {
	*ChecksumIndexInputImpl
	// contains filtered or unexported fields
}

Simple implementation of ChecksumIndexInput that wraps another input and delegates calls.

func (*BufferedChecksumIndexInput) Checksum 

func (in *BufferedChecksumIndexInput) Checksum() int64

func (*BufferedChecksumIndexInput) Clone 

func (in *BufferedChecksumIndexInput) Clone() IndexInput

func (*BufferedChecksumIndexInput) Close 

func (in *BufferedChecksumIndexInput) Close() error

func (*BufferedChecksumIndexInput) FilePointer 

func (in *BufferedChecksumIndexInput) FilePointer() int64

func (*BufferedChecksumIndexInput) Length 

func (in *BufferedChecksumIndexInput) Length() int64

func (*BufferedChecksumIndexInput) ReadByte 

func (in *BufferedChecksumIndexInput) ReadByte() (b byte, err error)

func (*BufferedChecksumIndexInput) ReadBytes 

func (in *BufferedChecksumIndexInput) ReadBytes(p []byte) (err error)

func (*BufferedChecksumIndexInput) Slice 

func (in *BufferedChecksumIndexInput) Slice(desc string, offset, length int64) (IndexInput, error)

type BufferedIndexInput 

type BufferedIndexInput struct {
	*IndexInputImpl
	// contains filtered or unexported fields
}

Base implementation class for buffered IndexInput.

func (*BufferedIndexInput) Clone 

func (in *BufferedIndexInput) Clone() *BufferedIndexInput

func (*BufferedIndexInput) FilePointer 

func (in *BufferedIndexInput) FilePointer() int64

func (*BufferedIndexInput) ReadByte 

func (in *BufferedIndexInput) ReadByte() (b byte, err error)

func (*BufferedIndexInput) ReadBytes 

func (in *BufferedIndexInput) ReadBytes(buf []byte) error

func (*BufferedIndexInput) ReadBytesBuffered 

func (in *BufferedIndexInput) ReadBytesBuffered(buf []byte, useBuffer bool) error

func (*BufferedIndexInput) ReadInt 

func (in *BufferedIndexInput) ReadInt() (n int32, err error)

func (*BufferedIndexInput) ReadLong 

func (in *BufferedIndexInput) ReadLong() (n int64, err error)

func (*BufferedIndexInput) ReadShort 

func (in *BufferedIndexInput) ReadShort() (n int16, err error)

func (*BufferedIndexInput) ReadVInt 

func (in *BufferedIndexInput) ReadVInt() (n int32, err error)

func (*BufferedIndexInput) ReadVLong 

func (in *BufferedIndexInput) ReadVLong() (n int64, err error)

func (*BufferedIndexInput) Seek 

func (in *BufferedIndexInput) Seek(pos int64) error

func (*BufferedIndexInput) Slice 

func (in *BufferedIndexInput) Slice(desc string, offset, length int64) (IndexInput, error)

type ByteArrayDataInput 

type ByteArrayDataInput struct {
	*util.DataInputImpl

	Pos int
	// contains filtered or unexported fields
}

DataInput backed by a byte array. Warning: this class omits all low-level checks.

func NewByteArrayDataInput 

func NewByteArrayDataInput(bytes []byte) *ByteArrayDataInput

func NewEmptyByteArrayDataInput 

func NewEmptyByteArrayDataInput() *ByteArrayDataInput

func (*ByteArrayDataInput) Length 

func (in *ByteArrayDataInput) Length() int

func (*ByteArrayDataInput) Position 

func (in *ByteArrayDataInput) Position() int

func (*ByteArrayDataInput) ReadByte 

func (in *ByteArrayDataInput) ReadByte() (b byte, err error)

func (*ByteArrayDataInput) ReadBytes 

func (in *ByteArrayDataInput) ReadBytes(buf []byte) error

func (*ByteArrayDataInput) ReadInt 

func (in *ByteArrayDataInput) ReadInt() (n int32, err error)

func (*ByteArrayDataInput) ReadLong 

func (in *ByteArrayDataInput) ReadLong() (n int64, err error)

func (*ByteArrayDataInput) ReadShort 

func (in *ByteArrayDataInput) ReadShort() (n int16, err error)

func (*ByteArrayDataInput) ReadVInt 

func (in *ByteArrayDataInput) ReadVInt() (n int32, err error)

func (*ByteArrayDataInput) ReadVLong 

func (in *ByteArrayDataInput) ReadVLong() (n int64, err error)

func (*ByteArrayDataInput) Reset 

func (in *ByteArrayDataInput) Reset(bytes []byte)

func (*ByteArrayDataInput) Rewind 

func (in *ByteArrayDataInput) Rewind()

NOTE: sets pos to 0, which is not right if you had called reset w/ non-zero offset!!

func (*ByteArrayDataInput) SkipBytes 

func (in *ByteArrayDataInput) SkipBytes(count int64)

type ByteArrayDataOutput 

type ByteArrayDataOutput struct {
	*util.DataOutputImpl
	// contains filtered or unexported fields
}

func NewByteArrayDataOutput 

func NewByteArrayDataOutput(data []byte) *ByteArrayDataOutput

func (*ByteArrayDataOutput) Position 

func (o *ByteArrayDataOutput) Position() int

func (*ByteArrayDataOutput) WriteByte 

func (o *ByteArrayDataOutput) WriteByte(b byte) error

func (*ByteArrayDataOutput) WriteBytes 

func (o *ByteArrayDataOutput) WriteBytes(b []byte) error

type ChecksumIndexInput 

type ChecksumIndexInput interface {
	IndexInput
	Checksum() int64
}

Extension of IndexInput, computing checksum as it goes. Callers can retrieve the checksum via Checksum().

type ChecksumIndexInputImpl 

type ChecksumIndexInputImpl struct {
	*IndexInputImpl
	// contains filtered or unexported fields
}

func NewChecksumIndexInput 

func NewChecksumIndexInput(desc string, spi ChecksumIndexInputImplSPI) *ChecksumIndexInputImpl

func (*ChecksumIndexInputImpl) Seek 

func (in *ChecksumIndexInputImpl) Seek(pos int64) error

type ChecksumIndexInputImplSPI 

type ChecksumIndexInputImplSPI interface {
	util.DataReader
	FilePointer() int64
}

type CompoundFileDirectory 

type CompoundFileDirectory struct {
	*DirectoryImpl
	*BaseDirectory
	sync.Locker
	// contains filtered or unexported fields
}

func NewCompoundFileDirectory 

func NewCompoundFileDirectory(directory Directory, fileName string, context IOContext, openForWrite bool) (d *CompoundFileDirectory, err error)

func (*CompoundFileDirectory) Close 

func (d *CompoundFileDirectory) Close() error

func (*CompoundFileDirectory) CreateOutput 

func (d *CompoundFileDirectory) CreateOutput(name string, context IOContext) (out IndexOutput, err error)

func (*CompoundFileDirectory) DeleteFile 

func (d *CompoundFileDirectory) DeleteFile(name string) error

func (*CompoundFileDirectory) FileExists 

func (d *CompoundFileDirectory) FileExists(name string) bool

func (*CompoundFileDirectory) FileLength 

func (d *CompoundFileDirectory) FileLength(name string) (n int64, err error)

Returns the length of a file in the directory.

func (*CompoundFileDirectory) ListAll 

func (d *CompoundFileDirectory) ListAll() (paths []string, err error)

func (*CompoundFileDirectory) MakeLock 

func (d *CompoundFileDirectory) MakeLock(name string) Lock

func (*CompoundFileDirectory) OpenInput 

func (d *CompoundFileDirectory) OpenInput(name string, context IOContext) (in IndexInput, err error)

func (*CompoundFileDirectory) String 

func (d *CompoundFileDirectory) String() string

func (*CompoundFileDirectory) Sync 

func (d *CompoundFileDirectory) Sync(names []string) error

type CompoundFileWriter 

type CompoundFileWriter struct {
	sync.Locker
	// contains filtered or unexported fields
}

Combines multiple files into a single compound file

func (*CompoundFileWriter) Close 

func (w *CompoundFileWriter) Close() (err error)

Closes all resouces and writes the entry table

type DirectCFSIndexOutput 

type DirectCFSIndexOutput struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

func (*DirectCFSIndexOutput) Checksum 

func (out *DirectCFSIndexOutput) Checksum() int64

func (*DirectCFSIndexOutput) Close 

func (out *DirectCFSIndexOutput) Close() error

func (*DirectCFSIndexOutput) FilePointer 

func (out *DirectCFSIndexOutput) FilePointer() int64

func (*DirectCFSIndexOutput) Flush 

func (out *DirectCFSIndexOutput) Flush() error

func (*DirectCFSIndexOutput) WriteByte 

func (out *DirectCFSIndexOutput) WriteByte(b byte) error

func (*DirectCFSIndexOutput) WriteBytes 

func (out *DirectCFSIndexOutput) WriteBytes(b []byte) error

type Directory 

type Directory interface {
	io.Closer
	// Files related methods
	ListAll() (paths []string, err error)
	// Returns true iff a file with the given name exists.
	// @deprecated This method will be removed in 5.0
	FileExists(name string) bool
	// Removes an existing file in the directory.
	DeleteFile(name string) error
	// Returns the length of a file in the directory. This method
	// follows the following contract:
	// 	- Must return error if the file doesn't exists.
	// 	- Returns a value >=0 if the file exists, which specifies its
	// length.
	FileLength(name string) (n int64, err error)
	// Creates a new, empty file in the directory with the given name.
	// Returns a stream writing this file.
	CreateOutput(name string, ctx IOContext) (out IndexOutput, err error)
	// Ensure that any writes to these files ar emoved to stable
	// storage. Lucene uses this to properly commit changes to the
	// index, to prevent a machine/OS crash from corrupting the index.
	//
	// NOTE: Clients may call this method for same files over and over
	// again, so some impls might optimize for that. For other impls
	// the operation can be a noop, for various reasons.
	Sync(names []string) error
	OpenInput(name string, context IOContext) (in IndexInput, err error)
	// Returns a stream reading an existing file, computing checksum as it reads
	OpenChecksumInput(name string, ctx IOContext) (ChecksumIndexInput, error)
	// Locks related methods
	MakeLock(name string) Lock
	ClearLock(name string) error
	SetLockFactory(lockFactory LockFactory)
	LockFactory() LockFactory
	LockID() string
	// Utilities
	Copy(to Directory, src, dest string, ctx IOContext) error

	EnsureOpen()
}

func OpenFSDirectory 

func OpenFSDirectory(path string) (d Directory, err error)

type DirectoryImpl 

type DirectoryImpl struct {
	// contains filtered or unexported fields
}

func NewDirectoryImpl 

func NewDirectoryImpl(spi DirectoryImplSPI) *DirectoryImpl

func (*DirectoryImpl) Copy 

func (d *DirectoryImpl) Copy(to Directory, src, dest string, ctx IOContext) (err error)

Copies the file src to 'to' under the new file name dest.

If you want to copy the entire source directory to the destination one, you can do so like this: 

var to Directory // the directory to copy to
for _, file := range dir.ListAll() {
	dir.Copy(to, file, newFile, IO_CONTEXT_DEFAULT)
	// newFile can be either file, or a new name
}

NOTE: this method does not check whether dest exists and will overwrite it if it does.

func (*DirectoryImpl) LockID 

func (d *DirectoryImpl) LockID() string

Return a string identifier that uniquely differentiates this Directory instance from other Directory instances. This ID should be the same if two Directory instances (even in different JVMs and/or on different machines) are considered "the same index". This is how locking "scopes" to the right index.

func (*DirectoryImpl) OpenChecksumInput 

func (d *DirectoryImpl) OpenChecksumInput(name string, ctx IOContext) (ChecksumIndexInput, error)

func (*DirectoryImpl) String 

func (d *DirectoryImpl) String() string

type DirectoryImplSPI 

type DirectoryImplSPI interface {
	OpenInput(string, IOContext) (IndexInput, error)
	LockFactory() LockFactory
}

type FSDirectory 

type FSDirectory struct {
	*DirectoryImpl
	*BaseDirectory
	FSDirectorySPI
	sync.Locker
	// contains filtered or unexported fields
}

func (*FSDirectory) Close 

func (d *FSDirectory) Close() error

func (*FSDirectory) CreateOutput 

func (d *FSDirectory) CreateOutput(name string, ctx IOContext) (out IndexOutput, err error)

Creates an IndexOutput for the file with the given name.

func (*FSDirectory) DeleteFile 

func (d *FSDirectory) DeleteFile(name string) (err error)

Removes an existing file in the directory.

func (*FSDirectory) FileExists 

func (d *FSDirectory) FileExists(name string) bool

func (*FSDirectory) FileLength 

func (d *FSDirectory) FileLength(name string) (n int64, err error)

Returns the length in bytes of a file in the directory.

func (*FSDirectory) ListAll 

func (d *FSDirectory) ListAll() (paths []string, err error)

func (*FSDirectory) LockID 

func (d *FSDirectory) LockID() string

func (*FSDirectory) SetLockFactory 

func (d *FSDirectory) SetLockFactory(lockFactory LockFactory)

func (*FSDirectory) String 

func (d *FSDirectory) String() string

func (*FSDirectory) Sync 

func (d *FSDirectory) Sync(names []string) (err error)

type FSDirectorySPI 

type FSDirectorySPI interface {
	OpenInput(string, IOContext) (IndexInput, error)
}

type FSIndexOutput 

type FSIndexOutput struct {
	*OutputStreamIndexOutput
	*FSDirectory
	// contains filtered or unexported fields
}

Writes output with File.Write([]byte) (int, error)

func (*FSIndexOutput) Close 

func (out *FSIndexOutput) Close() (err error)

type FSLockFactory 

type FSLockFactory struct {
	*LockFactoryImpl
	// contains filtered or unexported fields
}

func (*FSLockFactory) Clear 

func (f *FSLockFactory) Clear(name string) error

func (*FSLockFactory) Make 

func (f *FSLockFactory) Make(name string) Lock

func (*FSLockFactory) String 

func (f *FSLockFactory) String() string

type FileEntry 

type FileEntry struct {
	// contains filtered or unexported fields
}

type FileSlice 

type FileSlice struct {
	// contains filtered or unexported fields
}

type FilteredWriteCloser 

type FilteredWriteCloser struct {
	io.WriteCloser
	// contains filtered or unexported fields
}

func (*FilteredWriteCloser) Write 

func (w *FilteredWriteCloser) Write(p []byte) (int, error)

type FlushInfo 

type FlushInfo struct {
	NumDocs              int
	EstimatedSegmentSize int64
}

type IOContext 

type IOContext struct {
	MergeInfo *MergeInfo
	FlushInfo *FlushInfo
	// contains filtered or unexported fields
}

IOContext holds additional details on the merge/search context. A IOContext object can never be initialized as nil as passed as a parameter to either OpenInput() or CreateOutput()

func NewIOContextBool 

func NewIOContextBool(readOnce bool) IOContext

func NewIOContextForFlush 

func NewIOContextForFlush(flushInfo *FlushInfo) IOContext

func NewIOContextForMerge 

func NewIOContextForMerge(mergeInfo *MergeInfo) IOContext

func NewIOContextFromType 

func NewIOContextFromType(context IOContextType) IOContext

func (IOContext) String 

func (ctx IOContext) String() string

type IOContextType 

type IOContextType int

type IndexInput 

type IndexInput interface {
	util.DataInput
	IndexInputService
	ReadBytesBuffered(buf []byte, useBuffer bool) error
	// Clone
	Clone() IndexInput
	// Creates a slice of this index input, with the given description,
	// offset, and length. The slice is seeked to the beginning.
	Slice(desc string, offset, length int64) (IndexInput, error)
}

type IndexInputImpl 

type IndexInputImpl struct {
	*util.DataInputImpl
	// contains filtered or unexported fields
}

func NewIndexInputImpl 

func NewIndexInputImpl(desc string, r util.DataReader) *IndexInputImpl

func (*IndexInputImpl) String 

func (in *IndexInputImpl) String() string

type IndexInputService 

type IndexInputService interface {
	io.Closer
	// Returns the current position in this file, where the next read will occur.
	FilePointer() int64
	// Sets current position in this file, where the next read will occur.
	Seek(pos int64) error
	Length() int64
}

type IndexInputSub 

type IndexInputSub interface {
	io.Closer
	util.DataReader
	FilePointer() int64
	Seek(pos int64) error
	Length() int64
}

type IndexOutput 

type IndexOutput interface {
	io.Closer
	util.DataOutput
	// Forces any buffered output to be written.
	// Flush() error
	// Returns the current position in this file, where the next write will occur.
	FilePointer() int64
	// Returns the current checksum of bytes written so far
	Checksum() int64
}

type IndexOutputImpl 

type IndexOutputImpl struct {
	*util.DataOutputImpl
}

func NewIndexOutput 

func NewIndexOutput(part util.DataWriter) *IndexOutputImpl

type IndexOutputStream 

type IndexOutputStream struct {
	// contains filtered or unexported fields
}

Wrap IndexOutput to allow coding in flow style without worrying about error check. If error happens in the middle, following calls are just ignored.

func Stream 

func Stream(out IndexOutput) *IndexOutputStream

func (*IndexOutputStream) Close 

func (ios *IndexOutputStream) Close() error

func (*IndexOutputStream) WriteByte 

func (ios *IndexOutputStream) WriteByte(b byte) *IndexOutputStream

func (*IndexOutputStream) WriteBytes 

func (ios *IndexOutputStream) WriteBytes(b []byte) *IndexOutputStream

func (*IndexOutputStream) WriteInt 

func (ios *IndexOutputStream) WriteInt(i int32) *IndexOutputStream

func (*IndexOutputStream) WriteLong 

func (ios *IndexOutputStream) WriteLong(l int64) *IndexOutputStream

func (*IndexOutputStream) WriteString 

func (ios *IndexOutputStream) WriteString(s string) *IndexOutputStream

func (*IndexOutputStream) WriteStringSet 

func (ios *IndexOutputStream) WriteStringSet(m map[string]bool) *IndexOutputStream

func (*IndexOutputStream) WriteStringStringMap 

func (ios *IndexOutputStream) WriteStringStringMap(m map[string]string) *IndexOutputStream

func (*IndexOutputStream) WriteVInt 

func (ios *IndexOutputStream) WriteVInt(i int32) *IndexOutputStream

type Lock 

type Lock interface {
	// Releases exclusive access.
	io.Closer
	// Attempts to obtain exclusive access and immediately return
	// upon success or failure. Use Close() to release the lock.
	Obtain() (ok bool, err error)
	// Attempts to obtain an exclusive lock within amount of time
	// given. Pools once per LOCK_POLL_INTERVAL (currently 1000)
	// milliseconds until lockWaitTimeout is passed.
	ObtainWithin(lockWaitTimeout int64) (ok bool, err error)
	// Returns true if the resource is currently locked. Note that one
	// must still call obtain() before using the resource.
	IsLocked() bool
}

An interprocess mutex lock.

Typical use might look like: 

WithLock(directory.MakeLock("my.lock"), func() interface{} {
	// code to execute while locked
})

type LockFactory 

type LockFactory interface {
	Make(name string) Lock
	Clear(name string) error
	SetLockPrefix(prefix string)
	LockPrefix() string
}

type LockFactoryImpl 

type LockFactoryImpl struct {
	// contains filtered or unexported fields
}

func (*LockFactoryImpl) LockPrefix 

func (f *LockFactoryImpl) LockPrefix() string

func (*LockFactoryImpl) SetLockPrefix 

func (f *LockFactoryImpl) SetLockPrefix(prefix string)

type LockImpl 

type LockImpl struct {
	// contains filtered or unexported fields
}

func NewLockImpl 

func NewLockImpl(self Lock) *LockImpl

func (*LockImpl) ObtainWithin 

func (lock *LockImpl) ObtainWithin(lockWaitTimeout int64) (locked bool, err error)

type MergeInfo 

type MergeInfo struct {
	TotalDocCount       int
	EstimatedMergeBytes int64
	IsExternal          bool
	MergeMaxNumSegments int
}

type MultiLevelSkipListWriter 

type MultiLevelSkipListWriter struct {
	// contains filtered or unexported fields
}

TODO: migrate original comment.

Note: this class was moved from package codec to store since it caused cyclic dependency (store<->codec).

func NewMultiLevelSkipListWriter 

func NewMultiLevelSkipListWriter(spi MultiLevelSkipListWriterSPI,
	skipInterval, skipMultiplier, maxSkipLevels, df int) *MultiLevelSkipListWriter

Creates a MultiLevelSkipListWriter.

func (*MultiLevelSkipListWriter) BufferSkip 

func (w *MultiLevelSkipListWriter) BufferSkip(df int) error

Writes the current skip data to the buffers. The current document frequency determines the max level is skip ddata is to be written to.

func (*MultiLevelSkipListWriter) ResetSkip 

func (w *MultiLevelSkipListWriter) ResetSkip()

Creates new buffers or empties the existing ones

func (*MultiLevelSkipListWriter) WriteSkip 

func (w *MultiLevelSkipListWriter) WriteSkip(output IndexOutput) (int64, error)

Writes the buffered skip lists to the given output.

type MultiLevelSkipListWriterSPI 

type MultiLevelSkipListWriterSPI interface {
	WriteSkipData(level int, skipBuffer IndexOutput) error
}

type NRTCachingDirectory 

type NRTCachingDirectory struct {
	Directory
	sync.Locker
	// contains filtered or unexported fields
}

Wraps a RAMDirectory around any provided delegate directory, to be used during NRT search.

This class is likely only useful in a near-real-time context, where indexing rate is lowish but reopen rate is highish, resulting in many tiny files being written. This directory keeps such segments (as well as the segments produced by merging them, as long as they are small enough), in RAM.

This is safe to use: when you app calls IndexWriter.Commit(), all cached files will be flushed from the cached and sync'd.

Here's a simple example usage: 

fsDir, _ := OpenFSDirectory("/path/to/index")
cachedFSDir := NewNRTCachingDirectory(fsDir, 5.0, 60.0)
conf := NewIndexWriterConfig(VERSION_49, analyzer)
writer := NewIndexWriter(cachedFSDir, conf)

This will cache all newly flushed segments, all merged whose expected segment size is <= 5 MB, unless the net cached bytes exceeds 60 MB at which point all writes will not be cached (until the net bytes falls below 60 MB).

func NewNRTCachingDirectory 

func NewNRTCachingDirectory(delegate Directory, maxMergeSizeMB, maxCachedMB float64) (nrt *NRTCachingDirectory)

We will cache a newly created output if 1) it's a flush or a merge and the estimated size of the merged segment is <= maxMergedSizeMB, and 2) the total cached bytes is <= maxCachedMB.

func (*NRTCachingDirectory) Close 

func (nrt *NRTCachingDirectory) Close() error

Close this directory, which flushes any cached files to the delegate and then closes the delegate.

func (*NRTCachingDirectory) CreateOutput 

func (nrt *NRTCachingDirectory) CreateOutput(name string, context IOContext) (out IndexOutput, err error)

func (*NRTCachingDirectory) DeleteFile 

func (nrt *NRTCachingDirectory) DeleteFile(name string) error

func (*NRTCachingDirectory) FileExists 

func (nrt *NRTCachingDirectory) FileExists(name string) bool

func (*NRTCachingDirectory) FileLength 

func (nrt *NRTCachingDirectory) FileLength(name string) (length int64, err error)

func (*NRTCachingDirectory) ListAll 

func (nrt *NRTCachingDirectory) ListAll() (all []string, err error)

func (*NRTCachingDirectory) OpenInput 

func (nrt *NRTCachingDirectory) OpenInput(name string, context IOContext) (in IndexInput, err error)

func (*NRTCachingDirectory) String 

func (nrt *NRTCachingDirectory) String() string

func (*NRTCachingDirectory) Sync 

func (nrt *NRTCachingDirectory) Sync(fileNames []string) (err error)

type NoSuchDirectoryError 

type NoSuchDirectoryError struct {
	// contains filtered or unexported fields
}

func (*NoSuchDirectoryError) Error 

func (err *NoSuchDirectoryError) Error() string

type OutputStreamIndexOutput 

type OutputStreamIndexOutput struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

Implementation class for buffered IndexOutput that writes to a WriterCloser.

func (*OutputStreamIndexOutput) Checksum 

func (out *OutputStreamIndexOutput) Checksum() int64

func (*OutputStreamIndexOutput) Close 

func (out *OutputStreamIndexOutput) Close() error

func (*OutputStreamIndexOutput) FilePointer 

func (out *OutputStreamIndexOutput) FilePointer() int64

func (*OutputStreamIndexOutput) WriteByte 

func (out *OutputStreamIndexOutput) WriteByte(b byte) error

func (*OutputStreamIndexOutput) WriteBytes 

func (out *OutputStreamIndexOutput) WriteBytes(p []byte) error

type RAMDirectory 

type RAMDirectory struct {
	*DirectoryImpl
	*BaseDirectory
	// contains filtered or unexported fields
}

A memory-resident Directory implementation. Locking implementation is by default the SingleInstanceLockFactory but can be changed with SetLockFactory().

Warning: This class is not intended to work with huge indexes. Everything beyond several hundred megabytes will waste resources (GC cycles), becaues it uses an internal buffer size of 1024 bytes, producing millions of byte[1024] arrays. This class is optimized for small memory-resident indexes. It also has bad concurrency on multithreaded environments.

It is recommended to materialze large indexes on disk and use MMapDirectory, which is a high-performance directory implementation working diretly on the file system cache of the operating system, so copying dat to Java heap space is not useful.

func NewRAMDirectory 

func NewRAMDirectory() *RAMDirectory

func (*RAMDirectory) ChangeSize 

func (rd *RAMDirectory) ChangeSize(diff int64)

test-only

func (*RAMDirectory) Close 

func (rd *RAMDirectory) Close() error

Closes the store to future operations, releasing associated memroy.

func (*RAMDirectory) CreateOutput 

func (rd *RAMDirectory) CreateOutput(name string, context IOContext) (out IndexOutput, err error)

Creates a new, empty file in the directory with the given name. Returns a stream writing this file:

func (*RAMDirectory) DeleteFile 

func (rd *RAMDirectory) DeleteFile(name string) error

Removes an existing file in the directory

func (*RAMDirectory) FileExists 

func (rd *RAMDirectory) FileExists(name string) bool

Returns true iff the named file exists in this directory

func (*RAMDirectory) FileLength 

func (rd *RAMDirectory) FileLength(name string) (length int64, err error)

Returns the length in bytes of a file in the directory.

func (*RAMDirectory) GetRAMFile 

func (rd *RAMDirectory) GetRAMFile(name string) *RAMFile

test-only

func (*RAMDirectory) ListAll 

func (rd *RAMDirectory) ListAll() (names []string, err error)

func (*RAMDirectory) LockID 

func (d *RAMDirectory) LockID() string

func (*RAMDirectory) OpenInput 

func (rd *RAMDirectory) OpenInput(name string, context IOContext) (in IndexInput, err error)

Returns a stream reading an existing file.

func (*RAMDirectory) PutRAMFile 

func (d *RAMDirectory) PutRAMFile(name string, file *RAMFile)

test-only

func (*RAMDirectory) RamBytesUsed 

func (rd *RAMDirectory) RamBytesUsed() int64

Return total size in bytes of all files in this directory. This is currently quantized to BUFFER_SIZE.

func (*RAMDirectory) String 

func (rd *RAMDirectory) String() string

func (*RAMDirectory) Sync 

func (rd *RAMDirectory) Sync(names []string) error

type RAMFile 

type RAMFile struct {
	sync.Locker
	// contains filtered or unexported fields
}

Represents a file in RAM as a list of []byte buffers.

func NewRAMFile 

func NewRAMFile(directory *RAMDirectory) *RAMFile

func NewRAMFileBuffer 

func NewRAMFileBuffer() *RAMFile

func (*RAMFile) Buffer 

func (rf *RAMFile) Buffer(index int) []byte

func (*RAMFile) Length 

func (rf *RAMFile) Length() int64

func (*RAMFile) RamBytesUsed 

func (rf *RAMFile) RamBytesUsed() int64

func (*RAMFile) SetLength 

func (rf *RAMFile) SetLength(length int64)

type RAMInputStream 

type RAMInputStream struct {
	*IndexInputImpl
	// contains filtered or unexported fields
}

A memory-resident IndexInput implementation.

func (*RAMInputStream) Clone 

func (in *RAMInputStream) Clone() IndexInput

func (*RAMInputStream) Close 

func (in *RAMInputStream) Close() error

func (*RAMInputStream) FilePointer 

func (in *RAMInputStream) FilePointer() int64

func (*RAMInputStream) Length 

func (in *RAMInputStream) Length() int64

func (*RAMInputStream) ReadByte 

func (in *RAMInputStream) ReadByte() (byte, error)

func (*RAMInputStream) ReadBytes 

func (in *RAMInputStream) ReadBytes(buf []byte) error

func (*RAMInputStream) Seek 

func (in *RAMInputStream) Seek(pos int64) error

func (*RAMInputStream) Slice 

func (in *RAMInputStream) Slice(desc string, offset, length int64) (IndexInput, error)

func (*RAMInputStream) String 

func (in *RAMInputStream) String() string

type RAMOutputStream 

type RAMOutputStream struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

A memory-resident IndexOutput implementation

func NewRAMOutputStream 

func NewRAMOutputStream(f *RAMFile, checksum bool) *RAMOutputStream

func NewRAMOutputStreamBuffer 

func NewRAMOutputStreamBuffer() *RAMOutputStream

Construct an empty output buffer.

func (*RAMOutputStream) Checksum 

func (out *RAMOutputStream) Checksum() int64

func (*RAMOutputStream) Close 

func (out *RAMOutputStream) Close() error

func (*RAMOutputStream) FilePointer 

func (out *RAMOutputStream) FilePointer() int64

func (*RAMOutputStream) Flush 

func (out *RAMOutputStream) Flush() error

func (*RAMOutputStream) Reset 

func (out *RAMOutputStream) Reset()

Resets this to an empty file.

func (*RAMOutputStream) WriteByte 

func (out *RAMOutputStream) WriteByte(b byte) error

func (*RAMOutputStream) WriteBytes 

func (out *RAMOutputStream) WriteBytes(buf []byte) error

func (*RAMOutputStream) WriteTo 

func (out *RAMOutputStream) WriteTo(output util.DataOutput) error

Copy the current contents of this buffer to the named output.

func (*RAMOutputStream) WriteToBytes 

func (out *RAMOutputStream) WriteToBytes(bytes []byte) error

Copy the current contents of this buffer to output byte slice

type RateLimitedDirectoryWrapper 

type RateLimitedDirectoryWrapper struct {
	Directory
	// contains filtered or unexported fields
}

A Directory wrapper that allows IndexOutput rate limiting using IO context specific rate limiters.

func NewRateLimitedDirectoryWrapper 

func NewRateLimitedDirectoryWrapper(wrapped Directory) *RateLimitedDirectoryWrapper

func (*RateLimitedDirectoryWrapper) CreateOutput 

func (w *RateLimitedDirectoryWrapper) CreateOutput(name string, ctx IOContext) (IndexOutput, error)

func (*RateLimitedDirectoryWrapper) MaxWriteMBPerSec 

func (w *RateLimitedDirectoryWrapper) MaxWriteMBPerSec(context int)

func (*RateLimitedDirectoryWrapper) SetMaxWriteMBPerSec 

func (w *RateLimitedDirectoryWrapper) SetMaxWriteMBPerSec(mbPerSec float64, context int)

Sets the maximum (approx) MB/sec allowed by all write IO performed by IndexOutput created with the given context. Pass non-positve value to have no limit.

NOTE: For already created IndexOutput instances there is no guarantee this new rate will apply to them; it will only be guaranteed to apply for new created IndexOutput instances.

NOTE: this is an optional operation and might not be respected by all Directory implementations. Currently only buffered Directory implementations use rate-limiting.

type RateLimitedIndexOutput 

type RateLimitedIndexOutput struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

A rate limiting IndexOutput

func (*RateLimitedIndexOutput) Checksum 

func (out *RateLimitedIndexOutput) Checksum() int64

func (*RateLimitedIndexOutput) Close 

func (out *RateLimitedIndexOutput) Close() error

func (*RateLimitedIndexOutput) FilePointer 

func (out *RateLimitedIndexOutput) FilePointer() int64

func (*RateLimitedIndexOutput) WriteByte 

func (out *RateLimitedIndexOutput) WriteByte(b byte) error

func (*RateLimitedIndexOutput) WriteBytes 

func (out *RateLimitedIndexOutput) WriteBytes(p []byte) error

type RateLimiter 

type RateLimiter interface {
	// Sets an updated mb per second rate limit.
	SetMbPerSec(mbPerSec float64)
	// The current mb per second rate limit.
	MbPerSec() float64
	/*
		Pause, if necessary, to keep the instantaneous IO rate at or below
		the target.

		Note: the implementation is thread-safe
	*/
	Pause(bytes int64) int64
}

Abstract base class to rate limit IO. Typically implementations are shared across multiple IndexInputs or IndexOutputs (for example those involved all merging). Those IndexInputs and IndexOutputs would call pause() whenever they have read or written more than minPauseCheckBytes() bytes.

type SeekReader 

type SeekReader interface {
	Length() int64
	// contains filtered or unexported methods
}

type SimpleFSDirectory 

type SimpleFSDirectory struct {
	*FSDirectory
}

func NewSimpleFSDirectory 

func NewSimpleFSDirectory(path string) (d *SimpleFSDirectory, err error)

func (*SimpleFSDirectory) OpenInput 

func (d *SimpleFSDirectory) OpenInput(name string, context IOContext) (IndexInput, error)

type SimpleFSIndexInput 

type SimpleFSIndexInput struct {
	*BufferedIndexInput
	// contains filtered or unexported fields
}

func (*SimpleFSIndexInput) Clone 

func (in *SimpleFSIndexInput) Clone() IndexInput

func (*SimpleFSIndexInput) Close 

func (in *SimpleFSIndexInput) Close() error

func (*SimpleFSIndexInput) Length 

func (in *SimpleFSIndexInput) Length() int64

func (*SimpleFSIndexInput) Slice 

func (in *SimpleFSIndexInput) Slice(desc string, offset, length int64) (IndexInput, error)

type SimpleFSLock 

type SimpleFSLock struct {
	*LockImpl
	// contains filtered or unexported fields
}

func (*SimpleFSLock) Close 

func (lock *SimpleFSLock) Close() error

func (*SimpleFSLock) IsLocked 

func (lock *SimpleFSLock) IsLocked() bool

func (*SimpleFSLock) Obtain 

func (lock *SimpleFSLock) Obtain() (ok bool, err error)

func (*SimpleFSLock) String 

func (lock *SimpleFSLock) String() string

type SimpleFSLockFactory 

type SimpleFSLockFactory struct {
	*FSLockFactory
}

Implements LockFactory using os.Create().

NOTE: This API may has the same issue as the one in Lucene Java that the write lock may not be released when Go program exists abnormally.

When this happens, an error is returned when trying to create a writer, in which case you need to explicitly clear the lock file first. You can either manually remove the file, or use UnlockDirectory() API. But, first be certain that no writer is in fact writing to the index otherwise you can easily corrupt your index.

If you suspect that this or any other LockFactory is not working properly in your environment, you can easily test it by using VerifyingLockFactory, LockVerifyServer and LockStressTest.

func NewSimpleFSLockFactory 

func NewSimpleFSLockFactory(path string) *SimpleFSLockFactory

func (*SimpleFSLockFactory) Clear 

func (f *SimpleFSLockFactory) Clear(name string) error

func (*SimpleFSLockFactory) Make 

func (f *SimpleFSLockFactory) Make(name string) Lock

type SimpleRateLimiter 

type SimpleRateLimiter struct {
	// contains filtered or unexported fields
}

Simple class to rate limit IO Ian: volatile is not supported

func (*SimpleRateLimiter) MbPerSec 

func (srl *SimpleRateLimiter) MbPerSec() float64

func (*SimpleRateLimiter) Pause 

func (srl *SimpleRateLimiter) Pause(bytes int64) int64

Pause, if necessary, to keep the instantaneous IO rate at or below the target. Be sure to only call this method when bytes > minPauseCheckBytes(), otherwise it will pause way too long!

func (*SimpleRateLimiter) SetMbPerSec 

func (srl *SimpleRateLimiter) SetMbPerSec(mbPerSec float64)

type SingleInstanceLock 

type SingleInstanceLock struct {
	*LockImpl
	// contains filtered or unexported fields
}

func (*SingleInstanceLock) Close 

func (lock *SingleInstanceLock) Close() error

func (*SingleInstanceLock) IsLocked 

func (lock *SingleInstanceLock) IsLocked() bool

func (*SingleInstanceLock) Obtain 

func (lock *SingleInstanceLock) Obtain() (ok bool, err error)

func (*SingleInstanceLock) String 

func (lock *SingleInstanceLock) String() string

type SingleInstanceLockFactory 

type SingleInstanceLockFactory struct {
	*LockFactoryImpl
	// contains filtered or unexported fields
}

Implements LockFactory for a single in-process instance, meaning all locking will take place through this one instance. Only use this LockFactory when you are certain all IndexReaders and IndexWriters for a given index are running against a single shared in-process Directory instance. This is currently the default locking for RAMDirectory.

func (*SingleInstanceLockFactory) Clear 

func (fac *SingleInstanceLockFactory) Clear(name string) error

func (*SingleInstanceLockFactory) Make 

func (fac *SingleInstanceLockFactory) Make(name string) Lock

func (*SingleInstanceLockFactory) String 

func (fac *SingleInstanceLockFactory) String() string

type TrackingDirectoryWrapper 

type TrackingDirectoryWrapper struct {
	Directory
	sync.Locker
	// contains filtered or unexported fields
}

A delegating Directory that records which files were written to and deleted.

func NewTrackingDirectoryWrapper 

func NewTrackingDirectoryWrapper(other Directory) *TrackingDirectoryWrapper

func (*TrackingDirectoryWrapper) ContainsFile 

func (w *TrackingDirectoryWrapper) ContainsFile(name string) bool

func (*TrackingDirectoryWrapper) Copy 

func (w *TrackingDirectoryWrapper) Copy(to Directory, src, dest string, ctx IOContext) error

func (*TrackingDirectoryWrapper) CreateOutput 

func (w *TrackingDirectoryWrapper) CreateOutput(name string, ctx IOContext) (IndexOutput, error)

func (*TrackingDirectoryWrapper) DeleteFile 

func (w *TrackingDirectoryWrapper) DeleteFile(name string) error

func (*TrackingDirectoryWrapper) EachCreatedFiles 

func (w *TrackingDirectoryWrapper) EachCreatedFiles(f func(name string))

func (*TrackingDirectoryWrapper) String 

func (w *TrackingDirectoryWrapper) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.