bfs

package
v1.3.9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 20, 2024 License: BSD-3-Clause Imports: 26 Imported by: 0

Documentation

Index

Constants

View Source 
const BFileExt = ".b"
View Source 
const MFileExt = ".m"
View Source 
const Version1 = 1

Variables

View Source 
var DefaultBlockFileOptions = &BlockFileOptions{
	BytesPerSync: 1 << 20,
}
View Source 
var DefaultFSOptions = &FSOptions{
	MaxOpenFiles: 1 << 10,
	BytesPerSync: 512 << 10,
	SyncTimeout:  1 * time.Second,
	MaxSyncFiles: 32,
}
View Source 
var ErrClosed = errors.New("the file closed")
View Source 
var ErrFileIsWriting = errors.New("the file is writing")
View Source 
var ErrInvalidHash = errors.New("invalid hash")
View Source 
var HashLen = 32
View Source 
var SharedCompressPool = NewGzipWriterPool()
View Source 
var SharedDecompressPool = NewGzipReaderPool()

Functions

func AckReadThread 

func AckReadThread()

func AckWriteThread 

func AckWriteThread()

func CheckHash 

func CheckHash(hash string) bool

CheckHash check hash string format

func CheckHashErr 

func CheckHashErr(hash string) error

func EncodeMetaBlock 

func EncodeMetaBlock(action MetaAction, hash string, data []byte) ([]byte, error)

func Hash 

func Hash(s string) string

func IsEnabled 

func IsEnabled() bool

func IsNotExist 

func IsNotExist(err error) bool

func IsWritingErr 

func IsWritingErr(err error) bool

func ReleaseReadThread 

func ReleaseReadThread()

func ReleaseWriteThread 

func ReleaseWriteThread()

Types

type BlockFileOptions 

type BlockFileOptions struct {
	BytesPerSync int64
}

func (*BlockFileOptions) EnsureDefaults 

func (this *BlockFileOptions) EnsureDefaults()

type BlockInfo 

type BlockInfo struct {
	OriginOffsetFrom int64 `json:"1,omitempty"`
	OriginOffsetTo   int64 `json:"2,omitempty"`

	BFileOffsetFrom int64 `json:"3,omitempty"`
	BFileOffsetTo   int64 `json:"4,omitempty"`
}

func (BlockInfo) Contains 

func (this BlockInfo) Contains(offset int64) bool

type BlockType 

type BlockType string
const (
	BlockTypeHeader BlockType = "header"
	BlockTypeBody   BlockType = "body"
)

type BlocksFile 

type BlocksFile struct {
	// contains filtered or unexported fields
}

func NewBlocksFileWithRawFile 

func NewBlocksFileWithRawFile(fp *os.File, options *BlockFileOptions) (*BlocksFile, error)

func OpenBlocksFile 

func OpenBlocksFile(filename string, options *BlockFileOptions) (*BlocksFile, error)

func (*BlocksFile) CanClose 

func (this *BlocksFile) CanClose() bool

CanClose 检查是否可以关闭

func (*BlocksFile) Close 

func (this *BlocksFile) Close() error

Close 关闭当前文件

func (*BlocksFile) CloseFileReader 

func (this *BlocksFile) CloseFileReader(reader *FileReader) error

func (*BlocksFile) Compact 

func (this *BlocksFile) Compact() error

func (*BlocksFile) DecrRef 

func (this *BlocksFile) DecrRef()

func (*BlocksFile) ExistFile 

func (this *BlocksFile) ExistFile(fileHash string) bool

func (*BlocksFile) Filename 

func (this *BlocksFile) Filename() string

func (*BlocksFile) ForceSync 

func (this *BlocksFile) ForceSync() error

func (*BlocksFile) IncrRef 

func (this *BlocksFile) IncrRef()

func (*BlocksFile) IsClosing 

func (this *BlocksFile) IsClosing() bool

IsClosing 判断当前文件是否正在关闭或者已关闭

func (*BlocksFile) OpenFileReader 

func (this *BlocksFile) OpenFileReader(fileHash string, isPartial bool) (*FileReader, error)

func (*BlocksFile) OpenFileWriter 

func (this *BlocksFile) OpenFileWriter(fileHash string, bodySize int64, isPartial bool) (writer *FileWriter, err error)

func (*BlocksFile) RemoveAll 

func (this *BlocksFile) RemoveAll() error

func (*BlocksFile) RemoveFile 

func (this *BlocksFile) RemoveFile(fileHash string) error

func (*BlocksFile) Sync 

func (this *BlocksFile) Sync() error

func (*BlocksFile) SyncAt 

func (this *BlocksFile) SyncAt() time.Time

func (*BlocksFile) TestReaderPool 

func (this *BlocksFile) TestReaderPool() chan *FileReader

func (*BlocksFile) Write 

func (this *BlocksFile) Write(hash string, blockType BlockType, b []byte, originOffset int64) (n int, err error)

type FS 

type FS struct {
	// contains filtered or unexported fields
}

FS 文件系统对象

func OpenFS 

func OpenFS(dir string, options *FSOptions) (*FS, error)

OpenFS 打开文件系统

func (*FS) Close 

func (this *FS) Close() error

func (*FS) ExistFile 

func (this *FS) ExistFile(hash string) (bool, error)

func (*FS) OpenFileReader 

func (this *FS) OpenFileReader(hash string, isPartial bool) (*FileReader, error)

OpenFileReader 打开文件读取器

func (*FS) OpenFileWriter 

func (this *FS) OpenFileWriter(hash string, bodySize int64, isPartial bool) (*FileWriter, error)

OpenFileWriter 打开文件写入器

func (*FS) RemoveFile 

func (this *FS) RemoveFile(hash string) error

func (*FS) TestBList 

func (this *FS) TestBList() *linkedlist.List[string]

func (*FS) TestBMap 

func (this *FS) TestBMap() map[string]*BlocksFile

type FSOptions 

type FSOptions struct {
	MaxOpenFiles int
	BytesPerSync int64
	SyncTimeout  time.Duration
	MaxSyncFiles int
}

func (*FSOptions) EnsureDefaults 

func (this *FSOptions) EnsureDefaults()

type FileHeader 

type FileHeader struct {
	Version         int         `json:"1,omitempty"`
	ModifiedAt      int64       `json:"2,omitempty"`
	ExpiresAt       int64       `json:"3,omitempty"`
	Status          int         `json:"4,omitempty"`
	HeaderSize      int64       `json:"5,omitempty"`
	BodySize        int64       `json:"6,omitempty"`
	ExpiredBodySize int64       `json:"7,omitempty"`
	HeaderBlocks    []BlockInfo `json:"8,omitempty"`
	BodyBlocks      []BlockInfo `json:"9,omitempty"`
	IsCompleted     bool        `json:"10,omitempty"`
	IsWriting       bool        `json:"11,omitempty"`
}

func (*FileHeader) BlockAt 

func (this *FileHeader) BlockAt(offset int64) (blockInfo BlockInfo, ok bool)

func (*FileHeader) Clone 

func (this *FileHeader) Clone() *FileHeader

Clone current header

func (*FileHeader) Compact 

func (this *FileHeader) Compact()

Compact blocks

func (*FileHeader) Encode 

func (this *FileHeader) Encode(hash string) ([]byte, error)

func (*FileHeader) MaxOffset 

func (this *FileHeader) MaxOffset() int64

type FileReader 

type FileReader struct {
	// contains filtered or unexported fields
}

func NewFileReader 

func NewFileReader(bFile *BlocksFile, fp *os.File, fileHeader *FileHeader) *FileReader

func (*FileReader) Close 

func (this *FileReader) Close() error

func (*FileReader) FileHeader 

func (this *FileReader) FileHeader() *FileHeader

func (*FileReader) Free 

func (this *FileReader) Free() error

func (*FileReader) Read 

func (this *FileReader) Read(b []byte) (n int, err error)

func (*FileReader) ReadAt 

func (this *FileReader) ReadAt(b []byte, offset int64) (n int, err error)

func (*FileReader) Reset 

func (this *FileReader) Reset(fileHeader *FileHeader)

type FileWriter 

type FileWriter struct {
	// contains filtered or unexported fields
}

FileWriter file writer not thread-safe

func NewFileWriter 

func NewFileWriter(bFile *BlocksFile, hash string, bodySize int64, isPartial bool) (*FileWriter, error)

func (*FileWriter) Close 

func (this *FileWriter) Close() error

func (*FileWriter) Discard 

func (this *FileWriter) Discard() error

func (*FileWriter) WriteBody 

func (this *FileWriter) WriteBody(b []byte) (n int, err error)

func (*FileWriter) WriteBodyAt 

func (this *FileWriter) WriteBodyAt(b []byte, offset int64) (n int, err error)

func (*FileWriter) WriteHeader 

func (this *FileWriter) WriteHeader(b []byte) (n int, err error)

func (*FileWriter) WriteMeta 

func (this *FileWriter) WriteMeta(status int, expiresAt int64, expectedFileSize int64) error

type GzipReaderPool 

type GzipReaderPool struct {
	// contains filtered or unexported fields
}

func NewGzipReaderPool 

func NewGzipReaderPool() *GzipReaderPool

func (*GzipReaderPool) Get 

func (this *GzipReaderPool) Get(rawReader io.Reader) (*gzip.Reader, error)

func (*GzipReaderPool) Put 

func (this *GzipReaderPool) Put(reader *gzip.Reader)

type GzipWriterPool 

type GzipWriterPool struct {
	// contains filtered or unexported fields
}

func NewGzipWriterPool 

func NewGzipWriterPool() *GzipWriterPool

func (*GzipWriterPool) Get 

func (this *GzipWriterPool) Get(rawWriter io.Writer) (*gzip.Writer, error)

func (*GzipWriterPool) Put 

func (this *GzipWriterPool) Put(writer *gzip.Writer)

type LazyFileHeader 

type LazyFileHeader struct {
	// contains filtered or unexported fields
}

LazyFileHeader load file header lazily to save memory

func NewLazyFileHeader 

func NewLazyFileHeader(fileHeader *FileHeader) *LazyFileHeader

func NewLazyFileHeaderFromData 

func NewLazyFileHeaderFromData(rawData []byte) *LazyFileHeader

func (*LazyFileHeader) FileHeaderUnsafe 

func (this *LazyFileHeader) FileHeaderUnsafe() (*FileHeader, error)

type MetaAction 

type MetaAction = byte
const (
	MetaActionNew    MetaAction = '+'
	MetaActionRemove MetaAction = '-'
)

func DecodeMetaBlock 

func DecodeMetaBlock(blockBytes []byte) (action MetaAction, hash string, data []byte, err error)

type MetaFile 

type MetaFile struct {
	// contains filtered or unexported fields
}

func OpenMetaFile 

func OpenMetaFile(filename string, mu *sync.RWMutex) (*MetaFile, error)

func (*MetaFile) CloneFileHeader 

func (this *MetaFile) CloneFileHeader(hash string) (header *FileHeader, ok bool)

func (*MetaFile) Close 

func (this *MetaFile) Close() error

Close 关闭当前文件

func (*MetaFile) Compact 

func (this *MetaFile) Compact() error

Compact the meta file TODO 考虑自动Compact的时机（脏数据比例？）

func (*MetaFile) ExistFile 

func (this *MetaFile) ExistFile(hash string) bool

func (*MetaFile) FileHeader 

func (this *MetaFile) FileHeader(hash string) (header *FileHeader, ok bool)

func (*MetaFile) FileHeaderUnsafe 

func (this *MetaFile) FileHeaderUnsafe(hash string) (header *FileHeader, ok bool)

func (*MetaFile) FileHeaders 

func (this *MetaFile) FileHeaders() map[string]*LazyFileHeader

func (*MetaFile) RemoveAll 

func (this *MetaFile) RemoveAll() error

RemoveAll 删除所有数据

func (*MetaFile) RemoveFile 

func (this *MetaFile) RemoveFile(hash string) error

func (*MetaFile) SyncUnsafe 

func (this *MetaFile) SyncUnsafe() error

func (*MetaFile) WriteBodyBlockUnsafe 

func (this *MetaFile) WriteBodyBlockUnsafe(hash string, bOffsetFrom int64, bOffsetTo int64, originOffsetFrom int64, originOffsetTo int64) error

func (*MetaFile) WriteClose 

func (this *MetaFile) WriteClose(hash string, headerSize int64, bodySize int64) error

func (*MetaFile) WriteHeaderBlockUnsafe 

func (this *MetaFile) WriteHeaderBlockUnsafe(hash string, bOffsetFrom int64, bOffsetTo int64) error

func (*MetaFile) WriteMeta 

func (this *MetaFile) WriteMeta(hash string, status int, expiresAt int64, expectedFileSize int64) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.