block_cache

Documentation

Index

• Constants
• func NewBlockCacheComponent() internal.Component
• type Block
  • func AllocateBlock(size uint64) (*Block, error)
  • func (b *Block) Delete() error
  • func (b *Block) ReUse()
  • func (b *Block) ReadyForReading()
  • func (b *Block) Unblock()
• type BlockCache
  • func (bc *BlockCache) CloseFile(options internal.CloseFileOptions) error
  • func (bc *BlockCache) Configure(_ bool) error
  • func (bc *BlockCache) Name() string
  • func (bc *BlockCache) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error)
  • func (bc *BlockCache) ReadInBuffer(options internal.ReadInBufferOptions) (int, error)
  • func (bc *BlockCache) SetName(name string)
  • func (bc *BlockCache) SetNextComponent(nc internal.Component)
  • func (bc *BlockCache) Start(ctx context.Context) error
  • func (bc *BlockCache) Stop() error
  • func (bc *BlockCache) TempCacheCleanup() error
• type BlockCacheOptions
• type BlockPool
  • func NewBlockPool(blockSize uint64, memSize uint64) *BlockPool
  • func (pool *BlockPool) MustGet() *Block
  • func (pool *BlockPool) Release(b *Block)
  • func (pool *BlockPool) Terminate()
  • func (pool *BlockPool) TryGet() *Block
  • func (pool *BlockPool) Usage() uint32
• type ThreadPool
  • func (t *ThreadPool) Do(priority bool)
  • func (t *ThreadPool) Schedule(urgent bool, item *workItem)
  • func (t *ThreadPool) Start()
  • func (t *ThreadPool) Stop()

Constants

const (
	MAX_POOL_USAGE uint32 = 80
	MIN_POOL_USAGE uint32 = 50
	MIN_PREFETCH          = 5
	MIN_RANDREAD          = 10
	MAX_FAIL_CNT          = 3
)

Functions

func NewBlockCacheComponent

func NewBlockCacheComponent() internal.Component

------------------------- Factory ------------------------------------------- Pipeline will call this method to create your object, initialize your variables here << DO NOT DELETE ANY AUTO GENERATED CODE HERE >>

Types

type Block

type Block struct {
	// contains filtered or unexported fields
}

Block is a memory mapped buffer with its state to hold data

func AllocateBlock

func AllocateBlock(size uint64) (*Block, error)

AllocateBlock creates a new memory mapped buffer for the given size

func (*Block) Delete

func (b *Block) Delete() error

Delete cleans up the memory mapped buffer

func (*Block) ReUse

func (b *Block) ReUse()

ReUse reinits the Block by recreating its channel

func (*Block) ReadyForReading

func (b *Block) ReadyForReading()

ReadyForReading marks this Block is now ready for reading by its first reader (data download completed)

func (*Block) Unblock

func (b *Block) Unblock()

Unblock marks this Block is ready to be read in parllel now

type BlockCache

type BlockCache struct {
	internal.BaseComponent
	// contains filtered or unexported fields
}

Common structure for Component

func (*BlockCache) CloseFile

func (bc *BlockCache) CloseFile(options internal.CloseFileOptions) error

CloseFile: File is closed by application so release all the blocks and submit back to blockPool

func (*BlockCache) Configure

func (bc *BlockCache) Configure(_ bool) error

Configure : Pipeline will call this method after constructor so that you can read config and initialize yourself

Return failure if any config is not valid to exit the process

func (*BlockCache) Name

func (bc *BlockCache) Name() string

func (*BlockCache) OpenFile

func (bc *BlockCache) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error)

OpenFile: Create a handle for the file user has requested to open

func (*BlockCache) ReadInBuffer

func (bc *BlockCache) ReadInBuffer(options internal.ReadInBufferOptions) (int, error)

ReadInBuffer: Read the file into a buffer

func (*BlockCache) SetName

func (bc *BlockCache) SetName(name string)

func (*BlockCache) SetNextComponent

func (bc *BlockCache) SetNextComponent(nc internal.Component)

func (*BlockCache) Start

func (bc *BlockCache) Start(ctx context.Context) error

Start : Pipeline calls this method to start the component functionality

this shall not Block the call otherwise pipeline will not start

func (*BlockCache) Stop

func (bc *BlockCache) Stop() error

Stop : Stop the component functionality and kill all threads started

func (*BlockCache) TempCacheCleanup

func (bc *BlockCache) TempCacheCleanup() error

TempCacheCleanup cleans up the local cached contents

type BlockCacheOptions

type BlockCacheOptions struct {
	BlockSize      uint64 `config:"block-size-mb" yaml:"block-size-mb,omitempty"`
	MemSize        uint64 `config:"mem-size-mb" yaml:"mem-size-mb,omitempty"`
	TmpPath        string `config:"path" yaml:"path,omitempty"`
	DiskSize       uint64 `config:"disk-size-mb" yaml:"disk-size-mb,omitempty"`
	DiskTimeout    uint32 `config:"disk-timeout-sec" yaml:"timeout-sec,omitempty"`
	PrefetchCount  uint32 `config:"prefetch" yaml:"prefetch,omitempty"`
	Workers        uint32 `config:"parallelism" yaml:"parallelism,omitempty"`
	PrefetchOnOpen bool   `config:"prefetch-on-open" yaml:"prefetch-on-open,omitempty"`
}

Structure defining your config parameters

type BlockPool

type BlockPool struct {
	// contains filtered or unexported fields
}

BlockPool is a pool of Blocks

func NewBlockPool

func NewBlockPool(blockSize uint64, memSize uint64) *BlockPool

NewBlockPool allocates a new pool of blocks

func (*BlockPool) MustGet

func (pool *BlockPool) MustGet() *Block

MustGet a Block from the pool, wait until something is free

func (*BlockPool) Release

func (pool *BlockPool) Release(b *Block)

Release back the Block to the pool

func (*BlockPool) Terminate

func (pool *BlockPool) Terminate()

Terminate ends the block pool life

func (*BlockPool) TryGet

func (pool *BlockPool) TryGet() *Block

TryGet a Block from the pool, return back if nothing is available

func (*BlockPool) Usage

func (pool *BlockPool) Usage() uint32

Usage provides % usage of this block pool

type ThreadPool

type ThreadPool struct {
	// contains filtered or unexported fields
}

ThreadPool is a group of workers that can be used to execute a task

func (*ThreadPool) Do

func (t *ThreadPool) Do(priority bool)

Do is the core task to be executed by each worker thread

func (*ThreadPool) Schedule

func (t *ThreadPool) Schedule(urgent bool, item *workItem)

Schedule the download of a block

func (*ThreadPool) Start

func (t *ThreadPool) Start()

Start all the workers and wait till they start receiving requests

func (*ThreadPool) Stop

func (t *ThreadPool) Stop()

Stop all the workers threads