compressor

Documentation

Index

• Constants
• Variables
• func CLIFlags(envPrefix string) []cli.Flag
• func NewNonCompressor(config Config) (derive.Compressor, error)
• func NewRatioCompressor(config Config) (derive.Compressor, error)
• func NewShadowCompressor(config Config) (derive.Compressor, error)
• type CLIConfig
  • func ReadCLIConfig(ctx *cli.Context) CLIConfig
  • func (c *CLIConfig) Config() Config
• type Config
  • func (c Config) NewCompressor() (derive.Compressor, error)
• type FactoryFunc
• type NonCompressor
  • func (t *NonCompressor) Close() error
  • func (t *NonCompressor) Flush() error
  • func (t *NonCompressor) FullErr() error
  • func (t *NonCompressor) Len() int
  • func (t *NonCompressor) Read(p []byte) (int, error)
  • func (t *NonCompressor) Reset()
  • func (t *NonCompressor) Write(p []byte) (int, error)
• type RatioCompressor
  • func (t *RatioCompressor) Close() error
  • func (t *RatioCompressor) Flush() error
  • func (t *RatioCompressor) FullErr() error
  • func (t *RatioCompressor) InputThreshold() uint64
  • func (t *RatioCompressor) Len() int
  • func (t *RatioCompressor) Read(p []byte) (int, error)
  • func (t *RatioCompressor) Reset()
  • func (t *RatioCompressor) Write(p []byte) (int, error)
• type ShadowCompressor
  • func (t *ShadowCompressor) Close() error
  • func (t *ShadowCompressor) Flush() error
  • func (t *ShadowCompressor) FullErr() error
  • func (t *ShadowCompressor) Len() int
  • func (t *ShadowCompressor) Read(p []byte) (int, error)
  • func (t *ShadowCompressor) Reset()
  • func (t *ShadowCompressor) Write(p []byte) (int, error)

Constants

const (
	TargetL1TxSizeBytesFlagName = "target-l1-tx-size-bytes"
	TargetNumFramesFlagName     = "target-num-frames"
	ApproxComprRatioFlagName    = "approx-compr-ratio"
	KindFlagName                = "compressor"
)

const (
	RatioKind  = "ratio"
	ShadowKind = "shadow"
	NoneKind   = "none"
)

Variables

var KindKeys []string

var Kinds = map[string]FactoryFunc{
	RatioKind:  NewRatioCompressor,
	ShadowKind: NewShadowCompressor,
	NoneKind:   NewNonCompressor,
}

Functions

func CLIFlags

func CLIFlags(envPrefix string) []cli.Flag

func NewNonCompressor

func NewNonCompressor(config Config) (derive.Compressor, error)

NewNonCompressor creates a new derive.Compressor implementation that doesn't compress by using zlib.NoCompression. It flushes to the underlying buffer any data from a prior write call. This is very unoptimal behavior and should only be used in tests. The NonCompressor can be used in tests to create a partially flushed channel. If the output buffer size after a write exceeds TargetFrameSize*TargetNumFrames, the compressor is marked as full, but the write succeeds.

func NewRatioCompressor

func NewRatioCompressor(config Config) (derive.Compressor, error)

NewRatioCompressor creates a new derive.Compressor implementation that uses the target size and a compression ratio parameter to determine how much data can be written to the compressor before it's considered full. The full calculation is as follows:

full = uncompressedLength * approxCompRatio >= targetFrameSize * targetNumFrames

func NewShadowCompressor

func NewShadowCompressor(config Config) (derive.Compressor, error)

NewShadowCompressor creates a new derive.Compressor implementation that contains two compression buffers: one used for size estimation, and one used for the final compressed output. The first is flushed on every write, the second isn't, which means the final compressed data is always slightly smaller than the target. There is one exception to this rule: the first write to the buffer is not checked against the target, which allows individual blocks larger than the target to be included (and will be split across multiple channel frames).

Types

type CLIConfig

type CLIConfig struct {
	// TargetL1TxSizeBytes to target when creating channel frames. Note that if the
	// realized compression ratio is worse than the approximate, more frames may
	// actually be created. This also depends on how close the target is to the
	// max frame size.
	TargetL1TxSizeBytes uint64
	// TargetNumFrames to create in this channel. If the realized compression ratio
	// is worse than approxComprRatio, additional leftover frame(s) might get created.
	TargetNumFrames int
	// ApproxComprRatio to assume. Should be slightly smaller than average from
	// experiments to avoid the chances of creating a small additional leftover frame.
	ApproxComprRatio float64
	// Type of compressor to use. Must be one of KindKeys.
	Kind string
}

func ReadCLIConfig

func ReadCLIConfig(ctx *cli.Context) CLIConfig

func (*CLIConfig) Config

func (c *CLIConfig) Config() Config

type Config

type Config struct {
	// TargetFrameSize to target when creating channel frames. Note that if the
	// realized compression ratio is worse than the approximate, more frames may
	// actually be created. This also depends on how close the target is to the
	// max frame size.
	TargetFrameSize uint64
	// TargetNumFrames to create in this channel. If the realized compression ratio
	// is worse than approxComprRatio, additional leftover frame(s) might get created.
	TargetNumFrames int
	// ApproxComprRatio to assume. Should be slightly smaller than average from
	// experiments to avoid the chances of creating a small additional leftover frame.
	ApproxComprRatio float64
	// Kind of compressor to use. Must be one of KindKeys. If unset, NewCompressor
	// will default to RatioKind.
	Kind string
}

func (Config) NewCompressor

func (c Config) NewCompressor() (derive.Compressor, error)

type FactoryFunc

type FactoryFunc func(Config) (derive.Compressor, error)

type NonCompressor

type NonCompressor struct {
	// contains filtered or unexported fields
}

func (*NonCompressor) Close

func (t *NonCompressor) Close() error

func (*NonCompressor) Flush

func (t *NonCompressor) Flush() error

func (*NonCompressor) FullErr

func (t *NonCompressor) FullErr() error

func (*NonCompressor) Len

func (t *NonCompressor) Len() int

func (*NonCompressor) Read

func (t *NonCompressor) Read(p []byte) (int, error)

func (*NonCompressor) Reset

func (t *NonCompressor) Reset()

func (*NonCompressor) Write

func (t *NonCompressor) Write(p []byte) (int, error)

type RatioCompressor

type RatioCompressor struct {
	// contains filtered or unexported fields
}

func (*RatioCompressor) Close

func (t *RatioCompressor) Close() error

func (*RatioCompressor) Flush

func (t *RatioCompressor) Flush() error

func (*RatioCompressor) FullErr

func (t *RatioCompressor) FullErr() error

func (*RatioCompressor) InputThreshold

func (t *RatioCompressor) InputThreshold() uint64

InputThreshold calculates the input data threshold in bytes from the given parameters.

func (*RatioCompressor) Len

func (t *RatioCompressor) Len() int

func (*RatioCompressor) Read

func (t *RatioCompressor) Read(p []byte) (int, error)

func (*RatioCompressor) Reset

func (t *RatioCompressor) Reset()

func (*RatioCompressor) Write

func (t *RatioCompressor) Write(p []byte) (int, error)

type ShadowCompressor

type ShadowCompressor struct {
	// contains filtered or unexported fields
}

func (*ShadowCompressor) Close

func (t *ShadowCompressor) Close() error

func (*ShadowCompressor) Flush

func (t *ShadowCompressor) Flush() error

func (*ShadowCompressor) FullErr

func (t *ShadowCompressor) FullErr() error

func (*ShadowCompressor) Len

func (t *ShadowCompressor) Len() int

func (*ShadowCompressor) Read

func (t *ShadowCompressor) Read(p []byte) (int, error)

func (*ShadowCompressor) Reset

func (t *ShadowCompressor) Reset()

func (*ShadowCompressor) Write

func (t *ShadowCompressor) Write(p []byte) (int, error)