httpuploader

Documentation

Index

• Constants
• func MarshalCItoJSON(row abstract.ChangeItem, rules *MarshallingRules, buf *bytes.Buffer) error
• type GFMap
  • func NewGrishaFMap(names []string, input map[string]int) *GFMap
  • func (g *GFMap) Lookup(key string, expectedPos int) (int, bool)
• type MarshallingRules
  • func NewRules(names []string, colSchema []abstract.ColSchema, colNameToIndex map[string]int, ...) *MarshallingRules
  • func (r *MarshallingRules) SetColType(name string, description *columntypes.TypeDescription)
• type UploadStats
  • func UploadCIBatch(batch []abstract.ChangeItem, rules *MarshallingRules, ...) (*UploadStats, error)

Constants

const MemReserveFactor = 1.2

Functions

func MarshalCItoJSON

func MarshalCItoJSON(row abstract.ChangeItem, rules *MarshallingRules, buf *bytes.Buffer) error

Types

type GFMap

type GFMap struct {
	// contains filtered or unexported fields
}

func NewGrishaFMap

func NewGrishaFMap(names []string, input map[string]int) *GFMap

NewGrishaFMap is a lookuper for name indexes, we accept expected order of names the idea is to have map lookup only when it's needed, i.e. when order of names is not a same as key this give guite a huge perf boost, map lookup on certain json-marshal workloads may consume up to 35% of CPU map named in honor to @GrigoryPervakov, since he was the one who suggest this hack.

func (*GFMap) Lookup

func (g *GFMap) Lookup(key string, expectedPos int) (int, bool)

type MarshallingRules

type MarshallingRules struct {
	ColSchema      []abstract.ColSchema
	ColNameToIndex map[string]int
	ColTypes       columntypes.TypeMapping
	AnyAsString    bool
	// contains filtered or unexported fields
}

func NewRules

func NewRules(names []string, colSchema []abstract.ColSchema, colNameToIndex map[string]int, colTypes columntypes.TypeMapping, anyAsString bool) *MarshallingRules

func (*MarshallingRules) SetColType

func (r *MarshallingRules) SetColType(name string, description *columntypes.TypeDescription)

type UploadStats

type UploadStats struct {
	Bytes           int
	StartTime       time.Time
	UploadStartTime time.Time
}

func UploadCIBatch

func UploadCIBatch(
	batch []abstract.ChangeItem,
	rules *MarshallingRules,
	config model.ChSinkServerParams,
	table string,
	avgRowSize int,
	lgr log.Logger) (*UploadStats, error)