inference

Documentation

Index

• Variables
• type BatchOpRewriteConfig
  • func (*BatchOpRewriteConfig) Descriptor() ([]byte, []int)
  • func (x *BatchOpRewriteConfig) GetEnableAdaptiveSharedBatchingThreadPool() bool
  • func (x *BatchOpRewriteConfig) GetModelSchedulerOptions() map[string]*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption
  • func (*BatchOpRewriteConfig) ProtoMessage()
  • func (x *BatchOpRewriteConfig) ProtoReflect() protoreflect.Message
  • func (x *BatchOpRewriteConfig) Reset()
  • func (x *BatchOpRewriteConfig) String() string
• type BatchOpRewriteConfig_AdaptiveBatchSchedulerOption
  • func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Descriptor() ([]byte, []int)
  • func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetBatchesToAverageOver() *wrapperspb.UInt32Value
  • func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetInitialInflightBatchesLimit() *wrapperspb.UInt32Value
  • func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMaxInflightBatchesLimit() *wrapperspb.UInt32Value
  • func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMinInflightBatchesLimit() *wrapperspb.UInt32Value
  • func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoMessage()
  • func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoReflect() protoreflect.Message
  • func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Reset()
  • func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) String() string

Variables

var File_tensorflow_core_grappler_optimizers_inference_batch_op_rewriter_proto protoreflect.FileDescriptor

Types

type BatchOpRewriteConfig

type BatchOpRewriteConfig struct {
	EnableAdaptiveSharedBatchingThreadPool bool `` /* 184-byte string literal not displayed */
	// Keyed by model name, meaning all batch-ops in one saved model would use the
	// same adaptive-batch-scheduler option.
	ModelSchedulerOptions map[string]*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption `` /* 214-byte string literal not displayed */
	// contains filtered or unexported fields
}

Config for the batch op rewriter. This should be serialized and set a param in RewriterConfig with key kBatchOpRewriteParamKey.

func (*BatchOpRewriteConfig) Descriptor

func (*BatchOpRewriteConfig) Descriptor() ([]byte, []int)

Deprecated: Use BatchOpRewriteConfig.ProtoReflect.Descriptor instead.

func (*BatchOpRewriteConfig) GetEnableAdaptiveSharedBatchingThreadPool

func (x *BatchOpRewriteConfig) GetEnableAdaptiveSharedBatchingThreadPool() bool

func (*BatchOpRewriteConfig) GetModelSchedulerOptions

func (x *BatchOpRewriteConfig) GetModelSchedulerOptions() map[string]*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption

func (*BatchOpRewriteConfig) ProtoMessage

func (*BatchOpRewriteConfig) ProtoMessage()

func (*BatchOpRewriteConfig) ProtoReflect

func (x *BatchOpRewriteConfig) ProtoReflect() protoreflect.Message

func (*BatchOpRewriteConfig) Reset

func (x *BatchOpRewriteConfig) Reset()

func (*BatchOpRewriteConfig) String

func (x *BatchOpRewriteConfig) String() string

type BatchOpRewriteConfig_AdaptiveBatchSchedulerOption

type BatchOpRewriteConfig_AdaptiveBatchSchedulerOption struct {
	MinInflightBatchesLimit     *wrapperspb.UInt32Value `` /* 134-byte string literal not displayed */
	InitialInflightBatchesLimit *wrapperspb.UInt32Value `` /* 146-byte string literal not displayed */
	MaxInflightBatchesLimit     *wrapperspb.UInt32Value `` /* 134-byte string literal not displayed */
	// You can use QPS as a reference to decide how quickly to react to workload
	// changes.
	BatchesToAverageOver *wrapperspb.UInt32Value `protobuf:"bytes,4,opt,name=batches_to_average_over,json=batchesToAverageOver,proto3" json:"batches_to_average_over,omitempty"`
	// contains filtered or unexported fields
}

The options for tensorflow::serving::AdaptiveSharedBatchScheduler. See AdaptiveSharedBatchScheduler::Options for meaning of each field.

NOTE: Leave this unset to pick up default settings which should work for most scenarios.

Example scenarios when tuning helps: * Latency sensitive

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Descriptor

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Descriptor() ([]byte, []int)

Deprecated: Use BatchOpRewriteConfig_AdaptiveBatchSchedulerOption.ProtoReflect.Descriptor instead.

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetBatchesToAverageOver

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetBatchesToAverageOver() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetInitialInflightBatchesLimit

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetInitialInflightBatchesLimit() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMaxInflightBatchesLimit

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMaxInflightBatchesLimit() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMinInflightBatchesLimit

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMinInflightBatchesLimit() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoMessage

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoMessage()

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoReflect

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoReflect() protoreflect.Message

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Reset

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Reset()

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) String

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) String() string