cmd

README

Evaluation Commands

go build main.go

Run ./main -h for help.

To get help on a subcommand, accuracy for example, run go run main.go accuracy -h.

Model

• Model information across different batch sizes

  ./main model info --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --format=csv

Layer

• Layer information

  ./main layer info --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• Layer duration

  ./main layer duration --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Layer duration variance

  ./main layer duration --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --box_plot

• Layer memory

  ./main layer memory --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Layer occurrence

  ./main layer occurrence --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --pie_plot

• Layer aggregated duration based on operator type

  ./main layer aggre_duration --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --pie_plot

• Layer theoretical flops calculation using the layer operator type and shape

  TODO

GPU

• GPU kernel information

  ./main gpu_kernel info --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• GPU kernel information aggregated within each layer

  ./main gpu_kernel layer_aggre --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• GPU kernel information aggregated within the model

  ./main gpu_kernel model_aggre --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• Total flops of GPU kernels per layer

  ./main gpu_kernel layer_flops --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Total dram read of GPU kernels per layer

  ./main gpu_kernel layer_dram_read --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Total dram write of GPU kernels per layer

  ./main gpu_kernel layer_dram_write --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Layer GPU vs CPU time

  Use the information from gpu_kernel layer_aggre

• GPU kernel roofline analysis

  Use the information from gpu_kernel info

• Layer roofline analysis

  Use the information from gpu_kernel layer_aggre

• Model roofline analysis

  Use the information from gpu_kernel model_aggre

Documentation

Index

• Variables
• type Rower
• type Rowers
• type Writer
  • func NewWriter(rower Rower, opts ...writer.Option) *Writer
  • func (w *Writer) Close()
  • func (w *Writer) Flush()
  • func (w *Writer) Header(rower Rower) error
  • func (w *Writer) Row(rower Rower) error
  • func (w *Writer) Rows(rower Rowers) error

Variables

var (
	AllCmds = []*cobra.Command{
		modelCmd,
		layerCmd,
		gpuKernelCmd,
		eventflowCmd,
		accuracyCmd,
	}
)

var EvaluationCmd = &cobra.Command{
	Use:   "evaluation",
	Short: "Get various information about the evaluation",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		fmt.Println("Running " + cmd.Name())
		return nil
	},
	PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
		safeClose := func(cls ...io.Closer) {
			for _, c := range cls {
				if c == nil {
					return
				}
				c.Close()
			}
		}
		safeClose(
			evaluationCollection,
			performanceCollection,
			inputPredictionCollection,
			modelAccuracyCollection,
			divergenceCollection,
			db,
		)

		return nil
	},
}

Types

type Rower

type Rower interface {
	Header(...writer.Option) []string
	Row(...writer.Option) []string
}

type Rowers

type Rowers interface {
	Rower
	Rows(...writer.Option) [][]string
}

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func NewWriter

func NewWriter(rower Rower, opts ...writer.Option) *Writer

func (*Writer) Close

func (w *Writer) Close()

func (*Writer) Flush

func (w *Writer) Flush()

func (*Writer) Header

func (w *Writer) Header(rower Rower) error

func (*Writer) Row

func (w *Writer) Row(rower Rower) error

func (*Writer) Rows

func (w *Writer) Rows(rower Rowers) error