file

package
v1.3.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 19, 2021 License: CC-BY-4.0 Imports: 11 Imported by: 0

Documentation

Overview

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information. Package file implements file related structures, methods, and functions

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Index

Constants

View Source 
const (
	// ReadOperation represents read file
	ReadOperation = "read"
	// WriteOperation represents write file
	WriteOperation = "write"
	// NoIOBytes means that no bytes were read or written
	NoIOBytes = -1
	// NoDuration means that no duration was recorded
	NoDuration = time.Duration(-1)
)
View Source 
const (
	KB             = 1024
	MB             = KB * KB
	JobReaderLabel = "JobReader"
)

Variables

This section is empty.

Functions

func GetSummaryHeader 

func GetSummaryHeader() []string

GetSummaryHeader returns the header for the summary file

Types

type DirectoryManager 

type DirectoryManager struct {
	// contains filtered or unexported fields
}

DirectoryManager ensures directories are created, and ensuring only a single create ever gets sent to filesystem

func InitializeDirectoryManager 

func InitializeDirectoryManager() *DirectoryManager

InitializeDirectoryManager initilizes the directory manager

func (*DirectoryManager) EnsureDirectory 

func (d *DirectoryManager) EnsureDirectory(path string) error

EnsureDirectory ensures the directory exists, and if already created returns the directory

type IOStatistics 

type IOStatistics struct {
	StartTime       time.Time
	Hostname        string
	UniqueName      string
	RunName         string
	Label           string
	Operation       Operation
	Path            string
	FileOpenTimeNS  time.Duration
	FileCloseTimeNS time.Duration
	IOTimeNS        time.Duration
	IOBytes         int
	IsSuccess       bool
	FailureMessage  string
}

IOStatistics provides statistics on the file

func InitializeIOStatistics 

func InitializeIOStatistics(
	startTime time.Time,
	uniqueName string,
	runName string,
	label string,
	operation Operation,
	path string,
	fileOpenTimeNS time.Duration,
	closeTimeNS time.Duration,
	ioTimeNS time.Duration,
	ioBytes int,
	err error) *IOStatistics

InitializeIOStatistics initializes the IO Statistics

func InitializeIOStatisticsFromString 

func InitializeIOStatisticsFromString(jsonString string) (*IOStatistics, error)

InitializeIOStatisticsFromString initializes the object from a json string

func (*IOStatistics) CSVHeader 

func (i *IOStatistics) CSVHeader() []string

CSVHeader returns the CSV header

func (*IOStatistics) GetCategoryKey 

func (i *IOStatistics) GetCategoryKey() string

GetCategoryKey returns a key to represent label and operation

func (*IOStatistics) GetJSON 

func (i *IOStatistics) GetJSON() ([]byte, error)

GetJSON returns the JSON representation of the object

func (*IOStatistics) ToStringArray 

func (i *IOStatistics) ToStringArray() []string

ToStringArray returns a csv formatted string

type IOStatsCollector 

type IOStatsCollector struct {
	UniqueName string
	// BatchMap maps the read and write operations
	BatchMap map[string]map[string]*IOStatsRows
	// IOMap maps the read and write operations
	IOMap    map[string]map[string]*IOStatsRows
	JobCount map[string]map[string]int
	// contains filtered or unexported fields
}

IOStatsCollector holds a collection of events

func InitializeIOStatsCollector 

func InitializeIOStatsCollector(uniqueName string) *IOStatsCollector

InitializeIOStatsCollector initializes IOStatsCollector

func (*IOStatsCollector) RecordEvent 

func (i *IOStatsCollector) RecordEvent(eMsg string)

RecordEvent records the event

func (*IOStatsCollector) WriteBatchSummaryFiles 

func (i *IOStatsCollector) WriteBatchSummaryFiles(statsPath string, uniqueName string)

WriteBatchSummaryFiles writes out a summary file for each batch run

func (*IOStatsCollector) WriteIOSummaryFiles 

func (i *IOStatsCollector) WriteIOSummaryFiles(statsPath string, uniqueName string)

WriteIOSummaryFiles writes out a summary file for each batch run

func (*IOStatsCollector) WriteRAWFiles 

func (i *IOStatsCollector) WriteRAWFiles(statsPath string, uniqueName string)

WriteRAWFiles writes out all the files

type IOStatsRows 

type IOStatsRows struct {
	// contains filtered or unexported fields
}

IOStatsRows represents rows of statistics for the same category

func InitializeIOStatsRows 

func InitializeIOStatsRows() *IOStatsRows

InitializeIOStatsRows initializes the io statistics rows structure

func (*IOStatsRows) AddIOStats 

func (i *IOStatsRows) AddIOStats(ios *IOStatistics)

AddIOStats adds a statistics row

func (*IOStatsRows) GetRowCount 

func (i *IOStatsRows) GetRowCount() int

GetRowCount returns the number of rows

func (*IOStatsRows) GetRows 

func (i *IOStatsRows) GetRows() []*IOStatistics

GetRows returns the rows

func (*IOStatsRows) GetSuccessCount 

func (i *IOStatsRows) GetSuccessCount() int

GetSuccessCount returns the count of successful rows

func (*IOStatsRows) WriteCSVFile 

func (i *IOStatsRows) WriteCSVFile(filename string)

WriteCSVFile writes the rows out to a file

func (*IOStatsRows) WriteSummaryLines 

func (i *IOStatsRows) WriteSummaryLines(writer *csv.Writer)

WriteSummaryLines writes summary rows for each label, for each operation of the batch

func (*IOStatsRows) WriteSummaryRow 

func (i *IOStatsRows) WriteSummaryRow(
	writer *csv.Writer,
	batchName string,
	label string,
	sampleSize int,
	percentSuccess float64,
	percentileArray []int,
	fileOp string,
	less func(i, j int) bool)

WriteSummaryRow writes a percentile summary row

type Operation 

type Operation string

Operation represents the type of file io operation

type ReaderWriter 

type ReaderWriter struct {
	// contains filtered or unexported fields
}

ReaderWriter records the time statistics for reading and writing files to an event hub

func InitializeReaderWriter 

func InitializeReaderWriter(label string, profiler log.Profiler) *ReaderWriter

InitializeReaderWriter initializes the file reader / writer

func (*ReaderWriter) ReadFile 

func (r *ReaderWriter) ReadFile(filename string, uniqueName string, runName string) ([]byte, error)

ReadFile reads the file bytes from the file name

func (*ReaderWriter) WriteFile 

func (r *ReaderWriter) WriteFile(filename string, data []byte, uniqueName string, runName string) error

WriteFile writes file bytes to the file

type RoundRobinPathManager 

type RoundRobinPathManager struct {
	// contains filtered or unexported fields
}

RoundRobinPathManager round robins among the available paths

func InitializeRoundRobinPathManager 

func InitializeRoundRobinPathManager(paths []string) *RoundRobinPathManager

InitializeRoundRobinPathManager initializes the directory manager

func (*RoundRobinPathManager) GetNextPath 

func (r *RoundRobinPathManager) GetNextPath() string

GetNextPath retrieves the next path

type SimpleProfiler 

type SimpleProfiler struct {
	// contains filtered or unexported fields
}

EventHubSender sends messages to Azure Event Hub

func InitializeSimpleProfiler 

func InitializeSimpleProfiler() *SimpleProfiler

InitializeSimpleProfiler initializes a simple profiler

func (*SimpleProfiler) GetSummary 

func (s *SimpleProfiler) GetSummary() string

func (*SimpleProfiler) RecordTiming 

func (s *SimpleProfiler) RecordTiming(bytes []byte)

RecordTiming implements interface Profiler

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.