lib

package module
v0.0.0-...-8d05fb3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 4, 2019 License: GPL-3.0 Imports: 13 Imported by: 0

README

Welcome to SANgo!

GoSAN is the discrete-event based simulator with reinforcement learning support.

What it simulates?

  • Resource usage
    • CPU
    • Links
    • Storage
      • SSD
      • HDD
  • Load generation
    • Read mode
    • Write mode

Installation

Execute the following command:

go get -u gitlab.com/lambda-hse/tatlin-hse/gosan

Example

To demonstrate how GoSAN can be used we implemented a toy version of actual storage array network. It consists of:

  • Load generator
  • 2 controllers
  • 8 storage units:
    • 4 SSD
    • 4 HDD

Client sends data blocks to controllers. They, in own turn, write data to storage units in the round-robin algorithm.

enter image description here

Reproducible run

To play around with the simulator check out https://codeocean.com/capsule/9707185.

Available flags

The following flags are available at the current moment:

  1. Input configuration files:
-platform — Tatlin storage array controller (hard-drives, controllers, etc.)
-packet — how much time packets (data blocks) consume on the different Tatlin components depending on their size with self-explainable field tags
-atm_dep - atmosphere (temperature, humidity, pressure, vibration) dependencies
-atm_control - file with atmospheric time-series data in JSON format
-output — file where measured data will be written to
  1. Load generation:
-file_amount — the amount of W/R files
-file_size — file size interval [lower, upper)
-load_range — file read/write frequency [minTime, maxTime)
  1. Failures:
-anomaly_type — failure type; VESNIN1ANOMALY, VESNIN2ANOMALY, VESNIN1CLIENTLINK, VESNIN2CLIENTLINK are available
-anomaly_amount — the number of failures
-anomaly_time_range — time interval [minTime, maxTime). A random number from this interval is chosen, which will be the time of the next failure 
-anomaly_duration —  time interval [minTime, maxTime). Repair time is sampled from this interval.
  1. Hard-drive amount:
-disk_amount — the number of disks in the JBOD. Overwrite disk amount, given in the -platform flag

DeepController synchronization:

Turning on/off DC connection.
--controling_mode=true/false(or 1/0), by default = false
Protocol:
--protocol={tcp,udp}, by default tcp
Connection hostname:
--host, by default localhost
Connection port:
--port, by default 1337
DC-GoTatlin communication delay:
--json_delay, by default 1(sec)

Additional

The below command makes the output file well-formed:

sed -i -e '1i \[' -e 's/}{/},{/g' -e "\$a]" $(jsonfilename)

Description of each field in json format can be found here.

For profiling (benchmarking) run:

go tool pprof http://localhost:6060/debug/pprof/profile

Documentation

Index

Constants

View Source 
const (
	KB float64 = 1 << (10 * iota)
	MB
	GB
	TB
	PB
	EB
	ZB
	YB
)

Variables

View Source 
var CIDNext = PIDNextFactory()
View Source 
var GetAllJBODs func() map[string]*JBOD
View Source 
var GetAllJBODsSlice func() []*JBOD
View Source 
var GetAllLinksMap func() map[string]*Link
View Source 
var GetAllVolumes func() map[string]*Volume

============================ Volume functions ============================

View Source 
var GetCacheJbod func() *JBOD
View Source 
var GetCallbacks func(EventType) []func(*Event)
View Source 
var GetFunctionByName func(string) BaseBehaviourFunction
View Source 
var GetHostByName func(string) *Host
View Source 
var GetHosts func() map[string]*Host
View Source 
var GetLinkBetweenHosts func(*Host, *Host) *Link
View Source 
var GetLinkByName func(name string) *Link
View Source 
var GetLinks func(*Host) []*Link
View Source 
var GetPacketByName func(RequestType, string) *Packet
View Source 
var GetRandomBlockSize func() string
View Source 
var GetRandomPacket func(RequestType) *Packet
View Source 
var NewProcess func(string, *Host) *Process
View Source 
var PIDNext = PIDNextFactory()
View Source 
var SIM_function_register func(string, BaseBehaviourFunction, interface{})
View Source 
var SIM_get_clock func() float64
View Source 
var SIM_run func(float64)
View Source 
var UnitToFloatParser = UnitToFloatFactory()

Functions

func InitPacketInfo 

func InitPacketInfo(filename string)

func Maximum 

func Maximum(a, b float64) float64

func NewProcessFactory 

func NewProcessFactory(env *Environment) func(string, *Host) *Process

func PIDNextFactory 

func PIDNextFactory() func() uint64

func ProcWrapper 

func ProcWrapper(processStrategy BaseBehaviourFunction, pi ProcessInterface, data interface{})

func ProcWrapperTemp 

func ProcWrapperTemp(processStrategy func(*Process, interface{}), p *Process, data interface{})

func RangeParser 

func RangeParser(flag string) (x1, x2 float64)

func SIM_get_clockFactory 

func SIM_get_clockFactory(env *Environment) func() float64

func SIM_init 

func SIM_init(packet_config string)

func SIM_launch_application 

func SIM_launch_application(FilePath string)

func SIM_platform_init 

func SIM_platform_init(FilePath string, flag_disk_amount int)

func SIM_runFactory 

func SIM_runFactory(env *Environment) func(float64)

func UnitToFloatFactory 

func UnitToFloatFactory() func(string) float64

Types

type AnomalyFlags 

type AnomalyFlags struct {
	AnomalyType        string
	AnomalyAmount      int
	MinAnomalyTime     float64
	MaxAnomalyTime     float64
	MinAnomalyDuration float64
	MaxAnomalyDuration float64
	// contains filtered or unexported fields
}

type BLOCKSIZE 

type BLOCKSIZE float64
const (
	K4 BLOCKSIZE = 1 << (10 + iota)
	K8
	K16
	K32
	K64
	K128
	K256
	K512
	K1024
	K2048
)

type BaseBehaviourFunction 

type BaseBehaviourFunction func(ProcessInterface, interface{})

type BlockParams 

type BlockParams struct {
	// Storage
	RateTime      *NormalParam `json:"rate_time"`
	SeekTime      *NormalParam `json:"seek_time"`
	OverheadsTime *NormalParam `json:"overheads_time"`
}

type ClientFlags 

type ClientFlags struct {
	FileAmount   int
	MinFileSize  float64
	MaxFileSize  float64
	MinPauseTime float64
	MaxPauseTime float64
	RequestType  RequestType
	// contains filtered or unexported fields
}

type Closeable 

type Closeable interface {
	Close()
}

type CommonPacketInfo 

type CommonPacketInfo struct {
	Size  float64 `json:"size"`
	Index int     `json:"-"`
	// Network
	TransmissionTime    *NormalParam `json:"transmission_time"`
	LatencyTime         *NormalParam `json:"latency_time"`
	ReadProcessingTime  *NormalParam `json:"read_processing_time"`
	WriteProcessingTime *NormalParam `json:"write_processing_time"`
	RecoveryTime        *NormalParam `json:"recovery_time"`

	RandReadParams  *BlockParams `json:"rand_read"`
	RandWriteParams *BlockParams `json:"rand_write"`
	SeqReadParams   *BlockParams `json:"seq_read"`
	SeqWriteParams  *BlockParams `json:"seq_write"`
	ReadWriteParams *BlockParams `json:"read_write"`
	// contains filtered or unexported fields
}

type ControllerParams 

type ControllerParams struct {
	ProcessingTime *NormalParam `json:"processing_time"`
	RecoveryTime   *NormalParam `json:"recovery_time"`
}

type Core 

type Core struct {
	// contains filtered or unexported fields
}

func (*Core) Put 

func (core *Core) Put(e *Event, globalQueue *globalEventQueue)

type CoreManager 

type CoreManager struct {
	// contains filtered or unexported fields
}

func NewCoreManager 

func NewCoreManager(start uint64, end uint64, speed float64, state float64) *CoreManager

func (*CoreManager) Next 

func (cid *CoreManager) Next() *Core

type DCAble 

type DCAble interface {
	Break(*Process, float64, float64)
	Repair(*Process, float64)
	Update(map[string]float64)
	Reset()

	GetCurrentState() string
	SetCurrentState(string)
}

type DeepControllerFlags 

type DeepControllerFlags struct {
	DeepControllingMode bool
	DeepHost            string
	DeepPort            string
	DeepProtocol        string
	DeepJsonDelay       float64

	SimRun float64
}

type Environment 

type Environment struct {
	// contains filtered or unexported fields
}

func NewEnvironment 

func NewEnvironment() *Environment

func (*Environment) GetGlobalQueue 

func (env *Environment) GetGlobalQueue() *globalEventQueue

func (*Environment) GetStepEnd 

func (env *Environment) GetStepEnd() chan interface{}

func (*Environment) SendResumeSignalToWorkers 

func (env *Environment) SendResumeSignalToWorkers()

func (*Environment) Step 

func (env *Environment) Step()

type Event 

type Event struct {
	// contains filtered or unexported fields
}

func NewEvent 

func NewEvent(timeEnd float64, process *Process, eventType EventType, packet *Packet) *Event

(link, task, process, destinationProcess)

type EventQueue 

type EventQueue []*Event

func (EventQueue) Len 

func (eq EventQueue) Len() int

func (EventQueue) Less 

func (eq EventQueue) Less(i, j int) bool

func (*EventQueue) Pop 

func (eq *EventQueue) Pop() interface{}

func (*EventQueue) Push 

func (eq *EventQueue) Push(e interface{})

func (EventQueue) Swap 

func (eq EventQueue) Swap(i, j int)

type EventType 

type EventType int
const (
	SyncNetworkEvent EventType = iota
	AsyncNetworkEvent
	SyncStorageEvent
	AsyncStorageEvent
	WaitEvent
	ExecuteEvent
	RecoveryEvent

	LinkAnomalyEvent
	HostAnomalyEvent

	LinkRepairEvent
	HostRepairEvent

	DiskAnomalyEvent

	StopEvent
	SingleNetworkEvent
)

type File 

type File struct {
	Size        float64
	Filename    string
	RequestType RequestType
	Packet1     *Packet
	Packet2     *Packet
	Latency     float64
	NumJobs     int
}

func NewFile 

func NewFile(filename string, size float64, requestType RequestType, numJobs int, blockSize string,
	latency float64) *File

type Host 

type Host struct {
	Name  string
	Type  string
	Id    string
	Speed float64
	// contains filtered or unexported fields
}

func NewHost 

func NewHost(name, typeID, ID string, speed float64, nCore uint64) *Host

func (*Host) GetAmountOfTasks 

func (host *Host) GetAmountOfTasks() uint64

func (*Host) GetDevTemp 

func (host *Host) GetDevTemp() float64

func (*Host) GetName 

func (host *Host) GetName() string

func (*Host) GetSpeed 

func (host *Host) GetSpeed() float64

func (*Host) GetType 

func (host *Host) GetType() string

func (*Host) Put 

func (host *Host) Put(e *Event, globalQueue *globalEventQueue)

type JBOD 

type JBOD struct {
	Name    string
	Disks   JBODDisks
	DiskArr map[int]*Storage
}

func NewJBOD 

func NewJBOD(JBODId string, amount int, sType string, storageTypes map[string]*StorageType) *JBOD

func (*JBOD) GetName 

func (jbod *JBOD) GetName() string

Getters and setters

type JBODDisks 

type JBODDisks []*Storage

func (JBODDisks) Len 

func (JD JBODDisks) Len() int

func (JBODDisks) Less 

func (JD JBODDisks) Less(i, j int) bool

func (*JBODDisks) Pop 

func (JD *JBODDisks) Pop() interface{}

func (*JBODDisks) Push 

func (JD *JBODDisks) Push(e interface{})

func (JBODDisks) Swap 

func (JD JBODDisks) Swap(i, j int)
type Link struct {
	Name      string
	Bandwidth float64
	Latency   float64
	State     float64

	Src, Dst *Host
	// contains filtered or unexported fields
}
func NewLink(bandwidth float64, name string) *Link

func (*Link) GetQueueSize 

func (link *Link) GetQueueSize() int

func (*Link) Put 

func (link *Link) Put(e *Event, globalQueue *globalEventQueue)

type MachineInterface 

type MachineInterface interface {
}

type NetworkPacket 

type NetworkPacket int

type NetworkParams 

type NetworkParams struct {
	TransmissionTime *NormalParam `json:"transmission_time"`
	LatencyTime      *NormalParam `json:"latency_time"`
}

type NormalParam 

type NormalParam struct {
	Mean   float64 `json:"mean"`
	StdDev float64 `json:"std_dev"`
}

type Packet 

type Packet struct {
	Type  RequestType `json:"-"`
	Index int         `json:"-"`
	Size  float64     `json:"size"`

	NetworkTime *NetworkParams    `json:"transmission_time"`
	ContrTime   *ControllerParams `json:"recovery_time"`
	DiskParams  *BlockParams      `json:"disk_params"`
}
var (
	//PACKET_4K    *Packet
	//PACKET_8K    *Packet
	//PACKET_16K   *Packet
	//PACKET_32K   *Packet
	//PACKET_64K   *Packet
	//PACKET_128K  *Packet
	//PACKET_256K  *Packet
	//PACKET_512K  *Packet
	//PACKET_1024K *Packet
	//PACKET_2048K *Packet
	PACKET_ACK      *Packet
	PACKET_FINALIZE *Packet
	PACKET_RECOVERY *Packet
)

type Process 

type Process struct {
	// contains filtered or unexported fields
}

func (*Process) Daemonize 

func (process *Process) Daemonize(resource ...Closeable)

func (*Process) DetachedSendPacket 

func (process *Process) DetachedSendPacket(packet *Packet, destinationProcess string) interface{}

func (*Process) Execute 

func (process *Process) Execute(packet *Packet) (STATUS, float64)

func (*Process) GetData 

func (process *Process) GetData() interface{}

func (*Process) GetEnv 

func (process *Process) GetEnv() *Environment

func (*Process) GetHost 

func (process *Process) GetHost() *Host

func (*Process) GetName 

func (process *Process) GetName() string

func (*Process) GetPID 

func (process *Process) GetPID() uint64

func (*Process) GetResumeChan 

func (process *Process) GetResumeChan() chan STATUS

func (*Process) ReadAsync 

func (process *Process) ReadAsync(storage *Storage, packet *Packet) (*Packet, STATUS)

func (*Process) ReadSync 

func (process *Process) ReadSync(storage *Storage, packet *Packet) (*Packet, STATUS)

func (*Process) ReceivePacket 

func (process *Process) ReceivePacket(address string) (*Packet, STATUS)

func (*Process) RecoveryExecutePacket 

func (process *Process) RecoveryExecutePacket(packet *Packet) (*Packet, STATUS)

func (*Process) SIM_wait 

func (process *Process) SIM_wait(waitTime float64) STATUS

func (*Process) SendPacket 

func (process *Process) SendPacket(packet *Packet, destinationProcess string) STATUS

"component_filename"

func (*Process) SendToHostWithoutReceive 

func (process *Process) SendToHostWithoutReceive(destinationHost *Host, packet *Packet) STATUS

func (*Process) SetHost 

func (process *Process) SetHost(host *Host)

func (*Process) WriteAsync 

func (process *Process) WriteAsync(storage *Storage, packet *Packet) STATUS

func (*Process) WriteSync 

func (process *Process) WriteSync(storage *Storage, packet *Packet) STATUS

type ProcessID 

type ProcessID struct {
	// contains filtered or unexported fields
}

type ProcessInterface 

type ProcessInterface interface {
	SetParent(*Process)
	GetParent() *Process
}

func SIM_subprocess_create_async 

func SIM_subprocess_create_async(name string, f BaseBehaviourFunction, host *Host, pi ProcessInterface,
	data interface{}) ProcessInterface

func SIM_subprocess_create_sync 

func SIM_subprocess_create_sync(name string, f BaseBehaviourFunction, host *Host, pi ProcessInterface, data interface{}) ProcessInterface

type RequestType 

type RequestType int
const (
	RANDREAD RequestType = iota
	RANDWRITE
	READ
	WRITE
)

type Resource 

type Resource interface {
	Put(e *Event, globalQueue *globalEventQueue)
}

type STATUS 

type STATUS int
const (
	OK STATUS = iota
	FAIL
)

type Storage 

type Storage struct {
	*StorageType
	Name string
	ID   string
	Type string

	JBOD      *JBOD
	BlockSize BLOCKSIZE
	ReadLink  *Link
	WriteLink *Link
	// contains filtered or unexported fields
}

func NewStorage 

func NewStorage(storageType *StorageType, name string, jbod *JBOD) *Storage

func (*Storage) GetJBOD 

func (storage *Storage) GetJBOD() *JBOD

func (*Storage) Put 

func (storage *Storage) Put(e *Event, globalQueue *globalEventQueue)

type StoragePacket 

type StoragePacket int

type StorageType 

type StorageType struct {
	TypeId    string
	WriteRate float64
	ReadRate  float64
	Size      float64
}

type SystemFlags 

type SystemFlags struct {
	PlatformFileName        string
	PacketFileName          string
	AtmControlFileName      string
	AtmDependenciesFileName string
	ClientFileName          string
	NumJobsFileName         string

	SimRun  float64
	NDisks  int
	NumJobs int

	WriteClientFlags *ClientFlags
	ReadClientFlags  *ClientFlags
	AnomalyFlags     *AnomalyFlags
	TracerFlags      *TracerFlags
	DCFlags          *DeepControllerFlags
}

func InitFlags 

func InitFlags(sf *SystemFlags) *SystemFlags

func NewSystemFlags 

func NewSystemFlags() *SystemFlags

func ParseFlags 

func ParseFlags(sf *SystemFlags) *SystemFlags

type TracerFlags 

type TracerFlags struct {
	OutputFileName string
}

type Volume 

type Volume struct {
	Id     string       `xml:"id,attr"`
	Mounts []VolumePart `xml:"mount"`
}

type VolumePart 

type VolumePart struct {
	XMLName xml.Name `xml:"mount"`
	Id      string   `xml:"id,attr"`
}

Source Files

View all Source files

Directories

Path Synopsis
examples
SAN
SAN/src
Created by kenenbek on 18.12.17.
 Created by kenenbek on 18.12.17.
file_mode/anomaly_host
file_mode/anomaly_link
file_mode/asyncsenderreceiver
file_mode/host_task_execute
file_mode/logical_storage
file_mode/multirouting
file_mode/sender_receiver
file_mode/sender_receiver_fail
file_mode/storage_read
file_mode/storage_write
file_mode/subprocessing
file_mode/timeout
packet_mode/anomaly_host
packet_mode/anomaly_link
packet_mode/execute
packet_mode/send_1_file
packet_mode/send_2_file
packet_mode/storage_write_1_file
packet_mode/storage_write_1_packet
packet_mode/wait

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.