Versions in this module Expand all Collapse all v1 v1.3.3 Jul 5, 2022 v1.3.2 Jul 19, 2021 v1.3.1 Jul 19, 2021 v1.3.0 Jul 16, 2021 v1.2.0 Jul 14, 2021 Changes in this version + const DefaultFileSizeKB + const DefaultJobCount + const DefaultJobEndFiles + const DefaultJobSubmitterThreadCount + const DefaultOrchestratorThreads + const DefaultWorkStartFiles + const JobCompleteReaderLabel + const JobCompleteWriterLabel + const JobDir + const JobReaderLabel + const JobWriterLabel + const KB + const MB + const QueueJobComplete + const QueueJobProcess + const QueueJobReady + const QueueMessageCount + const QueueUploader + const StatsDir + const WorkCompleteFileReaderLabel + const WorkCompleteFileWriterLabel + const WorkDir + const WorkStartFileReaderLabel + const WorkStartFileWriterLabel + var JobCompleteReader *file.ReaderWriter + var JobCompleteWriter *file.ReaderWriter + var JobReader *file.ReaderWriter + var JobWriter *file.ReaderWriter + var WorkCompleteFileReader *file.ReaderWriter + var WorkCompleteFileWriter *file.ReaderWriter + var WorkStartFileReader *file.ReaderWriter + var WorkStartFileWriter *file.ReaderWriter + func GenerateBatchName(jobCount int) string + func GenerateBatchName2(jobCount int) string + func GenerateBatchNameFromJobRun(uniqueName string, jobRunName string, batchid int) string + func GetBatchName(fullFilePath string) string + func GetBatchNamePartsFromJobRun(fullFilePath string) (string, string) + func GetEventHubName(uniqueName string) string + func GetJobCompleteQueueName(uniqueName string) string + func GetJobRunQueueName(uniqueName string) string + func GetJobStartQueueName(uniqueName string) string + func GetWorkCompleteQueueName(uniqueName string) string + func GetWorkStartQueueName(uniqueName string) string + func InitializeReaderWriters(ctx context.Context, eventHubSenderName string, eventHubSenderKey string, ...) *azure.EventHubSender + func SetStatsChannel(ctx context.Context) context.Context + func StatsCollector(ctx context.Context, syncWaitGroup *sync.WaitGroup) + type EdasimFile struct + FullPath string + MountParity bool + MountPath string + func InitializeEdasimFileFromString(edasimFileString string) (*EdasimFile, error) + func (e *EdasimFile) GetEdasimFileString() (string, error) + type JobConfigFile struct + IsCompleteFile bool + JobRun JobRun + Name string + PaddedString string + func InitializeJobCompleteFile(name string, jobRun *JobRun) *JobConfigFile + func InitializeJobConfigFile(name string, jobRun *JobRun) *JobConfigFile + func ReadJobConfigFile(reader *file.ReaderWriter, filename string) (*JobConfigFile, error) + func (j *JobConfigFile) WriteJobConfigFile(writer *file.ReaderWriter, filepath string, fileSize int) (string, error) + type JobRun struct + BatchCount int + BatchID int + DeleteFiles bool + JobCount int + JobFileConfigSizeKB int + JobRunName string + JobRunStartQueueName string + MountParity bool + UniqueName string + WorkCompleteFailedFileSizeKB int + WorkCompleteFileCount int + WorkCompleteFileSizeKB int + WorkFailedProbability float64 + WorkStartFileCount int + WorkStartFileSizeKB int + func InitializeJobRunFromString(jobRunString string) (*JobRun, error) + func (j *JobRun) SubmitBatches(ctx context.Context, storageAccount string, storageKey string) + type JobSubmitter struct + Context context.Context + DirManager *file.DirectoryManager + JobRunQueue *azure.Queue + JobStartQueue *azure.Queue + PathManager *file.RoundRobinPathManager + ThreadCount int + UniqueName string + func InitializeJobSubmitter(ctx context.Context, storageAccount string, storageKey string, ...) *JobSubmitter + func (j *JobSubmitter) JobSubmitterWorkerRun(syncWaitGroup *sync.WaitGroup, id int, batchName string, jobRun *JobRun, ...) + func (j *JobSubmitter) Run(syncWaitGroup *sync.WaitGroup) + type Orchestrator struct + Context context.Context + DirManager *file.DirectoryManager + JobComplete *azure.Queue + JobStartQueue *azure.Queue + MsgCh chan *azqueue.DequeuedMessage + OrchestratorThreads int + PathManager *file.RoundRobinPathManager + ReadyCh chan struct{} + UniqueName string + WorkComplete *azure.Queue + WorkStartQueue *azure.Queue + func InitializeOrchestrator(ctx context.Context, storageAccount string, storageKey string, ...) *Orchestrator + func (o *Orchestrator) JobDispatcher(syncWaitGroup *sync.WaitGroup) + func (o *Orchestrator) Run(syncWaitGroup *sync.WaitGroup) + func (o *Orchestrator) StartJobWorker(syncWaitGroup *sync.WaitGroup) + type StatsChannels struct + ChError chan struct{} + ChJobCompleted chan struct{} + ChJobProcessed chan struct{} + ChProcessedFilesWritten chan struct{} + ChUpload chan struct{} + func GetStatsChannel(ctx context.Context) *StatsChannels + func InitializeStatsChannels() *StatsChannels + func (s *StatsChannels) Error() + func (s *StatsChannels) JobCompleted() + func (s *StatsChannels) JobProcessed() + func (s *StatsChannels) ProcessedFilesWritten() + func (s *StatsChannels) Upload() + type WorkFileWriter struct + JobConfigName string + JobRun JobRun + PaddedString string + func InitializeWorkerFileWriter(jobConfigName string, jobRun *JobRun) *WorkFileWriter + func ReadWorkFile(reader *file.ReaderWriter, filename string) (*WorkFileWriter, error) + func (w *WorkFileWriter) FirstStartFile(filepath string) string + func (w *WorkFileWriter) WriteStartFiles(writer *file.ReaderWriter, filepath string, fileSize int, fileCount int) error
Go to main page