upload_utils

Documentation

Index

• func FindMinMissingInteger(arr []int) int
• type AddFileMsg
  • func (msg AddFileMsg) UpdateStatus(m *UploadStatusMonitor)
• type CanUpdateStatus
• type FileInfo
• type IntHeap
  • func NewHeap(s []int) *IntHeap
  • func (h IntHeap) Len() int
  • func (h IntHeap) Less(i, j int) bool
  • func (h *IntHeap) Peek() int
  • func (h *IntHeap) Pop() any
  • func (h *IntHeap) Push(x any)
  • func (h *IntHeap) Remove(x any)
  • func (h IntHeap) Swap(i, j int)
• type MultipartOpts
  • func (opt *MultipartOpts) Valid() error
• type UpdateStatusMsg
  • func (msg UpdateStatusMsg) UpdateStatus(m *UploadStatusMonitor)
• type UploadDB
  • func NewUploadDB(filename string, recordId string, hash string) (*UploadDB, error)
  • func (db *UploadDB) BatchPut(kvs map[string][]byte) error
  • func (db *UploadDB) Delete() error
  • func (db *UploadDB) Get(key string) ([]byte, error)
  • func (db *UploadDB) Reset() error
• type UploadManager
  • func NewUploadManager(client *minio.Client, hideMonitor bool, opts *MultipartOpts) *UploadManager
  • func NewUploadManagerFromConfig(pm *config.ProfileManager, proj *name.Project, timeout time.Duration, ...) (*UploadManager, error)
  • func (um *UploadManager) AddErr(path string, err error)
  • func (um *UploadManager) Debugf(format string, args ...interface{})
  • func (um *UploadManager) FMultipartPutObject(ctx context.Context, bucket string, key string, filePath string, ...) (err error)
  • func (um *UploadManager) FPutObject(absPath string, bucket string, key string, userTags map[string]string)
  • func (um *UploadManager) PrintErrs()
  • func (um *UploadManager) UpdateMonitor(msg interface{})
  • func (um *UploadManager) UploadFileThroughUrl(file string, uploadUrl string) error
  • func (um *UploadManager) Wait()
• type UploadStatusEnum
• type UploadStatusMonitor
  • func NewUploadStatusMonitor(startSignal *sync.WaitGroup) *UploadStatusMonitor
  • func (m *UploadStatusMonitor) Init() tea.Cmd
  • func (m *UploadStatusMonitor) Update(msg tea.Msg) (tea.Model, tea.Cmd)
  • func (m *UploadStatusMonitor) View() string

Functions

func FindMinMissingInteger

func FindMinMissingInteger(arr []int) int

FindMinMissingInteger finds the minimum missing integer in a sorted array

Types

type AddFileMsg

type AddFileMsg struct {
	Name string
}

AddFileMsg is a message that is used to add a file to the upload status monitor

func (AddFileMsg) UpdateStatus

func (msg AddFileMsg) UpdateStatus(m *UploadStatusMonitor)

type CanUpdateStatus

type CanUpdateStatus interface {
	UpdateStatus(m *UploadStatusMonitor)
}

type FileInfo

type FileInfo struct {
	Path   string
	Size   int64
	Sha256 string
}

FileInfo contains the path, size and sha256 of a file.

type IntHeap

type IntHeap []int

An IntHeap is a min-heap of ints.

func NewHeap

func NewHeap(s []int) *IntHeap

func (IntHeap) Len

func (h IntHeap) Len() int

func (IntHeap) Less

func (h IntHeap) Less(i, j int) bool

func (*IntHeap) Peek

func (h *IntHeap) Peek() int

func (*IntHeap) Pop

func (h *IntHeap) Pop() any

func (*IntHeap) Push

func (h *IntHeap) Push(x any)

func (*IntHeap) Remove

func (h *IntHeap) Remove(x any)

func (IntHeap) Swap

func (h IntHeap) Swap(i, j int)

type MultipartOpts

type MultipartOpts struct {
	Threads uint
	Size    string
}

func (*MultipartOpts) Valid

func (opt *MultipartOpts) Valid() error

type UpdateStatusMsg

type UpdateStatusMsg struct {
	Name     string
	Total    int64
	Uploaded int64
	Status   UploadStatusEnum
}

func (UpdateStatusMsg) UpdateStatus

func (msg UpdateStatusMsg) UpdateStatus(m *UploadStatusMonitor)

type UploadDB

type UploadDB struct {
	*bolt.DB
}

func NewUploadDB

func NewUploadDB(filename string, recordId string, hash string) (*UploadDB, error)

func (*UploadDB) BatchPut

func (db *UploadDB) BatchPut(kvs map[string][]byte) error

BatchPut puts a batch of key-value pairs into the database.

func (*UploadDB) Delete

func (db *UploadDB) Delete() error

Delete removes the database file from the filesystem.

func (*UploadDB) Get

func (db *UploadDB) Get(key string) ([]byte, error)

Get retrieves the value of a key from the database. if the key does not exist, it returns nil.

func (*UploadDB) Reset

func (db *UploadDB) Reset() error

Reset removes all the keys from the database multipart_uploads bucket.

type UploadManager

type UploadManager struct {
	StatusMonitor *tea.Program

	FileInfos map[string]FileInfo
	Errs      map[string]error
	sync.WaitGroup
	// contains filtered or unexported fields
}

UploadManager is a manager for uploading files through minio client. Note that it's user's responsibility to check the Errs field after Wait() to see if there's any error.

func NewUploadManager

func NewUploadManager(client *minio.Client, hideMonitor bool, opts *MultipartOpts) *UploadManager

func NewUploadManagerFromConfig

func NewUploadManagerFromConfig(pm *config.ProfileManager, proj *name.Project, timeout time.Duration, hideMonitor bool, multiOpts *MultipartOpts) (*UploadManager, error)

func (*UploadManager) AddErr

func (um *UploadManager) AddErr(path string, err error)

AddErr adds an error to the manager.

func (*UploadManager) Debugf

func (um *UploadManager) Debugf(format string, args ...interface{})

func (*UploadManager) FMultipartPutObject

func (um *UploadManager) FMultipartPutObject(ctx context.Context, bucket string, key string, filePath string, fileSize int64, fileSha256 string, opts minio.PutObjectOptions) (err error)

func (*UploadManager) FPutObject

func (um *UploadManager) FPutObject(absPath string, bucket string, key string, userTags map[string]string)

FPutObject uploads a file to a bucket with a key and sha256. If the file size is larger than minPartSize, it will use multipart upload.

func (*UploadManager) PrintErrs

func (um *UploadManager) PrintErrs()

PrintErrs prints all errors.

func (*UploadManager) UpdateMonitor

func (um *UploadManager) UpdateMonitor(msg interface{})

func (*UploadManager) UploadFileThroughUrl

func (um *UploadManager) UploadFileThroughUrl(file string, uploadUrl string) error

UploadFileThroughUrl uploads a single file to the given uploadUrl. um is the upload manager to use. file is the absolute path of the file to be uploaded. uploadUrl is the pre-signed url to upload the file to.

func (*UploadManager) Wait

func (um *UploadManager) Wait()

Wait waits for all uploads to finish. And wait for status monitor to finish.

type UploadStatusEnum

type UploadStatusEnum int

UploadStatusEnum is used to keep track of the state of a file upload

const (
	// Unprocessed is used to indicate that the file has not been processed yet
	Unprocessed UploadStatusEnum = iota

	// PreviouslyUploaded is used to indicate that the file has been uploaded before
	PreviouslyUploaded

	// UploadInProgress is used to indicate that the file upload is in progress
	UploadInProgress

	// UploadCompleted is used to indicate that the file upload has completed
	UploadCompleted

	// MultipartCompletionInProgress is used to indicate that the multipart upload completion is in progress
	MultipartCompletionInProgress

	// UploadFailed is used to indicate that the file upload has failed
	UploadFailed
)

type UploadStatusMonitor

type UploadStatusMonitor struct {
	ManualQuit bool
	// contains filtered or unexported fields
}

UploadStatusMonitor is a bubbletea model that is used to monitor the progress of file uploads

func NewUploadStatusMonitor

func NewUploadStatusMonitor(startSignal *sync.WaitGroup) *UploadStatusMonitor

func (*UploadStatusMonitor) Init

func (m *UploadStatusMonitor) Init() tea.Cmd

func (*UploadStatusMonitor) Update

func (m *UploadStatusMonitor) Update(msg tea.Msg) (tea.Model, tea.Cmd)

func (*UploadStatusMonitor) View

func (m *UploadStatusMonitor) View() string