operations

Documentation

Index

• func BatchUpload(cfg *iqshell.Config, info BatchUploadInfo)
• func BatchUpload2(cfg *iqshell.Config, info BatchUpload2Info)
• func BatchUploadConfigMould(cfg *iqshell.Config, info BatchUploadConfigMouldInfo)
• func SyncFile(cfg *iqshell.Config, info SyncInfo)
• func UploadFile(cfg *iqshell.Config, info UploadInfo)
• type BatchUpload2Info
  • func (info *BatchUpload2Info) Check() *data.CodeError
• type BatchUploadConfigMouldInfo
• type BatchUploadInfo
  • func (info *BatchUploadInfo) Check() *data.CodeError
• type Metric
  • func (m *Metric) AddNotOverwriteCount(count int64)
  • func (m *Metric) AddOverwriteCount(count int64)
• type SyncInfo
  • func (info *SyncInfo) Check() *data.CodeError
• type UploadConfig
  • func DefaultUploadConfig() UploadConfig
  • func (up *UploadConfig) Check() *data.CodeError
  • func (up *UploadConfig) HitByFilePrefixes(localFileRelativePath string) (hit bool, filePrefix string)
  • func (up *UploadConfig) HitByFixesString(localFileRelativePath string) (hit bool, hitFixedStr string)
  • func (up *UploadConfig) HitByPathPrefixes(localFileRelativePath string) (hit bool, pathPrefix string)
  • func (up *UploadConfig) HitBySuffixes(localFileRelativePath string) (hit bool, hitSuffix string)
  • func (up *UploadConfig) IsCheckExists() bool
  • func (up *UploadConfig) IsCheckHash() bool
  • func (up *UploadConfig) IsCheckSize() bool
  • func (up *UploadConfig) IsDeleteOnSuccess() bool
  • func (up *UploadConfig) IsDisableForm() bool
  • func (up *UploadConfig) IsDisableResume() bool
  • func (up *UploadConfig) IsIgnoreDir() bool
  • func (up *UploadConfig) IsOverwrite() bool
  • func (up *UploadConfig) IsRescanLocal() bool
  • func (up *UploadConfig) IsResumeAPIV2() bool
  • func (up *UploadConfig) JobId() string
• type UploadInfo
  • func (info *UploadInfo) Check() *data.CodeError
  • func (info *UploadInfo) WorkId() string

Functions

func BatchUpload

func BatchUpload(cfg *iqshell.Config, info BatchUploadInfo)

BatchUpload 该命令会读取配置文件， 上传本地文件系统的文件到七牛存储中; 可以设置多线程上传，默认的线程区间在[iqshell.min_upload_thread_count, iqshell.max_upload_thread_count]

func BatchUpload2

func BatchUpload2(cfg *iqshell.Config, info BatchUpload2Info)

func BatchUploadConfigMould

func BatchUploadConfigMould(cfg *iqshell.Config, info BatchUploadConfigMouldInfo)

func SyncFile

func SyncFile(cfg *iqshell.Config, info SyncInfo)

func UploadFile

func UploadFile(cfg *iqshell.Config, info UploadInfo)

Types

type BatchUpload2Info

type BatchUpload2Info struct {
	flow.Info
	export.FileExporterConfig
	UploadConfig

	// 工作数据源
	InputFile    string // 工作数据源：文件
	ItemSeparate string // 工作数据源：每行元素按分隔符分的分隔符
	EnableStdin  bool   // 工作数据源：stdin, 当 InputFile 不存在时使用 stdin
}

func (*BatchUpload2Info) Check

func (info *BatchUpload2Info) Check() *data.CodeError

type BatchUploadConfigMouldInfo

type BatchUploadConfigMouldInfo struct {
}

type BatchUploadInfo

type BatchUploadInfo struct {
	flow.Info
	export.FileExporterConfig

	Overwrite bool // 是否覆盖

	// 工作数据源
	InputFile    string // 工作数据源：文件
	ItemSeparate string // 工作数据源：每行元素按分隔符分的分隔符
	EnableStdin  bool   // 工作数据源：stdin, 当 InputFile 不存在时使用 stdin

	UploadConfigFile string
	CallbackHost     string
	CallbackUrl      string
}

func (*BatchUploadInfo) Check

func (info *BatchUploadInfo) Check() *data.CodeError

type Metric

type Metric struct {
	batch.Metric

	OverwriteCount    int64 `json:"overwrite_count"`
	NotOverwriteCount int64 `json:"not_overwrite_count"`
}

func (*Metric) AddNotOverwriteCount

func (m *Metric) AddNotOverwriteCount(count int64)

func (*Metric) AddOverwriteCount

func (m *Metric) AddOverwriteCount(count int64)

type SyncInfo

type SyncInfo UploadInfo

func (*SyncInfo) Check

func (info *SyncInfo) Check() *data.CodeError

type UploadConfig

type UploadConfig struct {
	UpHost    string `json:"up_host,omitempty"`
	BindUpIp  string `json:"bind_up_ip,omitempty"`
	BindRsIp  string `json:"bind_rs_ip,omitempty"`
	BindNicIp string `json:"bind_nic_ip,omitempty"` //local network interface card config

	SrcDir                 string `json:"src_dir,omitempty"`
	FileList               string `json:"file_list,omitempty"`
	IgnoreDir              bool   `json:"ignore_dir,omitempty"`
	SkipFilePrefixes       string `json:"skip_file_prefixes,omitempty"`
	SkipPathPrefixes       string `json:"skip_path_prefixes,omitempty"`
	SkipFixedStrings       string `json:"skip_fixed_strings,omitempty"`
	SkipSuffixes           string `json:"skip_suffixes,omitempty"`
	FileEncoding           string `json:"file_encoding,omitempty"`
	Bucket                 string `json:"bucket,omitempty"`
	ResumableAPIV2         bool   `json:"resumable_api_v2,omitempty"`
	ResumableAPIV2PartSize int64  `json:"resumable_api_v2_part_size,omitempty"`
	PutThreshold           int64  `json:"put_threshold,omitempty"`
	KeyPrefix              string `json:"key_prefix,omitempty"`
	Overwrite              bool   `json:"overwrite,omitempty"`
	CheckExists            bool   `json:"check_exists,omitempty"`
	CheckHash              bool   `json:"check_hash,omitempty"`
	CheckSize              bool   `json:"check_size,omitempty"`
	RescanLocal            bool   `json:"rescan_local,omitempty"`
	FileType               int    `json:"file_type,omitempty"`
	DeleteOnSuccess        bool   `json:"delete_on_success,omitempty"`
	DisableResume          bool   `json:"disable_resume,omitempty"`
	DisableForm            bool   `json:"disable_form,omitempty"`
	WorkerCount            int    `json:"work_count,omitempty"` // 分片上传并发数
	RecordRoot             string `json:"record_root,omitempty"`
	SequentialReadFile     bool   `json:"sequential_read_file"` // 文件顺序读

	Policy *storage.PutPolicy `json:"policy"`
}

func DefaultUploadConfig

func DefaultUploadConfig() UploadConfig

func (*UploadConfig) Check

func (up *UploadConfig) Check() *data.CodeError

func (*UploadConfig) HitByFilePrefixes

func (up *UploadConfig) HitByFilePrefixes(localFileRelativePath string) (hit bool, filePrefix string)

func (*UploadConfig) HitByFixesString

func (up *UploadConfig) HitByFixesString(localFileRelativePath string) (hit bool, hitFixedStr string)

func (*UploadConfig) HitByPathPrefixes

func (up *UploadConfig) HitByPathPrefixes(localFileRelativePath string) (hit bool, pathPrefix string)

func (*UploadConfig) HitBySuffixes

func (up *UploadConfig) HitBySuffixes(localFileRelativePath string) (hit bool, hitSuffix string)

func (*UploadConfig) IsCheckExists

func (up *UploadConfig) IsCheckExists() bool

func (*UploadConfig) IsCheckHash

func (up *UploadConfig) IsCheckHash() bool

func (*UploadConfig) IsCheckSize

func (up *UploadConfig) IsCheckSize() bool

func (*UploadConfig) IsDeleteOnSuccess

func (up *UploadConfig) IsDeleteOnSuccess() bool

func (*UploadConfig) IsDisableForm

func (up *UploadConfig) IsDisableForm() bool

func (*UploadConfig) IsDisableResume

func (up *UploadConfig) IsDisableResume() bool

func (*UploadConfig) IsIgnoreDir

func (up *UploadConfig) IsIgnoreDir() bool

func (*UploadConfig) IsOverwrite

func (up *UploadConfig) IsOverwrite() bool

func (*UploadConfig) IsRescanLocal

func (up *UploadConfig) IsRescanLocal() bool

func (*UploadConfig) IsResumeAPIV2

func (up *UploadConfig) IsResumeAPIV2() bool

func (*UploadConfig) JobId

func (up *UploadConfig) JobId() string

type UploadInfo

type UploadInfo struct {
	upload.ApiInfo
	RelativePathToSrcPath string // 相对与上传文件夹的路径信息
	Policy                *storage.PutPolicy
	DeleteOnSuccess       bool
}

func (*UploadInfo) Check

func (info *UploadInfo) Check() *data.CodeError

func (*UploadInfo) WorkId

func (info *UploadInfo) WorkId() string