gofile

README

文件操作相关模块

• 文件下载功能

方法 1

func (this FileDownload) DoHandle(ctx *gin.Context) *goserver.Response {
    ds := gofile.NewGoDownload(ctx, "downloaded_file.csv", ctx.Writer, ctx.Request)
    err := ds.SetFileHeaders()
    if err != nil {
        return nil
    }
    
    filePath := "file.csv"
    err = gofile.ReadLines(filePath, func(chunk string) error {
    err = ds.Write([]byte(chunk + "\n"))
        return err
    })
    
    if err != nil {
        ds.Error(err)
        return nil
    }
    return nil
}

方法 2

func (this FileDownload) DoHandle(ctx *gin.Context) *goserver.Response {
	ds := gofile.NewGoDownload(ctx, "downloaded_file.csv", ctx.Writer, ctx.Request)
	err := ds.SetFileHeaders()
	file := "file.csv"
	err = ds.Output(file)
	if err != nil {
		http.Error(ctx.Writer, "Streaming unsupported!", http.StatusInternalServerError)
		return nil
	}
	return nil
}

Documentation

Index

• Constants
• func CalculateFileMD5(filePath string) (string, error)
• func CopyFile(src, dst string) error
• func CreateSavePath(dst string, perm os.FileMode) error
• func DIR() string
• func Exist(path string) (bool, error)
• func Exists(filename string) bool
• func FILE() string
• func GetFileContent(filePath string) ([]byte, error)
• func GetFileContentString(filePath string) (string, error)
• func GetFileHeaderMd5Name(fileHeader *multipart.FileHeader) (string, error)
• func GetFileInfo(filename string) (size int64, mode os.FileMode, err error)
• func GetFileList(path string) []string
• func LINE() int
• func MD5(file string) (string, error)
• func MergeChunk(url string, fileMergeReq *FileMergeReq) (*gohttp.Response, error)
• func ReadByLine(filename string, cb func(b []byte, end bool) error) error
• func ReadEntireFile(filePath string) ([]byte, error)
• func ReadFileAt(filePath string, offset int64, length int) ([]byte, error)
• func ReadFileChunks(filename string, chunkSize int, callback func(chunk []byte) error) error
• func ReadLines(filePath string, lineFunc func(line string) error) error
• func RemoveFile(file string) error
• func SaveFile(file *multipart.FileHeader, dst string) error
• func SaveToLocal(savePath string, chunk *FileChunk) error
• func Trace(skip int) (arr []string)
• func UploadChunk(url string, chunk *FileChunk) (*gohttp.Response, error)
• func WriteToFile(filename string, b []byte) error
• func WriteWithBuffer(w http.ResponseWriter, data []byte) error
• type Belt
• type BigFile
  • func (b *BigFile) IsSuccess() bool
  • func (b *BigFile) Start() error
• type FileChunk
• type FileLock
  • func (fl *FileLock) Lock() (err error)
  • func (fl *FileLock) UnLock() (err error)
• type FileMergeReq
• type FileReceiveResult
  • func MergeFileForChunks(filePath string, fileName string, fileMd5 string, totalChunks int64, ...) (*FileReceiveResult, error)
  • func ReceiveChunkHandler(assetsDir string, chunkIndex int64, chunkMd5 string, fileMd5 string, ...) (*FileReceiveResult, error)
  • func ReceiveFile(assetsDir string, file *multipart.FileHeader) (*FileReceiveResult, error)
• type GoDownload
  • func NewGoDownload(ctx context.Context, filename string, w http.ResponseWriter, r *http.Request) *GoDownload
  • func (g *GoDownload) Error(err error)
  • func (g *GoDownload) Output(filePath string) error
  • func (g *GoDownload) OutputByLine(filePath string) error
  • func (g *GoDownload) SetFileHeaders() error
  • func (g *GoDownload) Write(data []byte) error
  • func (g *GoDownload) WriteString(data string) error

Constants

const (
	UPLOAD_TYPE_LOCAL = 1
	UPLOAD_TYPE_OSS   = 2
	UPLOAD_TYPE_CHUNK = 3
)

const (
	EL = "\n"
)

Functions

func CalculateFileMD5

func CalculateFileMD5(filePath string) (string, error)

计算文件md5(支持超大文件)

func CopyFile

func CopyFile(src, dst string) error

复制文件

func CreateSavePath

func CreateSavePath(dst string, perm os.FileMode) error

func DIR

func DIR() string

目录名称

func Exist

func Exist(path string) (bool, error)

func Exists

func Exists(filename string) bool

func FILE

func FILE() string

文件名

func GetFileContent

func GetFileContent(filePath string) ([]byte, error)

ReadEntireFile 读取整个文件到字节切片一样

func GetFileContentString

func GetFileContentString(filePath string) (string, error)

func GetFileHeaderMd5Name

func GetFileHeaderMd5Name(fileHeader *multipart.FileHeader) (string, error)

func GetFileInfo

func GetFileInfo(filename string) (size int64, mode os.FileMode, err error)

GetFileInfo 获取文件信息

func GetFileList

func GetFileList(path string) []string

获取当前目录下所有文件

func LINE

func LINE() int

行号

func MD5

func MD5(file string) (string, error)

计算文件MD5 支持大文件

func MergeChunk

func MergeChunk(url string, fileMergeReq *FileMergeReq) (*gohttp.Response, error)

分片全部上传完毕后，再请求文件分片合并请求（作为客户端请求时验证非法请求认证逻辑需要加，如authToken sign 等等）

func ReadByLine

func ReadByLine(filename string, cb func(b []byte, end bool) error) error

func ReadEntireFile

func ReadEntireFile(filePath string) ([]byte, error)

ReadEntireFile 读取整个文件到字节切片

func ReadFileAt

func ReadFileAt(filePath string, offset int64, length int) ([]byte, error)

ReadFileAt 从指定位置读取文件

func ReadFileChunks

func ReadFileChunks(filename string, chunkSize int, callback func(chunk []byte) error) error

ReadFileChunks 分块读取大文件，避免内存占用过大

func ReadLines

func ReadLines(filePath string, lineFunc func(line string) error) error

func RemoveFile

func RemoveFile(file string) error

func SaveFile

func SaveFile(file *multipart.FileHeader, dst string) error

func SaveToLocal

func SaveToLocal(savePath string, chunk *FileChunk) error

//////////////////////////////////////////////////////////////local save 把分片存在指定目录

func Trace

func Trace(skip int) (arr []string)

追踪信息

func UploadChunk

func UploadChunk(url string, chunk *FileChunk) (*gohttp.Response, error)

////////////////////////////////////////////////////////////// http server upload and merge 供参考 上传一个文件分片，（作为客户端请求时验证非法请求认证逻辑需要加，如authToken sign 等等）

func WriteToFile

func WriteToFile(filename string, b []byte) error

写文件，支持路径创建

func WriteWithBuffer

func WriteWithBuffer(w http.ResponseWriter, data []byte) error

Types

type Belt

type Belt struct {
	DataChan  chan []byte
	CloseChan chan interface{}
}

type BigFile

type BigFile struct {
	ChunkSize           int64                        // 分片大小 M
	MaxWorkers          int                          // 同时处理最大分块数量，合理用防止超大文件内存益处
	File                string                       // 文件路径
	FileMd5             string                       // 文件Md5
	ChunkCount          int64                        // 分片数量
	FileChunkCallback   func(chunk *FileChunk) error // 分片处理消息
	SuccessChunkIndexes []int64                      //处理成功的碎片index

	ErrorGroup goutils.ErrorGroup
	// contains filtered or unexported fields
}

func (*BigFile) IsSuccess

func (b *BigFile) IsSuccess() bool

func (*BigFile) Start

func (b *BigFile) Start() error

type FileChunk

type FileChunk struct {
	Data             []byte //分片数据
	Hash             string //分片Hash
	Index            int64  //分片顺序号
	OriginalFileName string //原文件名
	OriginalFileMd5  string //原文件Md5
	FileName         string //分片文件名称
}

fmt.Sprintf("%s.part%d", fileName, i) mapreduce big file 大文件逻辑 for 把大文件并发分片处理，为了防止OOM超大文件边分片边处理的策略

type FileLock

type FileLock struct {
	Filename string
	// contains filtered or unexported fields
}

func (*FileLock) Lock

func (fl *FileLock) Lock() (err error)

func (*FileLock) UnLock

func (fl *FileLock) UnLock() (err error)

type FileMergeReq

type FileMergeReq struct {
	FileMd5     string `json:"fileMd5"`
	FileName    string `json:"fileName"`
	TotalChunks int64  `json:"totalChunks"`
}

type FileReceiveResult

type FileReceiveResult struct {
	OriginalFile     string
	FileName         string
	OriginalFileName string
	ChunkCount       int64
}

func MergeFileForChunks

func MergeFileForChunks(filePath string, fileName string, fileMd5 string, totalChunks int64, isNotRemoveChunk bool) (*FileReceiveResult, error)

服务器合并所有文件分片，并验证md5, isNotRemoveChunk =true 合并后时不会删除分片

func ReceiveChunkHandler

func ReceiveChunkHandler(assetsDir string, chunkIndex int64, chunkMd5 string, fileMd5 string, file *multipart.FileHeader) (*FileReceiveResult, error)

服务器接受文件分片

func ReceiveFile

func ReceiveFile(assetsDir string, file *multipart.FileHeader) (*FileReceiveResult, error)

服务器接受file文件到assetsDir目录下，assetsDir 目录不存在则自动创建,返回存储位置

type GoDownload

type GoDownload struct {
	Logger logx.Logger
	// contains filtered or unexported fields
}

func NewGoDownload

func NewGoDownload(ctx context.Context, filename string, w http.ResponseWriter, r *http.Request) *GoDownload

func (*GoDownload) Error

func (g *GoDownload) Error(err error)

func (*GoDownload) Output

func (g *GoDownload) Output(filePath string) error

二进制读取输出 每次读取4096字节 输出

func (*GoDownload) OutputByLine

func (g *GoDownload) OutputByLine(filePath string) error

一行一行输出

func (*GoDownload) SetFileHeaders

func (g *GoDownload) SetFileHeaders() error

func (*GoDownload) Write

func (g *GoDownload) Write(data []byte) error

func (*GoDownload) WriteString

func (g *GoDownload) WriteString(data string) error