goseaweedfs

package module
v0.6.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 29, 2024 License: Apache-2.0 Imports: 20 Imported by: 6

README

goseaweedfs

Build Status Go Report Card Coverage Status godoc license

A complete Golang client for SeaweedFS (version 1.44+). Inspired by:

Installation

go get -u github.com/linxGnu/goseaweedfs

Usage

Please refer to Test Cases for sample code.

Supported

  • Grow
  • Status
  • Cluster Status
  • Filer
  • Upload
  • Submit
  • Delete
  • Replace
  • Upload large file with builtin manifest handler, auto file split and chunking
  • Admin Operations (mount, unmount, delete volumn, etc)

Contributing

Please issue me for things gone wrong or:

  1. Fork it!
  2. Create your feature branch: git checkout -b my-new-feature
  3. Commit your changes: git commit -am 'Add some feature'
  4. Push to the branch: git push origin my-new-feature
  5. Submit a pull request :D

Documentation

Index

Constants

View Source 
const (
	// ParamCollection http param to specify collection which files belong. According to SeaweedFS API.
	ParamCollection = "collection"

	// ParamTTL http param to specify time to live. According to SeaweedFS API.
	ParamTTL = "ttl"

	// ParamCount http param to specify how many file ids to reserve. According to SeaweedFS API.
	ParamCount = "count"

	// ParamAssignReplication http param to assign files with a specific replication type.
	ParamAssignReplication = "replication"

	// ParamAssignCount http param to specify how many file ids to reserve.
	ParamAssignCount = "count"

	// ParamAssignDataCenter http param to assign a specific data center
	ParamAssignDataCenter = "dataCenter"

	// ParamLookupVolumeID http param to specify volume ID for looking up.
	ParamLookupVolumeID = "volumeId"

	// ParamLookupPretty http param to make json response prettified or not. Default should not be set.
	ParamLookupPretty = "pretty"

	// ParamLookupCollection http param to specify known collection, this would make file look up/search faster.
	ParamLookupCollection = "collection"

	// ParamVacuumGarbageThreshold if your system has many deletions, the deleted file's disk space will not be synchronously re-claimed.
	// There is a background job to check volume disk usage. If empty space is more than the threshold,
	// default to 0.3, the vacuum job will make the volume readonly, create a new volume with only existing files,
	// and switch on the new volume. If you are impatient or doing some testing, vacuum the unused spaces this way.
	ParamVacuumGarbageThreshold = "GarbageThreshold"

	// ParamGrowReplication http param to specify a specific replication.
	ParamGrowReplication = "replication"

	// ParamGrowCount http param to specify number of empty volume to grow.
	ParamGrowCount = "count"

	// ParamGrowDataCenter http param to specify datacenter of growing volume.
	ParamGrowDataCenter = "dataCenter"

	// ParamGrowCollection http param to specify collection of files for growing.
	ParamGrowCollection = "collection"

	// ParamGrowTTL specify time to live for growing api. Refers to: https://github.com/chrislusf/seaweedfs/wiki/Store-file-with-a-Time-To-Live
	// 3m: 3 minutes
	// 4h: 4 hours
	// 5d: 5 days
	// 6w: 6 weeks
	// 7M: 7 months
	// 8y: 8 years
	ParamGrowTTL = "ttl"
)

Variables

View Source 
var (
	// ErrFileNotFound return file not found error
	ErrFileNotFound = fmt.Errorf("File not found")
)

Functions

func GetBytesMd5sum 

func GetBytesMd5sum(data []byte) (md5sum string, err error)

func GetFileExtension added in v0.6.3 

func GetFileExtension(fileName string) (extension string)

func GetFileMd5sum 

func GetFileMd5sum(filePath string) (md5sum string, err error)

func GetFileName added in v0.6.3 

func GetFileName(fullPath string) (fileName string)

func IsDir added in v0.6.3 

func IsDir(path string) bool

func IsFileMarkBytes added in v0.6.3 

func IsFileMarkBytes(data []byte, m FileMark) bool

Types

type AssignResult 

type AssignResult struct {
	FileID    string `json:"fid,omitempty"`
	URL       string `json:"url,omitempty"`
	PublicURL string `json:"publicUrl,omitempty"`
	Count     uint64 `json:"count,omitempty"`
	Error     string `json:"error,omitempty"`
}

AssignResult contains assign result. Raw response: {"fid":"1,0a1653fd0f","url":"localhost:8899","publicUrl":"localhost:8899","count":1,"error":""}

type ChunkInfo 

type ChunkInfo struct {
	Fid    string `json:"fid"`
	Offset int64  `json:"offset"`
	Size   int64  `json:"size"`
}

ChunkInfo chunk information. According to https://github.com/chrislusf/seaweedfs/wiki/Large-File-Handling.

type ChunkManifest 

type ChunkManifest struct {
	Name   string       `json:"name,omitempty"`
	Mime   string       `json:"mime,omitempty"`
	Size   int64        `json:"size,omitempty"`
	Chunks []*ChunkInfo `json:"chunks,omitempty"`
}

ChunkManifest chunk manifest. According to https://github.com/chrislusf/seaweedfs/wiki/Large-File-Handling.

func (*ChunkManifest) Marshal 

func (c *ChunkManifest) Marshal() ([]byte, error)

Marshal marshal whole chunk manifest

type ClusterStatus 

type ClusterStatus struct {
	IsLeader bool
	Leader   string
	Peers    []string
}

ClusterStatus result of getting status of cluster

type DataCenter 

type DataCenter struct {
	Free  int
	Max   int
	Racks []*Rack
}

DataCenter stats of a datacenter

type DataNode 

type DataNode struct {
	Free      int
	Max       int
	PublicURL string `json:"PublicUrl"`
	URL       string `json:"Url"`
	Volumes   int
}

DataNode stats of data node

type FileInfo 

type FileInfo struct {
	// name of the file
	Name string `json:"name"`

	// absolute path of the file
	Path string `json:"path"`

	// MD5 hash
	Md5 string `json:"md5"`
}

func ListFilesRecursive added in v0.6.3 

func ListFilesRecursive(dirPath string) (files []FileInfo, err error)

type FileMark added in v0.6.3 

type FileMark rune
const (
	EmptyMark FileMark = '\U0000FFFF'
)

func (FileMark) Bytes added in v0.6.3 

func (m FileMark) Bytes() []byte

func (FileMark) String added in v0.6.3 

func (m FileMark) String() string

type FilePart 

type FilePart struct {
	Reader     io.ReadCloser
	FileName   string
	FileSize   int64
	MimeType   string
	ModTime    int64 //in seconds
	Collection string

	// TTL Time to live.
	// 3m: 3 minutes
	// 4h: 4 hours
	// 5d: 5 days
	// 6w: 6 weeks
	// 7M: 7 months
	// 8y: 8 years
	TTL string

	Server string
	FileID string
}

FilePart file wrapper with reader and some metadata

func NewFilePart 

func NewFilePart(fullPathFilename string) (*FilePart, error)

NewFilePart new file path from real file dir

func NewFilePartFromReader 

func NewFilePartFromReader(reader io.ReadCloser, fileName string, fileSize int64) *FilePart

NewFilePartFromReader new file part from file reader. fileName and fileSize must be known

func NewFileParts 

func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error)

NewFileParts create many file part at once.

func (*FilePart) Close 

func (f *FilePart) Close() (err error)

Close underlying openned file.

type Filer 

type Filer struct {
	// contains filtered or unexported fields
}

Filer client

func NewFiler 

func NewFiler(u string, client *http.Client, opts ...FilerOption) (f *Filer, err error)

NewFiler new filer with filer server's url

func (*Filer) Close 

func (f *Filer) Close() (err error)

Close underlying daemons.

func (*Filer) Delete 

func (f *Filer) Delete(path string, args url.Values) (err error)

Delete a file/dir.

func (*Filer) DeleteDir 

func (f *Filer) DeleteDir(path string) (err error)

DeleteDir a dir.

func (*Filer) DeleteFile 

func (f *Filer) DeleteFile(path string) (err error)

DeleteFile a file.

func (*Filer) Download 

func (f *Filer) Download(path string, args url.Values, callback func(io.Reader) error) (err error)

Download a file.

func (*Filer) Get 

func (f *Filer) Get(path string, args url.Values, header map[string]string) (data []byte, statusCode int, err error)

Get response data from filer.

func (*Filer) GetJson 

func (f *Filer) GetJson(path string, args url.Values) (data []byte, statusCode int, err error)

GetJson Get response data from filer.

func (*Filer) ListDir 

func (f *Filer) ListDir(path string) (files []FilerFileInfo, err error)

ListDir List a directory.

func (*Filer) ListDirRecursive 

func (f *Filer) ListDirRecursive(path string) (files []FilerFileInfo, err error)

ListDirRecursive List a directory recursively.

func (*Filer) Upload 

func (f *Filer) Upload(content io.Reader, fileSize int64, newPath, collection, ttl string) (result *FilerUploadResult, err error)

Upload content.

func (*Filer) UploadDir 

func (f *Filer) UploadDir(localDirPath, newPath, collection, ttl string) (results []*FilerUploadResult, err error)

UploadDir upload files from a directory.

func (*Filer) UploadFile 

func (f *Filer) UploadFile(localFilePath, newPath, collection, ttl string) (result *FilerUploadResult, err error)

UploadFile a file.

type FilerFileInfo 

type FilerFileInfo struct {
	FullPath        string
	Mtime           time.Time
	Crtime          time.Time
	Mode            int
	Uid             int
	Gid             int
	Mime            string
	Replication     string
	Collection      string
	TtlSec          int
	UserName        string
	GroupNames      string
	SymlinkTarget   string
	Md5             string
	FileSize        int64
	Extended        string
	HardLinkId      string
	HardLinkCounter int64
	Chunks          interface{}
	Children        []FilerFileInfo
	Name            string
	Extension       string
	IsDir           bool
}

func GetFileWithExtendedFields added in v0.6.3 

func GetFileWithExtendedFields(file FilerFileInfo) (res FilerFileInfo)

type FilerListDirResponse 

type FilerListDirResponse struct {
	Path    string
	Entries []FilerFileInfo
}

type FilerOption added in v0.2.0 

type FilerOption func(f *Filer)

func WithFilerAuthKey added in v0.2.0 

func WithFilerAuthKey(authKey string) FilerOption

type FilerUploadResult 

type FilerUploadResult struct {
	Name    string `json:"name,omitempty"`
	FileURL string `json:"url,omitempty"`
	FileID  string `json:"fid,omitempty"`
	Size    int64  `json:"size,omitempty"`
	Error   string `json:"error,omitempty"`
}

FilerUploadResult upload result of response from filer server. According to https://github.com/chrislusf/seaweedfs/wiki/Filer-Server-API.

type HttpClientOption added in v0.2.0 

type HttpClientOption func(client *httpClient)

func WithHttpClientAuthKey added in v0.2.0 

func WithHttpClientAuthKey(authKey string) HttpClientOption

type Layout 

type Layout struct {
	Replication string
	Writables   []uint64
}

Layout of replication/collection stats. According to https://github.com/chrislusf/seaweedfs/wiki/Master-Server-API

type LookupResult 

type LookupResult struct {
	VolumeLocations VolumeLocations `json:"locations,omitempty"`
	Error           string          `json:"error,omitempty"`
}

LookupResult the result of looking up volume. According to https://github.com/chrislusf/seaweedfs/wiki/Master-Server-API

type Rack 

type Rack struct {
	DataNodes []*DataNode
	Free      int
	Max       int
}

Rack stats of racks

type Seaweed 

type Seaweed struct {
	// contains filtered or unexported fields
}

Seaweed client containing almost features/operations to interact with SeaweedFS

func NewSeaweed 

func NewSeaweed(masterURL string, filers []string, chunkSize int64, client *http.Client) (c *Seaweed, err error)

NewSeaweed create new seaweed client. Master url must be a valid uri (which includes scheme).

func (*Seaweed) Assign 

func (c *Seaweed) Assign(args url.Values) (result *AssignResult, err error)

Assign do assign api.

func (*Seaweed) BatchUploadFileParts 

func (c *Seaweed) BatchUploadFileParts(files []*FilePart, collection string, ttl string) ([]*SubmitResult, error)

BatchUploadFileParts uploads multiple file parts at once.

func (*Seaweed) BatchUploadFiles 

func (c *Seaweed) BatchUploadFiles(files []string, collection, ttl string) (results []*SubmitResult, err error)

BatchUploadFiles batch uploads files.

func (*Seaweed) Close 

func (c *Seaweed) Close() (err error)

Close underlying daemons.

func (*Seaweed) ClusterStatus 

func (c *Seaweed) ClusterStatus() (result *ClusterStatus, err error)

ClusterStatus get cluster status.

func (*Seaweed) DeleteChunks 

func (c *Seaweed) DeleteChunks(cm *ChunkManifest, args url.Values) (err error)

DeleteChunks concurrently delete chunks.

func (*Seaweed) DeleteFile 

func (c *Seaweed) DeleteFile(fileID string, args url.Values) (err error)

DeleteFile by id.

func (*Seaweed) Download 

func (c *Seaweed) Download(fileID string, args url.Values, callback func(io.Reader) error) (fileName string, err error)

Download file by id.

func (*Seaweed) Filers 

func (c *Seaweed) Filers() []*Filer

Filers returns initialized filer(s).

func (*Seaweed) GC 

func (c *Seaweed) GC(threshold float64) (err error)

GC force Garbage Collection.

func (*Seaweed) Grow 

func (c *Seaweed) Grow(count int, collection, replication, dataCenter string) error

Grow pre-Allocate Volumes.

func (*Seaweed) GrowArgs 

func (c *Seaweed) GrowArgs(args url.Values) (err error)

GrowArgs pre-Allocate volumes with args.

func (*Seaweed) Lookup 

func (c *Seaweed) Lookup(volID string, args url.Values) (result *LookupResult, err error)

Lookup volume ID.

func (*Seaweed) LookupFileID 

func (c *Seaweed) LookupFileID(fileID string, args url.Values, readonly bool) (fullURL string, err error)

LookupFileID lookup file by id.

func (*Seaweed) LookupServerByFileID 

func (c *Seaweed) LookupServerByFileID(fileID string, args url.Values, readonly bool) (server string, err error)

LookupServerByFileID lookup server by file id.

func (*Seaweed) Replace 

func (c *Seaweed) Replace(fileID string, newContent io.Reader, fileName string, size int64, collection, ttl string, deleteFirst bool) (err error)

Replace file content with new one.

func (*Seaweed) ReplaceFile 

func (c *Seaweed) ReplaceFile(fileID, localFilePath string, deleteFirst bool) (err error)

ReplaceFile replaces file with local file.

func (*Seaweed) ReplaceFilePart 

func (c *Seaweed) ReplaceFilePart(f *FilePart, deleteFirst bool) (err error)

ReplaceFilePart replaces file part.

func (*Seaweed) Status 

func (c *Seaweed) Status() (result *SystemStatus, err error)

Status check System Status.

func (*Seaweed) Submit 

func (c *Seaweed) Submit(filePath string, collection, ttl string) (result *SubmitResult, err error)

Submit file directly to master.

func (*Seaweed) SubmitFilePart 

func (c *Seaweed) SubmitFilePart(f *FilePart, args url.Values) (result *SubmitResult, err error)

SubmitFilePart directly to master.

func (*Seaweed) Upload 

func (c *Seaweed) Upload(fileReader io.Reader, fileName string, size int64, collection, ttl string) (fp *FilePart, err error)

Upload file by reader.

func (*Seaweed) UploadFile 

func (c *Seaweed) UploadFile(filePath string, collection, ttl string) (cm *ChunkManifest, fp *FilePart, err error)

UploadFile with full file dir/path.

func (*Seaweed) UploadFilePart 

func (c *Seaweed) UploadFilePart(f *FilePart) (cm *ChunkManifest, err error)

UploadFilePart uploads a file part.

type SubmitResult 

type SubmitResult struct {
	FileName string `json:"fileName,omitempty"`
	FileURL  string `json:"fileUrl,omitempty"`
	FileID   string `json:"fid,omitempty"`
	Size     int64  `json:"size,omitempty"`
	Error    string `json:"error,omitempty"`
}

SubmitResult result of submit operation.

type SystemStatus 

type SystemStatus struct {
	Topology Topology
	Version  string
	Error    string
}

SystemStatus result of getting status of system

type Topology 

type Topology struct {
	DataCenters []*DataCenter
	Free        int
	Max         int
	Layouts     []*Layout
}

Topology result of topology stats request

type UploadResult 

type UploadResult struct {
	Name  string `json:"name,omitempty"`
	Size  int64  `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
}

UploadResult contains upload result after put file to SeaweedFS Raw response: {"name":"go1.8.3.linux-amd64.tar.gz","size":82565628,"error":""}

type VolumeLocation 

type VolumeLocation struct {
	URL       string `json:"url,omitempty"`
	PublicURL string `json:"publicUrl,omitempty"`
}

VolumeLocation location of volume responsed from master API. According to https://github.com/chrislusf/seaweedfs/wiki/Master-Server-API

type VolumeLocations 

type VolumeLocations []*VolumeLocation

VolumeLocations returned VolumeLocations (volumes)

func (VolumeLocations) Head 

func (c VolumeLocations) Head() *VolumeLocation

Head get first location in list

func (VolumeLocations) RandomPickForRead 

func (c VolumeLocations) RandomPickForRead() *VolumeLocation

RandomPickForRead random pick a location for further read request

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.