common

Documentation

Index

• type Config
• type FsClient
• type FsReader
• type HdfsConfig
• type IpfsConfig

Types

type Config

type Config struct {
	// dfs type, supported values: hdfs, ipfs
	Type     string
	HdfsConf *HdfsConfig
	IpfsConf *IpfsConfig
}

type FsClient

type FsClient interface {
	ReadDir(dirname string) ([]os.FileInfo, error)
	Stat(name string) (os.FileInfo, error)
	CopyToRemote(src string, dst string) error
	Open(name string) (FsReader, error)
	Close() error
}

type FsReader

type FsReader interface {
	io.ReaderAt
	io.Seeker
	io.Reader
	io.Closer
	Stat() os.FileInfo
}

type HdfsConfig

type HdfsConfig struct {
	// HDFS user
	User string

	// HDFS namenode addresses
	NameNodes []string

	// Docker DNS config is needed when this value is true
	UseDatanodeHostname bool
}

type IpfsConfig

type IpfsConfig struct {
	// IPFS api url, example: 127.0.0.1:5001
	Url string

	// IPFS cluster api url, example: 127.0.0.1:9094
	ClusterUrl string

	// Store dir for file name and cid mapping as index
	CidIndexDir string
}