hdfs

Documentation

Index

• Constants
• type ContentSummary
• type FileChecksum
• type FileStatus
• type FileStatuses
• type HdfsData
• type Path
• type RemoteException
  • func (re RemoteException) Error() string
• type Token
• type Tokens
• type WebHdfs
  • func NewWebHdfs(config *WebHdfsConfig) (*WebHdfs, error)
  • func (h *WebHdfs) Append(localfile string, destination string) error
  • func (h *WebHdfs) Delete(recursive bool, paths ...string) map[string]error
  • func (h *WebHdfs) Get(path string, server *colonycore.Server) ([]byte, error)
  • func (h *WebHdfs) GetToLocal(path string, destination string, permission string, server *colonycore.Server) error
  • func (h *WebHdfs) List(path string) (*HdfsData, error)
  • func (h *WebHdfs) MakeDir(path string, permission string) error
  • func (h *WebHdfs) MakeDirs(paths []string, permission string) map[string]error
  • func (h *WebHdfs) Put(localfile string, destination string, permission string, ...) error
  • func (h *WebHdfs) PutDir(dirname string, destination string, server *colonycore.Server) (error, map[string]error)
  • func (h *WebHdfs) Puts(paths []string, destinationFolder string, permission string, ...) map[string]error
  • func (h *WebHdfs) Rename(path string, destination string) error
  • func (h *WebHdfs) SetOwner(path string, owner string, group string) error
  • func (h *WebHdfs) SetPermission(path string, permission string) error
• type WebHdfsConfig
  • func NewHdfsConfig(host, userid string) *WebHdfsConfig

Constants

const (
	OP_OPEN                  = "OPEN"
	OP_CREATE                = "CREATE"
	OP_APPEND                = "APPEND"
	OP_CONCAT                = "CONCAT"
	OP_RENAME                = "RENAME"
	OP_DELETE                = "DELETE"
	OP_SETPERMISSION         = "SETPERMISSION"
	OP_SETOWNER              = "SETOWNER"
	OP_SETREPLICATION        = "SETREPLICATION"
	OP_SETTIMES              = "SETTIMES"
	OP_MKDIRS                = "MKDIRS"
	OP_CREATESYMLINK         = "CREATESYMLINK"
	OP_LISTSTATUS            = "LISTSTATUS"
	OP_GETFILESTATUS         = "GETFILESTATUS"
	OP_GETCONTENTSUMMARY     = "GETCONTENTSUMMARY"
	OP_GETFILECHECKSUM       = "GETFILECHECKSUM"
	OP_GETDELEGATIONTOKEN    = "GETDELEGATIONTOKEN"
	OP_GETDELEGATIONTOKENS   = "GETDELEGATIONTOKENS"
	OP_RENEWDELEGATIONTOKEN  = "RENEWDELEGATIONTOKEN"
	OP_CANCELDELEGATIONTOKEN = "CANCELDELEGATIONTOKEN"
)

const WebHdfsApi string = "/webhdfs/v1/"

Constant

Types

type ContentSummary

type ContentSummary struct {
	DirectoryCount int64
	FileCount      int64
	Length         int64
	Quota          int64
	SpaceConsumed  int64
	SpaceQuota     int64
}

Type for HDFS FileSystem content summary (FileSystem.getContentSummary())
See http://hadoop.apache.org/docs/r2.2.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#ContentSummary_JSON_Schema

Example:

{
  "ContentSummary":
  {
    "directoryCount": 2,
    "fileCount"     : 1,
    "length"        : 24930,
    "quota"         : -1,
    "spaceConsumed" : 24930,
    "spaceQuota"    : -1
  }
}

type FileChecksum

type FileChecksum struct {
	Algorithm string
	Bytes     string
	Length    int64
}

Type for HDFS FileSystem.getFileChecksum()
See http://hadoop.apache.org/docs/r2.2.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#FileChecksum_JSON_Schema

Example:

{
  "FileChecksum":
  {
    "algorithm": "MD5-of-1MD5-of-512CRC32",
    "bytes"    : "eadb10de24aa315748930df6e185c0d ...",
    "length"   : 28
  }
}

type FileStatus

type FileStatus struct {
	AccesTime        int64
	BlockSize        int64
	Group            string
	Length           int64
	ModificationTime int64
	Owner            string
	PathSuffix       string
	Permission       string
	Replication      int64
	Type             string
}

Represents HDFS FileStatus (FileSystem.getStatus()) See http://hadoop.apache.org/docs/r2.2.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#FileStatus_JSON_Schema

Example:

{
  "FileStatus":
  {
    "accessTime"      : 0, 				// integer
    "blockSize"       : 0, 				// integer
    "group"           : "grp",			// string
    "length"          : 0,             	// integer - zero for directories
    "modificationTime": 1320173277227,	// integer
    "owner"           : "webuser",		// string
    "pathSuffix"      : "",				// string
    "permission"      : "777",			// string
    "replication"     : 0,				// integer
    "type"            : "DIRECTORY"    	// string - enum {FILE, DIRECTORY, SYMLINK}
  }
}

type FileStatuses

type FileStatuses struct {
	FileStatus []FileStatus
}

Container type for multiple FileStatus for directory, etc (see HDFS FileSystem.listStatus()) NOTE: the confusing naming and Plurality is to match WebHDFS schema.

type HdfsData

type HdfsData struct {
	Boolean         bool
	FileStatus      FileStatus
	FileStatuses    FileStatuses
	FileChecksum    FileChecksum
	ContentSummary  ContentSummary
	Token           Token
	Tokens          Tokens
	Long            int64
	RemoteException RemoteException
}

Root level struct for data JSON data from WebHDFS.

type Path

type Path struct {
	Name       string  // Relative path representation (/root/leaf)
	RefererUrl url.URL // URL related to path (http://server:port/root/leaf)
}

Represents a remote webHDFS path

type RemoteException

type RemoteException struct {
	Exception     string
	JavaClassName string
	Message       string
}

Example:

{
  "RemoteException":
  {
    "exception"    : "FileNotFoundException",
    "javaClassName": "java.io.FileNotFoundException",
    "message"      : "File does not exist: /foo/a.patch"
  }
}

func (RemoteException) Error

func (re RemoteException) Error() string

Implementation of error type. Returns string representation of RemoteException.

type Token

type Token struct {
	UrlString string
}

Example:

{
  "Token":
  {
    "urlString": "JQAIaG9y..."
  }
}

type Tokens

type Tokens struct {
	Token []Token
}

Container type for Token

type WebHdfs

type WebHdfs struct {
	Config *WebHdfsConfig
	// contains filtered or unexported fields
}

func NewWebHdfs

func NewWebHdfs(config *WebHdfsConfig) (*WebHdfs, error)

func (*WebHdfs) Append

func (h *WebHdfs) Append(localfile string, destination string) error

func (*WebHdfs) Delete

func (h *WebHdfs) Delete(recursive bool, paths ...string) map[string]error

func (*WebHdfs) Get

func (h *WebHdfs) Get(path string, server *colonycore.Server) ([]byte, error)

func (*WebHdfs) GetToLocal

func (h *WebHdfs) GetToLocal(path string, destination string, permission string, server *colonycore.Server) error

func (*WebHdfs) List

func (h *WebHdfs) List(path string) (*HdfsData, error)

func (*WebHdfs) MakeDir

func (h *WebHdfs) MakeDir(path string, permission string) error

func (*WebHdfs) MakeDirs

func (h *WebHdfs) MakeDirs(paths []string, permission string) map[string]error

func (*WebHdfs) Put

func (h *WebHdfs) Put(localfile string, destination string, permission string, parms map[string]string, server *colonycore.Server) error

func (*WebHdfs) PutDir

func (h *WebHdfs) PutDir(dirname string, destination string, server *colonycore.Server) (error, map[string]error)

func (*WebHdfs) Puts

func (h *WebHdfs) Puts(paths []string, destinationFolder string, permission string, parms map[string]string, server *colonycore.Server) map[string]error

func (*WebHdfs) Rename

func (h *WebHdfs) Rename(path string, destination string) error

func (*WebHdfs) SetOwner

func (h *WebHdfs) SetOwner(path string, owner string, group string) error

func (*WebHdfs) SetPermission

func (h *WebHdfs) SetPermission(path string, permission string) error

type WebHdfsConfig

type WebHdfsConfig struct {
	Host     string
	UserId   string
	Password string
	Token    string
	Method   string
	TimeOut  time.Duration
	PoolSize int
}

func NewHdfsConfig

func NewHdfsConfig(host, userid string) *WebHdfsConfig