hdfsbackend

Documentation

Overview

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• type Client
  • func NewClient(config Config, opts ...Option) (*Client, error)
  • func (c *Client) Download(namespace, name string, dst io.Writer) error
  • func (c *Client) List(prefix string) ([]string, error)
  • func (c *Client) Stat(namespace, name string) (*core.BlobInfo, error)
  • func (c *Client) Upload(namespace, name string, src io.Reader) error
• type Config
• type Option
  • func WithWebHDFS(w webhdfs.Client) Option

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client is a backend.Client for HDFS.

func NewClient

func NewClient(config Config, opts ...Option) (*Client, error)

NewClient creates a new Client for HDFS.

func (*Client) Download

func (c *Client) Download(namespace, name string, dst io.Writer) error

Download downloads name into dst.

func (*Client) List

func (c *Client) List(prefix string) ([]string, error)

List lists names which start with prefix.

func (*Client) Stat

func (c *Client) Stat(namespace, name string) (*core.BlobInfo, error)

Stat returns blob info for name.

func (*Client) Upload

func (c *Client) Upload(namespace, name string, src io.Reader) error

Upload uploads src to name.

type Config

type Config struct {
	NameNodes     []string `yaml:"namenodes"`
	UserName      string   `yaml:"username"`
	RootDirectory string   `yaml:"root_directory"`

	// ListConcurrency is the number of threads used for listing.
	ListConcurrency int `yaml:"list_concurrency"`

	// NamePath identifies which namepath.Pather to use.
	NamePath string `yaml:"name_path"`

	// UploadDirectory is scratch space, relative to RootDirectory, used for
	// uploading files before moving them to content-addressable storage. Avoids
	// partial uploads corrupting the content-addressable storage space.
	UploadDirectory string `yaml:"upload_directory"`

	WebHDFS webhdfs.Config `yaml:"webhdfs"`
	// contains filtered or unexported fields
}

Config defines configuration for all HDFS clients.

type Option

type Option func(*Client)

Option allows setting optional Client parameters.

func WithWebHDFS

func WithWebHDFS(w webhdfs.Client) Option

WithWebHDFS configures a Client with a custom webhdfs implementation.