containerserver

Documentation

Index

• Variables
• func NewReplicator(serverconf conf.Config, flags *flag.FlagSet, cnf srv.ConfigLoader) (*srv.IpPort, srv.Server, srv.LowLevelLogger, error)
• func NewServer(serverconf conf.Config, flags *flag.FlagSet, cnf srv.ConfigLoader) (*srv.IpPort, srv.Server, srv.LowLevelLogger, error)
• func SendPriRepJob(job *PriorityRepJob, client *http.Client) (string, bool)
• type Container
• type ContainerEngine
• type ContainerInfo
• type ContainerServer
  • func (server *ContainerServer) AcquireDevice(next http.Handler) http.Handler
  • func (server *ContainerServer) Background(flags *flag.FlagSet) chan struct{}
  • func (server *ContainerServer) ContainerDeleteHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) ContainerGetHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) ContainerPostHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) ContainerPutHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) ContainerReplicateHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) ContainerTmpUploadHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) DiskUsageHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) Finalize()
  • func (server *ContainerServer) GetHandler(config conf.Config, metricsPrefix string) http.Handler
  • func (server *ContainerServer) HealthcheckHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) LogRequest(next http.Handler) http.Handler
  • func (server *ContainerServer) ObjDeleteHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) ObjPutHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) OptionsHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) ReconHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *ContainerServer) Type() string
• type ObjectListingRecord
• type ObjectRecord
• type PriorityRepJob
• type ReplicableContainer
• type Replicator
  • func (server *Replicator) Background(flags *flag.FlagSet) chan struct{}
  • func (server *Replicator) Finalize()
  • func (server *Replicator) GetHandler(config conf.Config, metricsPrefix string) http.Handler
  • func (server *Replicator) HealthcheckHandler(writer http.ResponseWriter, request *http.Request)
  • func (server *Replicator) LogRequest(next http.Handler) http.Handler
  • func (r *Replicator) Run()
  • func (r *Replicator) RunForever()
  • func (server *Replicator) Type() string
• type SubdirListingRecord
• type SyncRecord

Variables

var (
	// ErrorNoSuchContainer is returned when a requested container doesn't exist.
	ErrorNoSuchContainer = fmt.Errorf("No such container.")
	// ErrorInvalidMetadata is returned for errors that violate the API metadata constraints.
	ErrorInvalidMetadata = fmt.Errorf("Invalid metadata value")
	// ErrorPolicyConflict is returned when an operation conflicts with the container's existing policy.
	ErrorPolicyConflict = fmt.Errorf("Policy conflicts with existing value")
)

var (

	// GetRing is a local pointer to the hummingbird function, for overriding in tests.
	GetRing = ring.GetRing
)

Functions

func NewReplicator

func NewReplicator(serverconf conf.Config, flags *flag.FlagSet, cnf srv.ConfigLoader) (*srv.IpPort, srv.Server, srv.LowLevelLogger, error)

NewReplicator uses the config settings and command-line flags to configure and return a replicator daemon struct.

func NewServer

func NewServer(serverconf conf.Config, flags *flag.FlagSet, cnf srv.ConfigLoader) (*srv.IpPort, srv.Server, srv.LowLevelLogger, error)

NewServer parses configs and command-line flags, returning a configured server object and the ip and port it should bind on.

func SendPriRepJob

func SendPriRepJob(job *PriorityRepJob, client *http.Client) (string, bool)

TODO

Types

type Container

type Container interface {
	// GetInfo returns the ContainerInfo struct for the container.
	GetInfo() (*ContainerInfo, error)
	// IsDeleted returns true if the container has been deleted.
	IsDeleted() (bool, error)
	// Delete deletes the container.
	Delete(timestamp string) error
	// ListObjects lists the container's object entries.
	ListObjects(limit int, marker string, endMarker string, prefix string, delimiter string, path *string, reverse bool, storagePolicyIndex int) ([]interface{}, error)
	// GetMetadata returns the container's current metadata.
	GetMetadata() (map[string]string, error)
	// UpdateMetadata applies updates to the container's metadata.
	UpdateMetadata(updates map[string][]string, timestamp string) error
	// PutObject adds a new object to the container.
	PutObject(name string, timestamp string, size int64, contentType string, etag string, storagePolicyIndex int) error
	// DeleteObject deletes an object from the container.
	DeleteObject(name string, timestamp string, storagePolicyIndex int) error
	// ID returns a unique identifier for the container.
	ID() string
	// Close frees any resources associated with the container.
	Close() error
}

Container is the interface implemented by a container.

type ContainerEngine

type ContainerEngine interface {
	// GET returns a container, given a vars mapping.
	Get(vars map[string]string) (c Container, err error)
	// Return returns a Container to the engine, where it can close or retain them as it sees fit.
	Return(c Container)
	// Create creates a new container.  Returns true if the container was created and a pointer to the container.
	Create(vars map[string]string, putTimestamp string, metadata map[string][]string, policyIndex, defaultPolicyIndex int) (bool, Container, error)
	// Close releases all cached containers and any other retained resources.
	Close()

	// GetByHash returns a replicable database given its hash.  This will probably move from this interface once we
	// have replicator->replicator communication.
	GetByHash(device, hash, partition string) (c ReplicableContainer, err error)
	// Invalidate removes a container from the cache entirely.  This will probably also move, since it's only used by replication.
	Invalidate(c Container)

	// Number of containers that have been Get()ted but not Return()ed
	OpenCount() (count int)
}

ContainerEngine is the interface of an object that creates and returns containers.

type ContainerInfo

type ContainerInfo struct {
	Account                 string              `json:"account"`
	Container               string              `json:"container"`
	CreatedAt               string              `json:"created_at"`
	PutTimestamp            string              `json:"put_timestamp"`
	DeleteTimestamp         string              `json:"delete_timestamp"`
	StatusChangedAt         string              `json:"status_changed_at"`
	ObjectCount             int64               `json:"count"`
	BytesUsed               int64               `json:"bytes_used"`
	ReportedPutTimestamp    string              `json:"-"`
	ReportedDeleteTimestamp string              `json:"-"`
	ReportedObjectCount     int64               `json:"-"`
	ReportedBytesUsed       int64               `json:"-"`
	Hash                    string              `json:"hash"`
	ID                      string              `json:"id"`
	XContainerSyncPoint1    string              `json:"-"`
	XContainerSyncPoint2    string              `json:"-"`
	StoragePolicyIndex      int                 `json:"storage_policy_index"`
	RawMetadata             string              `json:"metadata"`
	Metadata                map[string][]string `json:"-"`
	MaxRow                  int64               `json:"max_row"`

	// This row isn't populated by GetInfo, it only exists for the times this is
	// serialized during replication.
	Point int64 `json:"point"`
	// contains filtered or unexported fields
}

ContainerInfo represents the container_info database record - basic information about the container.

type ContainerServer

type ContainerServer struct {
	// contains filtered or unexported fields
}

ContainerServer contains all of the information for a running container server.

func (*ContainerServer) AcquireDevice

func (server *ContainerServer) AcquireDevice(next http.Handler) http.Handler

AcquireDevice is a middleware that makes sure the device is available - mounted and not beyond its max concurrency.

func (*ContainerServer) Background

func (server *ContainerServer) Background(flags *flag.FlagSet) chan struct{}

func (*ContainerServer) ContainerDeleteHandler

func (server *ContainerServer) ContainerDeleteHandler(writer http.ResponseWriter, request *http.Request)

ContainerDeleteHandler handles DELETE requests for the container.

func (*ContainerServer) ContainerGetHandler

func (server *ContainerServer) ContainerGetHandler(writer http.ResponseWriter, request *http.Request)

ContainerGetHandler handles GET and HEAD requests for a container.

func (*ContainerServer) ContainerPostHandler

func (server *ContainerServer) ContainerPostHandler(writer http.ResponseWriter, request *http.Request)

ContainerPostHandler handles POST requests for a container.

func (*ContainerServer) ContainerPutHandler

func (server *ContainerServer) ContainerPutHandler(writer http.ResponseWriter, request *http.Request)

ContainerPutHandler handles PUT requests for a container.

func (*ContainerServer) ContainerReplicateHandler

func (server *ContainerServer) ContainerReplicateHandler(writer http.ResponseWriter, request *http.Request)

ContainerReplicateHandler handles the REPLICATE call for containers.

func (*ContainerServer) ContainerTmpUploadHandler

func (server *ContainerServer) ContainerTmpUploadHandler(writer http.ResponseWriter, request *http.Request)

ContainerTmpUploadHandler handles uploading container files to the tmp directory for various replication strategies. This replaces the swift replicator's use of rsync.

func (*ContainerServer) DiskUsageHandler

func (server *ContainerServer) DiskUsageHandler(writer http.ResponseWriter, request *http.Request)

DiskUsageHandler returns information on the current outstanding HTTP requests per-disk.

func (*ContainerServer) Finalize

func (server *ContainerServer) Finalize()

func (*ContainerServer) GetHandler

func (server *ContainerServer) GetHandler(config conf.Config, metricsPrefix string) http.Handler

GetHandler returns the server's http handler - it sets up routes and instantiates middleware.

func (*ContainerServer) HealthcheckHandler

func (server *ContainerServer) HealthcheckHandler(writer http.ResponseWriter, request *http.Request)

HealthcheckHandler implements a basic health check, that just returns "OK".

func (*ContainerServer) LogRequest

func (server *ContainerServer) LogRequest(next http.Handler) http.Handler

LogRequest is a middleware that logs requests and also sets up a logger in the request context.

func (*ContainerServer) ObjDeleteHandler

func (server *ContainerServer) ObjDeleteHandler(writer http.ResponseWriter, request *http.Request)

ObjDeleteHandler handles the DELETE of object records in a container.

func (*ContainerServer) ObjPutHandler

func (server *ContainerServer) ObjPutHandler(writer http.ResponseWriter, request *http.Request)

ObjPutHandler handles the PUT of object records to a container.

func (*ContainerServer) OptionsHandler

func (server *ContainerServer) OptionsHandler(writer http.ResponseWriter, request *http.Request)

OptionsHandler delegates incoming OPTIONS calls to the common options handler.

func (*ContainerServer) ReconHandler

func (server *ContainerServer) ReconHandler(writer http.ResponseWriter, request *http.Request)

ReconHandler delegates incoming /recon calls to the common recon handler.

func (*ContainerServer) Type

func (server *ContainerServer) Type() string

type ObjectListingRecord

type ObjectListingRecord struct {
	XMLName      xml.Name `xml:"object" json:"-"`
	Name         string   `xml:"name" json:"name"`
	LastModified string   `xml:"last_modified" json:"last_modified"`
	Size         int64    `xml:"bytes" json:"bytes"`
	ContentType  string   `xml:"content_type" json:"content_type"`
	ETag         string   `xml:"hash" json:"hash"`
}

ObjectListingRecord is the struct used for serializing objects in json and xml container listings.

type ObjectRecord

type ObjectRecord struct {
	Rowid              int64  `json:"ROWID"`
	Name               string `json:"name"`
	CreatedAt          string `json:"created_at"`
	Size               int64  `json:"size"`
	ContentType        string `json:"content_type"`
	ETag               string `json:"etag"`
	Deleted            int    `json:"deleted"`
	StoragePolicyIndex int    `json:"storage_policy_index"`
}

ObjectRecord represents the object's data in-databaee, it is used by replication.

type PriorityRepJob

type PriorityRepJob struct {
	Partition  uint64       `json:"partition"`
	FromDevice *ring.Device `json:"from_device"`
	ToDevice   *ring.Device `json:"to_device"`
}

type ReplicableContainer

type ReplicableContainer interface {
	Container
	// MergeItems merges object records into the container, with an optional remoteId.
	MergeItems(records []*ObjectRecord, remoteID string) error
	// ItemsSince returns count object records with a ROWID greater than start.
	ItemsSince(start int64, count int) ([]*ObjectRecord, error)
	// MergeSyncTable updates the container's incoming sync tables with new data.
	MergeSyncTable(records []*SyncRecord) error
	// SyncTable returns the container's current sync table.
	SyncTable() ([]*SyncRecord, error)
	// SyncRemoteData accepts a remote container's current status information and updates local records accordingly.
	SyncRemoteData(maxRow int64, hash, id, createdAt, putTimestamp, deleteTimestamp, metadata string) (*ContainerInfo, error)
	// NewID gives the database a new unique identifier, which is used for incoming_sync entries.
	NewID() error
	// OpenDatabaseFile returns a consistent reader for the underlying database file.
	OpenDatabaseFile() (*os.File, func(), error)
	// CleanupTombstones removes any metadata and object tombstones older than reclaimAge seconds.
	CleanupTombstones(reclaimAge int64) error
	// CheckSyncLinks makes sure container sync symlinks are correct for the database.
	CheckSyncLink() error
	// RingHash returns the container's ring hash.
	RingHash() string
	// Reported records the information as having been reported to an account database.
	Reported(putTimestamp, deleteTimestamp string, objectCount, bytesUsed int64) error
}

ReplicableContainer is a container that also implements the replication API.

type Replicator

type Replicator struct {
	Ring ring.Ring
	// contains filtered or unexported fields
}

Replicator is the container replicator daemon object

func (*Replicator) Background

func (server *Replicator) Background(flags *flag.FlagSet) chan struct{}

func (*Replicator) Finalize

func (server *Replicator) Finalize()

func (*Replicator) GetHandler

func (server *Replicator) GetHandler(config conf.Config, metricsPrefix string) http.Handler

func (*Replicator) HealthcheckHandler

func (server *Replicator) HealthcheckHandler(writer http.ResponseWriter, request *http.Request)

func (*Replicator) LogRequest

func (server *Replicator) LogRequest(next http.Handler) http.Handler

func (*Replicator) Run

func (r *Replicator) Run()

Run runs a pass of the replicator once.

func (*Replicator) RunForever

func (r *Replicator) RunForever()

RunForever runs the replicator in a forever-loop.

func (*Replicator) Type

func (server *Replicator) Type() string

type SubdirListingRecord

type SubdirListingRecord struct {
	XMLName xml.Name `xml:"subdir" json:"-"`
	Name2   string   `xml:"name,attr" json:"-"`
	Name    string   `xml:"name" json:"subdir"`
}

SubdirListingRecord is the struct used for serializing subdirs in json and xml container listings.

type SyncRecord

type SyncRecord struct {
	SyncPoint int64  `json:"sync_point"`
	RemoteID  string `json:"remote_id"`
}

SyncRecord represents a row in the incoming_sync table. It is used by replication.