v3io

package
v0.2.28 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 9, 2022 License: Apache-2.0 Imports: 10 Imported by: 27

Documentation

Overview

Copyright 2019 Iguazio Systems Ltd.

Licensed under the Apache License, Version 2.0 (the "License") with an addition restriction as set forth herein. You may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0.

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

In addition, you may not use the software for any purposes that are illegal under applicable law, and the grant of the foregoing license under the Apache 2.0 license is conditioned upon your compliance with such restriction.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type CheckPathExistsInput 

type CheckPathExistsInput struct {
	DataPlaneInput
	Path string
}

type ChunkMetadata 

type ChunkMetadata struct {
	ChunkSeqNumber       uint64 `json:"ChunkSequenceNumber"`
	LengthInBytes        uint64 `json:"LengthInBytes"`
	FirstRecordSeqNumber uint64 `json:"FirstRecordSequenceNumber"`
	FirstRecordTimeSecs  uint64 `json:"FirstRecordTimeSec"`
	FirstRecordTimeNSecs uint64 `json:"FirstRecordTimeNSec"`
}

type CommonPrefix 

type CommonPrefix struct {
	Prefix                 string   `xml:"Prefix"`               // directory name
	LastModified           string   `xml:"LastModified"`         // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z"
	AccessTime             string   `xml:"AccessTime"`           // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z"
	CreatingTime           string   `xml:"CreatingTime"`         // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z"
	Mode                   FileMode `xml:"Mode"`                 // octal number, e.g. 040775
	GID                    string   `xml:"GID"`                  // Hexadecimal representation of GID (e.g. "3e8" -> i.e. "0x3e8" == 1000)
	UID                    string   `xml:"UID"`                  // Hexadecimal representation of UID (e.g. "3e8" -> i.e. "0x3e8" == 1000)
	InodeNumber            *uint64  `xml:"InodeNumber"`          // iNode number
	ShardCount             int      `xml:"ShardCount"`           // For stream-dirs only - the number of shards in the stream
	RetentionPeriodHours   int      `xml:"RetentionPeriodHours"` // For stream-dirs only - the shard retention (in hours)
	RetentionPeriodSeconds int      `xml:"RetentionPeriodSec"`   // For stream-dirs only - the shard retention (in seconds)
}

type Container 

type Container interface {

	// GetContainers
	GetClusterMD(*GetClusterMDInput, interface{}, chan *Response) (*Request, error)

	// GetContainersSync
	GetClusterMDSync(*GetClusterMDInput) (*Response, error)

	// GetContainers
	GetContainers(*GetContainersInput, interface{}, chan *Response) (*Request, error)

	// GetContainersSync
	GetContainersSync(*GetContainersInput) (*Response, error)

	// GetContainers
	GetContainerContents(*GetContainerContentsInput, interface{}, chan *Response) (*Request, error)

	// GetContainerContentsSync
	GetContainerContentsSync(*GetContainerContentsInput) (*Response, error)

	//
	// Object
	//
	// CheckPathExists
	CheckPathExists(*CheckPathExistsInput, interface{}, chan *Response) (*Request, error)

	// CheckPathExistsSync
	CheckPathExistsSync(*CheckPathExistsInput) error

	// GetObject
	GetObject(*GetObjectInput, interface{}, chan *Response) (*Request, error)

	// GetObjectSync
	GetObjectSync(*GetObjectInput) (*Response, error)

	// PutObject
	PutObject(*PutObjectInput, interface{}, chan *Response) (*Request, error)

	// PutObjectSync
	PutObjectSync(*PutObjectInput) error

	// UpdateObjectSync
	UpdateObjectSync(*UpdateObjectInput) error

	// DeleteObject
	DeleteObject(*DeleteObjectInput, interface{}, chan *Response) (*Request, error)

	// DeleteObjectSync
	DeleteObjectSync(*DeleteObjectInput) error

	// GetItem
	GetItem(*GetItemInput, interface{}, chan *Response) (*Request, error)

	// GetItemSync
	GetItemSync(*GetItemInput) (*Response, error)

	// GetItems
	GetItems(*GetItemsInput, interface{}, chan *Response) (*Request, error)

	// GetItemSync
	GetItemsSync(*GetItemsInput) (*Response, error)

	// PutItem
	PutItem(*PutItemInput, interface{}, chan *Response) (*Request, error)

	// PutItemSync
	PutItemSync(*PutItemInput) (*Response, error)

	// PutItems
	PutItems(*PutItemsInput, interface{}, chan *Response) (*Request, error)

	// PutItemsSync
	PutItemsSync(*PutItemsInput) (*Response, error)

	// UpdateItem
	UpdateItem(*UpdateItemInput, interface{}, chan *Response) (*Request, error)

	// UpdateItemSync
	UpdateItemSync(*UpdateItemInput) (*Response, error)

	// CreateStream
	CreateStream(*CreateStreamInput, interface{}, chan *Response) (*Request, error)

	// CreateStreamSync
	CreateStreamSync(*CreateStreamInput) error

	// DescribeStream
	DescribeStream(*DescribeStreamInput, interface{}, chan *Response) (*Request, error)

	// DescribeStreamSync
	DescribeStreamSync(*DescribeStreamInput) (*Response, error)

	// DeleteStream
	DeleteStream(*DeleteStreamInput, interface{}, chan *Response) (*Request, error)

	// DeleteStreamSync
	DeleteStreamSync(*DeleteStreamInput) error

	// SeekShard
	SeekShard(*SeekShardInput, interface{}, chan *Response) (*Request, error)

	// SeekShardSync
	SeekShardSync(*SeekShardInput) (*Response, error)

	// PutRecords
	PutRecords(*PutRecordsInput, interface{}, chan *Response) (*Request, error)

	// PutRecordsSync
	PutRecordsSync(*PutRecordsInput) (*Response, error)

	// PutChunk
	PutChunk(*PutChunkInput, interface{}, chan *Response) (*Request, error)

	// PutChunkSync
	PutChunkSync(input *PutChunkInput) error

	// GetRecords
	GetRecords(*GetRecordsInput, interface{}, chan *Response) (*Request, error)

	// GetRecordsSync
	GetRecordsSync(*GetRecordsInput) (*Response, error)

	// PutOOSObject
	PutOOSObject(*PutOOSObjectInput, interface{}, chan *Response) (*Request, error)

	// PutOOSObjectSync
	PutOOSObjectSync(*PutOOSObjectInput) error
}

A container interface allows perform actions against a container

type ContainerInfo 

type ContainerInfo struct {
	BucketName   xml.Name `xml:"Bucket"`
	Name         string   `xml:"Name"`
	CreationDate string   `xml:"CreationDate"`
	ID           int      `xml:"Id"`
}

type Containers 

type Containers struct {
	Name       xml.Name        `xml:"Buckets"`
	Containers []ContainerInfo `xml:"Bucket"`
}

type Content 

type Content struct {
	Key            string `xml:"Key"`
	Size           *int   `xml:"Size"`           // file size in bytes
	LastSequenceID *int   `xml:"LastSequenceId"` // greater than zero for shard files
	LastModified   string `xml:"LastModified"`   // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z"

	Mode         FileMode `xml:"Mode"`         // octal (ListDir) or decimal (GetItems) base, depends on API, e.g. 33204 or 0100664
	AccessTime   string   `xml:"AccessTime"`   // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z"
	CreatingTime string   `xml:"CreatingTime"` // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z"
	GID          string   `xml:"GID"`          // Hexadecimal representation of GID (e.g. "3e8" -> i.e. "0x3e8" == 1000)
	UID          string   `xml:"UID"`          // Hexadecimal representation of UID (e.g. "3e8" -> i.e. "0x3e8" == 1000)
	InodeNumber  *uint32  `xml:"InodeNumber"`  // iNode number
}

type Context 

type Context interface {
	Container

	// create a new session
	NewSession(*NewSessionInput) (Session, error)
}

type CreateStreamInput 

type CreateStreamInput struct {
	DataPlaneInput
	Path                 string
	ShardCount           int
	RetentionPeriodHours int
}

type CurrentChunkMetadata 

type CurrentChunkMetadata struct {
	ChunkSeqNumber       uint32 `json:"ChunkSequenceNumber"`
	OffsetAfterJob       uint64 `json:"OffsetAfterJob"`
	SeqNumberAfterJob    uint64 `json:"SequenceNumberAfterJob"`
	FirstRecordTimeSec   uint32 `json:"FirstRecordTimeSec"`
	LatestRecordTimeSec  uint64 `json:"LatestRecordTimeSec"`
	LatestRecordTimeNSec uint64 `json:"LatestRecordTimeNSec"`
}

type DataPlaneInput 

type DataPlaneInput struct {
	Ctx                    context.Context
	URL                    string
	ContainerName          string
	AuthenticationToken    string
	AccessKey              string
	MtimeSec               string
	MtimeNsec              string
	Timeout                time.Duration
	IncludeResponseInError bool
}

type DataPlaneOutput 

type DataPlaneOutput struct {
	// contains filtered or unexported fields
}

type DeleteObjectInput 

type DeleteObjectInput struct {
	DataPlaneInput
	Path string
}

type DeleteStreamInput 

type DeleteStreamInput struct {
	DataPlaneInput
	Path string
}

type DescribeStreamInput 

type DescribeStreamInput struct {
	DataPlaneInput
	Path string
}

type DescribeStreamOutput 

type DescribeStreamOutput struct {
	DataPlaneOutput
	ShardCount           int
	RetentionPeriodHours int
}

type DirAttributes 

type DirAttributes struct {
	Mode      int `json:"mode,omitempty"`
	UID       int `json:"uid"`
	GID       int `json:"gid"`
	AtimeSec  int `json:"atime.sec,omitempty"`
	AtimeNSec int `json:"atime.nsec"`
	CtimeSec  int `json:"ctime.sec,omitempty"`
	CtimeNSec int `json:"ctime.nsec"`
	MtimeSec  int `json:"mtime.sec,omitempty"`
	MtimeNSec int `json:"mtime.nsec"`
}

type FileMode 

type FileMode string

func (FileMode) FileMode 

func (vfm FileMode) FileMode() (os.FileMode, error)

func (FileMode) String 

func (vfm FileMode) String() string

type GetClusterMDInput 

type GetClusterMDInput struct {
	DataPlaneInput
}

type GetClusterMDOutput 

type GetClusterMDOutput struct {
	DataPlaneOutput
	NumberOfVNs int
}

type GetContainerContentsInput 

type GetContainerContentsInput struct {
	DataPlaneInput
	Path             string
	GetAllAttributes bool   // if "true" return ALL available attributes
	DirectoriesOnly  bool   // if "true" return directory entries only, otherwise return children of any kind
	Limit            int    // max number of entries per request
	Marker           string // start from specific entry (e.g. to get next chunk)
}

type GetContainerContentsOutput 

type GetContainerContentsOutput struct {
	Name           string         `xml:"Name"`           // Bucket name
	NextMarker     string         `xml:"NextMarker"`     // if not empty and isTruncated="true" - has more children (need another fetch to get them)
	MaxKeys        string         `xml:"MaxKeys"`        // max number of entries in single batch
	Contents       []Content      `xml:"Contents"`       // files
	CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` // directories
	IsTruncated    bool           `xml:"IsTruncated"`    // "true" if has more content. Note, "NextMarker" should not be empty if "true"
}

type GetContainersInput 

type GetContainersInput struct {
	DataPlaneInput
}

type GetContainersOutput 

type GetContainersOutput struct {
	DataPlaneOutput
	XMLName xml.Name    `xml:"ListBucketResult"`
	Owner   interface{} `xml:"Owner"`
	Results Containers  `xml:"Buckets"`
}

type GetItemInput 

type GetItemInput struct {
	DataPlaneInput
	Path           string
	AttributeNames []string
}

type GetItemOutput 

type GetItemOutput struct {
	DataPlaneOutput
	Item Item
}

type GetItemsInput 

type GetItemsInput struct {
	DataPlaneInput
	Path                string
	TableName           string
	AttributeNames      []string
	Filter              string
	Marker              string
	ShardingKey         string
	Limit               int
	Segment             int
	TotalSegments       int
	SortKeyRangeStart   string
	SortKeyRangeEnd     string
	AllowObjectScatter  string
	ReturnData          string
	ReturnAllInodes     bool
	DataMaxSize         int
	RequestJSONResponse bool `json:"RequestJsonResponse"`
	ChokeGetItemsMS     int
}

type GetItemsOutput 

type GetItemsOutput struct {
	DataPlaneOutput
	Last       bool
	NextMarker string
	Scattered  bool
	Items      []Item
}

type GetObjectInput 

type GetObjectInput struct {
	DataPlaneInput
	Path      string
	Offset    int
	NumBytes  int
	CtimeSec  int
	CtimeNsec int
}

type GetRecordsInput 

type GetRecordsInput struct {
	DataPlaneInput
	Path     string
	Location string
	Limit    int
}

type GetRecordsOutput 

type GetRecordsOutput struct {
	DataPlaneOutput
	NextLocation        string
	MSecBehindLatest    int
	RecordsBehindLatest int
	Records             []GetRecordsResult
}

type GetRecordsResult 

type GetRecordsResult struct {
	ArrivalTimeSec  int
	ArrivalTimeNSec int
	SequenceNumber  uint64
	ClientInfo      []byte
	PartitionKey    string
	Data            []byte
}

type Item 

type Item map[string]interface{}

func (Item) GetField 

func (i Item) GetField(name string) interface{}

func (Item) GetFieldInt 

func (i Item) GetFieldInt(name string) (int, error)

func (Item) GetFieldString 

func (i Item) GetFieldString(name string) (string, error)

func (Item) GetFieldUint64 

func (i Item) GetFieldUint64(name string) (uint64, error)

func (Item) GetShard 

func (i Item) GetShard() (map[int]*ItemChunk, *ItemCurrentChunkMetadata, error)

For internal use only - DO NOT USE!

type ItemChunk 

type ItemChunk struct {
	Metadata *ItemChunkMetadata
	Data     []*ItemChunkData
}

type ItemChunkData 

type ItemChunkData struct {
	Offset uint64
	Data   *[]byte
}

type ItemChunkMetadata 

type ItemChunkMetadata struct {
	OSSID                uint32
	OSDID                uint32
	StoredHandle         uint64
	ChunkSeqNumber       uint64
	FirstRecordSeqNumber uint64
	FirstRecordTsSec     uint64
	FirstRecordTsNSec    uint64
	LengthInBytes        uint64
}

type ItemCurrentChunkMetadata 

type ItemCurrentChunkMetadata struct {
	NextRecordSeqNumber         uint64
	CurrentChunkLengthBytes     uint64
	CurrentChunkSeqNumber       uint32
	ChunkGranularity            uint32
	NumStoredChunks             uint16
	FirstRecordOnChunkSec       uint32
	LatestRecordArrivalTimeSec  uint64
	LatestRecordArrivalTimeNSec uint64
}

type ItemsCursor 

type ItemsCursor struct {
	// contains filtered or unexported fields
}

func NewItemsCursor 

func NewItemsCursor(container Container, getItemsInput *GetItemsInput) (*ItemsCursor, error)

func (*ItemsCursor) AllSync 

func (ic *ItemsCursor) AllSync() ([]Item, error)

gets all items

func (*ItemsCursor) Err 

func (ic *ItemsCursor) Err() error

Err returns the last error

func (*ItemsCursor) GetField 

func (ic *ItemsCursor) GetField(name string) interface{}

func (*ItemsCursor) GetFieldInt 

func (ic *ItemsCursor) GetFieldInt(name string) (int, error)

func (*ItemsCursor) GetFieldString 

func (ic *ItemsCursor) GetFieldString(name string) (string, error)

func (*ItemsCursor) GetFields 

func (ic *ItemsCursor) GetFields() map[string]interface{}

func (*ItemsCursor) GetItem 

func (ic *ItemsCursor) GetItem() Item

func (*ItemsCursor) NextItemSync 

func (ic *ItemsCursor) NextItemSync() (Item, error)

NextItem gets the next matching item. this may potentially block as this lazy loads items from the collection

func (*ItemsCursor) NextSync 

func (ic *ItemsCursor) NextSync() bool

Next gets the next matching item. this may potentially block as this lazy loads items from the collection

func (*ItemsCursor) Release 

func (ic *ItemsCursor) Release()

Release releases a cursor and its underlying resources

func (*ItemsCursor) Scattered 

func (ic *ItemsCursor) Scattered() bool

type NewContainerInput 

type NewContainerInput struct {
	ContainerName string
}

type NewSessionInput 

type NewSessionInput struct {
	URL       string
	Username  string
	Password  string
	AccessKey string
}

type PutChunkInput 

type PutChunkInput struct {
	DataPlaneInput       `json:"-"`
	Path                 string                `json:"-"`
	ChunkSeqNumber       int                   `json:"ChunkSequenceNumber,omitempty"`
	Offset               uint64                `json:"Offset,omitempty"`
	Data                 []byte                `json:"Data,omitempty"`
	ChunksMetadata       []*ChunkMetadata      `json:"Metadata,omitempty"`
	CurrentChunkMetadata *CurrentChunkMetadata `json:"CurrentMetadata,omitempty"`
}

type PutItemInput 

type PutItemInput struct {
	DataPlaneInput
	Path       string
	Condition  string
	Attributes map[string]interface{}
	UpdateMode string
}

type PutItemOutput 

type PutItemOutput struct {
	DataPlaneInput
	MtimeSecs  int
	MtimeNSecs int
}

type PutItemsInput 

type PutItemsInput struct {
	DataPlaneInput
	Path      string
	Condition string
	Items     map[string]map[string]interface{}
}

type PutItemsOutput 

type PutItemsOutput struct {
	DataPlaneOutput
	Success bool
	Errors  map[string]error
}

type PutOOSObjectInput 

type PutOOSObjectInput struct {
	DataPlaneInput
	Path   string
	Header []byte
	Data   [][]byte
}

type PutObjectInput 

type PutObjectInput struct {
	DataPlaneInput
	Path   string
	Offset int
	Body   []byte
	Append bool
}

type PutRecordResult 

type PutRecordResult struct {
	SequenceNumber uint64
	ShardID        int `json:"ShardId"`
	ErrorCode      int
	ErrorMessage   string
}

type PutRecordsInput 

type PutRecordsInput struct {
	DataPlaneInput
	Path    string
	Records []*StreamRecord
}

type PutRecordsOutput 

type PutRecordsOutput struct {
	DataPlaneOutput
	FailedRecordCount int
	Records           []PutRecordResult
}

type Request 

type Request struct {
	ID uint64

	// holds the input (e.g. ListBucketInput, GetItemInput)
	Input interface{}

	// a user supplied context
	Context interface{}

	// the channel to which the response must be posted
	ResponseChan chan *Response

	// pointer to container
	RequestResponse *RequestResponse

	// Request time
	SendTimeNanoseconds int64
}

type RequestResponse 

type RequestResponse struct {
	Request  Request
	Response Response
}

holds both a request and response

type Response 

type Response struct {

	// hold a decoded output, if any
	Output interface{}

	// Equal to the ID of request
	ID uint64

	// holds the error for async responses
	Error error

	// a user supplied context
	Context interface{}

	// pointer to container
	RequestResponse *RequestResponse

	// HTTP
	HTTPResponse *fasthttp.Response
}

func (*Response) Body 

func (r *Response) Body() []byte

func (*Response) HeaderPeek 

func (r *Response) HeaderPeek(key string) []byte

func (*Response) Release 

func (r *Response) Release()

func (*Response) Request 

func (r *Response) Request() *Request

type SeekShardInput 

type SeekShardInput struct {
	DataPlaneInput
	Path                   string
	Type                   SeekShardInputType
	StartingSequenceNumber uint64
	Timestamp              int
}

type SeekShardInputType 

type SeekShardInputType int
const (
	SeekShardInputTypeTime SeekShardInputType = iota
	SeekShardInputTypeSequence
	SeekShardInputTypeLatest
	SeekShardInputTypeEarliest
)

type SeekShardOutput 

type SeekShardOutput struct {
	DataPlaneOutput
	Location string
}

type Session 

type Session interface {

	// NewContainer creates a container
	NewContainer(*NewContainerInput) (Container, error)
}

type StreamRecord 

type StreamRecord struct {
	ShardID        *int
	Data           []byte
	ClientInfo     []byte
	PartitionKey   string
	SequenceNumber uint64
}

type UpdateItemInput 

type UpdateItemInput struct {
	DataPlaneInput
	Path       string
	Attributes map[string]interface{}
	Expression *string
	Condition  string
	UpdateMode string
}

type UpdateItemOutput 

type UpdateItemOutput struct {
	DataPlaneInput
	MtimeSecs  int
	MtimeNSecs int
}

type UpdateObjectInput 

type UpdateObjectInput struct {
	DataPlaneInput
	Path          string
	DirAttributes *DirAttributes
}

Source Files

View all Source files

Directories

Path Synopsis
schemas
node/common

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.