node

package
v2.0.2-alpha+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 28, 2019 License: BSD-2-Clause Imports: 47 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var DocumentMaxByte = 16777216

mongodb has hard limit of 16 MB docuemnt size

View Source 
var TransitMapMutex = sync.RWMutex{}

TransitMapLock lock write access to the CacheMap

Functions

func AsyncIndexer 

func AsyncIndexer(idxType string, nid string, colNum int, ctx context.Context)

func AzureDownload 

func AzureDownload(uuid string, nodeInstance *Node, location *conf.LocationConfig) (err error, md5sum string)

AzureDownload support for downloading off https://github.com/daos-stack

func DaosDownload 

func DaosDownload(uuid string, nodeInstance *Node) (err error, md5sum string)

DaosDownload support for downloading off https://github.com/daos-stack

func DbFindDistinct 

func DbFindDistinct(q bson.M, d string) (results interface{}, err error)

func FMOpen 

func FMOpen(filepath string) (f *os.File, err error)

FMOpen drop in replacement for os.Open that attempts to download files from locations stored in MongoDB

func GCloudStoreDownload 

func GCloudStoreDownload(uuid string, nodeInstance *Node, location *conf.LocationConfig) (err error, md5sum string)

GCloudStoreDownload support for downloading off https://github.com/daos-stack

func HasAttributeField 

func HasAttributeField(a string) bool

func InitReaper 

func InitReaper()

func Initialize 

func Initialize()

Initialize creates a copy of the mongodb connection and then uses that connection to create the Nodes collection in mongodb. Then, it ensures that there is a unique index on the id key in this collection, creating the index if necessary.

func IsInMappy 

func IsInMappy(item string, mp mappy) bool

func LocationsRequest 

func LocationsRequest(ctx context.Context)

GET, PUT, DELETE: /node/{nid}/locations/{loc} , for PUT send body: {loc}, specify -H "Content-Type: application/json"

func Path2uuid 

func Path2uuid(filepath string) string

Path2uuid extract uuid from path

func ReloadFromDisk 

func ReloadFromDisk(path string) (err error)

func RestoreRequest 

func RestoreRequest(ctx context.Context)

GET, PUT, DELETE: /node/{nid}/restore/ , for PUT send body: {loc}, specify -H "Content-Type: application/json"

func S3Download 

func S3Download(uuid string, nodeInstance *Node, location *conf.LocationConfig) (err error, md5sum string)

S3Download download a file and its indices from an S3 source

func SetFileFromParts 

func SetFileFromParts(id string, compress string, numParts int, allowEmpty bool)

this runs asynchronously and uses FileLockMgr

func ShockDownload 

func ShockDownload(uuid string, nodeInstance *Node, location *conf.LocationConfig) (err error, md5sum string)

ShockDownload download a file from a Shock server

func SortByteArray 

func SortByteArray(b []byte) []byte

func TSMDownload 

func TSMDownload(uuid string, nodeInstance *Node) (err error)

TSMDownload support for downloading files of an existing IBM Tivoli service

func Unzip 

func Unzip(src string, dest string) ([]string, error)

Unzip will decompress a zip archive, moving all files and folders q within the zip file (parameter 1) to an output directory (parameter 2).

Types

type IdxInfo 

type IdxInfo struct {
	Type        string           `bson:"index_type" json:"-"`
	TotalUnits  int64            `bson:"total_units" json:"total_units"`
	AvgUnitSize int64            `bson:"average_unit_size" json:"average_unit_size"`
	Format      string           `bson:"format" json:"-"`
	CreatedOn   time.Time        `bson:"created_on" json:"created_on"`
	Locked      *locker.LockInfo `bson:"-" json:"locked"`
}

type Indexes 

type Indexes map[string]*IdxInfo

type Location 

type Location struct {
	ID            string     `bson:"id" json:"id"`                                           // name of the location, if present data is verified to exist in said location
	Stored        bool       `bson:"stored,omitempty" json:"stored,omitempty"`               //
	RequestedDate *time.Time `bson:"requestedDate,omitempty" json:"requestedDate,omitempty"` // what is the date the data item was send on its way
}

Location a data type to represent storage locations (defined in LocationConfig) and status of data in flight

type Node 

type Node struct {
	Id           string            `bson:"id" json:"id"`
	Version      string            `bson:"version" json:"version"`
	File         file.File         `bson:"file" json:"file"`
	Attributes   interface{}       `bson:"attributes" json:"attributes"`
	Indexes      Indexes           `bson:"indexes" json:"indexes"`
	Acl          acl.Acl           `bson:"acl" json:"-"`
	VersionParts map[string]string `bson:"version_parts" json:"version_parts"`
	Tags         []string          `bson:"tags" json:"tags"`
	Revisions    []Node            `bson:"revisions" json:"-"`
	Linkages     []linkage         `bson:"linkage" json:"linkage"`
	Priority     int               `bson:"priority" json:"priority"`
	CreatedOn    time.Time         `bson:"created_on" json:"created_on"`
	LastModified time.Time         `bson:"last_modified" json:"last_modified"`
	Expiration   time.Time         `bson:"expiration" json:"expiration"` // 0 means no expiration of Node
	Type         string            `bson:"type" json:"type"`
	Subset       Subset            `bson:"subset" json:"-"`
	Parts        *PartsList        `bson:"parts" json:"parts"`
	Locations    []Location        `bson:"locations" json:"locations"` // see below
	Restore      bool              `bson:"restore" json:"restore"`     // has a restore request been observed
}

func CreateNodeUpload 

func CreateNodeUpload(u *user.User, params map[string]string, files file.FormFiles) (node *Node, err error)

func CreateNodesFromArchive 

func CreateNodesFromArchive(u *user.User, params map[string]string, files file.FormFiles, archiveId string) (nodes []*Node, err error)

func Load 

func Load(id string) (n *Node, err error)

func New 

func New(uuid string) (node *Node)

func (*Node) AddLocation 

func (node *Node) AddLocation(loc Location) (err error)

AddLocation _

func (*Node) CheckTransit 

func (node *Node) CheckTransit() (locked bool)

CheckTransit - return true if Node is currently being uploaded to an external Location

func (*Node) DBInit 

func (node *Node) DBInit()

func (*Node) Delete 

func (node *Node) Delete() (deleted bool, err error)

ExpireNode -- delete the node from Mongo and Disk

func (*Node) DeleteIndex 

func (node *Node) DeleteIndex(indextype string) (err error)
************************ ************************ ************************ ************************ ************************ ************************ ************************ ************************
************************ ************************ ************************ ************************ ************************ ************************ ************************ ************************
************************ ************************ ************************ ************************ ************************ ************************ ************************ ************************

DeleteIndex for node

func (*Node) DeleteLocation 

func (node *Node) DeleteLocation(locID string) (err error)

DeleteLocation _

func (*Node) DeleteLocations 

func (node *Node) DeleteLocations() (err error)

DeleteLocations _

func (*Node) DynamicIndex 

func (node *Node) DynamicIndex(name string) (idx index.Index, err error)

func (*Node) ExpireNodeFiles 

func (n *Node) ExpireNodeFiles() (deleted bool, err error)
************************ ************************ ************************ ************************ ************************ ************************ ************************ ************************
************************ ************************ ************************ ************************ ************************ ************************ ************************ ************************
************************ ************************ ************************ ************************ ************************ ************************ ************************ ************************

ExpireNodeFiles -- remove files in DATA_PATH if present on >= conf.MIN_REPLICA_COUNT external Locations

func (*Node) FileExt 

func (node *Node) FileExt() string

func (*Node) FilePath 

func (node *Node) FilePath() string

func (*Node) FileReader 

func (node *Node) FileReader() (reader file.ReaderAt, err error)

func (*Node) GetLocation 

func (node *Node) GetLocation(locID string) (myLocation Location, err error)

GetLocation _

func (*Node) GetLocations 

func (node *Node) GetLocations() (locations []Location)

GetLocations _

func (*Node) GetRestore 

func (node *Node) GetRestore() (stat bool)

GetRestore return true if node has been marked for restoring from external Location

func (*Node) HasFile 

func (node *Node) HasFile() bool

has

func (*Node) HasFileLock 

func (node *Node) HasFileLock() bool

func (*Node) HasIndex 

func (node *Node) HasIndex(index string) bool

func (*Node) HasIndexLock 

func (node *Node) HasIndexLock(name string) bool

func (*Node) HasParent 

func (node *Node) HasParent() bool

func (*Node) IndexFiles 

func (node *Node) IndexFiles() []string

func (*Node) IndexPath 

func (node *Node) IndexPath() string

func (*Node) Mkdir 

func (node *Node) Mkdir() (err error)

func (*Node) Path 

func (node *Node) Path() string

path

func (*Node) Rmdir 

func (node *Node) Rmdir() (err error)

func (*Node) Save 

func (node *Node) Save() (err error)

func (*Node) SetAttributes 

func (node *Node) SetAttributes(attr file.FormFile) (err error)

func (*Node) SetAttributesFromString 

func (node *Node) SetAttributesFromString(attributes string) (err error)

func (*Node) SetExpiration 

func (node *Node) SetExpiration(expire string) (err error)

func (*Node) SetFile 

func (node *Node) SetFile(file file.FormFile) (err error)

func (*Node) SetFileFromPath 

func (node *Node) SetFileFromPath(path string, action string) (err error)

SetFileFromPath _

func (*Node) SetFileFromSubset 

func (node *Node) SetFileFromSubset(subsetIndices file.FormFile) (err error)

func (*Node) SetInTransit 

func (node *Node) SetInTransit()

SetInTransit - return true if Node is currently being uploaded to an external Location

func (*Node) SetIndexInfo 

func (node *Node) SetIndexInfo(indextype string, idxinfo *IdxInfo)

func (*Node) SetRestore 

func (node *Node) SetRestore()

SetRestore set Restore value to true to mark node for restoring from external Location

func (*Node) ToJson 

func (node *Node) ToJson() (s string, err error)

func (*Node) UnSetInTransitLocked 

func (node *Node) UnSetInTransitLocked()

UnSetInTransitLocked - return true if Node is currently being uploaded to an external Location

func (*Node) UnSetRestore 

func (node *Node) UnSetRestore()

UnsetRestore set Restore value to false, node restore has been requested e.g. via TSM client

func (*Node) Update 

func (node *Node) Update(params map[string]string, files file.FormFiles, hasLock bool) (err error)

Update has node Modification functions

func (*Node) UpdateDataTags 

func (node *Node) UpdateDataTags(types string)

func (*Node) UpdateLinkages 

func (node *Node) UpdateLinkages(ltype string, ids string, operation string)

func (*Node) UpdateVersion 

func (node *Node) UpdateVersion() (err error)

type NodeReaper 

type NodeReaper struct{}
var (
	Ttl         *NodeReaper
	ExpireRegex = regexp.MustCompile(`^(\d+)(M|H|D)$`)
)

func NewNodeReaper 

func NewNodeReaper() *NodeReaper

func (*NodeReaper) Handle 

func (nr *NodeReaper) Handle()

type Nodes 

type Nodes []*Node

Node array type

func LoadNodes 

func LoadNodes(ids []string) (n Nodes, err error)

func (*Nodes) DBInit 

func (n *Nodes) DBInit()

func (*Nodes) GetAll 

func (n *Nodes) GetAll(q bson.M) (err error)

func (*Nodes) GetAllD 

func (n *Nodes) GetAllD(q bson.D) (err error)

func (*Nodes) GetPaginated 

func (n *Nodes) GetPaginated(q bson.M, limit int, offset int, order string) (count int, err error)

type Parent 

type Parent struct {
	Id        string `bson:"id" json:"-"`
	IndexName string `bson:"index_name" json:"-"`
}

type PartsList 

type PartsList struct {
	Count       int         `bson:"count" json:"count"`
	Length      int         `bson:"length" json:"length"`
	VarLen      bool        `bson:"varlen" json:"varlen"`
	Parts       []partsFile `bson:"parts" json:"parts"`
	Compression string      `bson:"compression" json:"compression"`
}

type Subset 

type Subset struct {
	Parent Parent            `bson:"parent" json:"-"`
	Index  SubsetNodeIdxInfo `bson:"index" json:"-"`
}

Subset is used to store information about a subset node's parent and its index. A subset node's index defines the subset of the data file that this node represents. A subset node's index is immutable after it is defined.

type SubsetNodeIdxInfo 

type SubsetNodeIdxInfo struct {
	Path        string `bson:"path" json:"-"`
	TotalUnits  int64  `bson:"total_units" json:"-"`
	AvgUnitSize int64  `bson:"average_unit_size" json:"-"`
	Format      string `bson:"format" json:"-"`
}

Source Files

View all Source files

Directories

Path Synopsis
Package contains Node File struct and MultiReaderAt implementation
 Package contains Node File struct and MultiReaderAt implementation
format/fasta
Package to read and write FASTA format files
 Package to read and write FASTA format files
format/fastq
Package to read and write FASTQ format files
 Package to read and write FASTQ format files
format/line
Package to read and index lines of a file
 Package to read and index lines of a file
format/multi
Package to read and auto-detect format of fasta & fastq files
 Package to read and auto-detect format of fasta & fastq files
format/sam
Package to read and write SAM format files
 Package to read and write SAM format files
format/seq
Package contains interfaces for fasta & fastq & and multi packages
 Package contains interfaces for fasta & fastq & and multi packages
index
anonymize
fq2fa

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.