model

package
v5.0.0-...-b98a0e9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 3, 2024 License: AGPL-3.0 Imports: 17 Imported by: 0

Documentation

Overview

Package model gathers the main structs and interfaces used in sync library.

Index

Constants

View Source 
const (
	// InternalPathSeparator defines the unique internal path separator everywhere
	InternalPathSeparator = "/"
	GlobSeparator         = '/'
	// DefaultEtag is the minio default value when a new file is detected
	DefaultEtag = "00000000000000000000000000000000-1"

	DirectionLeft  DirectionType = 1
	DirectionRight DirectionType = 2
	DirectionBi    DirectionType = 3

	Interrupt SyncCmd = iota
	Pause
	Resume
)
View Source 
const (
	StatusScopeTask = iota
	StatusScopeProcessing
)

Variables

This section is empty.

Functions

func IgnoreMatcher 

func IgnoreMatcher(ignores ...glob.Glob) func(string) bool

func Ignores 

func Ignores(endpoint Endpoint, name string) bool

Ignores checks if a specific name should be ignored by the given Endpoint

func IsConfigError 

func IsConfigError(e error) bool

func IsFolderHiddenFile 

func IsFolderHiddenFile(name string) bool

IsFolderHiddenFile checks if file is .pydio

func IsIgnoredFile 

func IsIgnoredFile(path string, ignores ...glob.Glob) (ignored bool)

func NewConfigError 

func NewConfigError(err error) error

func NodeRequiresChecksum 

func NodeRequiresChecksum(node tree.N) bool

NodeRequiresChecksum checks whether the current checksum is usable as index ETag or not. This part is linked to the **structured datasource**, and we still have return True if checksum is {md5}-numberOfParts otherwise at the next object Move ( = Copy in s3 ), the resulting ETag (now a real md5) will differ from the original one The downside is that uploading huge files will trigger a full re-reading of the file afterward to compute the stable md5.

func Retry 

func Retry(f func() error, seconds ...time.Duration) error

Retry tries to apply an operation as many times as required

func RetryWithCtx 

func RetryWithCtx(ctx context.Context, f func(retry int) error, seconds ...time.Duration) error

RetryWithCtx does like Retry with an additional cancellable context

func StringContentToETag 

func StringContentToETag(uuid string) string

func ZapEndpoint 

func ZapEndpoint(key string, e Endpoint) zapcore.Field

Types

type AuthRequester 

type AuthRequester interface {
	RequiresAuth() bool
}

type BulkLoader 

type BulkLoader interface {
	BulkLoadNodes(ctx context.Context, nodes map[string]string) (map[string]interface{}, error)
}

A BulkLoader can stream calls to ReadNode - Better use CachedBranchProvider

type CachedBranchProvider 

type CachedBranchProvider interface {
	Endpoint
	GetCachedBranches(ctx context.Context, roots ...string) (PathSyncSource, error)
}

A CachedBranchProvider can quickly load a full branch recursively in memory and expose it as a PathSyncSource

type ChecksumProvider 

type ChecksumProvider interface {
	Endpoint
	ComputeChecksum(ctx context.Context, node tree.N) error
}

ChecksumProvider is able to compute a checksum for a given node (typically an Etag)

type Command 

type Command struct {
	sync.Mutex
	// contains filtered or unexported fields
}

Command is a pub/sub for dispatching SyncCmd

func NewCommand 

func NewCommand() *Command

NewCommand creates a new command

func (*Command) Publish 

func (c *Command) Publish(cmd SyncCmd)

func (*Command) Stop 

func (c *Command) Stop()

Stop closes internal channels

func (*Command) Subscribe 

func (c *Command) Subscribe() (chan SyncCmd, chan bool)

Subscribe creates a new listener to the Input and returns a chan for closing subscription

type ConfigError 

type ConfigError struct {
	// contains filtered or unexported fields
}

func (*ConfigError) HasConfigError 

func (c *ConfigError) HasConfigError() bool

type Conflicter 

type Conflicter interface {
	Conflict() (bool, interface{})
}

type DataSyncSource 

type DataSyncSource interface {
	PathSyncSource
	// GetReaderOn provides a ReadCloser for reading content of a node located at a given path
	GetReaderOn(ctx context.Context, path string) (out io.ReadCloser, err error)
}

DataSyncSource provides a way to read the actual content of the nodes

func AsDataSyncSource 

func AsDataSyncSource(endpoint Endpoint) (DataSyncSource, bool)

AsDataSyncSource tries to cast an Endpoint to a DataSyncSource

type DataSyncTarget 

type DataSyncTarget interface {
	PathSyncTarget
	// GetWriterOn provides a writeCloser for writing content to a given path.
	GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)
}

DataSyncTarget provides a way to write some actual content to the nodes

func AsDataSyncTarget 

func AsDataSyncTarget(endpoint Endpoint) (DataSyncTarget, bool)

AsDataSyncTarget tries to cast an Endpoint to a DataSyncTarget

type DirectionType 

type DirectionType int

type Endpoint 

type Endpoint interface {
	// LoadNode loads a given node by its path from this endpoint
	LoadNode(ctx context.Context, path string, extendedStats ...bool) (node tree.N, err error)
	// GetEndpointInfo returns static information about this endpoint
	GetEndpointInfo() EndpointInfo
}

Endpoint is the most basic interface for representing an endpoint for synchronization. It is just able to return some info and to load a node

type EndpointInfo 

type EndpointInfo struct {
	URI                   string
	RequiresNormalization bool
	RequiresFoldersRescan bool
	IsAsynchronous        bool
	EchoTime              time.Duration
	Ignores               []string
}

EndpointInfo provides static info about a given Endpoint (returned by GetEndpointInfo method)

type EndpointOptions 

type EndpointOptions struct {
	BrowseOnly bool
	Properties map[string]string
}

EndpointOptions is used to configure an Endpoint at creation time

type EndpointRootStat 

type EndpointRootStat struct {
	HasChildrenInfo bool
	HasSizeInfo     bool

	Size    int64
	Folders int64
	Files   int64

	PgSize     int64 `json:"-"`
	PgChildren int64 `json:"-"`
	PgFolders  int64 `json:"-"`
	PgFiles    int64 `json:"-"`

	LastPg float64 `json:"-"`
}

EndpointRootStat gives information about the size/files/folders of an endpoint

func (*EndpointRootStat) Children 

func (e *EndpointRootStat) Children() int64

Children returns the sum of Folders and Files

func (*EndpointRootStat) IsEmpty 

func (e *EndpointRootStat) IsEmpty() bool

IsEmpty checks if the given stat seems empty. Call to IsKnown should be done before otherwise it can be a false negative

func (*EndpointRootStat) IsKnown 

func (e *EndpointRootStat) IsKnown() bool

IsKnown returns true if either HasSizeInfo or HasChildrenInfo is set

type EndpointStatus 

type EndpointStatus struct {
	EndpointInfo
	WatchConnection WatchConnectionInfo
	Stats           *EndpointRootStat
}

type ErrorType 

type ErrorType int
const (
	SyncErrorUnknown ErrorType = iota
	SyncErrorNetworking
	SyncErrorAuthentication
	SyncErrorConflicts
)

type EventInfo 

type EventInfo struct {
	Time           string
	Size           int64
	Etag           string
	Folder         bool
	Path           string
	Source         PathSyncSource `json:"-"`
	Type           EventType
	Host           string
	Port           string
	UserAgent      string
	OperationId    string
	ScanEvent      bool
	ScanSourceNode tree.N
	Metadata       map[string]string
	MoveSource     tree.N
	MoveTarget     tree.N
}

EventInfo contains the information of the event that occurred and the source IP:PORT of the client which triggered the event.

func NodeToEventInfo 

func NodeToEventInfo(ctx context.Context, path string, node tree.N, eventType EventType) (eventInfo EventInfo)

func (EventInfo) CreateContext 

func (e EventInfo) CreateContext(ctx context.Context) context.Context

type EventType 

type EventType string

EventType represents the type of the event occurred. 

const (
	// EventCreate notifies when a new object is created
	EventCreate EventType = "ObjectCreated"
	// EventRename notifies when a new object is renamed
	EventRename EventType = "ObjectRenamed"
	// EventRemove notifies when a new object is deleted
	EventRemove EventType = "ObjectRemoved"
	// EventAccessed notifies when an object is accessed.
	EventAccessed EventType = "ObjectAccessed"
	// EventAccessedRead notifies when an object is accessed (specifically read).
	EventAccessedRead EventType = "ObjectAccessed:Read"
	// EventAccessedStat notifies when an object is accessed (specifically stat).
	EventAccessedStat EventType = "ObjectAccessed:Stat"
	// EventOther notifies any other events
	EventOther EventType = "ObjectOther"
	// EventSureMove notifies a known move
	EventSureMove EventType = "NodeMoved"
)

type HashStoreReader 

type HashStoreReader interface {
	// SetRefHashStore passes a reference to a loaded snapshot
	SetRefHashStore(source PathSyncSource)
}

HashStoreReader can maintain a reference to a snapshot to quickly find hashes for nodes directly from the snapshot if they have not been modified

type LockBranchProvider 

type LockBranchProvider interface {
	// LockBranch sets a lock on a branch, with a preset sessionUUID and an expiration time
	LockBranch(ctx context.Context, node tree.N, sessionUUID string, expireAfter time.Duration) error
	// UnlockBranch removes lock manually from this branch, if it was not expired already
	UnlockBranch(ctx context.Context, sessionUUID string) error
}

LockBranchProvider can set/remove a lock on a branch, with automatic expiration

type LockEvent 

type LockEvent struct {
	Type        LockEventType
	Source      PathSyncSource
	Path        string
	OperationId string
}

LockEvent is a simple struct for signaling lock/unlock operation

type LockEventType 

type LockEventType int
const (
	LockEventLock LockEventType = iota
	LockEventUnlock
)

type MetadataProvider 

type MetadataProvider interface {
	// ProvidesMetadataNamespaces returns a list of patterns to check on provider nodes MetaStore.
	ProvidesMetadataNamespaces() ([]glob.Glob, bool)
}

MetadataProvider declares metadata namespaces that may be mapped to target metadata

type MetadataReceiver 

type MetadataReceiver interface {
	// CreateMetadata add a metadata to the node
	CreateMetadata(ctx context.Context, node tree.N, namespace string, jsonValue string) error
	// UpdateMetadata updates an existing metadata value
	UpdateMetadata(ctx context.Context, node tree.N, namespace string, jsonValue string) error
	// DeleteMetadata deletes a metadata by namespace
	DeleteMetadata(ctx context.Context, node tree.N, namespace string) error
}

MetadataReceiver implements methods for updating nodes metadata

type MultiStater 

type MultiStater map[string]Stater

func NewMultiStater 

func NewMultiStater() MultiStater

func (MultiStater) Stats 

func (m MultiStater) Stats() map[string]interface{}

func (MultiStater) String 

func (m MultiStater) String() string

type Networking 

type Networking interface {
	NetworkError() error
}

type PathSyncSource 

type PathSyncSource interface {
	Endpoint
	// Walk walks the nodes with a callback
	Walk(ctx context.Context, walknFc WalkNodesFunc, root string, recursive bool) (err error)
	// Watch sets up an event watcher on the nodes
	Watch(recursivePath string) (*WatchObject, error)
}

PathSyncSource is a type of endpoint that can be used as a source of tree.Nodes for synchronization. It can browse and watch the nodes, but not get the nodes actual content (see DataSyncSource).

func AsPathSyncSource 

func AsPathSyncSource(endpoint Endpoint) (PathSyncSource, bool)

AsPathSyncSource tries to cast an Endpoint to a PathSyncSource

type PathSyncTarget 

type PathSyncTarget interface {
	Endpoint
	// CreateNode is used to create a node in the tree
	CreateNode(ctx context.Context, node tree.N, updateIfExists bool) (err error)
	// DeleteNode is used to remove a node (and all its children) from the tree
	DeleteNode(ctx context.Context, path string) (err error)
	// MoveNode is used to move a node (and all its children) from one place to another in the tree.
	MoveNode(ctx context.Context, oldPath string, newPath string) (err error)
}

PathSyncTarget is a type of endpoint that can be used as a target for synchronization, typically an Index. It can be updated with a tree of nodes, but cannot store actual data (see DataSyncTarget)

func AsPathSyncTarget 

func AsPathSyncTarget(endpoint Endpoint) (PathSyncTarget, bool)

AsPathSyncTarget tries to cast an Endpoint to a PathSyncTarget

type ProcessingStatus 

type ProcessingStatus struct {
	// contains filtered or unexported fields
}

func NewProcessingStatus 

func NewProcessingStatus(info string) *ProcessingStatus

func (*ProcessingStatus) AtomicProgress 

func (p *ProcessingStatus) AtomicProgress() bool

func (*ProcessingStatus) EndpointURI 

func (p *ProcessingStatus) EndpointURI() string

func (*ProcessingStatus) Error 

func (p *ProcessingStatus) Error() error

func (*ProcessingStatus) IsError 

func (p *ProcessingStatus) IsError() bool

func (*ProcessingStatus) MarshalJSON 

func (p *ProcessingStatus) MarshalJSON() ([]byte, error)

MarshalJSON implements custom JSON marshalling

func (*ProcessingStatus) Node 

func (p *ProcessingStatus) Node() tree.N

func (*ProcessingStatus) Progress 

func (p *ProcessingStatus) Progress() float32

func (*ProcessingStatus) SetEndpoint 

func (p *ProcessingStatus) SetEndpoint(uri string) *ProcessingStatus

func (*ProcessingStatus) SetError 

func (p *ProcessingStatus) SetError(e error) *ProcessingStatus

func (*ProcessingStatus) SetNode 

func (p *ProcessingStatus) SetNode(node tree.N) *ProcessingStatus

func (*ProcessingStatus) SetProgress 

func (p *ProcessingStatus) SetProgress(pg float32, atomic ...bool) *ProcessingStatus

func (*ProcessingStatus) Status 

func (p *ProcessingStatus) Status() int

func (*ProcessingStatus) String 

func (p *ProcessingStatus) String() string

func (*ProcessingStatus) Type 

func (p *ProcessingStatus) Type() StatusScope

func (*ProcessingStatus) UnmarshalJSON 

func (p *ProcessingStatus) UnmarshalJSON(data []byte) error

UnmarshalJSON implements custom JSON unmarshalling

type ProcessorEvent 

type ProcessorEvent struct {
	Type string
	Data interface{}
}

type SessionProvider 

type SessionProvider interface {
	// StartSession opens a new indexation session and returns it
	StartSession(ctx context.Context, rootNode tree.N, silent bool) (string, error)
	// FlushSession calls the Flush method on the underlying service without closing the session yet
	FlushSession(ctx context.Context, sessionUuid string) error
	// FinishSession closes the indexation session
	FinishSession(ctx context.Context, sessionUuid string) error
}

SessionProvider has internal mechanism to start/flush/finish an IndexationSession

func AsSessionProvider 

func AsSessionProvider(endpoint Endpoint) (SessionProvider, bool)

AsSessionProvider tries to cast an Endpoint to a SessionProvider

type SnapshotFactory 

type SnapshotFactory interface {
	// Load creates the snapshot and return it
	Load(source PathSyncSource) (Snapshoter, error)
	Close(ctx context.Context) error
	Reset(ctx context.Context) error
}

SnapshotFactory provides dependency injection for creating snapshots using a specific persistence layer.

type SnapshotUpdater 

type SnapshotUpdater interface {
	// SetUpdateSnapshot stores internal reference to a Snapshot
	SetUpdateSnapshot(PathSyncTarget)
	// PatchUpdateSnapshot applies a patch of operations to the internal snapshot
	PatchUpdateSnapshot(ctx context.Context, patch interface{})
}

SnapshotUpdater is an endpoint that can embed a reference to a snapshot and update it afterward

type Snapshoter 

type Snapshoter interface {
	PathSyncSource
	// IsEmpty indicates whether this snapshot is properly initialized or not (e.g. underlying DB not found)
	IsEmpty() bool
	// Capture walks the source and stores all paths inside snapshot. The paths parameter allows to filter the branches to capture.
	Capture(ctx context.Context, source PathSyncSource, paths ...string) error
}

Snapshoter is an extended version of PathSyncSource that can capture another source at once.

type Stater 

type Stater interface {
	fmt.Stringer
	Stats() map[string]interface{}
}

Stater can provide a json-serializable description of its content

type Status 

type Status interface {
	fmt.Stringer

	Type() StatusScope
	Status() int
	IsError() bool
	Error() error

	Progress() float32
	AtomicProgress() bool

	EndpointURI() string
	Node() tree.N
}

type StatusProvider 

type StatusProvider interface {
	// SetupChannels register channels for listening to status and done infos
	SetupChannels(status chan Status, done chan interface{}, cmd *Command)
	// Status notify of a new ProcessStatus
	Status(s Status)
	// Done notify the patch is processed, can send any useful info to the associated channel
	Done(info interface{})
}

StatusProvider can register channels to send status/done events during processing

type StatusScope 

type StatusScope int

type SyncCmd 

type SyncCmd int

type TaskStatus 

type TaskStatus int
const (
	TaskStatusIdle TaskStatus = iota
	TaskStatusPaused
	TaskStatusDisabled
	TaskStatusProcessing
	TaskStatusError
	TaskStatusRestarting
	TaskStatusStopping
	TaskStatusRemoved
)

type UuidFoldersRefresher 

type UuidFoldersRefresher interface {
	// ExistingFolders lists all folders with their UUID
	ExistingFolders(ctx context.Context) (map[string][]tree.N, error)
	// UpdateFolderUuid refreshes a given folder UUID and return it.
	UpdateFolderUuid(ctx context.Context, node tree.N) (tree.N, error)
}

UuidFoldersRefresher provides tools to detect UUID duplicates and update them if necessary

type UuidProvider 

type UuidProvider interface {
	// LoadNodeByUuid loads a node by UUID.
	LoadNodeByUuid(ctx context.Context, uuid string) (node tree.N, err error)
}

UuidProvider declares an endpoint to be able to load a node by its unique UUID

type UuidReceiver 

type UuidReceiver interface {
	// UpdateNodeUuid refresh node UUID and returns the new node
	UpdateNodeUuid(ctx context.Context, node tree.N) (tree.N, error)
}

UuidReceiver is able to update an existing node UUID

type Versioner 

type Versioner interface {
	Commit(node tree.N)
	ListVersions(node tree.N) (versions map[int]string, lastVersion int)
}

type WalkNodesFunc 

type WalkNodesFunc func(path string, node tree.N, err error) error

type WatchConnectionInfo 

type WatchConnectionInfo int
const (
	WatchConnected WatchConnectionInfo = iota
	WatchDisconnected
	WatchActive
	WatchIdle
	WatchStats
)

type WatchObject 

type WatchObject struct {
	// eventInfo will be put on this chan
	EventInfoChan chan EventInfo
	// errors will be put on this chan
	ErrorChan chan error
	// will stop the watcher goroutines
	DoneChan chan bool
	// Provides info about the internal watcher connection
	// Can be nil if the internal watcher does not support such notifications
	ConnectionInfo chan WatchConnectionInfo
}

func (*WatchObject) Close 

func (w *WatchObject) Close()

Close the watcher, will stop all goroutines

func (*WatchObject) ConnectionInfos 

func (w *WatchObject) ConnectionInfos() chan WatchConnectionInfo

func (*WatchObject) Done 

func (w *WatchObject) Done() <-chan bool

Done returns a channel that unblocks when Close has been called

func (*WatchObject) Errors 

func (w *WatchObject) Errors() chan error

Errors returns the chan receiving errors

func (*WatchObject) Events 

func (w *WatchObject) Events() chan EventInfo

Events returns the chan receiving events

func (*WatchObject) NextError 

func (w *WatchObject) NextError() error

NextError pops the next error from the ErrorChan

func (*WatchObject) NextEvent 

func (w *WatchObject) NextEvent() EventInfo

NextEvent pops the next event from the EventInfoChan

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.