director

Documentation

Overview

**************************************************************

*
* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License.  You may
* obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
**************************************************************

Index

• func AdvertiseOSDF(ctx context.Context) error
• func ConfigFilteredServers()
• func InitializeDB() error
• func InitializeGeoIPDB(ctx context.Context)
• func LaunchMapMetrics(ctx context.Context, egrp *errgroup.Group)
• func LaunchPeriodicDirectorTest(ctx context.Context, serverAd server_structs.ServerAd)
• func LaunchRegistryPeriodicQuery(ctx context.Context, egrp *errgroup.Group)
• func LaunchServerIOQuery(ctx context.Context, egrp *errgroup.Group)
• func LaunchTTLCache(ctx context.Context, egrp *errgroup.Group)
• func PeriodicCacheReload(ctx context.Context)
• func RegisterDirectorAPI(ctx context.Context, router *gin.RouterGroup)
• func RegisterDirectorOIDCAPI(router *gin.RouterGroup)
• func RegisterDirectorWebAPI(router *gin.RouterGroup)
• func SetStartupTime(t time.Time)
• func ShortcutMiddleware(defaultResponse string) gin.HandlerFunc
• func WithToken(tk string) queryOption
• type Coordinate
• type GeoIPOverride
• type GeoNetOverride
• type HealthTestStatus
• type NamespaceAdV2MappedResponse
• type NamespaceAdV2Response
• type ObjectStat
  • func NewObjectStat() *ObjectStat
• type ProjectContextKey
• type PromDiscoveryItem
• type ServerDowntime
• type SwapMap
• type SwapMaps
  • func (me SwapMaps) Len() int
  • func (me SwapMaps) Less(left, right int) bool
  • func (me SwapMaps) Swap(left, right int)
• type TokenGenResponse
• type TokenIssuerResponse

Functions

func AdvertiseOSDF

func AdvertiseOSDF(ctx context.Context) error

Populate internal cache with origin/cache ads

func ConfigFilteredServers

func ConfigFilteredServers()

Populate internal filteredServers map using Director.FilteredServers param and director db

func InitializeDB

func InitializeDB() error

Initialize the Director's sqlite database, which is used to persist information about server downtimes

func InitializeGeoIPDB

func InitializeGeoIPDB(ctx context.Context)

func LaunchMapMetrics

func LaunchMapMetrics(ctx context.Context, egrp *errgroup.Group)

Launch a goroutine to scrape metrics from various TTL caches and maps in the director

func LaunchPeriodicDirectorTest

func LaunchPeriodicDirectorTest(ctx context.Context, serverAd server_structs.ServerAd)

Run a periodic test file transfer against an origin to ensure it's talking to the director

func LaunchRegistryPeriodicQuery

func LaunchRegistryPeriodicQuery(ctx context.Context, egrp *errgroup.Group)

LaunchRegistryPeriodicQuery starts a new goroutine that periodically refreshes the allowed prefixes for caches data maintained by the director in memory. It queries the registry at the interval specified by the config parameter Director.RegistryQueryInterval. If the data is stale (older than 15 minutes) or uninitialized, it queries the registry at a shorter interval of 1 second and switches back to the regular interval upon successful retrieval of the information.

func LaunchServerIOQuery

func LaunchServerIOQuery(ctx context.Context, egrp *errgroup.Group)

Start a goroutine to query director's Prometheus endpoint for origin/cache server I/O stats and save the value to the corresponding serverAd

func LaunchTTLCache

func LaunchTTLCache(ctx context.Context, egrp *errgroup.Group)

Configure TTL caches to enable cache eviction and other additional cache events handling logic

The `ctx` is the context for listening to server shutdown event in order to cleanup internal cache eviction goroutine

func PeriodicCacheReload

func PeriodicCacheReload(ctx context.Context)

func RegisterDirectorAPI

func RegisterDirectorAPI(ctx context.Context, router *gin.RouterGroup)

func RegisterDirectorOIDCAPI

func RegisterDirectorOIDCAPI(router *gin.RouterGroup)

func RegisterDirectorWebAPI

func RegisterDirectorWebAPI(router *gin.RouterGroup)

func SetStartupTime

func SetStartupTime(t time.Time)

Used for testing, where the Director has pretty much _always_ been started in the last 5 minutes.

func ShortcutMiddleware

func ShortcutMiddleware(defaultResponse string) gin.HandlerFunc

Middleware sends GET /foo/bar to the RedirectToCache function, as if the original request had been made to /api/v1.0/director/object/foo/bar

func WithToken

func WithToken(tk string) queryOption

Issue the stat call with a token

Types

type Coordinate

type Coordinate struct {
	Lat  float64 `mapstructure:"lat"`
	Long float64 `mapstructure:"long"`
}

type GeoIPOverride

type GeoIPOverride struct {
	IP         string     `mapstructure:"IP"`
	Coordinate Coordinate `mapstructure:"Coordinate"`
}

type GeoNetOverride

type GeoNetOverride struct {
	IPNet      net.IPNet
	Coordinate Coordinate
}

type HealthTestStatus

type HealthTestStatus string

status of director-based health tests to origins and caches

const (
	HealthStatusDisabled HealthTestStatus = "Health Test Disabled"
	HealthStatusUnknown  HealthTestStatus = "Unknown"
	HealthStatusInit     HealthTestStatus = "Initializing"
	HealthStatusOK       HealthTestStatus = "OK"
	HealthStatusError    HealthTestStatus = "Error"
)

type NamespaceAdV2MappedResponse

type NamespaceAdV2MappedResponse struct {
	Path         string                      `json:"path"`
	Caps         server_structs.Capabilities `json:"capabilities"`
	Generation   []TokenGenResponse          `json:"tokenGeneration"`
	Issuer       []TokenIssuerResponse       `json:"tokenIssuer"`
	FromTopology bool                        `json:"fromTopology"`
	Origins      []string                    `json:"origins"`
	Caches       []string                    `json:"caches"`
}

NamespaceAdV2MappedResponse creates a response struct for NamespaceAdV2 with mapped origins and caches

type NamespaceAdV2Response

type NamespaceAdV2Response struct {
	Path         string                      `json:"path"`
	Caps         server_structs.Capabilities `json:"capabilities"`
	Generation   []TokenGenResponse          `json:"tokenGeneration"`
	Issuer       []TokenIssuerResponse       `json:"tokenIssuer"`
	FromTopology bool                        `json:"fromTopology"`
}

NamespaceAdV2Response creates a response struct for NamespaceAdV2

type ObjectStat

type ObjectStat struct {
	// Handle the request to test if an object exists on a server
	//
	// dataUrl: the base url to access data on the server. This is usually the url pointed at the XRootD instance on the server
	//
	// digest: request digest for object checksum. XRootD responds with 403 if digest feature is turned off on the server
	//
	// token: a bearer token to be used when issuing the request
	ReqHandler func(maxCancelCtx context.Context, objectName string, dataUrl url.URL, digest bool, token string, timeout time.Duration) (*objectMetadata, error)
	// Manage a `stat` request to origin servers given an objectName
	Query func(cancelContext context.Context, objectName string, sType server_structs.ServerType, minimum, maximum int, options ...queryOption) queryResult
}

A struct to implement `object stat`, by querying against origins/caches with namespaces match the prefix of an object name and return origins that have the object.

**Note**: Currently it only returns successful result when the file is under a public namespace.

func NewObjectStat

func NewObjectStat() *ObjectStat

Initialize a new stat instance and set default method implementations

type ProjectContextKey

type ProjectContextKey struct{}

Context key for the project name

type PromDiscoveryItem

type PromDiscoveryItem struct {
	Targets []string          `json:"targets"`
	Labels  map[string]string `json:"labels"`
}

Prometheus HTTP discovery endpoint struct, used by director to dynamically return available origin/cache servers for Prometheus to scrape

type ServerDowntime

type ServerDowntime struct {
	UUID       string     `gorm:"primaryKey"`
	Name       string     `gorm:"not null;unique"`
	FilterType filterType `gorm:"type:text;not null"`
	// We don't use gorm default gorm.Model to change ID type to string
	CreatedAt time.Time
	UpdatedAt time.Time
}

type SwapMap

type SwapMap struct {
	Weight float64
	Index  int
}

type SwapMaps

type SwapMaps []SwapMap

func (SwapMaps) Len

func (me SwapMaps) Len() int

func (SwapMaps) Less

func (me SwapMaps) Less(left, right int) bool

func (SwapMaps) Swap

func (me SwapMaps) Swap(left, right int)

type TokenGenResponse

type TokenGenResponse struct {
	Strategy         server_structs.StrategyType `json:"strategy"`
	VaultServer      string                      `json:"vaultServer"`
	MaxScopeDepth    uint                        `json:"maxScopeDepth"`
	CredentialIssuer string                      `json:"issuer"`
}

TokenGenResponse creates a response struct for TokenGen

type TokenIssuerResponse

type TokenIssuerResponse struct {
	BasePaths       []string `json:"basePaths"`
	RestrictedPaths []string `json:"restrictedPaths"`
	IssuerUrl       string   `json:"issuer"`
}

TokenIssuerResponse creates a response struct for TokenIssuer