config

Documentation

Overview

Package config provides types and functions to configure ishard executable.

• Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.

Package config provides types and functions to configure ishard executable.

• Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.

Index

• Variables
• type ClusterConfig
• type Config
  • func Load() (*Config, error)
• type DryRunFlag
  • func (d *DryRunFlag) IsBoolFlag() bool
  • func (d *DryRunFlag) Set(value string) error
  • func (d *DryRunFlag) String() string
• type EKMFlag
  • func (e *EKMFlag) Set(param string) error
  • func (e *EKMFlag) String() string
• type IshardConfig
• type MissingExtManager
  • func NewMissingExtManager(name string, sampleExts []string) (*MissingExtManager, error)
• type SampleKeyPattern
• type ShardSize
  • func (s *ShardSize) Set(value string) error
  • func (s *ShardSize) String() string
• type SortFlag
  • func (alg *SortFlag) Set(value string) error
  • func (alg *SortFlag) String() string

Variables

var (
	BaseFileNamePattern   = SampleKeyPattern{Regex: `.*/([^/]+)$`, CaptureGroup: "$1"}
	FullNamePattern       = SampleKeyPattern{Regex: `^(.*)$`, CaptureGroup: "$1"}
	CollapseAllDirPattern = SampleKeyPattern{Regex: `/`, CaptureGroup: ""}
)

Define some commonly used sample key patterns

var DefaultConfig = Config{
	ClusterConfig: ClusterConfig{URL: "http://" + defaultClusterIPv4 + ":" + defaultProxyPort},
	IshardConfig: IshardConfig{
		ShardSize:        ShardSize{Size: 102400},
		Ext:              ".tar",
		ShardTemplate:    "shard-%d",
		Collapse:         false,
		SampleKeyPattern: BaseFileNamePattern,
		MExtMgr:          nil,
	},
	SrcBck:     cmn.Bck{Name: "src_bck", Provider: apc.AIS},
	DstBck:     cmn.Bck{Name: "dst_bck", Provider: apc.AIS},
	Progress:   false,
	DryRunFlag: DryRunFlag{IsSet: false},
	SortFlag:   SortFlag{IsSet: false},
}

Types

type ClusterConfig

type ClusterConfig struct {
	URL string
}

type Config

type Config struct {
	ClusterConfig
	IshardConfig
	DryRunFlag
	SortFlag
	EKMFlag
	SrcBck    cmn.Bck
	SrcPrefix string
	DstBck    cmn.Bck
	Progress  bool
}

func Load

func Load() (*Config, error)

Load configuration for ishard from cli, or spec files (TODO)

type DryRunFlag

type DryRunFlag struct {
	IsSet bool
	Mode  string
}

func (*DryRunFlag) IsBoolFlag

func (d *DryRunFlag) IsBoolFlag() bool

func (*DryRunFlag) Set

func (d *DryRunFlag) Set(value string) error

func (*DryRunFlag) String

func (d *DryRunFlag) String() string

type EKMFlag

type EKMFlag struct {
	IsSet     bool
	Path      string
	JSONBytes []byte
	Ekm       shard.ExternalKeyMap
}

func (*EKMFlag) Set

func (e *EKMFlag) Set(param string) error

func (*EKMFlag) String

func (e *EKMFlag) String() string

type IshardConfig

type IshardConfig struct {
	ShardSize        ShardSize
	Ext              string
	ShardTemplate    string
	SampleKeyPattern SampleKeyPattern
	MExtMgr          *MissingExtManager
	Collapse         bool
}

type MissingExtManager

type MissingExtManager struct {
	Name             string
	EffectiveObjSize int64

	// Action to take on the given Records, returns the potentially updated Records and any error encountered
	React func(*shard.Records) (*shard.Records, error)
	// contains filtered or unexported fields
}

MissingExtManager contains the set of expected extensions for each sample, and corresponding reaction

func NewMissingExtManager

func NewMissingExtManager(name string, sampleExts []string) (*MissingExtManager, error)

type SampleKeyPattern

type SampleKeyPattern struct {
	Regex        string
	CaptureGroup string
}

type ShardSize

type ShardSize struct {
	IsCount bool
	Size    int64
	Count   int
}

func (*ShardSize) Set

func (s *ShardSize) Set(value string) error

func (*ShardSize) String

func (s *ShardSize) String() string

type SortFlag

type SortFlag struct {
	dsort.Algorithm
	IsSet bool
}

func (*SortFlag) Set

func (alg *SortFlag) Set(value string) error

func (*SortFlag) String

func (alg *SortFlag) String() string