config

README

These should be used so that enviroment variables, and other defaults can be read.

*Config.yaml are the config files

others can be submitted as subtress. Subtress cannot be extracted when they are arrays, aka v1.Sub('sources'), so passing in the root may be a better solution, but env substitute is more reliable without the dot path

Documentation

Index

• Variables
• func GetBucketName(v1 *viper.Viper) (string, error)
• func PruneSources(v1 *viper.Viper, useSources []string) (*viper.Viper, error)
• func ReadGleanerConfig(filename string, cfgDir string) (*viper.Viper, error)
• func ReadServersConfig(filename string, cfgDir string) (*viper.Viper, error)
• func SourceToNabuPrefix(sources []Sources, includeProv bool) []string
• type Minio
  • func ReadMinioConfig(minioSubtress *viper.Viper) (Minio, error)
• type Objects
  • func ReadS3Config(viperSubtree *viper.Viper) (Objects, error)
• type Sources
  • func GetActiveSourceByType(sources []Sources, key string) []Sources
  • func GetActiveSources(g1 *viper.Viper) ([]Sources, error)
  • func GetSourceByType(sources []Sources, key string) []Sources
  • func GetSources(g1 *viper.Viper) ([]Sources, error)
  • func ReadSourcesCSV(filename string, cfgPath string) ([]Sources, error)
• type SourcesConfig
• type Sparql
  • func ReadSparqlConfig(viperSubtree *viper.Viper) (Sparql, error)
• type Summoner
  • func ReadSummmonerConfig(viperSubtree *viper.Viper) (Summoner, error)

Variables

var MinioTemplate = map[string]interface{}{
	"minio": map[string]string{
		"address":   "localhost",
		"port":      "9000",
		"accesskey": "",
		"secretkey": "",
		"bucket":    "",
	},
}

var ObjectTemplate = map[string]interface{}{
	"objects": map[string]string{
		"bucket":    "gleaner",
		"domain":    "us-east-1",
		"endpoint":  "http://localhost/blazegraph/namespace/nabu/sparql",
		"prefix":    "",
		"prefixoff": "",
	},
}

var SourcesTemplate = map[string]interface{}{
	"sources": map[string]string{
		"sourcetype":      "sitemap",
		"name":            "",
		"url":             "",
		"logo":            "",
		"headless":        "",
		"pid":             "",
		"propername":      "",
		"domain":          "",
		"credentialsfile": "",
	},
}

var SummonerTemplate = map[string]interface{}{
	"summoner": map[string]string{
		"after":    "",
		"mode":     "full",
		"threads":  "5",
		"delay":    "10000",
		"headless": "http://127.0.0.1:9222",
	},
}

Functions

func GetBucketName

func GetBucketName(v1 *viper.Viper) (string, error)

func PruneSources

func PruneSources(v1 *viper.Viper, useSources []string) (*viper.Viper, error)

func ReadGleanerConfig

func ReadGleanerConfig(filename string, cfgDir string) (*viper.Viper, error)

func ReadServersConfig

func ReadServersConfig(filename string, cfgDir string) (*viper.Viper, error)

func SourceToNabuPrefix

func SourceToNabuPrefix(sources []Sources, includeProv bool) []string

Types

type Minio

type Minio struct {
	Address   string // `mapstructure:"MINIO_ADDRESS"`
	Port      int    //`mapstructure:"MINIO_PORT"`
	Ssl       bool   //`mapstructure:"MINIO_USE_SSL"`
	Accesskey string //`mapstructure:"MINIO_ACCESS_KEY"`
	Secretkey string // `mapstructure:"MINIO_SECRET_KEY"`
	Bucket    string
}

frig frig... do not use lowercase... those are private variables

func ReadMinioConfig

func ReadMinioConfig(minioSubtress *viper.Viper) (Minio, error)

use config.Sub("minio)

type Objects

type Objects struct {
	Bucket    string //`mapstructure:"MINIO_BUCKET"`
	Domain    string //`mapstructure:"MINIO_DOMAIN"`
	Prefix    []string
	PrefixOff []string
}

func ReadS3Config

func ReadS3Config(viperSubtree *viper.Viper) (Objects, error)

type Sources

type Sources struct {
	SourceType      string `default:"sitemap"`
	Name            string
	Logo            string
	URL             string
	Headless        bool
	PID             string
	ProperName      string
	Domain          string
	Active          bool                   `default:"true"`
	CredentialsFile string                 // do not want someones google api key exposed.
	Other           map[string]interface{} `mapstructure:",remain"`
}

as read from csv

func GetActiveSourceByType

func GetActiveSourceByType(sources []Sources, key string) []Sources

func GetActiveSources

func GetActiveSources(g1 *viper.Viper) ([]Sources, error)

func GetSourceByType

func GetSourceByType(sources []Sources, key string) []Sources

func GetSources

func GetSources(g1 *viper.Viper) ([]Sources, error)

func ReadSourcesCSV

func ReadSourcesCSV(filename string, cfgPath string) ([]Sources, error)

type SourcesConfig

type SourcesConfig struct {
	Name       string
	Logo       string
	URL        string
	Headless   bool
	PID        string
	ProperName string
	Domain     string
}

add needed for file

type Sparql

type Sparql struct {
	Endpoint     string
	Authenticate bool
	Username     string
	Password     string
}

func ReadSparqlConfig

func ReadSparqlConfig(viperSubtree *viper.Viper) (Sparql, error)

type Summoner

type Summoner struct {
	After    string
	Mode     string // full || diff:  If diff compare what we have currently in gleaner to sitemap, get only new, delete missing
	Threads  int
	Delay    int64  // milliseconds (1000 = 1 second) to delay between calls (will FORCE threads to 1)
	Headless string // URL for headless see docs/headless
}

func ReadSummmonerConfig

func ReadSummmonerConfig(viperSubtree *viper.Viper) (Summoner, error)