data

Documentation

Index

• Constants
• Variables

Constants

const (
	GroupTag = "data-file-group"
)

Variables

var DataCmd = &cobra.Command{
	Use:   "data",
	Short: "subcommand for managing data files",
}

DataCmd manages data files

var RefreshCmd = &cobra.Command{
	Use:   "refresh",
	Short: "updates data files in S3 storage",
	Args:  cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		repo, _ := cmd.Flags().GetString("repository")
		branch, _ := cmd.Flags().GetString("branch")
		subf, _ := cmd.Flags().GetString("subfolder")
		bucketPath, _ := cmd.Flags().GetString("s3-bucket-path")
		dir, err := git.CloneRepo(repo, branch)
		if err != nil {
			return err
		}
		fmap, err := newdataFileManager(subf).allFileReaders(dir)
		if err != nil {
			return err
		}
		logger := registry.GetLogger()
		for path, rd := range fmap {
			_, err := registry.GetS3Client().PutObject(
				viper.GetString("s3-bucket"),
				fmt.Sprintf("%s/%s", bucketPath, path),
				rd, -1, minio.PutObjectOptions{
					UserMetadata: map[string]string{
						GroupTag: viper.GetString("group"),
					},
				},
			)
			if err != nil {
				return err
			}
			logger.Debugf("uploaded file to path %s", path)
		}
		logger.Infof("refreshed %d data files", len(fmap))
		return nil
	},
}

RefreshCmd updates dictybase data files in S3(minio) storage