nestcsv

package module
v0.0.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 10, 2025 License: MIT Imports: 30 Imported by: 0

README

nestcsv

A Go CLI tool that analyzes CSV data based on a predefined schema and converts it into a nested JSON structure.

Installation

go install github.com/unsafe9/nestcsv/cmd/nestcsv@latest

Usage

Compose your configurations:

# nestcsv.yaml
datasources:
  - spreadsheet_gas:
      url: <YOUR_GOOGLE_APPS_SCRIPT_WEB_APP_ENDPOINT>
      password: <YOUR_GOOGLE_APPS_SCRIPT_WEB_APP_PASSWORD>
      google_drive_folder_ids:
        - <YOUR_GOOGLE_DRIVE_FOLDER_ID>
      spreadsheet_file_ids:
        - <YOUR_GOOGLE_SPREADSHEET_FILE_ID>
      debug_save_dir: ./debug
  - excel:
      patterns:
        - ./datasource/*.xlsx
      debug_save_dir: ./debug
  - csv:
      patterns:
        - ./datasource/*.csv
        #- ./debug/*.csv

outputs:
  - tags: [server, client]
    json:
      root_dir: ./output
      indent: "  "

codegens:
  - tags: [server]
    go:
      root_dir: ./go
      package_name: table
  - tags: [client]
    ue5:
      root_dir: ./ue5
      prefix: Nest

Run the following command:

nestcsv

# specify your config file
nestcsv -c ../config/config.yaml

How to structure the schema

See examples

Roadmap

Docs
  • Add a csv(datasource) schema structure guide
  • Add an example of UE5 json file loading
Datasource
  • Implement Google OAuth2 authentication for Google Apps Script
  • Integrate spreadsheet datasource using Sheets API
Config
  • Extract time format settings into the configuration file
Output
  • Generate SQL dump file
Code generation
  • Generate Protobuf schema
  • Generate Rust code
  • Generate Node.js code with type definitions
  • Generate PostgreSQL DDL
  • Generate MySQL DDL

Documentation

Index

Constants

View Source 
const (
	TableMetadataRow  = 0
	TableFieldTagRow  = 1
	TableFieldNameRow = 2
	TableFieldTypeRow = 3
	TableDataStartRow = 5

	TableFieldIndexCol = 0
)

Variables

View Source 
var ErrSkipTable = fmt.Errorf("skip table")

Functions

func SetCommandArgs 

func SetCommandArgs(argsStr string)

Types

type Code 

type Code struct {
	Tables       []*CodeFile
	NamedStructs []*CodeFile
}

func AnalyzeTableCode 

func AnalyzeTableCode(tableDatas []*TableData, tags []string) (*Code, error)

func (*Code) Files 

func (c *Code) Files(yield func(*CodeFile) bool)

type CodeFile 

type CodeFile struct {
	IsTable          bool
	IsMap            bool
	Name             string
	Struct           *CodeStruct
	AnonymousStructs []*CodeStruct
	FileRefs         []*CodeFile
	FieldTypes       []FieldType
	IDField          *CodeStructField
	IDFieldType      FieldType // this will be set even if IDField is nil
}

type CodeStruct 

type CodeStruct struct {
	Name   string
	Fields []*CodeStructField
}

type CodeStructField 

type CodeStructField struct {
	Name      string
	Type      FieldType
	IsArray   bool
	StructRef *CodeStruct
}

type Codegen 

type Codegen interface {
	Generate(code *Code) error
}

type CodegenConfig 

type CodegenConfig struct {
	When *When    `yaml:"when,omitempty"`
	Tags []string `yaml:"tags"`

	Go  *CodegenGo  `yaml:"go,omitempty"`
	UE5 *CodegenUE5 `yaml:"ue5,omitempty"`
	// contains filtered or unexported fields
}

func (*CodegenConfig) Generate 

func (c *CodegenConfig) Generate(tableDatas []*TableData) error

func (*CodegenConfig) UnmarshalYAML 

func (c *CodegenConfig) UnmarshalYAML(node *yaml.Node) error

type CodegenGo 

type CodegenGo struct {
	RootDir     string `yaml:"root_dir"`
	PackageName string `yaml:"package_name"`
	Singleton   bool   `yaml:"singleton"`
	Context     bool   `yaml:"context"`
}

func (*CodegenGo) Generate 

func (c *CodegenGo) Generate(code *Code) error

type CodegenUE5 

type CodegenUE5 struct {
	RootDir string `yaml:"root_dir"`
	Prefix  string `yaml:"prefix"`
}

func (*CodegenUE5) Generate 

func (c *CodegenUE5) Generate(code *Code) error

type Config 

type Config struct {
	Datasources []DatasourceConfig `yaml:"datasources"`
	Outputs     []OutputConfig     `yaml:"outputs"`
	Codegens    []CodegenConfig    `yaml:"codegens"`
}

func ParseConfig 

func ParseConfig(configPath string) (*Config, error)

type Datasource 

type Datasource interface {
	Collect(out chan<- *TableData) error
}

type DatasourceCSV 

type DatasourceCSV struct {
	Patterns []string `yaml:"patterns"`
}

func (*DatasourceCSV) Collect 

func (d *DatasourceCSV) Collect(out chan<- *TableData) error

type DatasourceConfig 

type DatasourceConfig struct {
	When *When `yaml:"when,omitempty"`

	SpreadsheetGAS *DatasourceSpreadsheetGAS `yaml:"spreadsheet_gas,omitempty"`
	Excel          *DatasourceExcel          `yaml:"excel,omitempty"`
	CSV            *DatasourceCSV            `yaml:"csv,omitempty"`
	// contains filtered or unexported fields
}

func (*DatasourceConfig) Collect 

func (c *DatasourceConfig) Collect(out chan<- *TableData) error

func (*DatasourceConfig) UnmarshalYAML 

func (c *DatasourceConfig) UnmarshalYAML(node *yaml.Node) error

type DatasourceExcel 

type DatasourceExcel struct {
	Patterns     []string `yaml:"patterns"`
	DebugSaveDir *string  `yaml:"debug_save_dir,omitempty"`
}

func (*DatasourceExcel) Collect 

func (d *DatasourceExcel) Collect(out chan<- *TableData) error

type DatasourceSpreadsheetGAS 

type DatasourceSpreadsheetGAS struct {
	URL                  string   `yaml:"url"`
	Password             string   `yaml:"password"`
	GoogleDriveFolderIDs []string `yaml:"google_drive_folder_ids"`
	SpreadsheetFileIDs   []string `yaml:"spreadsheet_file_ids"`
	DebugSaveDir         *string  `yaml:"debug_save_dir,omitempty"`
}

func (*DatasourceSpreadsheetGAS) Collect 

func (d *DatasourceSpreadsheetGAS) Collect(out chan<- *TableData) error

type FieldType 

type FieldType string
const (
	FieldTypeInt    FieldType = "int"
	FieldTypeLong   FieldType = "long"
	FieldTypeFloat  FieldType = "float"
	FieldTypeBool   FieldType = "bool"
	FieldTypeString FieldType = "string"
	FieldTypeTime   FieldType = "time"
	FieldTypeJSON   FieldType = "json"
	FieldTypeStruct FieldType = "struct"
)

func (FieldType) Array 

func (t FieldType) Array() FieldType

func (FieldType) String 

func (t FieldType) String() string

type OutputConfig 

type OutputConfig struct {
	When *When    `yaml:"when,omitempty"`
	Tags []string `yaml:"tags"`

	JSON *TableWriterJSON `yaml:"json,omitempty"`
	Bin  *TableWriterBin  `yaml:"bin,omitempty"`
	// contains filtered or unexported fields
}

func (*OutputConfig) UnmarshalYAML 

func (c *OutputConfig) UnmarshalYAML(node *yaml.Node) error

func (*OutputConfig) Write 

func (c *OutputConfig) Write(tableData *TableData) error

type StructMap 

type StructMap map[string]string

StructMap - you can map a field id to a named struct 

You can match the field id with a regex by enclosing it with / (e.g. /regex/),
If the field is part of a named struct, you should omit the parent struct's name.
ex. RewardWin.Weight,RewardWin.RewardPool.Reward.Type,RewardWin.RewardPool.Reward.Param
	struct=RewardWin:GameReward&struct=/RewardPool\.Reward$/:GameRewardPool

func (StructMap) Get 

func (t StructMap) Get(id string) (string, bool)

type TableData 

type TableData struct {
	Name       string
	Metadata   *TableMetadata
	Columns    int
	FieldTags  [][]string
	FieldNames []string
	FieldTypes []string
	DataRows   [][]string
}

func ParseTableData 

func ParseTableData(name string, csvData [][]string) (*TableData, error)

func (*TableData) CSV 

func (d *TableData) CSV() [][]string

type TableField 

type TableField struct {
	Name             string
	Type             FieldType
	IsMultiLineArray bool
	IsCellArray      bool
	StructFields     []*TableField
	ParentField      *TableField
	// contains filtered or unexported fields
}

func (*TableField) Clone 

func (f *TableField) Clone() *TableField

func (*TableField) GetMultiLineArrayField 

func (f *TableField) GetMultiLineArrayField() *TableField

func (*TableField) Identifier 

func (f *TableField) Identifier() string

func (*TableField) IsArray 

func (f *TableField) IsArray() bool

func (*TableField) Iterate 

func (f *TableField) Iterate(yield func(*TableField) bool)

func (*TableField) StructEqual 

func (f *TableField) StructEqual(other *TableField) bool

type TableMetadata 

type TableMetadata struct {
	AsMap      bool      `query:"as_map"`
	SortAscBy  string    `query:"sort_asc_by"`
	SortDescBy string    `query:"sort_desc_by"`
	Structs    StructMap `query:"struct"`
}

func (*TableMetadata) Validate 

func (m *TableMetadata) Validate(td *TableData) error

type TableMetadataQuery 

type TableMetadataQuery string

func (TableMetadataQuery) Decode 

func (q TableMetadataQuery) Decode() (*TableMetadata, error)

type TableParser 

type TableParser struct {
	// contains filtered or unexported fields
}

func NewTableParser 

func NewTableParser(td *TableData) *TableParser

func (*TableParser) Marshal 

func (p *TableParser) Marshal(fields []*TableField) (any, error)

func (*TableParser) ParseTableFields 

func (p *TableParser) ParseTableFields(tags []string) ([]*TableField, error)

type TableWriter 

type TableWriter interface {
	Write(name string, value any) error
}

type TableWriterBin 

type TableWriterBin struct {
	RootDir string `yaml:"root_dir"`
}

func (*TableWriterBin) Write 

func (e *TableWriterBin) Write(name string, value any) error

type TableWriterJSON 

type TableWriterJSON struct {
	RootDir string `yaml:"root_dir"`
	Indent  string `yaml:"indent"`
}

func (*TableWriterJSON) Write 

func (e *TableWriterJSON) Write(name string, value any) error

type When 

type When struct {
	// Not - if true, the condition is negated
	Not  bool              `yaml:"not"`
	Env  map[string]string `yaml:"env,omitempty"`
	Args []string          `yaml:"args,omitempty"`
}

func (*When) Match 

func (w *When) Match() bool

Source Files

View all Source files

Directories

Path Synopsis
cmd
nestcsv
examples
functions/go

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.