csvutil

Documentation

Index

• Variables
• func FilterCSVFile(inPath, outPath string, inComma rune, inStripBom bool, ...) error
• func MergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, ...) error
• func MergeFilterCSVFilesToJSONL(inPaths []string, outPath string, inComma rune, inStripBom bool, ...) error
• func NewReader(path string, comma rune, stripBom bool) (*csv.Reader, *os.File, error)
• func NewWriterFile(filename string) (*csv.Writer, *os.File, error)
• func ReadCSVFileSingleColumnValuesString(filename string, sep rune, stripBOM, hasHeader, trimSpace bool, colIdx uint, ...) ([]string, error)
• func ReadCSVFilesSingleColumnValuesString(files []string, sep rune, stripBOM, hasHeader, trimSpace bool, colIdx uint, ...) ([]string, error)
• func WriteCSVFiltered(reader *csv.Reader, writer *csv.Writer, ...) error
• type CSVHeader
  • func (ch *CSVHeader) Index(want string) int
  • func (ch *CSVHeader) RecordMatch(row []string, andFilter map[string]stringsutil.MatchInfo) (bool, error)
  • func (ch *CSVHeader) RecordToMSS(row []string) map[string]string
• type Writer
  • func NewWriter(filepath, sep string, replaceSeparator bool, alt string) (Writer, error)
  • func (w *Writer) AddLine(cells []interface{}) error
  • func (w *Writer) Close() error

Variables

var DebugReadCSV = false // should not need to edit

Functions

func FilterCSVFile

func FilterCSVFile(inPath, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error

func MergeFilterCSVFiles

func MergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error

MergeFilterCSVFiles can merge and filter multiple CSV files. It expects row definitions to be the same across all input files.

func MergeFilterCSVFilesToJSONL

func MergeFilterCSVFilesToJSONL(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error

type JsonRecordsInfo struct {
	Meta    JsonRecordsInfoMeta `json:"meta"`
	Records []map[string]string `json:"records"`
}

type JsonRecordsInfoMeta struct {
	Count int `json:"count"`
}

func ReadMergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) (table.DocumentsSet, error) {
	//data := JsonRecordsInfo{Records: []map[string]string{}}
	data := table.NewDocumentsSet()

	for _, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return data, err
		}

		csvHeader := CSVHeader{}
		j := -1

		for {
			line, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return data, err
			}
			j += 1

			if j == 0 {
				csvHeader.Columns = line
				continue
			}
			match, err := csvHeader.RecordMatch(line, andFilter)
			if err != nil {
				return data, err
			}
			if !match {
				continue
			}

			mss := csvHeader.RecordToMSS(line)
			data.Documents = append(data.Documents, mss)
		}
		err = inFile.Close()
		if err != nil {
			return data, err
		}
	}
	data.Inflate()
	return data, nil
}

func MergeFilterCSVFilesToJSON(inPaths []string, outPath string, inComma rune, inStripBom bool, perm os.FileMode, andFilter map[string]stringsutil.MatchInfo) error {
	data, err := ReadMergeFilterCSVFiles(inPaths, outPath, inComma, inStripBom, andFilter)
	if err != nil {
		return err
	}
	bytes, err := jsonutil.MarshalSimple(data, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(outPath, bytes, perm)
}

func NewReader

func NewReader(path string, comma rune, stripBom bool) (*csv.Reader, *os.File, error)

NewReader will create a csv.Reader and optionally strip off the byte order mark (BOM) if requested. Close file reader with `defer f.Close()`.

func NewWriterFile

func NewWriterFile(filename string) (*csv.Writer, *os.File, error)

func ReadCSVFileSingleColumnValuesString

func ReadCSVFileSingleColumnValuesString(filename string, sep rune, stripBOM, hasHeader, trimSpace bool, colIdx uint, condenseUniqueSort bool) ([]string, error)

func ReadCSVFilesSingleColumnValuesString

func ReadCSVFilesSingleColumnValuesString(files []string, sep rune, stripBOM, hasHeader, trimSpace bool, colIdx uint, condenseUniqueSort bool) ([]string, error)

func WriteCSVFiltered

func WriteCSVFiltered(reader *csv.Reader, writer *csv.Writer, andFilter map[string]stringsutil.MatchInfo, writeHeader bool) error

WriteCSVFiltered filters an existing CSV and writes the matching lines to a *csv.Writer.

Types

type CSVHeader

type CSVHeader struct {
	Columns []string
}

func (*CSVHeader) Index

func (ch *CSVHeader) Index(want string) int

func (*CSVHeader) RecordMatch

func (ch *CSVHeader) RecordMatch(row []string, andFilter map[string]stringsutil.MatchInfo) (bool, error)

func (*CSVHeader) RecordToMSS

func (ch *CSVHeader) RecordToMSS(row []string) map[string]string

type Writer

type Writer struct {
	Separator        string
	StripRepeatedSep bool
	ReplaceSeparator bool
	SeparatorAlt     string
	File             *os.File
}

Writer is a struct for a CSV/TSV writer.

func NewWriter

func NewWriter(filepath, sep string, replaceSeparator bool, alt string) (Writer, error)

NewWriter returns a Writer with the separator params set.

func (*Writer) AddLine

func (w *Writer) AddLine(cells []interface{}) error

AddLine adds an []interface{} to the file.

func (*Writer) Close

func (w *Writer) Close() error

Close closes the file.