copier

Documentation

Overview

Package copier provides tools and utilities for copying and modifying GTFS feeds.

Index

• func Copy(reader adapters.Reader, writer adapters.Writer, optfns ...func(*Options)) error
• func QuietCopy(reader adapters.Reader, writer adapters.Writer, optfns ...func(*Options)) error
• type AfterValidator
• type AfterWrite
• type Copier
  • func NewCopier(reader adapters.Reader, writer adapters.Writer, opts Options) (*Copier, error)
  • func (copier *Copier) AddExtension(ext interface{}) error
  • func (copier *Copier) AddValidator(ext Validator, level int) error
  • func (copier *Copier) Copy() *Result
  • func (copier *Copier) CopyEntities(ents []tt.Entity) error
  • func (copier *Copier) CopyEntity(ent tt.Entity) (error, error)
  • func (copier *Copier) SetLogger(g zerolog.Logger)
• type ErrorHandler
• type ExpandFilter
• type Extension
• type Filter
• type Marker
• type Options
  • func (opts *Options) AddExtension(ext Extension)
• type Prepare
• type Result
  • func NewResult(errorLimit int) *Result
  • func (cr *Result) DisplayErrors()
  • func (cr *Result) DisplaySummary()
  • func (cr *Result) DisplayWarnings()
  • func (cr *Result) HandleEntityErrors(ent tt.Entity, errs []error, warns []error)
  • func (cr *Result) HandleError(fn string, errs []error)
  • func (cr *Result) HandleSourceErrors(fn string, errs []error, warns []error)
• type ValidationError
  • func (e ValidationError) Error() string
• type ValidationErrorGroup
  • func NewValidationErrorGroup(err error, limit int) *ValidationErrorGroup
  • func (e *ValidationErrorGroup) Add(err error)
  • func (eg *ValidationErrorGroup) Key() string
• type Validator

Functions

func Copy

func Copy(reader adapters.Reader, writer adapters.Writer, optfns ...func(*Options)) error

Copy with options builder

func QuietCopy

func QuietCopy(reader adapters.Reader, writer adapters.Writer, optfns ...func(*Options)) error

Quiet copy

Types

type AfterValidator

type AfterValidator interface {
	AfterValidator(tt.Entity, *tt.EntityMap) error
}

AfterValidator is called for each fully validated entity before writing.

type AfterWrite

type AfterWrite interface {
	AfterWrite(string, tt.Entity, *tt.EntityMap) error
}

AfterWrite is called for after writing each entity.

type Copier

type Copier struct {
	// Default options
	Options
	// Reader and writer
	Reader adapters.Reader
	Writer adapters.Writer
	// Entity selection strategy
	Marker Marker
	// Error handler, called for each entity
	ErrorHandler ErrorHandler

	EntityMap *tt.EntityMap
	// contains filtered or unexported fields
}

Copier copies from Reader to Writer

func NewCopier

func NewCopier(reader adapters.Reader, writer adapters.Writer, opts Options) (*Copier, error)

NewCopier creates and initializes a new Copier.

func (*Copier) AddExtension

func (copier *Copier) AddExtension(ext interface{}) error

AddExtension adds an Extension to the copy process.

func (*Copier) AddValidator

func (copier *Copier) AddValidator(ext Validator, level int) error

AddValidator adds an additional entity validator.

func (*Copier) Copy

func (copier *Copier) Copy() *Result

Copy copies Base GTFS entities from the Reader to the Writer, returning the summary as a Result.

func (*Copier) CopyEntities

func (copier *Copier) CopyEntities(ents []tt.Entity) error

CopyEntities validates a slice of entities and writes those that pass validation.

func (*Copier) CopyEntity

func (copier *Copier) CopyEntity(ent tt.Entity) (error, error)

CopyEntity performs validation and saves errors and warnings. An entity error means the entity was not not written because it had an error or was filtered out; not fatal. A write error should be considered fatal and should stop any further write attempts. Any errors and warnings are added to the copier result.

func (*Copier) SetLogger

func (copier *Copier) SetLogger(g zerolog.Logger)

type ErrorHandler

type ErrorHandler interface {
	HandleEntityErrors(tt.Entity, []error, []error)
	HandleSourceErrors(string, []error, []error)
}

ErrorHandler is called on each source file and entity; errors can be nil

type ExpandFilter

type ExpandFilter interface {
	Expand(tt.Entity, *tt.EntityMap) ([]tt.Entity, bool, error)
}

type Extension

type Extension interface {
	Copy(*Copier) error
}

Extension is run after normal copying has completed.

type Filter

type Filter interface {
	Filter(tt.Entity, *tt.EntityMap) error
}

Filter is called before validation.

type Marker

type Marker interface {
	IsMarked(string, string) bool
	IsVisited(string, string) bool
}

Marker visits and marks entities.

type Options

type Options struct {
	// Batch size
	BatchSize int
	// Skip most validation filters
	NoValidators bool
	// Skip shape cache
	NoShapeCache bool
	// Attempt to save an entity that returns validation errors
	AllowEntityErrors    bool
	AllowReferenceErrors bool
	// Interpolate any missing StopTime values: ArrivalTime/DepartureTime/ShapeDistTraveled
	InterpolateStopTimes bool
	// Create a stop-to-stop Shape for Trips without a ShapeID.
	CreateMissingShapes bool
	// Create missing Calendar entries
	NormalizeServiceIDs bool
	// Normalize timezones, e.g. US/Pacific -> America/Los_Angeles
	NormalizeTimezones bool
	// Simplify Calendars that use mostly CalendarDates
	SimplifyCalendars bool
	// Convert extended route types to primitives
	UseBasicRouteTypes bool
	// Copy extra files (requires CSV input)
	CopyExtraFiles bool
	// Simplify shapes
	SimplifyShapes float64
	// DeduplicateStopTimes
	DeduplicateJourneyPatterns bool
	// Default error handler
	ErrorHandler ErrorHandler
	// Journey Pattern Key Function
	JourneyPatternKey func(*gtfs.Trip) string
	// Named extensions
	Extensions []string

	// Error limit
	ErrorLimit int

	// Sub-logger
	Quiet bool
	// contains filtered or unexported fields
}

Options defines the settable options for a Copier.

func (*Options) AddExtension

func (opts *Options) AddExtension(ext Extension)

type Prepare

type Prepare interface {
	Prepare(adapters.Reader, *tt.EntityMap) error
}

Prepare is called before general copying begins.

type Result

type Result struct {
	WriteError                error
	InterpolatedStopTimeCount int
	EntityCount               map[string]int
	GeneratedCount            map[string]int
	SkipEntityErrorCount      map[string]int
	SkipEntityReferenceCount  map[string]int
	SkipEntityFilterCount     map[string]int
	SkipEntityMarkedCount     map[string]int
	Errors                    map[string]*ValidationErrorGroup
	Warnings                  map[string]*ValidationErrorGroup
	ErrorLimit                int
}

Result stores Copier results and statistics.

func NewResult

func NewResult(errorLimit int) *Result

NewResult returns a new Result.

func (*Result) DisplayErrors

func (cr *Result) DisplayErrors()

DisplayErrors shows individual errors in log.Info

func (*Result) DisplaySummary

func (cr *Result) DisplaySummary()

DisplaySummary shows entity and error counts in log.Info

func (*Result) DisplayWarnings

func (cr *Result) DisplayWarnings()

DisplayWarnings shows individual warnings in log.Info

func (*Result) HandleEntityErrors

func (cr *Result) HandleEntityErrors(ent tt.Entity, errs []error, warns []error)

HandleEntityErrors .

func (*Result) HandleError

func (cr *Result) HandleError(fn string, errs []error)

HandleError .

func (*Result) HandleSourceErrors

func (cr *Result) HandleSourceErrors(fn string, errs []error, warns []error)

HandleSourceErrors .

type ValidationError

type ValidationError struct {
	Filename   string `db:"-"`
	Field      string `db:"-"`
	ErrorCode  string `db:"-"`
	Line       int
	GroupKey   string
	Message    string
	EntityID   string
	Value      string
	Geometry   tt.Geometry
	EntityJson tt.Map
}

func (ValidationError) Error

func (e ValidationError) Error() string

type ValidationErrorGroup

type ValidationErrorGroup struct {
	Filename  string
	Field     string
	ErrorType string
	ErrorCode string
	GroupKey  string
	Level     int
	Count     int
	Limit     int               `db:"-"`
	Errors    []ValidationError `db:"-"`
}

ValidationErrorGroup helps group errors together with a maximum limit on the number stored.

func NewValidationErrorGroup

func NewValidationErrorGroup(err error, limit int) *ValidationErrorGroup

func (*ValidationErrorGroup) Add

func (e *ValidationErrorGroup) Add(err error)

Add an error to the error group.

func (*ValidationErrorGroup) Key

func (eg *ValidationErrorGroup) Key() string

type Validator

type Validator interface {
	Validate(tt.Entity) []error
}

Validator is called for each entity.