file

README

Conduit Connector File

General

The File plugin is one of Conduit builtin plugins. It provides both source and destination File connectors, allowing for a file to be either a source, or a destination in a Conduit pipeline.

How to build it

Run make.

Testing

Run make test to run all the tests.

How it works

The Source connector listens for changes appended to the source file and sends records with the changes. The Destination connector receives records and writes them to a file.

Source

The Source connector only cares to have a valid path, even if the file doesn't exist, it will still run and wait until a file with the configured name is there, then it will start listening to changes and sending records.

Destination

The Destination connector will create the file if it doesn't exist, and records with changes will be appended to the destination file when received.

Configuration

name	part of	description	required	default value
path	destination, source	The path to the file	true

Limitations

• The Source connector only detects appended changes to the file, so it doesn't detect deletes or edits.
• The connectors can only access local files on the machine where Conduit is running. So, running Conduit on a server means it can't access a file on your local machine.
• Currently, only works reliably with text files (may work with non-text files, but not guaranteed)

Documentation

Index

• Constants
• Variables
• func NewDestination() sdk.Destination
• func NewSource() sdk.Source
• func Specification() sdk.Specification
• type Config
  • func (c Config) Validate() error
• type Destination
  • func (d *Destination) Configure(ctx context.Context, cfg config.Config) error
  • func (d *Destination) Open(context.Context) error
  • func (d *Destination) Parameters() config.Parameters
  • func (d *Destination) Teardown(context.Context) error
  • func (d *Destination) Write(_ context.Context, recs []opencdc.Record) (int, error)
• type DestinationConfig
  • func (DestinationConfig) Parameters() map[string]config.Parameter
  • func (c DestinationConfig) Validate() error
• type Source
  • func (s *Source) Ack(context.Context, opencdc.Position) error
  • func (s *Source) Configure(ctx context.Context, cfg config.Config) error
  • func (s *Source) Open(ctx context.Context, position opencdc.Position) error
  • func (s *Source) Parameters() config.Parameters
  • func (s *Source) Read(ctx context.Context) (opencdc.Record, error)
  • func (s *Source) Teardown(context.Context) error
• type SourceConfig
  • func (SourceConfig) Parameters() map[string]config.Parameter
  • func (c SourceConfig) Validate() error

Constants

const (
	DestinationConfigPath = "path"
)

const MetadataFilePath = "file.path"

const (
	SourceConfigPath = "path"
)

Variables

var Connector = sdk.Connector{
	NewSpecification: Specification,
	NewSource:        NewSource,
	NewDestination:   NewDestination,
}

Functions

func NewDestination

func NewDestination() sdk.Destination

func NewSource

func NewSource() sdk.Source

func Specification

func Specification() sdk.Specification

Specification returns the connector's specification.

Types

type Config

type Config struct {
	// Path is the file path used by the connector to read/write records.
	Path string `json:"path" validate:"required"`
}

func (Config) Validate

func (c Config) Validate() error

type Destination

type Destination struct {
	sdk.UnimplementedDestination
	// contains filtered or unexported fields
}

func (*Destination) Configure

func (d *Destination) Configure(ctx context.Context, cfg config.Config) error

func (*Destination) Open

func (d *Destination) Open(context.Context) error

func (*Destination) Parameters

func (d *Destination) Parameters() config.Parameters

func (*Destination) Teardown

func (d *Destination) Teardown(context.Context) error

func (*Destination) Write

func (d *Destination) Write(_ context.Context, recs []opencdc.Record) (int, error)

type DestinationConfig

type DestinationConfig struct {
	Config // embed the global config
}

func (DestinationConfig) Parameters

func (DestinationConfig) Parameters() map[string]config.Parameter

func (DestinationConfig) Validate

func (c DestinationConfig) Validate() error

type Source

type Source struct {
	sdk.UnimplementedSource
	// contains filtered or unexported fields
}

func (*Source) Ack

func (s *Source) Ack(context.Context, opencdc.Position) error

func (*Source) Configure

func (s *Source) Configure(ctx context.Context, cfg config.Config) error

func (*Source) Open

func (s *Source) Open(ctx context.Context, position opencdc.Position) error

func (*Source) Parameters

func (s *Source) Parameters() config.Parameters

func (*Source) Read

func (s *Source) Read(ctx context.Context) (opencdc.Record, error)

func (*Source) Teardown

func (s *Source) Teardown(context.Context) error

type SourceConfig

type SourceConfig struct {
	Config // embed the global config
}

func (SourceConfig) Parameters

func (SourceConfig) Parameters() map[string]config.Parameter

func (SourceConfig) Validate

func (c SourceConfig) Validate() error