ingest

Dec 4, 2023

Changes in this version

type FileOption

+ func CompressionType(compressionType ingestoptions.CompressionType) FileOption

Aug 10, 2023

Changes in this version

type FileOption

+ func RawDataSize(size int64) FileOption

May 9, 2023

Changes in this version

type FileOption

+ func IgnoreFirstRecord() FileOption

May 1, 2023

Changes in this version

type DataFormat

+ func InferFormatFromFileName(fName string) DataFormat

Feb 14, 2023

Changes in this version

type QueryClient

+ ClientDetails func() *kusto.ClientDetails

May 8, 2022

Changes in this version

type Ingestion

+ func (i *Ingestion) Close() error

type Managed

+ func (m *Managed) Close() error

type QueryClient

+ HttpClient func() *http.Client

type Streaming

+ func (i *Streaming) Close() error

Apr 12, 2022

Changes in this version

+ const FromBlob

+ const FromFile

+ const FromReader

+ const ManagedClient

+ const QueuedClient

+ const StreamingClient

+ var FileIsBlobErr = errors.ES(errors.OpIngestStream, errors.KClientArgs, ...)

+ func GetErrorCode(err error) (string, error)

+ func IsStatusRecord(err error) bool

+ func StatusFromMapForTests(data map[string]interface{}) error

+ type ClientScope uint

+ func (s ClientScope) String() string

type FileOption

+ func ClientRequestId(clientRequestId string) FileOption

+ func Database(name string) FileOption

+ func DontCompress() FileOption

+ func Table(name string) FileOption

+ ClientScopes func() ClientScope

+ Run func(p *properties.All, clientType ClientScope, sourceType SourceScope) error

+ SourceScopes func() SourceScope

+ type Ingestor interface

+ FromFile func(ctx context.Context, fPath string, options ...FileOption) (*Result, error)

+ FromReader func(ctx context.Context, reader io.Reader, options ...FileOption) (*Result, error)

+ type Managed struct

+ func NewManaged(client QueryClient, db, table string, options ...Option) (*Managed, error)

+ func (m *Managed) FromFile(ctx context.Context, fPath string, options ...FileOption) (*Result, error)

+ func (m *Managed) FromReader(ctx context.Context, reader io.Reader, options ...FileOption) (*Result, error)

+ type SourceScope uint

+ func (s SourceScope) String() string

+ type Streaming struct

+ func NewStreaming(client QueryClient, db, table string) (*Streaming, error)

+ func (i *Streaming) FromFile(ctx context.Context, fPath string, options ...FileOption) (*Result, error)

+ func (i *Streaming) FromReader(ctx context.Context, reader io.Reader, options ...FileOption) (*Result, error)

Feb 7, 2022

Changes in this version

+ type QueryClient interface

+ Auth func() kusto.Authorization

+ Endpoint func() string

+ Mgmt func(ctx context.Context, db string, query kusto.Stmt, options ...kusto.MgmtOption) (*kusto.RowIterator, error)

+ Query func(ctx context.Context, db string, query kusto.Stmt, options ...kusto.QueryOption) (*kusto.RowIterator, error)

Jan 23, 2022

Changes in this version

type DataFormat

+ const SingleJSON

+ type Option func(s *Ingestion)

+ func WithStaticBuffer(bufferSize int, maxBuffers int) Option

Jul 13, 2021

Changes in this version

type FileOption

+ func SetCreationTime(t time.Time) FileOption

Nov 26, 2020

Changes in this version

+ func IsRetryable(err error) bool

+ type FailureStatusCode string

+ const Exhausted

+ const Permanent

+ const Transient

+ const Unknown

+ func GetIngestionFailureStatus(err error) (FailureStatusCode, error)

+ func (i FailureStatusCode) IsRetryable() bool

type FileOption

+ func ReportResultToTable() FileOption

+ type Result struct

+ func (r *Result) Wait(ctx context.Context) chan error

+ type StatusCode string

+ const Failed

+ const PartiallySucceeded

+ const Pending

+ const Queued

+ const Skipped

+ const StatusRetrievalCanceled

+ const StatusRetrievalFailed

+ const Succeeded

+ func GetIngestionStatus(err error) (StatusCode, error)

+ func (i StatusCode) IsFinal() bool

+ func (i StatusCode) IsSuccess() bool

Jul 1, 2020

Changes in this version

type DataFormat

+ const ApacheAVRO

+ const MultiJSON

+ const SStream

+ const TSVE

+ const W3CLogFile

type Ingestion

+ func (i *Ingestion) FromReader(ctx context.Context, reader io.Reader, options ...FileOption) error

Apr 16, 2020

Changes in this version

+ var ErrTooLarge = errors.ES(errors.OpIngestStream, errors.KClientArgs, "cannot add data larger than 4MiB")

+ type DataFormat = properties.DataFormat

+ const AVRO

+ const CSV

+ const DFUnknown

+ const JSON

+ const ORC

+ const PSV

+ const Parquet

+ const Raw

+ const SCSV

+ const SOHSV

+ const TSV

+ const TXT

+ type FileOption interface

+ func DeleteSource() FileOption

+ func FileFormat(et DataFormat) FileOption

+ func FlushImmediately() FileOption

+ func IfNotExists(ingestByTag string) FileOption

+ func IgnoreSizeLimit() FileOption

+ func IngestionMapping(mapping interface{}, mappingKind DataFormat) FileOption

+ func IngestionMappingRef(refName string, mappingKind DataFormat) FileOption

+ func Tags(tags []string) FileOption

+ func ValidationPolicy(policy ValPolicy) FileOption

+ type Ingestion struct

+ func New(client *kusto.Client, db, table string) (*Ingestion, error)

+ func (i *Ingestion) FromFile(ctx context.Context, fPath string, options ...FileOption) error

+ func (i *Ingestion) Stream(ctx context.Context, payload []byte, format DataFormat, mappingName string) error

+ type ValPolicy struct

+ Implications ValidationImplication

+ Options ValidationOption

+ type ValidationImplication int8

+ const FailIngestion

+ const IgnoreFailures

+ type ValidationOption int8

+ const IgnoreNonDoubleQuotedFields

+ const SameNumberOfFields

+ const VOUnknown