properties

package
v0.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 1, 2020 License: MIT Imports: 8 Imported by: 0

Documentation

Overview

Package properties provides Kusto REST properties that will need to be serialized and sent to Kusto based upon the type of ingestion we are doing.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Additional

type Additional struct {
	// AuthContext is the authorization string that we get from resources.Manager.AuthContext().
	AuthContext string `json:"authorizationContext,omitempty"`
	// IngestionMapping is a json string that maps the data being imported to the table's columns.
	// See: https://docs.microsoft.com/en-us/azure/kusto/management/data-ingestion/
	IngestionMapping string `json:"ingestionMapping,omitempty"`
	// IngestionMappingRef is a string representing a mapping reference that has been uploaded to the server
	// via a Mgmt() call. See: https://docs.microsoft.com/en-us/azure/kusto/management/create-ingestion-mapping-command
	IngestionMappingRef string `json:"ingestionMappingReference,omitempty"`
	// IngestionMappingType is what the mapping reference is encoded in: csv, json, avro, ...
	IngestionMappingType DataFormat `json:"ingestionMappingType,omitempty"`
	// ValidationPolicy is a JSON encoded string that tells our ingestion action what policies we want on the
	// data being ingested and what to do when that is violated.
	ValidationPolicy string     `json:"validationPolicy,omitempty"`
	Format           DataFormat `json:"format,omitempty"`
	// Tags is a list of tags to associated with the ingested data.
	Tags []string `json:"tags,omitempty"`
	// IngestIfNotExists is a string value that, if specified, prevents ingestion from succeeding if the table already
	// has data tagged with an ingest-by: tag with the same value. This ensures idempotent data ingestion.
	IngestIfNotExists string `json:"ingestIfNotExists,omitempty"`
}

Additional is additional properites.

func (Additional) MarshalJSON

func (a Additional) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaller. This is for use only by the SDK and may be removed at any time.

type All

type All struct {
	// Ingestion is a set of properties that are used across all ingestion methods.
	Ingestion Ingestion
	// Source provides options that are used are used when doing an ingestion on a filesystem.
	Source SourceOptions
}

All holds the complete set of properties that might be used.

type CompressionType

type CompressionType int8

CompressionType is a file's compression type.

const (
	// CTUnknown indicates that that the compression type was unset.
	CTUnknown CompressionType = 0
	// CTNone indicates that the file was not compressed.
	CTNone CompressionType = 1
	// GZIP indicates that the file is GZIP compressed.
	GZIP CompressionType = 2
	// ZIP indicates that the file is ZIP compressed.
	ZIP CompressionType = 3
)

func (CompressionType) MarshalJSON

func (c CompressionType) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler.MarshalJSON.

func (CompressionType) String

func (c CompressionType) String() string

String implements fmt.Stringer.

type DataFormat

type DataFormat int

DataFormat indicates what type of encoding format was used for source data. Note: This is very similar to ingest.DataFormat, except this supports more formats. We are not using a shared list, because this list is used only internally and is for the data itself, not the mapping reference. Structure prevents packages such as filesystem from importing ingest, because ingest imports filesystem. We would end up with recursive imports. More info here: https://docs.microsoft.com/en-us/azure/kusto/management/data-ingestion/

const (
	// DFUnknown indicates the EncodingType is not set.
	DFUnknown DataFormat = 0
	// AVRO indicates the source is encoded in Apache Avro format.
	AVRO DataFormat = 1
	// ApacheAVRO indicates the source is encoded in Apache avro2json format.
	ApacheAVRO DataFormat = 2
	// CSV indicates the source is encoded in comma seperated values.
	CSV DataFormat = 3
	// JSON indicates the source is encoded as one or more lines, each containing a record in Javscript Object Notation.
	JSON DataFormat = 4
	// MultiJSON indicates the source is encoded in JSON-Array of individual records in Javscript Object Notation.
	MultiJSON DataFormat = 5
	// ORC indicates the source is encoded in Apache Optimized Row Columnar format.
	ORC DataFormat = 6
	// Parquet indicates the source is encoded in Apache Parquet format.
	Parquet DataFormat = 7
	// PSV is pipe "|" separated values.
	PSV DataFormat = 8
	// Raw is a text file that has only a single string value.
	Raw DataFormat = 9
	// SCSV is a file containing semicolon ";" separated values.
	SCSV DataFormat = 10
	// SOHSV is a file containing SOH-separated values(ASCII codepont 1).
	SOHSV DataFormat = 11
	// SStream indicats the source is encoded as a Microsoft Cosmos Structured Streams format
	SStream DataFormat = 12
	// TSV is a file containing tab seperated values ("\t").
	TSV DataFormat = 13
	// TSVE is a file containing escaped-tab seperated values ("\t").
	TSVE DataFormat = 14
	// TXT is a text file with lines deliminated by "\n".
	TXT DataFormat = 15
	// W3CLogFile indicates the source is encoded using W3C Extended Log File format
	W3CLogFile DataFormat = 16
)

func DataFormatDiscovery added in v0.2.0

func DataFormatDiscovery(fName string) DataFormat

DataFormatDiscovery looks at the file name and tries to discern what the file format is.

func (DataFormat) CamelCase

func (d DataFormat) CamelCase() string

CamelCase returns the CamelCase version. This is for internal use, do not use. This can be removed in future versions.

func (DataFormat) IsValidMappingKind added in v0.2.0

func (d DataFormat) IsValidMappingKind() bool

IsValidMappingKind returns true if a dataformat can be used as a MappingKind

func (DataFormat) MarshalJSON

func (d DataFormat) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler.MarshalJSON.

func (DataFormat) RequiresMapping added in v0.2.0

func (d DataFormat) RequiresMapping() bool

RequiresMapping indicates whether a data format must be provided with valid mapping file information.

func (DataFormat) String

func (d DataFormat) String() string

String implements fmt.Stringer.

type Ingestion

type Ingestion struct {
	// ID is the unqique UUID for this upload.
	ID uuid.UUID `json:"Id"`
	// BlobPath is the URI representing the blob.
	BlobPath string
	// DatabaseName is the name of the Kusto database the data will ingest into.
	DatabaseName string
	// TableName is the name of the Kusto table the the data will ingest into.
	TableName string
	// RawDataSize is the size of the file on the filesystem, if it was provided.
	RawDataSize int64 `json:",omitempty"`
	// RetainBlobOnSuccess indicates if the source blob should be retained or deleted.
	RetainBlobOnSuccess bool `json:",omitempty"`
	// Daniel:
	// FlushImmediately ... I know what flushing means, but in terms of here, do we not return until the Kusto
	// table is updated, does this mean we do....  This is really a duplicate comment on the options in ingest.go
	FlushImmediately bool
	// Daniel:
	// IgnoreSizeLimit
	IgnoreSizeLimit bool `json:",omitempty"`
	ReportLevel     int  `json:",omitempty"`
	ReportMethod    int  `json:",omitempty"`
	// SourceMessageCreationTime is when we created the blob.
	SourceMessageCreationTime time.Time  `json:",omitempty"`
	Additional                Additional `json:"AdditionalProperties"`
}

Ingestion is a JSON serializable set of options that must be provided to the service.

func (Ingestion) MarshalJSONString

func (i Ingestion) MarshalJSONString() (base64String string, err error)

MarshalJSONString will marshal Ingestion into a base64 encoded string.

type SourceOptions

type SourceOptions struct {
	// ID allows someone to set the UUID for upload themselves. We aren't providing this option at this time, but here
	// when we do.
	ID uuid.UUID

	// DeleteLocalSource indicates to delete the local file after it has been consumed.
	DeleteLocalSource bool
}

SourceOptions are options that the user provides about the source file that is going to be uploaded.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL