importutilv2

Documentation

Index

• Constants
• Variables
• func GetCSVNullKey(options Options) (string, error)
• func GetCSVSep(options Options) (rune, error)
• func GetTimeoutTs(options Options) (uint64, error)
• func IsBackup(options Options) bool
• func IsL0Import(options Options) bool
• func ParseTimeRange(options Options) (uint64, uint64, error)
• func SkipDiskQuotaCheck(options Options) bool
• type FileType
  • func GetFileType(file *internalpb.ImportFile) (FileType, error)
  • func (f FileType) String() string
• type MockReader
  • func NewMockReader(t interface{ ... }) *MockReader
  • func (_m *MockReader) Close()
  • func (_m *MockReader) EXPECT() *MockReader_Expecter
  • func (_m *MockReader) Read() (*storage.InsertData, error)
  • func (_m *MockReader) Size() (int64, error)
• type MockReader_Close_Call
  • func (_c *MockReader_Close_Call) Return() *MockReader_Close_Call
  • func (_c *MockReader_Close_Call) Run(run func()) *MockReader_Close_Call
  • func (_c *MockReader_Close_Call) RunAndReturn(run func()) *MockReader_Close_Call
• type MockReader_Expecter
  • func (_e *MockReader_Expecter) Close() *MockReader_Close_Call
  • func (_e *MockReader_Expecter) Read() *MockReader_Read_Call
  • func (_e *MockReader_Expecter) Size() *MockReader_Size_Call
• type MockReader_Read_Call
  • func (_c *MockReader_Read_Call) Return(_a0 *storage.InsertData, _a1 error) *MockReader_Read_Call
  • func (_c *MockReader_Read_Call) Run(run func()) *MockReader_Read_Call
  • func (_c *MockReader_Read_Call) RunAndReturn(run func() (*storage.InsertData, error)) *MockReader_Read_Call
• type MockReader_Size_Call
  • func (_c *MockReader_Size_Call) Return(_a0 int64, _a1 error) *MockReader_Size_Call
  • func (_c *MockReader_Size_Call) Run(run func()) *MockReader_Size_Call
  • func (_c *MockReader_Size_Call) RunAndReturn(run func() (int64, error)) *MockReader_Size_Call
• type Options
• type Reader
  • func NewReader(ctx context.Context, cm storage.ChunkManager, ...) (Reader, error)

Constants

const (
	// Timeout specifies the timeout duration for import, such as "300s", "1.5h" or "1h45m".
	Timeout = "timeout"

	// SkipDQC indicates whether to bypass the disk quota check, default to false.
	SkipDQC = "skip_disk_quota_check"

	// CSVSep specifies the delimiter used for importing CSV files.
	CSVSep = "sep"

	// CSVNullKey specifies the null key used when importing CSV files.
	CSVNullKey = "nullkey"
)

const (
	// BackupFlag indicates whether the import is in backup-restore mode, default to false.
	BackupFlag = "backup"

	// L0Import indicates whether to import l0 segments only.
	L0Import = "l0_import"

	// StartTs StartTs2 EndTs EndTs2 are used to filter data during backup-restore import.
	StartTs  = "start_ts"
	StartTs2 = "startTs"
	EndTs    = "end_ts"
	EndTs2   = "endTs"
)

Options for backup-restore mode.

const (
	Invalid FileType = 0
	JSON    FileType = 1
	Numpy   FileType = 2
	Parquet FileType = 3
	CSV     FileType = 4

	JSONFileExt    = ".json"
	NumpyFileExt   = ".npy"
	ParquetFileExt = ".parquet"
	CSVFileExt     = ".csv"
)

Variables

var FileTypeName = map[int]string{
	0: "Invalid",
	1: "JSON",
	2: "Numpy",
	3: "Parquet",
	4: "CSV",
}

Functions

func GetCSVNullKey

func GetCSVNullKey(options Options) (string, error)

func GetCSVSep

func GetCSVSep(options Options) (rune, error)

func GetTimeoutTs

func GetTimeoutTs(options Options) (uint64, error)

func IsBackup

func IsBackup(options Options) bool

func IsL0Import

func IsL0Import(options Options) bool

func ParseTimeRange

func ParseTimeRange(options Options) (uint64, uint64, error)

func SkipDiskQuotaCheck

func SkipDiskQuotaCheck(options Options) bool

SkipDiskQuotaCheck indicates whether the import skips the disk quota check. This option should only be enabled during backup restoration.

Types

type FileType

type FileType int

func GetFileType

func GetFileType(file *internalpb.ImportFile) (FileType, error)

func (FileType) String

func (f FileType) String() string

type MockReader

type MockReader struct {
	mock.Mock
}

MockReader is an autogenerated mock type for the Reader type

func NewMockReader

func NewMockReader(t interface {
	mock.TestingT
	Cleanup(func())
}) *MockReader

NewMockReader creates a new instance of MockReader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. The first argument is typically a *testing.T value.

func (*MockReader) Close

func (_m *MockReader) Close()

Close provides a mock function with given fields:

func (*MockReader) EXPECT

func (_m *MockReader) EXPECT() *MockReader_Expecter

func (*MockReader) Read

func (_m *MockReader) Read() (*storage.InsertData, error)

Read provides a mock function with given fields:

func (*MockReader) Size

func (_m *MockReader) Size() (int64, error)

Size provides a mock function with given fields:

type MockReader_Close_Call

type MockReader_Close_Call struct {
	*mock.Call
}

MockReader_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'

func (*MockReader_Close_Call) Return

func (_c *MockReader_Close_Call) Return() *MockReader_Close_Call

func (*MockReader_Close_Call) Run

func (_c *MockReader_Close_Call) Run(run func()) *MockReader_Close_Call

func (*MockReader_Close_Call) RunAndReturn

func (_c *MockReader_Close_Call) RunAndReturn(run func()) *MockReader_Close_Call

type MockReader_Expecter

type MockReader_Expecter struct {
	// contains filtered or unexported fields
}

func (*MockReader_Expecter) Close

func (_e *MockReader_Expecter) Close() *MockReader_Close_Call

Close is a helper method to define mock.On call

func (*MockReader_Expecter) Read

func (_e *MockReader_Expecter) Read() *MockReader_Read_Call

Read is a helper method to define mock.On call

func (*MockReader_Expecter) Size

func (_e *MockReader_Expecter) Size() *MockReader_Size_Call

Size is a helper method to define mock.On call

type MockReader_Read_Call

type MockReader_Read_Call struct {
	*mock.Call
}

MockReader_Read_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Read'

func (*MockReader_Read_Call) Return

func (_c *MockReader_Read_Call) Return(_a0 *storage.InsertData, _a1 error) *MockReader_Read_Call

func (*MockReader_Read_Call) Run

func (_c *MockReader_Read_Call) Run(run func()) *MockReader_Read_Call

func (*MockReader_Read_Call) RunAndReturn

func (_c *MockReader_Read_Call) RunAndReturn(run func() (*storage.InsertData, error)) *MockReader_Read_Call

type MockReader_Size_Call

type MockReader_Size_Call struct {
	*mock.Call
}

MockReader_Size_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Size'

func (*MockReader_Size_Call) Return

func (_c *MockReader_Size_Call) Return(_a0 int64, _a1 error) *MockReader_Size_Call

func (*MockReader_Size_Call) Run

func (_c *MockReader_Size_Call) Run(run func()) *MockReader_Size_Call

func (*MockReader_Size_Call) RunAndReturn

func (_c *MockReader_Size_Call) RunAndReturn(run func() (int64, error)) *MockReader_Size_Call

type Options

type Options []*commonpb.KeyValuePair

type Reader

type Reader interface {
	// Size returns the size of the underlying file/files in bytes.
	// It returns an error if the size cannot be determined.
	Size() (int64, error)

	// Read reads data from the underlying file/files.
	// It returns the storage.InsertData and an error, if any.
	Read() (*storage.InsertData, error)

	// Close closes the underlying file reader.
	Close()
}

func NewReader

func NewReader(ctx context.Context,
	cm storage.ChunkManager,
	schema *schemapb.CollectionSchema,
	importFile *internalpb.ImportFile,
	options Options,
	bufferSize int,
) (Reader, error)