index

package
v0.0.0-...-3852a4b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 29, 2024 License: GPL-2.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	OffsetBytesLen = 8 // the storage size of i.IndexFileOffset(int64) is always 8 bytes
)

Variables

View Source 
var (
	REGMySQL8 = regexp.MustCompile(`^8\.`)
	REGMySQL5 = regexp.MustCompile(`^5\.[1,5-7]\.`)
)

Functions

func CheckFieldExists 

func CheckFieldExists(
	tableName, fieldName string,
	db *gorm.DB,
) (exists bool, err error)

func CheckTableExists 

func CheckTableExists(tableName string, db *gorm.DB) (exists bool, err error)

func CloseConnection 

func CloseConnection(db *gorm.DB)

func DecodeChunkHeader 

func DecodeChunkHeader(xr *xbstream.Reader) (header *xbstream.ChunkHeader, err error)

func ExtractFile 

func ExtractFile(
	rsp *readseekerpool.ReadSeekerPool,
	cis chan *ChunkIndex,
	filePath,
	targetDIR string,
) (n int64, err error)

func ExtractFiles 

func ExtractFiles(r io.Reader, targetDIR string) (n int64, err error)

func ExtractSchemaByPayload 

func ExtractSchemaByPayload(
	schemaChan chan *TableSchema,
	ci *ChunkIndex,
	r io.Reader,
	payLen int64,
) (n int64, err error)

func ExtractSingleSchema 

func ExtractSingleSchema(
	ci *ChunkIndex,
	schemaChan chan *TableSchema,
	r io.ReadSeeker,
) (err error)

func NewConnection 

func NewConnection(dbPath string) (db *gorm.DB, err error)

func SanitizeString 

func SanitizeString(input string) string

Types

type ChunkIndex 

type ChunkIndex struct {
	gorm.Model
	Filepath             string          `gorm:"column:filepath;type:varchar(4096);index:idx_filepath_start_position;index:idx_filepath_pay_offset"`
	StartPosition        int64           `gorm:"column:start_position;type:bigint;index:idx_filepath_start_position"`
	EndPosition          int64           `gorm:"column:end_position;type:bigint"`
	PayOffset            uint64          `gorm:"column:pay_offset;type:bigint;index:idx_filepath_pay_offset"`
	EncryptKey           []byte          `gorm:"-"`
	DecryptedFileType    string          `gorm:"-"`
	DecryptMethod        string          `gorm:"-"`
	DecompressedFileType string          `gorm:"-"`
	DecompressMethod     string          `gorm:"-"`
	ExtractLimitSize     int64           `gorm:"-"`
	Chunk                *xbstream.Chunk `gorm:"-"`
}

func (*ChunkIndex) DecodeFilepath 

func (ci *ChunkIndex) DecodeFilepath()

func (ChunkIndex) TableName 

func (ChunkIndex) TableName() string

type IndexStream 

type IndexStream struct {
	CTX                               context.Context
	Cancel                            context.CancelFunc
	Offset                            atomic.Int64
	ChunkIndexChan                    chan *ChunkIndex
	SchemaFileChan                    chan *TableSchema
	TableSchemaChan                   chan *TableSchema
	IsParseTableSchema                bool
	IsIndexDone                       bool
	IndexFilePath                     string
	IndexFilename                     string
	IndexFileSize                     int64
	IndexFileOffsetStart              int64 // [IndexFileOffsetStart
	IndexFileOffsetEnd                int64 // IndexFileOffsetEnd]
	IndexFileOffsetFilename           string
	IndexFileOffsetFileChunkTotalSize int64
	IndexDB                           *gorm.DB
	IndexTableDone                    chan struct{}
	IndexTableBatchSize               int
	ParserSchemaFileDone              chan struct{}
	SchemaTableDone                   chan struct{}
	SchemaTableBatchSize              int
	ParseTargetFileType               string
	DefaultLikePaths                  []string
	DefaultNotLikePaths               []string
	OpenFilesCatch                    map[string]*os.File
	RegSkipPattern                    *regexp.Regexp
	Err                               error
	GormLogger                        logger.Interface
	IsRemoveLocalIndexFile            bool
	MeilisearchIndex                  meilisearch.IndexManager
	MeilisearchDefaultDoc             map[string]interface{}
	EncryptKey                        []byte
	ExtractLimitSize                  int64
	Conn                              *client.Conn
	*MySQLServer
}

func NewIndexStream 

func NewIndexStream(
	ctx context.Context,
	indexFilename string,
	baseDIR string,
	mysqlVersion string,
	isRemoveLocalIndexFile bool,
	encryptKey []byte,
	extractLimitSize int64,
	conn *client.Conn,
	meilisearchIndex meilisearch.IndexManager,
	meilisearchDefaultDoc map[string]interface{},
) *IndexStream

func (*IndexStream) CloseIndexDB 

func (i *IndexStream) CloseIndexDB()

func (*IndexStream) ConnectIndexDB 

func (i *IndexStream) ConnectIndexDB()

func (*IndexStream) DecodeChunk 

func (i *IndexStream) DecodeChunk(xr *xbstream.Reader, ci *ChunkIndex) *ChunkIndex

func (*IndexStream) DecodeChunkPayload 

func (i *IndexStream) DecodeChunkPayload(
	ci *ChunkIndex,
	r io.Reader,
	payLen int64,
) (n int64, err error)

func (*IndexStream) ExtractIndexFile 

func (i *IndexStream) ExtractIndexFile(rsp *readseekerpool.ReadSeekerPool, targetDIR string)

func (*IndexStream) ExtractSchemas 

func (i *IndexStream) ExtractSchemas(rsp *readseekerpool.ReadSeekerPool, targetDIR string, likePaths, notLikePaths []string)

func (*IndexStream) IndexHeader 

func (i *IndexStream) IndexHeader(header *xbstream.ChunkHeader, ci *ChunkIndex) (newCI *ChunkIndex)

func (*IndexStream) IndexStream 

func (i *IndexStream) IndexStream(r io.Reader, w io.WriteCloser)

func (*IndexStream) IsNeedParsSchema 

func (i *IndexStream) IsNeedParsSchema(ci *ChunkIndex) bool

func (*IndexStream) ParseSchemaFile 

func (i *IndexStream) ParseSchemaFile()

func (*IndexStream) StreamIndexFile 

func (i *IndexStream) StreamIndexFile(w io.WriteCloser)

StreamIndexFile write index file and index file offset to xbstream

func (*IndexStream) WriteIndexTable 

func (i *IndexStream) WriteIndexTable()

func (*IndexStream) WriteSchemaTable 

func (i *IndexStream) WriteSchemaTable(db *gorm.DB)

type MySQLServer 

type MySQLServer struct {
	MySQLVersion string `gorm:"column:mysql_version;type:varchar(64)"`
}

func (MySQLServer) TableName 

func (MySQLServer) TableName() string

type TableSchema 

type TableSchema struct {
	gorm.Model
	Filepath             string         `gorm:"column:filepath;type:varchar(4096);uniqueIndex:uk_filepath"`
	TableName            string         `gorm:"column:table_name;type:varchar(256);index:idx_table_schema"`
	SchemaName           string         `gorm:"column:schema_name;type:varchar(256);index:idx_table_schema"`
	CreateStatement      string         `gorm:"column:create_statement;type:text"`
	ParseWarn            string         `gorm:"column:parse_warn;type:text"`
	ParseErr             string         `gorm:"column:parse_err;type:text"`
	DecryptErr           string         `gorm:"column:decrypt_err;type:text"`
	DecompressErr        string         `gorm:"column:decompress_err;type:text"`
	ExtractLimitSize     int64          `gorm:"-"`
	DecryptMethod        string         `gorm:"-"`
	DecryptedFileType    string         `gorm:"-"`
	DecryptedFilepath    string         `gorm:"-"`
	DecompressMethod     string         `gorm:"-"`
	DecompressedFileType string         `gorm:"-"`
	DecompressedFilepath string         `gorm:"-"`
	StreamIn             *io.PipeWriter `gorm:"-"`
	StreamOut            *io.PipeReader `gorm:"-"`
	EncryptKey           []byte         `gorm:"-"`
	MidPipeIn            *io.PipeWriter `gorm:"-"`
	MidPipeOut           *io.PipeReader `gorm:"-"`
	ParseIn              *io.PipeWriter `gorm:"-"`
	ParseOut             *io.PipeReader `gorm:"-"`
	ParseDone            chan struct{}  `gorm:"-"`
	IsHidden             bool           `gorm:"-"`
}

func NewTableSchema 

func NewTableSchema(
	filepath string,
	limitSize int64,
	encryptKey []byte,
	decryptedFileType,
	decryptMethod,
	decompressedFileType,
	decompressMethod string,
) (ts *TableSchema, err error)

func (*TableSchema) GetMeiliSearchDoc 

func (ts *TableSchema) GetMeiliSearchDoc(
	defaultDoc map[string]interface{},
) (
	meilisearchDoc map[string]interface{},
	err error,
)

func (*TableSchema) ParseSchema 

func (ts *TableSchema) ParseSchema(conn *client.Conn)

type TableSchemaMap 

type TableSchemaMap struct {
	// contains filtered or unexported fields
}

func (*TableSchemaMap) Delete 

func (t *TableSchemaMap) Delete(key string)

func (*TableSchemaMap) Get 

func (t *TableSchemaMap) Get(key string) (*TableSchema, bool)

func (*TableSchemaMap) Set 

func (t *TableSchemaMap) Set(key string, value *TableSchema)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.