datasource

package
v0.2.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 12, 2024 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Overview

Copyright 2023 KubeAGI. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright 2024 KubeAGI. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

Constants

View Source 
const ObjectNotExistMsg = "The specified key does not exist."

Variables

View Source 
var (
	ErrUnknowDatasourceType = errors.New("unknown datasource type")
	ErrBucketNotProvided    = errors.New("no bucket provided")
	ErrOSSNoSuchBucket      = errors.New("no such bucket")
	ErrOSSNoSuchObject      = errors.New("no such object in bucket")
	ErrOSSNoConfig          = errors.New("no bucket or object config")
)

Functions

func RemovePostgreSQLPool added in v0.2.0 

func RemovePostgreSQLPool(datasource v1alpha1.Datasource)

Types

type ChunkUploader 

type ChunkUploader interface {
	// Queries for blocks that have already been uploaded.
	CompletedChunks(context.Context, ...ChunkUploaderOption) (any, error)

	// CompletedChunks(context.Context, ...chunkUploaderOption) (any, error)
	// Generate uploadID for uploaded files
	NewMultipartIdentifier(context.Context, ...ChunkUploaderOption) (string, error)

	// Generate URLs for uploading files
	GenMultipartSignedURL(context.Context, ...ChunkUploaderOption) (string, error)

	// After all the chunks are uploaded, this interface is called and the backend merges the files.
	Complete(context.Context, ...ChunkUploaderOption) error

	// To stop the upload, the user needs to destroy the chunked data.
	Abort(context.Context, ...ChunkUploaderOption) error

	// IncompleteUpload returns the number of times an object does not have a complete upload. Ideally,
	// it is guaranteed that there is only one uploadid per object.
	// If there is more than one uploadid, return the latest one.
	IncompleteUpload(context.Context, ...ChunkUploaderOption) (string, error)
}

Uploading files in chunks. 1. Get completed blocks 2. if there is no unique uploadid, you need to request a unique uploadid 3. request the upload file URL 4. Upload the file via the URL 5. Update the information of the uploaded block 6. merge files

type ChunkUploaderConf 

type ChunkUploaderConf struct {
	// contains filtered or unexported fields
}

type ChunkUploaderOption 

type ChunkUploaderOption func(*ChunkUploaderConf)

func WithAnnotations 

func WithAnnotations(annotations map[string]string) ChunkUploaderOption

func WithBucket 

func WithBucket(bucket string) ChunkUploaderOption

func WithBucketPath 

func WithBucketPath(relativeDir string) ChunkUploaderOption

func WithFileName 

func WithFileName(fileName string) ChunkUploaderOption

func WithMD5 

func WithMD5(md5 string) ChunkUploaderOption

func WithPartNumber 

func WithPartNumber(partNumber string) ChunkUploaderOption

func WithUploadID 

func WithUploadID(uploadID string) ChunkUploaderOption

type Datasource 

type Datasource interface {
	Stat(ctx context.Context, info any) error
	Remove(ctx context.Context, info any) error
	ReadFile(ctx context.Context, info any) (io.ReadCloser, error)
	StatFile(ctx context.Context, info any) (any, error)
	GetTags(ctx context.Context, info any) (map[string]string, error)
	ListObjects(ctx context.Context, source string, info any) (any, error)
}

type Local 

type Local struct {
	// contains filtered or unexported fields
}

Local is a special datasource which use the system datasource as oss to store user-uploaded local files - `oss` in `Local` represents the system datasource oss client along with the `Local`'s oss info

func NewLocal 

func NewLocal(ctx context.Context, c client.Client, endpoint *v1alpha1.Endpoint) (*Local, error)

func (*Local) GetTags 

func (local *Local) GetTags(ctx context.Context, info any) (map[string]string, error)

func (*Local) ListObjects 

func (local *Local) ListObjects(ctx context.Context, source string, info any) (any, error)

func (*Local) ReadFile 

func (local *Local) ReadFile(ctx context.Context, info any) (io.ReadCloser, error)

func (*Local) Remove 

func (local *Local) Remove(ctx context.Context, info any) error

Remove object from OSS

func (*Local) Stat 

func (local *Local) Stat(ctx context.Context, options any) (err error)

Stat `Local` with `OSS`

func (*Local) StatFile 

func (local *Local) StatFile(ctx context.Context, info any) (any, error)

type OSS 

type OSS struct {
	*minio.Client
	*minio.Core
}

OSS is a wrapper to object storage service

func NewOSS 

func NewOSS(ctx context.Context, c client.Client, endpoint *v1alpha1.Endpoint) (*OSS, error)

func (*OSS) Abort 

func (oss *OSS) Abort(ctx context.Context, options ...ChunkUploaderOption) error

func (*OSS) Complete 

func (oss *OSS) Complete(ctx context.Context, options ...ChunkUploaderOption) error

func (*OSS) CompletedChunks 

func (oss *OSS) CompletedChunks(ctx context.Context, options ...ChunkUploaderOption) (any, error)

func (*OSS) GenMultipartSignedURL 

func (oss *OSS) GenMultipartSignedURL(ctx context.Context, options ...ChunkUploaderOption) (string, error)

func (*OSS) GetTags 

func (oss *OSS) GetTags(ctx context.Context, info any) (map[string]string, error)

func (*OSS) IncompleteUpload 

func (oss *OSS) IncompleteUpload(ctx context.Context, options ...ChunkUploaderOption) (string, error)

func (*OSS) ListObjects 

func (oss *OSS) ListObjects(ctx context.Context, bucketName string, info any) (any, error)

func (*OSS) NewMultipartIdentifier 

func (oss *OSS) NewMultipartIdentifier(ctx context.Context, options ...ChunkUploaderOption) (string, error)

func (*OSS) ReadFile 

func (oss *OSS) ReadFile(ctx context.Context, info any) (io.ReadCloser, error)

func (*OSS) Remove 

func (oss *OSS) Remove(ctx context.Context, info any) error

func (*OSS) Stat 

func (oss *OSS) Stat(ctx context.Context, info any) error

Check oss against info()

func (*OSS) StatFile 

func (oss *OSS) StatFile(ctx context.Context, info any) (any, error)

type PostgreSQL added in v0.2.0 

type PostgreSQL struct {
	*pgxpool.Pool
	Ref *v1alpha1.Datasource
}

PostgreSQL is a wrapper to PostgreSQL

func GetPostgreSQLPool added in v0.2.0 

func GetPostgreSQLPool(ctx context.Context, c client.Client, datasource *v1alpha1.Datasource) (*PostgreSQL, error)

func (*PostgreSQL) GetTags added in v0.2.0 

func (p *PostgreSQL) GetTags(ctx context.Context, info any) (map[string]string, error)

func (*PostgreSQL) ListObjects added in v0.2.0 

func (p *PostgreSQL) ListObjects(ctx context.Context, source string, info any) (any, error)

func (*PostgreSQL) ReadFile added in v0.2.0 

func (p *PostgreSQL) ReadFile(ctx context.Context, info any) (io.ReadCloser, error)

func (*PostgreSQL) Remove added in v0.2.0 

func (p *PostgreSQL) Remove(ctx context.Context, info any) error

func (*PostgreSQL) Stat added in v0.2.0 

func (p *PostgreSQL) Stat(ctx context.Context, _ any) error

func (*PostgreSQL) StatFile added in v0.2.0 

func (p *PostgreSQL) StatFile(ctx context.Context, info any) (any, error)

type Unknown 

type Unknown struct {
}

func NewUnknown 

func NewUnknown(ctx context.Context, c client.Client) (*Unknown, error)

func (*Unknown) GetTags 

func (u *Unknown) GetTags(ctx context.Context, info any) (map[string]string, error)

func (*Unknown) ListObjects 

func (*Unknown) ListObjects(ctx context.Context, source string, info any) (any, error)

func (*Unknown) ReadFile 

func (u *Unknown) ReadFile(ctx context.Context, info any) (io.ReadCloser, error)

func (*Unknown) Remove 

func (u *Unknown) Remove(ctx context.Context, info any) error

func (*Unknown) Stat 

func (u *Unknown) Stat(ctx context.Context, info any) error

func (*Unknown) StatFile 

func (u *Unknown) StatFile(ctx context.Context, info any) (any, error)

type Web added in v0.2.0 

type Web struct {
	// contains filtered or unexported fields
}

func NewWeb added in v0.2.0 

func NewWeb(ctx context.Context, url string) (*Web, error)

func (*Web) GetTags added in v0.2.0 

func (w *Web) GetTags(ctx context.Context, info any) (map[string]string, error)

func (*Web) ListObjects added in v0.2.0 

func (w *Web) ListObjects(ctx context.Context, source string, info any) (any, error)

func (*Web) ReadFile added in v0.2.0 

func (w *Web) ReadFile(ctx context.Context, info any) (io.ReadCloser, error)

func (*Web) Remove added in v0.2.0 

func (w *Web) Remove(ctx context.Context, info any) error

func (*Web) Stat added in v0.2.0 

func (w *Web) Stat(ctx context.Context, info any) error

func (*Web) StatFile added in v0.2.0 

func (w *Web) StatFile(ctx context.Context, info any) (any, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.