lib

package
v0.0.0-...-8b27293 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 29, 2021 License: Apache-2.0, Apache-2.0 Imports: 32 Imported by: 0

Documentation

Overview

Copyright 2019 Google LLC

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

Constants

View Source 
const (
	LEHeader = "II\x2A\x00" // Header for little-endian files.
	BEHeader = "MM\x00\x2A" // Header for big-endian files.  4D 4D 00  2A

)
View Source 
const (
	MinPartSize = 1024 * 1014 * 5 // 5 MB
)

Variables

View Source 
var (
	MaxPage     int   //  Number of max pages to be concurrentlz processed
	MaxPageSize int64 //  maximum page size for multipart upload or dowbload
	MaxPartSize int64
	MaxCon      int   // Maximum number of concurrent upload /download
	PartNumber  int64 //number of  parts in a multi par upload /download
	Replace     bool
)

Functions

func BackupBlob 

func BackupBlob(pn string, np int, maxPage int, ctimeout time.Duration) ([]error, *documentpb.Document)

func CheckBlob 

func CheckBlob(pn string, np int, maxPage int) int

func CheckBlobs 

func CheckBlobs(request datatype.ListObjRequest, maxLoop int, maxPage int)

Check pn's returned by listObject of the meta bucket

func CheckBucketName 

func CheckBucketName(srcBucket string, tgtBucket string) (err error)

func CheckPdfAndP0 

func CheckPdfAndP0(pn string, usermd string) (pdf bool, p0 bool)

func CheckTargetSproxyd 

func CheckTargetSproxyd(request datatype.ListObjRequest, maxLoop int, maxPage int)

func CloneBlob 

func CloneBlob(pn string, np int, maxPage int, replace bool) (int, *documentpb.Document)

func CompletedUpload 

func CompletedUpload(svc *s3.S3, resp *s3.CreateMultipartUploadOutput, part []*s3.CompletedPart)

func CreateS3Session 

func CreateS3Session(op string, location string) *s3.S3

func DeleteDocMetadata 

func DeleteDocMetadata(request *sproxyd.HttpRequest, document *documentpb.Document) (int, int)

func GetBucketSuffix 

func GetBucketSuffix(bucket string, prefix string) (err error, s string)

func GetCurrentDate 

func GetCurrentDate() string

func GetDocMetaStatus 

func GetDocMetaStatus(request sproxyd.HttpRequest, env string, pn string) (err error, status int, usermd string)

func GetDocument 

func GetDocument(bytes []byte) (*documentpb.Document, error)

Document -> document structure

func GetDocumentMeta 

func GetDocumentMeta(key string) (*moses.DocumentMetadata, error, int)

func GetHeader 

func GetHeader(request sproxyd.HttpRequest) (err error, usermd string, contentLength int64)

func GetObject 

func GetObject(request sproxyd.HttpRequest, pn string) (error, string, *[]byte)

func GetPageNumber 

func GetPageNumber(key string) (int, error, int)

func GetPathName 

func GetPathName(pathId string) (err error, pathName string, status int)

func GetUserMeta 

func GetUserMeta(request sproxyd.HttpRequest, pn string) (err error, usermd string)

func GetUserMetaWithPathId 

func GetUserMetaWithPathId(pathId string) (err error, usermd string, status int)

func HasSuffix 

func HasSuffix(bucket string) bool

func HashKey 

func HashKey(key string, modulo int) (v int)

func IndexDocument 

func IndexDocument(document *documentpb.Document, bucket string, service *s3.S3, ctimeout time.Duration) (*s3.PutObjectOutput, error)

func InspectBlobs 

func InspectBlobs(document *documentpb.Document, maxPage int, verbose bool)

func ListBlob 

func ListBlob(pn string, np int, maxPage int) int

func ListBlobs 

func ListBlobs(request datatype.ListObjRequest, maxLoop int, maxPage int)

Check pn's returned by listObject of the meta bucket

func ListPn 

func ListPn(buf *bufio.Scanner, num int) (*s3.ListObjectsV2Output, error)

func Logit 

func Logit(logReq LogRequest)

func MigrateBlob 

func MigrateBlob(reqm datatype.Reqm, s3meta string, pn string, np int, ctimeout time.Duration) (int, *documentpb.Document)

func OpByIds 

func OpByIds(method string, request datatype.ListObjRequest, maxLoop int, replace bool, check bool)

Check pn's returned by listObject of the meta bucket

func ParseInputKey 

func ParseInputKey(key string) (error, string)

func ParseLog 

func ParseLog(line string) (error, string, string)

func ParseLoggerKey 

func ParseLoggerKey(key string) (error, string)

func Profiling 

func Profiling(profiling int)

func ReadDocument 

func ReadDocument(pn string, inDir string) (*documentpb.Document, error)

Read Document from a file into a document structure

func ReadMultipartS3 

func ReadMultipartS3(service *s3.S3, bucket string, key string) (int64, *aws.WriteAtBuffer, error)

func ReadS3 

func ReadS3(service *s3.S3, bucket string, key string) (*s3.GetObjectOutput, error)

func RepairIt 

func RepairIt(resp *http.Response, request *sproxyd.HttpRequest, replace bool) error

func RestoreBlobs 

func RestoreBlobs(document *documentpb.Document, ctimeout time.Duration) int

func Restores3Objects 

func Restores3Objects(service *s3.S3, bucket string, document *documentpb.Document, ctimeout time.Duration) int

func SetBucketName 

func SetBucketName(prefix string, bucket string) (name string)

func SetBucketName1 

func SetBucketName1(condition bool, pn string, bucket string) (buck1 string)

func SetLocationSproxyd 

func SetLocationSproxyd(op string, location string, srcUrl string, driver string, env string) error

func SetSourceSproxyd 

func SetSourceSproxyd(op string, srcUrl string, driver string, env string) error

func SetTargetSproxyd 

func SetTargetSproxyd(op string, targetUrl string, targetDriver string, targetEnv string) error

func UploadPart 

func UploadPart(upload datatype.UploadPartRequest) (*s3.CompletedPart, error)

func WriteDirectory 

func WriteDirectory(pn string, document *documentpb.Document, outdir string) (error, int)

Write document to file

func WriteDocMetadata 

func WriteDocMetadata(request *sproxyd.HttpRequest, document *documentpb.Document, replace bool) (int, int)

func WriteDocPage 

func WriteDocPage(request sproxyd.HttpRequest, pg *documentpb.Page, replace bool) (int, int)

func WriteDocPdf 

func WriteDocPdf(pd *documentpb.Pdf, replace bool) (int, int)

func WriteDocumentToFile 

func WriteDocumentToFile(document *documentpb.Document, pn string, outDir string)

Write the document to file( pages + meta dada) To be completed ( page 0 and pdf are not yet taken into account

func WriteS3 

func WriteS3(service *s3.S3, bucket string, document *documentpb.Document, ctimeout time.Duration) (*s3.PutObjectOutput, error)

func WriteS3Metadata 

func WriteS3Metadata(service *s3.S3, bucket string, document *documentpb.Document, ctimeout time.Duration) (*s3.PutObjectOutput, error)

func WriteS3Multipart 

func WriteS3Multipart(service *s3.S3, bucket string, maxPartSize int64, document *documentpb.Document, ctimeout time.Duration) ([]error, error)

func WriteS3Page 

func WriteS3Page(service *s3.S3, bucket string, page *documentpb.Page, ctimeout time.Duration) (*s3.PutObjectOutput, error)

func WriteS3Pdf 

func WriteS3Pdf(service *s3.S3, bucket string, pdf *documentpb.Pdf, ctimeout time.Duration) (*s3.PutObjectOutput, error)

Types

type GetBlobResponse 

type GetBlobResponse struct {
	Size       int     // blob size
	PageNumber int     //  number of pages
	Error      int     // number of errors
	Err        error   // error
	UserMd     string  // user meta data
	Body       *[]byte //  Body
}

type LogBackup 

type LogBackup struct {
	Method      string `json:"method,omitempty"`
	Incremental bool   `json:"incremental,omitempty"`
	Key         string `json:"document-id"`
	Bucket      string `json:"bucket-name"`
	Pages       int    `json:"number-pages"`
	Size        int64  `json:"document-size"`
	Pubdate     string `json:"publication-date,omitempty"`
	Loaddate    string `json:"load-date,omitempty"`
	Errors      int    `json:"number-errors"`
}

func (*LogBackup) GetDate 

func (logb *LogBackup) GetDate() string

func (*LogBackup) Logit 

func (logb *LogBackup) Logit(request datatype.PutObjRequest3, timeout time.Duration) (error, string)

func (*LogBackup) Start 

func (logb *LogBackup) Start()

type LogRequest 

type LogRequest struct {
	Service   *s3.S3
	Bucket    string
	LogBackup []*LogBackup
	Ctimeout  time.Duration
}

type Resp 

type Resp struct {
	Cp  *s3.CompletedPart
	Cl  int64
	Err error
}

type Ret 

type Ret struct {
	Ndocs  int
	Npages int
	Nerrs  int
	N404s  int
	Nreps  int
}

func CatchUpBlobs 

func CatchUpBlobs(pn string, np int, maxPage int, repair bool) (ret Ret)

func CheckTargetPages 

func CheckTargetPages(pn string, np int, maxPage int) (ret Ret)

type RingId 

type RingId struct {
	Err      error
	UserMeta string
	Key      string
	Object   *[]byte
}

func GetObjAndId 

func GetObjAndId(request sproxyd.HttpRequest, pn string) RingId

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.