handlers

Documentation

Index

• Constants
• func FindUploaderInManifest(object string, oo io.Reader) string
• func GetIndexdRecordRev(uuid, indexURL string) (string, error)
• func IndexS3Object(s3objectURL string)
• func IndexS3ObjectEmbedded(s3objectURL string, indexdInfo *IndexdInfo, awsConfig *aws.Config)
• func RunIndexS3Object(s3objectURL string, indexdInfo *IndexdInfo, client *AwsClient)
• func SearchRecordByURL(indexdInfo *IndexdInfo, url string) (searchResponse, error)
• func UpdateIndexdRecord(uuid, rev string, indexdInfo *IndexdInfo, body []byte) (*http.Response, error)
• type AwsClient
  • func CreateNewAwsClient() (*AwsClient, error)
  • func (client *AwsClient) CalculateBasicHashes(bucket string, key string) (*HashInfo, int64, error)
  • func (client *AwsClient) GetObjectSize(bucket string, key string) (*int64, error)
  • func (client *AwsClient) GetS3BucketOwner(bucket string) (string, error)
  • func (client *AwsClient) GetS3ObjectOutput(bucket string, key string) (*s3.GetObjectOutput, error)
• type HashCollection
  • func CreateNewHashCollection() *HashCollection
  • func UpdateBasicHashes(hashCollection *HashCollection, rd []byte) (*HashCollection, error)
  • func (h *HashCollection) Reset()
• type HashInfo
• type IndexdInfo
• type IndexdRecord
  • func CreateBlankIndexdRecord(indexdInfo *IndexdInfo, body []byte) (*IndexdRecord, error)

Constants

const ChunkSize = 1024 * 1024 * 64

const MaxRetries = 10

MaxRetries maximum number of retries

Functions

func FindUploaderInManifest

func FindUploaderInManifest(object string, oo io.Reader) string

func GetIndexdRecordRev

func GetIndexdRecordRev(uuid, indexURL string) (string, error)

GetIndexdRecordRev gets record rev

func IndexS3Object

func IndexS3Object(s3objectURL string)

IndexS3Object indexes s3 object The fuction does several things. It first downloads the object from S3, computes size and hashes, and update indexd

func IndexS3ObjectEmbedded

func IndexS3ObjectEmbedded(s3objectURL string, indexdInfo *IndexdInfo, awsConfig *aws.Config)

func RunIndexS3Object

func RunIndexS3Object(s3objectURL string, indexdInfo *IndexdInfo, client *AwsClient)

func SearchRecordByURL

func SearchRecordByURL(indexdInfo *IndexdInfo, url string) (searchResponse, error)

func UpdateIndexdRecord

func UpdateIndexdRecord(uuid, rev string, indexdInfo *IndexdInfo, body []byte) (*http.Response, error)

UpdateIndexdRecord updates the record with size, urls and hashes endcoded in body

Types

type AwsClient

type AwsClient struct {
	// contains filtered or unexported fields
}

AWS sesssion wrapper

func CreateNewAwsClient

func CreateNewAwsClient() (*AwsClient, error)

CreateNewSession creates an aws s3 session

func (*AwsClient) CalculateBasicHashes

func (client *AwsClient) CalculateBasicHashes(bucket string, key string) (*HashInfo, int64, error)

CalculateBasicHashes generates hashes of aws bucket object

func (*AwsClient) GetObjectSize

func (client *AwsClient) GetObjectSize(bucket string, key string) (*int64, error)

GetObjectSize returns object size in bytes

func (*AwsClient) GetS3BucketOwner

func (client *AwsClient) GetS3BucketOwner(bucket string) (string, error)

func (*AwsClient) GetS3ObjectOutput

func (client *AwsClient) GetS3ObjectOutput(bucket string, key string) (*s3.GetObjectOutput, error)

GetS3ObjectOutput gets object output from s3

type HashCollection

type HashCollection struct {
	Crc32c hash.Hash
	Md5    hash.Hash
	Sha1   hash.Hash
	Sha256 hash.Hash
	Sha512 hash.Hash
}

HashCollection contains hashes

func CreateNewHashCollection

func CreateNewHashCollection() *HashCollection

CreateNewHashCollection creates a new HashCollection

func UpdateBasicHashes

func UpdateBasicHashes(hashCollection *HashCollection, rd []byte) (*HashCollection, error)

UpdateBasicHashes updates a hashes collection

func (*HashCollection) Reset

func (h *HashCollection) Reset()

type HashInfo

type HashInfo struct {
	Crc32c string `json:"crc"`
	Md5    string `json:"md5"`
	Sha1   string `json:"sha1"`
	Sha256 string `json:"sha256"`
	Sha512 string `json:"sha512"`
}

type IndexdInfo

type IndexdInfo struct {
	URL              string `json:"url"`
	Username         string `json:"username"`
	Password         string `json:"password"`
	ExtramuralBucket bool   `json:"extramural_bucket"`

	ExtramuralUploader         *string `json:"extramural_uploader"`
	ExtramuralUploaderS3Owner  bool    `json:"extramural_uploader_s3owner"`
	ExtramuralUploaderManifest *string `json:"extramural_uploader_manifest"`
	ExtramuralInitialMode      bool    `json:"extramural_initial_mode"` // If true, skips hash updates if record is already found in index
	ExtramuralFastMode         bool    `json:"extramural_fast_mode"`    // If true, always creates a new record for the object
}

type IndexdRecord

type IndexdRecord struct {
	DID    string `json:"did"`
	BaseID string `json:"baseid"`
	Rev    string `json:"rev"`
}

func CreateBlankIndexdRecord

func CreateBlankIndexdRecord(indexdInfo *IndexdInfo, body []byte) (*IndexdRecord, error)