cryptoStorage

README

这个包是基于tidb的br/stroage包封装的，用来加密存储的，参考 OSS 和 S3 的密码机逻辑，使用的是对称加密和非对称加密结合的方式来加密数据。首先是使用 RSA 非对称加密的方式来加密随机生成的密钥，然后使用该随机密钥对文件做对称加密，在 Header 中会记录原始文件的 Hash 值，提供防篡改功能。
代码里支持两种 Header 文件格式: V1 版本将 Header 嵌入到文件头中，V2 版本是将 Header 单独保存在一个文件中(默认是file.crypto，可以通过CryptoStoreOption.Suffix来自定义)。不过 V1 版本已经是废弃版本了，原因是 S3 不支持在写入文件后再修改文件内容，当前只支持生成 V2 版本的 Header 文件。下面是一个 Header 文件的示例：

{"hash":"LqIvjzIHBEbjqITikWLMSA==","hash_type":"md5","enc_key":"iJr8xMsSeTya9g3xK11myqeNHIa2MuFUpjvBGqo93KluvA4SfcPaaD4+du1BsGMMpFzTzTCD4OqxiawUOZwDJA1htWgZLsmnWHwem8yQ55dhPuINjxzcLmpdmF9ZNF7CRu0AxhNDKF86AXrtb1iiTmzQzKYW+uVvK1pmo+V4eNJ+6AV1hFg8Wx+afCOYn2O52aXVEkr50as2RF1rqNC0PyWg4m8/LPxtUgSMhShV6ZBcFhU3s06JSfeBjgyuku8xlL/kdqiSldX6kMtA4laUeOJ1tDQY6joMCdyapkjKW0NveMgRVYFgf5ksknK0Lux/IXO4OI4q1wAiie1mMFh+Ww==","iv":"EDaMoUrp2j27r2KfQXFWAA=="}

使用

PS： 目录格式可以参考 TiDB 官方文档连接：

1. https://docs.pingcap.com/zh/tidb/stable/backup-and-restore-storages
2. https://docs.pingcap.com/zh/tidb/stable/external-storage-uri

	// 创建存储选项
	storageOption := &storage.BackendOptions{}
	
	// 创建加密存储选项
	cryptoOption := &cryptoStorage.CryptoStoreOption{
	}
	cryptoOption.PublicData = []byte("your-public-data")
	
	// 创建 CryptoStore 实例
	store, err := cryptoStorage.NewCryptoStore(storageOption, "local:///path/to/storage", cryptoOption)
	if err != nil {
		log.Fatalf("创建 CryptoStore 失败: %v", err)
	}
	defer store.Close()

Documentation

Index

• Constants
• Variables
• func NewHash(hashType string) hash.Hash
• func ReadLastBytes(reader storage.ExternalFileReader, numBytes int64) ([]byte, error)
• type CryptoReader
  • func NewCryptoReader(dataReader storage.ExternalFileReader, privateData []byte, header *Header) (*CryptoReader, error)
  • func (r *CryptoReader) CheckSum() error
  • func (r *CryptoReader) Close() error
  • func (r *CryptoReader) Header() *Header
  • func (r *CryptoReader) Read(p []byte) (int, error)
• type CryptoStore
  • func NewCryptoStore(storageOption *storage.BackendOptions, path string, ...) (*CryptoStore, error)
  • func (c *CryptoStore) Close()
  • func (c *CryptoStore) Create(ctx context.Context, path string, option *storage.WriterOption) (storage.ExternalFileWriter, error)
  • func (c *CryptoStore) FormatHeaderName(name string) string
  • func (c *CryptoStore) NewHash() hash.Hash
  • func (c *CryptoStore) Open(ctx context.Context, name string, opt *storage.ReaderOption) (storage.ExternalFileReader, error)
  • func (c *CryptoStore) ReadFile(ctx context.Context, name string) ([]byte, error)
  • func (c *CryptoStore) Rename(ctx context.Context, src, dst string) error
  • func (c *CryptoStore) WriteFile(ctx context.Context, name string, data []byte) error
• type CryptoStoreOption
  • func NewCryptoStoreOption(privateData []byte, publicData []byte, suffix string, hashType string) (*CryptoStoreOption, error)
• type CryptoWriter
  • func NewCryptoWriter(publicData []byte, hashType string, ...) (nw *CryptoWriter, err error)
  • func (w *CryptoWriter) Close(ctx context.Context) error
  • func (w *CryptoWriter) Write(ctx context.Context, p []byte) (int, error)
  • func (c *CryptoWriter) WriteHeaderJson(ctx context.Context) error
  • func (c *CryptoWriter) WriteHeaderV1(ctx context.Context) error
• type Header
  • func NewHeader(key, encKey, iv []byte, hashType string) (*Header, error)
  • func NewRandHeader(publicData []byte, hashType string) (*Header, error)
  • func ParseHeaderV1(data storage.ExternalFileReader) (header *Header, err error)
  • func ReadHeaderV1(headerReader storage.ExternalFileReader) (*Header, error)
  • func ReadHeaderV2(headerReader storage.ExternalFileReader) (*Header, error)
  • func (header *Header) BytesV1() []byte
  • func (header Header) Map() map[string]string
  • func (header Header) Size() int
  • func (header Header) String() string
• type IoWriter
  • func NewIoWriter(w storage.ExternalFileWriter) *IoWriter
  • func (w *IoWriter) Write(p []byte) (int, error)

Constants

const (
	MaxHashSize   = 64 // aka: 512bit. md5: 128bit, sha256: 256bit, sha1: 160bit, sum32: 32 bit
	MaxEncKeySize = 1 << 16
	MaxIVSize     = 1 << 8
	KeySize       = 16        // aka: 128bit
	HeaderLenSize = 1 + 2 + 1 // hashSize: 1bit, encKeySize: 2bit, ivSize: 1bit
	MagicKeyV1    = "Encrypted&Hashed"
	MagicKeyV2    = "EncryptVersion-2"
)

Variables

var DefaultCryptoStoreOption = &CryptoStoreOption{
	Suffix:   "crypto",
	HashType: "md5",
}

Functions

func NewHash

func NewHash(hashType string) hash.Hash

func ReadLastBytes

func ReadLastBytes(reader storage.ExternalFileReader, numBytes int64) ([]byte, error)

Types

type CryptoReader

type CryptoReader struct {
	storage.ExternalFileReader
	// contains filtered or unexported fields
}

func NewCryptoReader

func NewCryptoReader(dataReader storage.ExternalFileReader, privateData []byte, header *Header) (*CryptoReader, error)

func (*CryptoReader) CheckSum

func (r *CryptoReader) CheckSum() error

func (*CryptoReader) Close

func (r *CryptoReader) Close() error

func (*CryptoReader) Header

func (r *CryptoReader) Header() *Header

func (*CryptoReader) Read

func (r *CryptoReader) Read(p []byte) (int, error)

Read: 实现Reader接口

type CryptoStore

type CryptoStore struct {
	storage.ExternalStorage
	// contains filtered or unexported fields
}

func NewCryptoStore

func NewCryptoStore(storageOption *storage.BackendOptions, path string, cryptoOption *CryptoStoreOption) (*CryptoStore, error)

func (*CryptoStore) Close

func (c *CryptoStore) Close()

func (*CryptoStore) Create

func (c *CryptoStore) Create(ctx context.Context, path string, option *storage.WriterOption) (storage.ExternalFileWriter, error)

func (*CryptoStore) FormatHeaderName

func (c *CryptoStore) FormatHeaderName(name string) string

func (*CryptoStore) NewHash

func (c *CryptoStore) NewHash() hash.Hash

func (*CryptoStore) Open

func (c *CryptoStore) Open(ctx context.Context, name string, opt *storage.ReaderOption) (storage.ExternalFileReader, error)

func (*CryptoStore) ReadFile

func (c *CryptoStore) ReadFile(ctx context.Context, name string) ([]byte, error)

func (*CryptoStore) Rename

func (c *CryptoStore) Rename(ctx context.Context, src, dst string) error

func (*CryptoStore) WriteFile

func (c *CryptoStore) WriteFile(ctx context.Context, name string, data []byte) error

type CryptoStoreOption

type CryptoStoreOption struct {
	PrivateData []byte // 只有Reader接口才需要私钥，如果只需要上传数据，可以不设置
	PublicData  []byte // 只有Writer接口才需要公钥，如果只需要下载数据，可以不设置
	Suffix      string // 文件后缀，默认是crypto
	HashType    string // 哈希类型，默认是md5
}

func NewCryptoStoreOption

func NewCryptoStoreOption(privateData []byte, publicData []byte, suffix string, hashType string) (*CryptoStoreOption, error)

type CryptoWriter

type CryptoWriter struct {
	// contains filtered or unexported fields
}

func NewCryptoWriter

func NewCryptoWriter(publicData []byte, hashType string, dataWriter, headerWriter storage.ExternalFileWriter) (nw *CryptoWriter, err error)

NewCryptoWriter : 创建加密写入器， 只需要公钥即可

func (*CryptoWriter) Close

func (w *CryptoWriter) Close(ctx context.Context) error

func (*CryptoWriter) Write

func (w *CryptoWriter) Write(ctx context.Context, p []byte) (int, error)

Write : 覆盖 ExternalFileWriter 的 Write 方法

func (*CryptoWriter) WriteHeaderJson

func (c *CryptoWriter) WriteHeaderJson(ctx context.Context) error

func (*CryptoWriter) WriteHeaderV1

func (c *CryptoWriter) WriteHeaderV1(ctx context.Context) error

type Header

type Header struct {
	Hash     []byte `json:"hash"`
	HashType string `json:"hash_type"`
	EncKey   []byte `json:"enc_key"`
	IV       []byte `json:"iv"`
	// contains filtered or unexported fields
}

func NewHeader

func NewHeader(key, encKey, iv []byte, hashType string) (*Header, error)

func NewRandHeader

func NewRandHeader(publicData []byte, hashType string) (*Header, error)

func ParseHeaderV1

func ParseHeaderV1(data storage.ExternalFileReader) (header *Header, err error)

func ReadHeaderV1

func ReadHeaderV1(headerReader storage.ExternalFileReader) (*Header, error)

func ReadHeaderV2

func ReadHeaderV2(headerReader storage.ExternalFileReader) (*Header, error)

func (*Header) BytesV1

func (header *Header) BytesV1() []byte

Bytes 生成Header的字节流

func (Header) Map

func (header Header) Map() map[string]string

func (Header) Size

func (header Header) Size() int

func (Header) String

func (header Header) String() string

type IoWriter

type IoWriter struct {
	// contains filtered or unexported fields
}

func NewIoWriter

func NewIoWriter(w storage.ExternalFileWriter) *IoWriter

func (*IoWriter) Write

func (w *IoWriter) Write(p []byte) (int, error)