soci

Documentation

Index

• func ExtractCheckpoint(ctx context.Context, from *flate.Checkpoint, bs BlobSeeker, start, end int64) (io.ReadCloser, error)
• func ExtractFile(ctx context.Context, t Index, bs BlobSeeker, tf *TOCFile) (io.ReadCloser, error)
• func Peek(rc io.ReadCloser) (string, io.ReadCloser, io.ReadCloser, error)
• func TarHeader(header *TOCFile) *tar.Header
• type BlobSeeker
• type Checkpoint
• type Checkpointer
• type Index
  • func NewIndex(bs BlobSeeker, toc *TOC, sub Index) (Index, error)
• type Indexer
  • func NewIndexer(rc io.ReadCloser, w io.Writer, span int64, mediaType string) (*Indexer, string, io.ReadCloser, io.ReadCloser, error)
  • func (i *Indexer) Close() error
  • func (i *Indexer) Next() (*tar.Header, error)
  • func (i *Indexer) Read(p []byte) (int, error)
  • func (i *Indexer) Size() int64
  • func (i *Indexer) TOC() (*TOC, error)
  • func (i *Indexer) Type() string
• type MultiFS
  • func NewMultiFS(fss []*SociFS, prefix string, ref string, size int64, mt types.MediaType, ...) *MultiFS
  • func (s *MultiFS) Everything() ([]fs.DirEntry, error)
  • func (s *MultiFS) Open(original string) (fs.File, error)
  • func (s *MultiFS) RenderHeader(w http.ResponseWriter, r *http.Request, fname string, f httpserve.File, ...) error
• type PeekReader
• type RenderDir
• type RenderFunc
• type SociFS
  • func FS(index Index, bs BlobSeeker, prefix string, ref string, maxSize int64, ...) *SociFS
  • func (s *SociFS) Everything() ([]fs.DirEntry, error)
  • func (s *SociFS) Open(original string) (fs.File, error)
  • func (s *SociFS) ReadDir(original string) ([]fs.DirEntry, error)
  • func (s *SociFS) RenderHeader(w http.ResponseWriter, r *http.Request, fname string, f httpserve.File, ...) error
• type TOC
  • func (toc *TOC) Checkpoint(tf *TOCFile) *Checkpointer
• type TOCFile
  • func FromTar(header *tar.Header) *TOCFile

Functions

func ExtractCheckpoint

func ExtractCheckpoint(ctx context.Context, from *flate.Checkpoint, bs BlobSeeker, start, end int64) (io.ReadCloser, error)

func ExtractFile

func ExtractFile(ctx context.Context, t Index, bs BlobSeeker, tf *TOCFile) (io.ReadCloser, error)

func Peek

func Peek(rc io.ReadCloser) (string, io.ReadCloser, io.ReadCloser, error)

Peek detects streams of: tar tar+gzip tar+zstd gzip zstd

func TarHeader

func TarHeader(header *TOCFile) *tar.Header

Types

type BlobSeeker

type BlobSeeker interface {
	Reader(ctx context.Context, off int64, end int64) (io.ReadCloser, error)
}

type Checkpoint

type Checkpoint interface {
	BytesRead() int64
	BytesWritten() int64
	History() []byte
	SetHistory([]byte)
	IsEmpty() bool
}

type Checkpointer

type Checkpointer struct {
	Checkpoint *flate.Checkpoint
	File       *TOCFile

	Discard int64
	// contains filtered or unexported fields
}

type Index

type Index interface {
	Dict(cp *Checkpointer) ([]byte, error)
	Locate(name string) (*TOCFile, error)
	TOC() *TOC
}

func NewIndex

func NewIndex(bs BlobSeeker, toc *TOC, sub Index) (Index, error)

type Indexer

type Indexer struct {
	// contains filtered or unexported fields
}

func NewIndexer

func NewIndexer(rc io.ReadCloser, w io.Writer, span int64, mediaType string) (*Indexer, string, io.ReadCloser, io.ReadCloser, error)

Returns: Indexer (if non-nil, everything else is nil) Original stream (buffered rc to Peek) Unwrapped stream (ungzip or unzstd, nil if we could not unwrap or err != nil) Error (maybe nil)

func (*Indexer) Close

func (i *Indexer) Close() error

func (*Indexer) Next

func (i *Indexer) Next() (*tar.Header, error)

func (*Indexer) Read

func (i *Indexer) Read(p []byte) (int, error)

func (*Indexer) Size

func (i *Indexer) Size() int64

func (*Indexer) TOC

func (i *Indexer) TOC() (*TOC, error)

func (*Indexer) Type

func (i *Indexer) Type() string

type MultiFS

type MultiFS struct {
	// contains filtered or unexported fields
}

func NewMultiFS

func NewMultiFS(fss []*SociFS, prefix string, ref string, size int64, mt types.MediaType, render RenderDir) *MultiFS

func (*MultiFS) Everything

func (s *MultiFS) Everything() ([]fs.DirEntry, error)

func (*MultiFS) Open

func (s *MultiFS) Open(original string) (fs.File, error)

func (*MultiFS) RenderHeader

func (s *MultiFS) RenderHeader(w http.ResponseWriter, r *http.Request, fname string, f httpserve.File, ctype string) error

type PeekReader

type PeekReader interface {
	io.Reader
	Peek(n int) ([]byte, error)
}

type RenderDir

type RenderDir func(w http.ResponseWriter, fname string, prefix string, mediaType types.MediaType, size int64, ref string, f httpserve.File, ctype string) error

type RenderFunc

type RenderFunc func(w http.ResponseWriter, r *http.Request, fname string, prefix string, ref string, kind string, mediaType types.MediaType, size int64, f httpserve.File, ctype string) error

type SociFS

type SociFS struct {
	// contains filtered or unexported fields
}

func FS

func FS(index Index, bs BlobSeeker, prefix string, ref string, maxSize int64, mt types.MediaType, render RenderFunc) *SociFS

func (*SociFS) Everything

func (s *SociFS) Everything() ([]fs.DirEntry, error)

func (*SociFS) Open

func (s *SociFS) Open(original string) (fs.File, error)

func (*SociFS) ReadDir

func (s *SociFS) ReadDir(original string) ([]fs.DirEntry, error)

func (*SociFS) RenderHeader

func (s *SociFS) RenderHeader(w http.ResponseWriter, r *http.Request, fname string, f httpserve.File, ctype string) error

type TOC

type TOC struct {
	// TODO: Move these so files/checkpoints can be streamingly parsed.
	// metadata.json?
	Csize       int64  `json:"csize,omitempty"`
	Usize       int64  `json:"usize,omitempty"`
	Ssize       int64  `json:"ssize,omitempty"`
	ArchiveSize int64  `json:"asize,omitempty"`
	Size        int64  `json:"size,omitempty"`
	Type        string `json:"type,omitempty"`
	MediaType   string `json:"mediaType,omitempty"`

	// TODO: Checkpoints as jsonlines in separate file.
	Checkpoints []*flate.Checkpoint `json:"checkpoints,omitempty"`

	// TODO: Files as jsonlines in separate file.
	Files []TOCFile `json:"files,omitempty"`
}

func (*TOC) Checkpoint

func (toc *TOC) Checkpoint(tf *TOCFile) *Checkpointer

type TOCFile

type TOCFile struct {
	// The tar stuff we actually care about.
	// TODO: Just include all of it, for completeness?
	Typeflag   byte              `json:"typeflag,omitempty"`
	Name       string            `json:"name,omitempty"`
	Linkname   string            `json:"linkname,omitempty"`
	Size       int64             `json:"size,omitempty"`
	Mode       int64             `json:"mode,omitempty"`
	ModTime    time.Time         `json:"mod,omitempty"`
	Uid        int               `json:"uid,omitempty"`
	Gid        int               `json:"gid,omitempty"`
	PAXRecords map[string]string `json:"pax,omitempty"`

	// Our uncompressed offset so we can seek ahead.
	Offset int64 `json:"offset,omitempty"`
}

func FromTar

func FromTar(header *tar.Header) *TOCFile