Documentation ¶
Overview ¶
Package adds an ability create ZSTD files in seekable format and randomly access them using uncompressed offsets.
Example ¶
package main import ( "fmt" "io" "log" "os" "github.com/klauspost/compress/zstd" seekable "github.com/SaveTheRbtz/zstd-seekable-format-go" ) func main() { f, err := os.CreateTemp("", "example") if err != nil { log.Fatal(err) } defer os.Remove(f.Name()) enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest)) if err != nil { log.Fatal(err) } defer enc.Close() w, err := seekable.NewWriter(f, enc) if err != nil { log.Fatal(err) } // Write data in chunks. for _, b := range [][]byte{[]byte("Hello"), []byte(" "), []byte("World!")} { _, err = w.Write(b) if err != nil { log.Fatal(err) } } // Close and flush seek table. err = w.Close() if err != nil { log.Fatal(err) } dec, err := zstd.NewReader(nil) if err != nil { log.Fatal(err) } defer dec.Close() r, err := seekable.NewReader(f, dec) if err != nil { log.Fatal(err) } ello := make([]byte, 4) // ReaderAt _, err = r.ReadAt(ello, 1) if err != nil { log.Fatal(err) } fmt.Printf("Offset: 1 from the start: %s\n", string(ello)) world := make([]byte, 5) // Seeker _, err = r.Seek(-6, io.SeekEnd) if err != nil { log.Fatal(err) } // Reader _, err = r.Read(world) if err != nil { log.Fatal(err) } fmt.Printf("Offset: -6 from the end: %s\n", string(world)) _, _ = f.Seek(0, io.SeekStart) // Standard ZSTD Reader. dec, err = zstd.NewReader(f) if err != nil { log.Fatal(err) } defer dec.Close() all, err := io.ReadAll(dec) if err != nil { log.Fatal(err) } fmt.Printf("Whole string: %s\n", string(all)) }
Output: Offset: 1 from the start: ello Offset: -6 from the end: World Whole string: Hello World!
Index ¶
Examples ¶
Constants ¶
const (
/*
The format consists of a number of frames (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table.
Seek Table Format
The structure of the seek table frame is as follows:
|`Skippable_Magic_Number`|`Frame_Size`|`[Seek_Table_Entries]`|`Seek_Table_Footer`|
|------------------------|------------|----------------------|-------------------|
| 4 bytes | 4 bytes | 8-12 bytes each | 9 bytes |
Skippable_Magic_Number
Value: 0x184D2A5E.
This is for compatibility with Zstandard skippable frames: https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#skippable-frames.
Since it is legal for other Zstandard skippable frames to use the same
magic number, it is not recommended for a decoder to recognize frames
solely on this.
Frame_Size
The total size of the skippable frame, not including the `Skippable_Magic_Number` or `Frame_Size`.
This is for compatibility with Zstandard skippable frames: https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#skippable-frames.
https://github.com/facebook/zstd/blob/dev/contrib/seekable_format/zstd_seekable_compression_format.md
*/
SkippableFrameMagic = 0x184D2A50
)
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Decoder ¶
type Decoder interface { // GetIndexByDecompOffset returns FrameOffsetEntry for an offset in the decompressed stream. // Will return nil if offset is greater or equal than Size(). GetIndexByDecompOffset(off uint64) *env.FrameOffsetEntry // GetIndexByID returns FrameOffsetEntry for a given frame id. // Will return nil if offset is greater or equal than NumFrames() or less than 0. GetIndexByID(id int64) *env.FrameOffsetEntry // Size returns the size of the uncompressed stream. Size() int64 // NumFrames returns number of frames in the compressed stream. NumFrames() int64 }
Decoder is a byte-oriented API that is useful for cases where wrapping io.ReadSeeker is not desirable.
func NewDecoder ¶
NewDecoder creates a byte-oriented Decode interface from a given seektable index. This index can either be produced by either Writer's WriteSeekTable or Encoder's EndStream. Decoder can be used concurrently.
type Encoder ¶
type Encoder interface { // Encode returns compressed data and appends a frame to in-memory seek table. Encode(src []byte) ([]byte, error) // EndStream returns in-memory seek table as a ZSTD's skippable frame. EndStream() ([]byte, error) }
Encoder is a byte-oriented API that is useful where wrapping io.Writer is not desirable.
func NewEncoder ¶
func NewEncoder(encoder ZSTDEncoder, opts ...options.WOption) (Encoder, error)
type Reader ¶
type Reader interface { // Seek implements io.Seeker interface to randomly access data. // This method is NOT goroutine-safe and CAN NOT be called // concurrently since it modifies the underlying offset. io.Seeker // Read implements io.Reader interface to sequentially access data. // This method is NOT goroutine-safe and CAN NOT be called // concurrently since it modifies the underlying offset. io.Reader // ReadAt implements io.ReaderAt interface to randomly access data. // This method is goroutine-safe and can be called concurrently ONLY if // the underlying reader supports io.ReaderAt interface. io.ReaderAt }
func NewReader ¶
func NewReader(rs io.ReadSeeker, decoder ZSTDDecoder, opts ...options.ROption) (Reader, error)
NewReader returns ZSTD stream reader that can be randomly accessed using uncompressed data offset. Ideally, passed io.ReadSeeker should implement io.ReaderAt interface.
type Writer ¶
type Writer interface { // Write writes a chunk of data as a separate frame into the datastream. // // Note that Write does not do any coalescing nor splitting of data, // so each write will map to a separate ZSTD Frame. io.Writer // Close implement io.Closer interface. It writes the seek table footer // and releases occupied memory. // // Caller is still responsible to Close the underlying writer. io.Closer }
type ZSTDDecoder ¶
type ZSTDEncoder ¶
ZSTDEncoder is the compressor. Tested with github.com/klauspost/compress/zstd.