Documentation ¶
Overview ¶
Package adds an ability create ZSTD files in seekable format and randomly access them using uncompressed offsets.
Example ¶
package main import ( "fmt" "io" "log" "os" "github.com/klauspost/compress/zstd" seekable "github.com/SaveTheRbtz/zstd-seekable-format-go" ) func main() { f, err := os.CreateTemp("", "example") if err != nil { log.Fatal(err) } defer os.Remove(f.Name()) enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest)) if err != nil { log.Fatal(err) } defer enc.Close() w, err := seekable.NewWriter(f, enc) if err != nil { log.Fatal(err) } // Write data in chunks. for _, b := range [][]byte{[]byte("Hello"), []byte(" "), []byte("World!")} { _, err = w.Write(b) if err != nil { log.Fatal(err) } } // Close and flush seek table. err = w.Close() if err != nil { log.Fatal(err) } dec, err := zstd.NewReader(nil) if err != nil { log.Fatal(err) } defer dec.Close() r, err := seekable.NewReader(f, dec) if err != nil { log.Fatal(err) } ello := make([]byte, 4) // ReaderAt _, err = r.ReadAt(ello, 1) if err != nil { log.Fatal(err) } fmt.Printf("Offset: 1 from the start: %s\n", string(ello)) world := make([]byte, 5) // Seeker _, err = r.Seek(-6, io.SeekEnd) if err != nil { log.Fatal(err) } // Reader _, err = r.Read(world) if err != nil { log.Fatal(err) } fmt.Printf("Offset: -6 from the end: %s\n", string(world)) _, _ = f.Seek(0, io.SeekStart) // Standard ZSTD Reader. dec, err = zstd.NewReader(f) if err != nil { log.Fatal(err) } defer dec.Close() all, err := io.ReadAll(dec) if err != nil { log.Fatal(err) } fmt.Printf("Whole string: %s\n", string(all)) }
Output: Offset: 1 from the start: ello Offset: -6 from the end: World Whole string: Hello World!
Index ¶
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Decoder ¶
type Decoder interface { // GetIndexByDecompOffset returns FrameOffsetEntry for an offset in the decompressed stream. // Will return nil if offset is greater or equal than Size(). GetIndexByDecompOffset(off uint64) *env.FrameOffsetEntry // GetIndexByID returns FrameOffsetEntry for a given frame id. // Will return nil if offset is greater or equal than NumFrames() or less than 0. GetIndexByID(id int64) *env.FrameOffsetEntry // Size returns the size of the uncompressed stream. Size() int64 // NumFrames returns number of frames in the compressed stream. NumFrames() int64 }
Decoder is a byte-oriented API that is useful for cases where wrapping io.ReadSeeker is not desirable.
func NewDecoder ¶
NewDecoder creates a byte-oriented Decode interface from a given seektable index. This index can either be produced by either Writer's WriteSeekTable or Encoder's EndStream. Decoder can be used concurrently.
type Encoder ¶
type Encoder interface { // Encode returns compressed data and appends a frame to in-memory seek table. Encode(src []byte) ([]byte, error) // EndStream returns in-memory seek table as a ZSTD's skippable frame. EndStream() ([]byte, error) }
Encoder is a byte-oriented API that is useful where wrapping io.Writer is not desirable.
func NewEncoder ¶
func NewEncoder(encoder ZSTDEncoder, opts ...options.WOption) (Encoder, error)
type Reader ¶
type Reader interface { // Seek implements io.Seeker interface to randomly access data. // This method is NOT goroutine-safe and CAN NOT be called // concurrently since it modifies the underlying offset. io.Seeker // Read implements io.Reader interface to sequentially access data. // This method is NOT goroutine-safe and CAN NOT be called // concurrently since it modifies the underlying offset. io.Reader // ReadAt implements io.ReaderAt interface to randomly access data. // This method is goroutine-safe and can be called concurrently ONLY if // the underlying reader supports io.ReaderAt interface. io.ReaderAt }
func NewReader ¶
func NewReader(rs io.ReadSeeker, decoder ZSTDDecoder, opts ...options.ROption) (Reader, error)
NewReader returns ZSTD stream reader that can be randomly accessed using uncompressed data offset. Ideally, passed io.ReadSeeker should implement io.ReaderAt interface.
type Writer ¶
type Writer interface { // Write writes a chunk of data as a separate frame into the datastream. // // Note that Write does not do any coalescing nor splitting of data, // so each write will map to a separate ZSTD Frame. io.Writer // Close implement io.Closer interface. It writes the seek table footer // and releases occupied memory. // // Caller is still responsible to Close the underlying writer. io.Closer }
type ZSTDDecoder ¶
ZSTDDecoder is the decompressor. Tested with github.com/klauspost/compress/zstd.
type ZSTDEncoder ¶
ZSTDEncoder is the compressor. Tested with github.com/klauspost/compress/zstd.