npyio

README

npyio

npyio provides read/write access to numpy data files.

Installation

Is done via go get:

$> go get github.com/sbinet/npyio

Documentation

Is available on godoc

npyio-ls

npyio-ls is a command using github.com/sbinet/npyio (located under github.com/sbinet/npyio/cmd/npyio-ls) to display the content of a (list of) NumPy data file(s).

$> npyio-ls testdata/data_float64_2x3_?order.npy 
================================================================================
file: testdata/data_float64_2x3_corder.npy
npy-header: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[2 3]}}
data = [0 1 2 3 4 5]

================================================================================
file: testdata/data_float64_2x3_forder.npy
npy-header: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:true, Shape:[2 3]}}
data = [0 1 2 3 4 5]

$> npyio-ls testdata/data_float64_2x3x4_corder.npy 
================================================================================
file: testdata/data_float64_2x3x4_corder.npy
npy-header: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[2 3 4]}}
data = [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23]

npyio-ls automatically detects .npz archive files and inspects them too:

$> npyio-ls testdata/data_float64_corder.npz 
================================================================================
file: testdata/data_float64_corder.npz
entry: arr1.npy
npy-header: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[6 1]}}
data = [0 1 2 3 4 5]

entry: arr0.npy
npy-header: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[2 3]}}
data = [0 1 2 3 4 5]

Example

Reading a .npy file

Consider a .npy file created with the following python code:

>>> import numpy as np
>>> arr = np.arange(6, dtype="float64").reshape(2,3)
>>> f = open("data.npy", "w")
>>> np.save(f, arr)
>>> f.close()

The (float64) data array can be loaded into a (float64) mat.Matrix by the following code:

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/sbinet/npyio"
	"gonum.org/v1/gonum/mat"
)

func main() {
	f, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	r, err := npyio.NewReader(f)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("npy-header: %v\n", r.Header)
	shape := r.Header.Descr.Shape
	raw := make([]float64, shape[0]*shape[1])

	err = r.Read(&raw)
	if err != nil {
		log.Fatal(err)
	}

	m := mat.NewDense(shape[0], shape[1], raw)
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))
}

$> my-binary data.npy
npy-header: Header{Major:1, Minor:0, Descr:{Type:<i8, Fortran:false, Shape:[2 3]}}
data = ⎡0  1  2⎤
       ⎣3  4  5⎦

Reading a .npy file with npyio.Read

Alternatively, one can use the convenience function npyio.Read:

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/sbinet/npyio"
)

func main() {
	f, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	var m []float64
	err = npyio.Read(f, &m)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("data = %v\n", m)
}

$> my-binary ./data.npy
data = [0 1 2 3 4 5]

Writing a .npy file with npyio.Write

package main

import (
	"log"
	"os"

	"github.com/sbinet/npyio"
)

func main() {
	f, err := os.Create("data.npy")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	m := []float64{0, 1, 2, 3, 4, 5}
	err = npyio.Write(f, m)
	if err != nil {
		log.Fatalf("error writing to file: %v\n", err)
	}

	err = f.Close()
	if err != nil {
		log.Fatalf("error closing file: %v\n", err)
	}
}

Reading a .npz file with npyio/npz

func ExampleOpen() {
	f, err := npz.Open("../testdata/data_float64_corder.npz")
	if err != nil {
		log.Fatalf("could not open npz file: %+v", err)
	}
	defer f.Close()

	for _, name := range f.Keys() {
		fmt.Printf("%s: %v\n", name, f.Header(name))
	}

	var f0 []float64
	err = f.Read("arr0.npy", &f0)
	if err != nil {
		log.Fatalf("could not read value from npz file: %+v", err)
	}

	var f1 []float64
	err = f.Read("arr1.npy", &f1)
	if err != nil {
		log.Fatalf("could not read value from npz file: %+v", err)
	}

	fmt.Printf("arr0: %v\n", f0)
	fmt.Printf("arr1: %v\n", f1)

	// Output:
	// arr0.npy: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[2 3]}}
	// arr1.npy: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[6 1]}}
	// arr0: [0 1 2 3 4 5]
	// arr1: [0 1 2 3 4 5]
}

func ExampleReader() {
	f, err := os.Open("../testdata/data_float64_corder.npz")
	if err != nil {
		log.Fatalf("could not open npz file: %+v", err)
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		log.Fatalf("could not stat npz file: %+v", err)
	}

	r, err := npz.NewReader(f, stat.Size())
	if err != nil {
		log.Fatalf("could not open npz archive: %+v", err)
	}

	for _, name := range r.Keys() {
		fmt.Printf("%s: %v\n", name, r.Header(name))
	}

	var f0 []float64
	err = r.Read("arr0.npy", &f0)
	if err != nil {
		log.Fatalf("could not read value from npz file: %+v", err)
	}

	var f1 []float64
	err = r.Read("arr1.npy", &f1)
	if err != nil {
		log.Fatalf("could not read value from npz file: %+v", err)
	}

	fmt.Printf("arr0: %v\n", f0)
	fmt.Printf("arr1: %v\n", f1)

	// Output:
	// arr0.npy: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[2 3]}}
	// arr1.npy: Header{Major:1, Minor:0, Descr:{Type:<f8, Fortran:false, Shape:[6 1]}}
	// arr0: [0 1 2 3 4 5]
	// arr1: [0 1 2 3 4 5]
}

func ExampleRead() {
	f, err := os.Open("../testdata/data_float64_corder.npz")
	if err != nil {
		log.Fatalf("could not open npz file: %+v", err)
	}
	defer f.Close()

	var f0 []float64
	err = npz.Read(f, "arr0.npy", &f0)
	if err != nil {
		log.Fatalf("could not read value from npz file: %+v", err)
	}

	var f1 []float64
	err = npz.Read(f, "arr1.npy", &f1)
	if err != nil {
		log.Fatalf("could not read value from npz file: %+v", err)
	}

	fmt.Printf("arr0: %v\n", f0)
	fmt.Printf("arr1: %v\n", f1)

	// Output:
	// arr0: [0 1 2 3 4 5]
	// arr1: [0 1 2 3 4 5]
}

Writing a .npz file with npyio/npz

func ExampleCreate() {
	f, err := npz.Create("out.npz")
	if err != nil {
		log.Fatalf("could not create npz file: %+v", err)
	}
	defer f.Close()

	err = f.Write("arr0.npy", []float64{0, 1, 2, 3, 4, 5})
	if err != nil {
		log.Fatalf("could not write value arr0.npy to npz file: %+v", err)
	}

	err = f.Write("arr1.npy", []float32{0, 1, 2, 3, 4, 5})
	if err != nil {
		log.Fatalf("could not write value arr1.npy to npz file: %+v", err)
	}

	err = f.Close()
	if err != nil {
		log.Fatalf("could not close npz file: %+v", err)
	}

	// Output:
}

func ExampleWriter() {
	f, err := os.Create("out.npz")
	if err != nil {
		log.Fatalf("could not create npz file: %+v", err)
	}
	defer f.Close()

	wz := npz.NewWriter(f)
	defer wz.Close()

	err = wz.Write("arr0.npy", []float64{0, 1, 2, 3, 4, 5})
	if err != nil {
		log.Fatalf("could not write value arr0.npy to npz file: %+v", err)
	}

	err = wz.Write("arr1.npy", []float32{0, 1, 2, 3, 4, 5})
	if err != nil {
		log.Fatalf("could not write value arr1.npy to npz file: %+v", err)
	}

	err = wz.Close()
	if err != nil {
		log.Fatalf("could not close npz archive: %+v", err)
	}

	err = f.Close()
	if err != nil {
		log.Fatalf("could not close npz file: %+v", err)
	}

	// Output:
}

func ExampleWrite() {
	err := npz.Write("out.npz", map[string]interface{}{
		"arr0.npy": []float64{0, 1, 2, 3, 4, 5},
		"arr1.npy": []float32{0, 1, 2, 3, 4, 5},
	})
	if err != nil {
		log.Fatalf("could not save to npz file: %+v", err)
	}

	// Output:
}

Documentation

Overview

Package npyio provides read/write access to files following the NumPy data file format:

https://numpy.org/neps/nep-0001-npy-format.html

Supported types

npyio supports r/w of scalars, arrays, slices and gonum/mat.Dense. Supported scalars are:

• bool,
• (u)int{8,16,32,64},
• float{32,64},
• complex{64,128}

Reading

Reading from a NumPy data file can be performed like so:

f, err := os.Open("data.npy")
var m mat.Dense
err = npyio.Read(f, &m)
fmt.Printf("data = %v\n", mat.Formatted(&m, mat.Prefix("       "))))

npyio can also read data directly into slices, arrays or scalars, provided the on-disk data type and the provided one match.

Example:

var data []float64
err = npyio.Read(f, &data)

var data uint64
err = npyio.Read(f, &data)

Writing

Writing into a NumPy data file can be done like so:

f, err := os.Create("data.npy")
var m mat.Dense = ...
err = npyio.Write(f, m)

Scalars, arrays and slices are also supported:

var data []float64 = ...
err = npyio.Write(f, data)

var data int64 = 42
err = npyio.Write(f, data)

var data [42]complex128 = ...
err = npyio.Write(f, data)

Example (PartialRead)

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/sbinet/npyio"
)

func main() {
	out, err := os.Create("data.npy")
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	f := []float64{0, 1, 2, 3, 4, 5}
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", f)
	err = npyio.Write(out, f)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}
	err = out.Close()
	if err != nil {
		log.Fatal(err)
	}

	in, err := os.Open("data.npy")
	if err != nil {
		log.Fatal(err)
	}
	defer in.Close()
	r, err := npyio.NewReader(in)
	if err != nil {
		log.Fatal(err)
	}

	data := make([]float64, 3)
	err = r.Read(&data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- partial data read back --\n")
	fmt.Printf("data = %v\n", data)

	err = r.Read(&data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- rest of data read back --\n")
	fmt.Printf("data = %v\n", data)

}

Output:

-- original data --
data = [0 1 2 3 4 5]
-- partial data read back --
data = [0 1 2]
-- rest of data read back --
data = [3 4 5]

Index

• Variables
• func Dump(o io.Writer, r io.ReaderAt) error
• func Read(r io.Reader, ptr interface{}) error
• func TypeFrom(dtype string) reflect.Type
• func Write(w io.Writer, val interface{}) error
• type Header
• type Reader
  • func NewReader(r io.Reader) (*Reader, error)

Examples

• Package (PartialRead)
• Read
• Write

Variables

var (
	// ErrInvalidNumPyFormat is the error returned by NewReader when
	// the underlying io.Reader is not a valid or recognized NumPy data
	// file format.
	ErrInvalidNumPyFormat = npy.ErrInvalidNumPyFormat

	// ErrTypeMismatch is the error returned by Reader when the on-disk
	// data type and the user provided one do NOT match.
	ErrTypeMismatch = npy.ErrTypeMismatch

	// ErrInvalidType is the error returned by Reader and Writer when
	// confronted with a type that is not supported or can not be
	// reliably (de)serialized.
	ErrInvalidType = npy.ErrInvalidType

	// Magic header present at the start of a NumPy data file format.
	// See https://numpy.org/neps/nep-0001-npy-format.html
	Magic = npy.Magic
)

Functions

func Dump

func Dump(o io.Writer, r io.ReaderAt) error

Dump dumps the content of the provided reader to the writer, in a human readable format

func Read

func Read(r io.Reader, ptr interface{}) error

Read reads the data from the r NumPy data file io.Reader, into the provided pointed at value ptr. Read returns an error if the on-disk data type and the one provided don't match.

If a *mat.Dense matrix is passed to Read, the numpy-array data is loaded into the Dense matrix, honouring Fortran/C-order and dimensions/shape parameters.

Only numpy-arrays with up to 2 dimensions are supported. Only numpy-arrays with elements convertible to float64 are supported.

Example

package main

import (
	"bytes"
	"fmt"
	"log"

	"gonum.org/v1/gonum/mat"

	"github.com/sbinet/npyio"
)

func main() {
	m := mat.NewDense(2, 3, []float64{0, 1, 2, 3, 4, 5})
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))
	buf := new(bytes.Buffer)

	err := npyio.Write(buf, m)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}

	// modify original data
	m.Set(0, 0, 6)

	var data mat.Dense
	err = npyio.Read(buf, &data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- data read back --\n")
	fmt.Printf("data = %v\n", mat.Formatted(&data, mat.Prefix("       ")))

	fmt.Printf("-- modified original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))

}

Output:

-- original data --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- data read back --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- modified original data --
data = ⎡6  1  2⎤
       ⎣3  4  5⎦

func TypeFrom

func TypeFrom(dtype string) reflect.Type

TypeFrom returns the reflect.Type corresponding to the numpy-dtype string, if any.

func Write

func Write(w io.Writer, val interface{}) error

Write writes 'val' into 'w' in the NumPy data format.

• if val is a scalar, it must be of a supported type (bools, (u)ints, floats and complexes)
• if val is a slice or array, it must be a slice/array of a supported type. the shape (len,) will be written out.
• if val is a mat.Dense, the correct shape will be transmitted. (ie: (nrows, ncols))

The data-array will always be written out in C-order (row-major).

Example

package main

import (
	"bytes"
	"fmt"
	"log"

	"gonum.org/v1/gonum/mat"

	"github.com/sbinet/npyio"
)

func main() {
	m := mat.NewDense(2, 3, []float64{0, 1, 2, 3, 4, 5})
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))
	buf := new(bytes.Buffer)

	err := npyio.Write(buf, m)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}

	// modify original data
	m.Set(0, 0, 6)

	var data mat.Dense
	err = npyio.Read(buf, &data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- data read back --\n")
	fmt.Printf("data = %v\n", mat.Formatted(&data, mat.Prefix("       ")))

	fmt.Printf("-- modified original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))

}

Output:

-- original data --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- data read back --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- modified original data --
data = ⎡6  1  2⎤
       ⎣3  4  5⎦

Types

type Header

type Header = npy.Header

Header describes the data content of a NumPy data file.

type Reader

type Reader = npy.Reader

Reader reads data from a NumPy data file.

func NewReader

func NewReader(r io.Reader) (*Reader, error)

NewReader creates a new NumPy data file format reader.