npy

package
v0.8.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 27, 2024 License: BSD-3-Clause Imports: 18 Imported by: 0

Documentation

Overview

Package npy provides read/write access to files following the NumPy data file format: 

https://numpy.org/neps/nep-0001-npy-format.html

Supported types

npy supports r/w of scalars, arrays, slices and gonum/mat.Dense. Supported scalars are:

  • bool,
  • (u)int{8,16,32,64},
  • float{32,64},
  • complex{64,128}

Reading

Reading from a NumPy data file can be performed like so: 

f, err := os.Open("data.npy")
var m mat.Dense
err = npy.Read(f, &m)
fmt.Printf("data = %v\n", mat.Formatted(&m, mat.Prefix("       "))))

npy can also read data directly into slices, arrays or scalars, provided the on-disk data type and the provided one match.

Example: 

var data []float64
err = npy.Read(f, &data)

var data uint64
err = npy.Read(f, &data)

Writing

Writing into a NumPy data file can be done like so: 

f, err := os.Create("data.npy")
var m mat.Dense = ...
err = npy.Write(f, m)

Scalars, arrays and slices are also supported: 

var data []float64 = ...
err = npy.Write(f, data)

var data int64 = 42
err = npy.Write(f, data)

var data [42]complex128 = ...
err = npy.Write(f, data)
Example (PartialRead) 
package main

import (
	"fmt"
	"log"
	"os"

	"github.com/Hyde46/mc-npyio/npy"
)

func main() {
	out, err := os.Create("data.npy")
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	f := []float64{0, 1, 2, 3, 4, 5}
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", f)
	err = npy.Write(out, f)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}
	err = out.Close()
	if err != nil {
		log.Fatal(err)
	}

	in, err := os.Open("data.npy")
	if err != nil {
		log.Fatal(err)
	}
	defer in.Close()
	r, err := npy.NewReader(in)
	if err != nil {
		log.Fatal(err)
	}

	data := make([]float64, 3)
	err = r.Read(&data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- partial data read back --\n")
	fmt.Printf("data = %v\n", data)

	err = r.Read(&data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- rest of data read back --\n")
	fmt.Printf("data = %v\n", data)

}

Output:

-- original data --
data = [0 1 2 3 4 5]
-- partial data read back --
data = [0 1 2]
-- rest of data read back --
data = [3 4 5]

Index

Examples

Constants

This section is empty.

Variables

View Source 
var (

	// ErrInvalidNumPyFormat is the error returned by NewReader when
	// the underlying io.Reader is not a valid or recognized NumPy data
	// file format.
	ErrInvalidNumPyFormat = errors.New("npy: not a valid NumPy file format")

	// ErrTypeMismatch is the error returned by Reader when the on-disk
	// data type and the user provided one do NOT match.
	ErrTypeMismatch = errors.New("npy: types don't match")

	// ErrInvalidType is the error returned by Reader and Writer when
	// confronted with a type that is not supported or can not be
	// reliably (de)serialized.
	ErrInvalidType = errors.New("npy: invalid or unsupported type")

	// Magic header present at the start of a NumPy data file format.
	// See https://numpy.org/neps/nep-0001-npy-format.html
	Magic = [6]byte{'\x93', 'N', 'U', 'M', 'P', 'Y'}
)

Functions

func ClassLoader 

func ClassLoader(module, name string) (any, error)

ClassLoader provides a python class loader mechanism for python pickles containing numpy.dtype and numpy.ndarray values.

func Read 

func Read(r io.Reader, ptr interface{}) error

Read reads the data from the r NumPy data file io.Reader, into the provided pointed at value ptr. Read returns an error if the on-disk data type and the one provided don't match.

If a *mat.Dense matrix is passed to Read, the numpy-array data is loaded into the Dense matrix, honouring Fortran/C-order and dimensions/shape parameters.

Only numpy-arrays with up to 2 dimensions are supported. Only numpy-arrays with elements convertible to float64 are supported.

Example 
package main

import (
	"bytes"
	"fmt"
	"log"

	"gonum.org/v1/gonum/mat"

	"github.com/Hyde46/mc-npyio/npy"
)

func main() {
	m := mat.NewDense(2, 3, []float64{0, 1, 2, 3, 4, 5})
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))
	buf := new(bytes.Buffer)

	err := npy.Write(buf, m)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}

	// modify original data
	m.Set(0, 0, 6)

	var data mat.Dense
	err = npy.Read(buf, &data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- data read back --\n")
	fmt.Printf("data = %v\n", mat.Formatted(&data, mat.Prefix("       ")))

	fmt.Printf("-- modified original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))

}

Output:

-- original data --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- data read back --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- modified original data --
data = ⎡6  1  2⎤
       ⎣3  4  5⎦

func TypeFrom 

func TypeFrom(dtype string) reflect.Type

TypeFrom returns the reflect.Type corresponding to the numpy-dtype string, if any.

func Write 

func Write(w io.Writer, val interface{}) error

Write writes 'val' into 'w' in the NumPy data format.

  • if val is a scalar, it must be of a supported type (bools, (u)ints, floats and complexes)
  • if val is a slice or array, it must be a slice/array of a supported type. the shape (len,) will be written out.
  • if val is a mat.Dense, the correct shape will be transmitted. (ie: (nrows, ncols))

The data-array will always be written out in C-order (row-major).

Example 
package main

import (
	"bytes"
	"fmt"
	"log"

	"gonum.org/v1/gonum/mat"

	"github.com/Hyde46/mc-npyio/npy"
)

func main() {
	m := mat.NewDense(2, 3, []float64{0, 1, 2, 3, 4, 5})
	fmt.Printf("-- original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))
	buf := new(bytes.Buffer)

	err := npy.Write(buf, m)
	if err != nil {
		log.Fatalf("error writing data: %v\n", err)
	}

	// modify original data
	m.Set(0, 0, 6)

	var data mat.Dense
	err = npy.Read(buf, &data)
	if err != nil {
		log.Fatalf("error reading data: %v\n", err)
	}

	fmt.Printf("-- data read back --\n")
	fmt.Printf("data = %v\n", mat.Formatted(&data, mat.Prefix("       ")))

	fmt.Printf("-- modified original data --\n")
	fmt.Printf("data = %v\n", mat.Formatted(m, mat.Prefix("       ")))

}

Output:

-- original data --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- data read back --
data = ⎡0  1  2⎤
       ⎣3  4  5⎦
-- modified original data --
data = ⎡6  1  2⎤
       ⎣3  4  5⎦

func WriteHeader added in v0.8.2 

func WriteHeader(w io.Writer, val interface{}, header []int) error

Write writes 'val' into 'w' in the NumPy data format.

  • if val is a scalar, it must be of a supported type (bools, (u)ints, floats and complexes)
  • if val is a slice or array, it must be a slice/array of a supported type. the shape (len,) will be written out.
  • if val is a mat.Dense, the correct shape will be transmitted. (ie: (nrows, ncols))

The data-array will always be written out in C-order (row-major).

Types

type Array 

type Array struct {
	// contains filtered or unexported fields
}

Array is a multidimensional, homogeneous array of fixed-size items.

func (Array) Data 

func (arr Array) Data() any

Data returns the array's underlying data.

func (Array) Descr 

func (arr Array) Descr() ArrayDescr

Descr returns the array's data type descriptor.

func (Array) Fortran 

func (arr Array) Fortran() bool

Fortran returns whether the array's data is stored in FORTRAN-order (ie: column-major) instead of C-order (ie: row-major.)

func (*Array) PyNew 

func (*Array) PyNew(args ...any) (any, error)

func (*Array) PySetState 

func (arr *Array) PySetState(arg any) error

func (Array) Shape 

func (arr Array) Shape() []int

Shape returns the array's shape.

func (Array) Strides 

func (arr Array) Strides() []int

Strides returns the array's strides in bytes.

func (Array) String 

func (arr Array) String() string

type ArrayDescr 

type ArrayDescr struct {
	// contains filtered or unexported fields
}

ArrayDescr describes a numpy data type.

func (*ArrayDescr) Call 

func (*ArrayDescr) Call(args ...any) (any, error)

func (*ArrayDescr) PySetState 

func (dt *ArrayDescr) PySetState(arg any) error

func (ArrayDescr) String 

func (dt ArrayDescr) String() string
type Header struct {
	Major byte // data file major version
	Minor byte // data file minor version
	Descr struct {
		Type    string // data type of array elements ('<i8', '<f4', ...)
		Fortran bool   // whether the array data is stored in Fortran-order (col-major)
		Shape   []int  // array shape (e.g. [2,3] a 2-rows, 3-cols array
	}
}

Header describes the data content of a NumPy data file.

func (Header) String 

func (h Header) String() string

type Reader 

type Reader struct {
	Header Header
	// contains filtered or unexported fields
}

Reader reads data from a NumPy data file.

func NewReader 

func NewReader(r io.Reader) (*Reader, error)

NewReader creates a new NumPy data file format reader.

func (*Reader) Read 

func (r *Reader) Read(ptr interface{}) error

Read reads the numpy-array data from the underlying NumPy file. Read returns an error if the on-disk data type and the provided one don't match.

See npy.Read() for documentation.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.