sparkl

Documentation

Overview

Package sparkl provides facilities to generate an Index for a WissKI

Package sparkl implements a very primitive graph index

Index

• func Export(pb *pathbuilder.Pathbuilder, index *igraph.Index, engine storages.BundleEngine, ...) error
• func LoadIndex(path string, predicates Predicates, engine igraph.Engine, opts IndexOptions, ...) (*igraph.Index, error)
• func LoadPathbuilder(pb *pathbuilder.Pathbuilder, index *igraph.Index, engine storages.BundleEngine, ...) (map[string][]wisski.Entity, error)
• func MakeIndex(source Source, predicates Predicates, engine igraph.Engine, opts IndexOptions, ...) (*igraph.Index, error)
• func NewEngine(path string) igraph.Engine
• func ParsePredicateString(target *[]impl.Label, value string)
• func StoreBundle(bundle *pathbuilder.Bundle, index *igraph.Index, engine storages.BundleEngine, ...) (storages.BundleStorage, func() error, error)
• func StoreBundles(bundles []*pathbuilder.Bundle, index *igraph.Index, ...) ([]storages.BundleStorage, func() error, error)
• type Cache
  • func NewCache(Data map[string][]wisski.Entity, SameAs imap.HashMap[impl.Label, impl.Label], ...) (c Cache, err error)
  • func (c Cache) Aliases(uri impl.Label) []impl.Label
  • func (c Cache) Bundle(uri impl.Label) (string, bool)
  • func (cache Cache) BundleNames() []string
  • func (c Cache) Canonical(uri impl.Label) impl.Label
  • func (cache *Cache) Close() error
  • func (cache *Cache) DecodeFrom(decoder *gob.Decoder) error
  • func (cache *Cache) EncodeTo(encoder *gob.Encoder) error
  • func (cache Cache) Entities(bundle_machine string) []wisski.Entity
  • func (c Cache) Entity(uri impl.Label, bundle string) (*wisski.Entity, bool)
  • func (c Cache) FirstBundle(uris ...impl.Label) (uri impl.Label, bundle string, ok bool)
• type Context
  • func (context *Context) Close() (err error)
  • func (context *Context) Open()
  • func (context *Context) Store(bundle *pathbuilder.Bundle, onFinish func()) storages.BundleStorage
  • func (context *Context) Wait() error
• type IndexOptions
  • func DefaultIndexOptions(pb *pathbuilder.Pathbuilder) IndexOptions
• type Predicates
• type QuadSource
  • func (qs *QuadSource) Close() error
  • func (qs *QuadSource) Next() Token
  • func (qs *QuadSource) Open() error
• type Source
• type Token

Functions

func Export

func Export(pb *pathbuilder.Pathbuilder, index *igraph.Index, engine storages.BundleEngine, exporter exporter.Exporter, stats *stats.Stats) error

Export loads all top-level paths from the given path-builder from the index into the given engine. Afterwards it is exported into the given exporter.

func LoadIndex

func LoadIndex(path string, predicates Predicates, engine igraph.Engine, opts IndexOptions, st *stats.Stats) (*igraph.Index, error)

LoadIndex is like MakeIndex, but reads nquads from the given path. When err != nil, the caller must eventually close the index.

func LoadPathbuilder

func LoadPathbuilder(pb *pathbuilder.Pathbuilder, index *igraph.Index, engine storages.BundleEngine, stats *stats.Stats) (map[string][]wisski.Entity, error)

LoadPathbuilder loads all paths in the given pathbuilder

func MakeIndex

func MakeIndex(source Source, predicates Predicates, engine igraph.Engine, opts IndexOptions, st *stats.Stats) (*igraph.Index, error)

MakeIndex creates a new Index from the given source. When err != nil, the caller must eventually close the index.

func NewEngine

func NewEngine(path string) igraph.Engine

NewEngine creates an engine that stores data at the specified path. When path is the empty string, stores data in memory.

func ParsePredicateString

func ParsePredicateString(target *[]impl.Label, value string)

ParsePredicateString parses a value of comma-or-newline-separated value into a list of impl.Labels Empty values are ignored.

func StoreBundle

func StoreBundle(bundle *pathbuilder.Bundle, index *igraph.Index, engine storages.BundleEngine, st *stats.Stats) (storages.BundleStorage, func() error, error)

StoreBundle loads all entities from the given bundle into a new storage, which is then returned.

Storages for any child bundles, and the bundle itself, are created using the makeStorage function. The storage for this bundle is returned.

func StoreBundles

func StoreBundles(bundles []*pathbuilder.Bundle, index *igraph.Index, engine storages.BundleEngine, st *stats.Stats) ([]storages.BundleStorage, func() error, error)

StoreBundles is like StoreBundle, but takes multiple bundles

Types

type Cache

type Cache struct {
	// contains filtered or unexported fields
}

Cache represents an easily accessible cache of WissKIObjects. It is held entirely in memory.

func NewCache

func NewCache(Data map[string][]wisski.Entity, SameAs imap.HashMap[impl.Label, impl.Label], st *stats.Stats) (c Cache, err error)

NewCache creates a new cache from a bundle-entity-map

func (Cache) Aliases

func (c Cache) Aliases(uri impl.Label) []impl.Label

Aliases returns the Aliases of the given impl.Label, excluding itself

func (Cache) Bundle

func (c Cache) Bundle(uri impl.Label) (string, bool)

Bundle returns the bundle of the given uri, if any

func (Cache) BundleNames

func (cache Cache) BundleNames() []string

func (Cache) Canonical

func (c Cache) Canonical(uri impl.Label) impl.Label

Canonical returns the canonical version of the given uri

func (*Cache) Close

func (cache *Cache) Close() error

func (*Cache) DecodeFrom

func (cache *Cache) DecodeFrom(decoder *gob.Decoder) error

func (*Cache) EncodeTo

func (cache *Cache) EncodeTo(encoder *gob.Encoder) error

EncodeTo encodes this cache object to the given gob.Encoder.

func (Cache) Entities

func (cache Cache) Entities(bundle_machine string) []wisski.Entity

func (Cache) Entity

func (c Cache) Entity(uri impl.Label, bundle string) (*wisski.Entity, bool)

Entity looks up the given entity

func (Cache) FirstBundle

func (c Cache) FirstBundle(uris ...impl.Label) (uri impl.Label, bundle string, ok bool)

FirstBundle returns the first bundle for which the given impl.Label exists

type Context

type Context struct {
	Engine storages.BundleEngine

	Index *igraph.Index // index being used
	// contains filtered or unexported fields
}

Context represents a context to extract bundle data from index into storages.

A Context must be opened, and eventually waited on. See [Open] and [Close].

func (*Context) Close

func (context *Context) Close() (err error)

Close closes this context

func (*Context) Open

func (context *Context) Open()

Open opens this context, and signals that multiple calls to Store() may follow.

Multiple calls to Open are invalid.

func (*Context) Store

func (context *Context) Store(bundle *pathbuilder.Bundle, onFinish func()) storages.BundleStorage

Store creates a new Storage for the given bundle and schedules entities to be loaded. Once onFinish is finished May only be called between calls [Open] and [Wait].

Any error that occurs is returned only by Wait.

func (*Context) Wait

func (context *Context) Wait() error

Wait signals this context that no more bundles will be loaded. And then waits for all bundle extracting to finish.

Multiple calls to Wait() are invalid.

type IndexOptions

type IndexOptions struct {
	Mask            *pathbuilder.Pathbuilder // Pathbuilder to use as a mask when indexing
	CompactInterval int                      // Interval during which to call internal compact. Set <= 0 to disable.
}

func DefaultIndexOptions

func DefaultIndexOptions(pb *pathbuilder.Pathbuilder) IndexOptions

type Predicates

type Predicates struct {
	SameAs    []impl.Label
	InverseOf []impl.Label
}

Predicates represent special predicates

type QuadSource

type QuadSource struct {
	Reader io.ReadSeeker
	// contains filtered or unexported fields
}

QuadSource reads triples from a quad file

func (*QuadSource) Close

func (qs *QuadSource) Close() error

func (*QuadSource) Next

func (qs *QuadSource) Next() Token

Next reads the next token from the QuadSource

func (*QuadSource) Open

func (qs *QuadSource) Open() error

type Source

type Source interface {
	// Open opens this data source.
	//
	// It is valid to call open more than once after Next() returns a token with err = io.EOF.
	// In this case the second call to open should reset the data source.
	Open() error

	// Close closes this source.
	// Close may only be called once a token with err != io.EOF is called.
	Close() error

	// Next scans the next token
	Next() Token
}

Source represents a source of triples

type Token

type Token struct {
	Datum     impl.Datum
	Err       error
	Subject   impl.Label
	Predicate impl.Label
	Object    impl.Label
	Language  impl.Language
	HasDatum  bool
}

Token represents a token read from a triplestore file.

It can represent one of three states:

1. an error token 1. a (subject, predicate, object) token 2. a (subject, predicate, datum) token

In the case of 1, Error != nil. In the case of 2, Error == nil && HasDatum = False In the case of 3, Error == nil && HasDatum = True