api

Documentation

Index

• Variables
• func ExtractQueryParameters(urlQuery url.Values, validDimensions []string) (map[string]string, error)
• func GetDimensionOffsetInHeaderRow(headerRow []string) (int, error)
• func GetListOfValidDimensionNames(dimensions []dataset.VersionDimension) []string
• type API
  • func Setup(ctx context.Context, r *mux.Router, cfg *config.Config, graphDB IGraph, ...) *API
  • func (*API) Close(ctx context.Context) error
• type CantabularClient
• type IAuthHandler
• type IDatasetClient
• type IGraph

Variables

var SortFilter = func(ctx context.Context, api *API, event *models.FilterSubmitted, dbFilter *observation.DimensionFilters) {
	nofDimensions := len(dbFilter.Dimensions)
	if nofDimensions <= 1 {
		return
	}
	// Create a slice of sorted dimension sizes
	type dim struct {
		index         int
		dimensionSize int
	}

	dimSizes := make([]dim, 0, nofDimensions)
	var dimSizesMutex sync.Mutex

	// get info from mongo
	var getErrorCount int32
	var concurrent = 10 // limit number of go routines so as to not put too much on heap
	var semaphoreChan = make(chan struct{}, concurrent)
	var wg sync.WaitGroup // number of working goroutines

	for i, dimension := range dbFilter.Dimensions {
		if atomic.LoadInt32(&getErrorCount) != 0 {
			break
		}
		semaphoreChan <- struct{}{}

		wg.Add(1)

		go func(i int, dimension *observation.Dimension) {
			defer func() {
				<-semaphoreChan
			}()

			defer wg.Done()

			options, err := api.datasetClient.GetOptions(ctx,
				"",
				api.cfg.ServiceAuthToken,
				"",
				event.DatasetID, event.Edition, event.Version, dimension.Name,
				&dataset.QueryParams{Offset: 0, Limit: 0})

			if err != nil {
				if atomic.AddInt32(&getErrorCount, 1) <= 2 {

					logData := log.Data{"dataset_id": event.DatasetID, "edition": event.Edition, "version": event.Version, "dimension name": dimension.Name}
					log.Info(ctx, "SortFilter: GetOptions failed for dataset and dimension", logData)
				}
			} else {
				d := dim{dimensionSize: options.TotalCount, index: i}
				dimSizesMutex.Lock()
				dimSizes = append(dimSizes, d)
				dimSizesMutex.Unlock()
			}
		}(i, dimension)
	}
	wg.Wait()

	if getErrorCount != 0 {
		logData := log.Data{"dataset_id": event.DatasetID, "edition": event.Edition, "version": event.Version}
		log.Info(ctx, fmt.Sprintf("SortFilter: GetOptions failed for dataset %d times, sorting by default of 'geography' first", getErrorCount), logData)

		dimSizes = dimSizes[:0]
		for i, dimension := range dbFilter.Dimensions {
			if strings.ToLower(dimension.Name) == "geography" {
				d := dim{dimensionSize: 999999, index: i}
				dimSizes = append(dimSizes, d)
			} else {

				d := dim{dimensionSize: nofDimensions - i, index: i}
				dimSizes = append(dimSizes, d)
			}
		}
	}

	sort.Slice(dimSizes, func(i, j int) bool {
		return dimSizes[i].dimensionSize < dimSizes[j].dimensionSize
	})

	sortedDimensions := make([]observation.Dimension, 0, nofDimensions)

	for i := nofDimensions - 1; i >= 0; i-- {
		sortedDimensions = append(sortedDimensions, *dbFilter.Dimensions[dimSizes[i].index])
	}

	for i, dimension := range sortedDimensions {
		*dbFilter.Dimensions[i] = dimension
	}
}

sortFilter by Dimension size, largest first, to make Neptune searches faster The sort is done here because the sizes are retrieved from Mongo and its best not to have the dp-graph library acquiring such coupling to its caller.

Functions

func ExtractQueryParameters

func ExtractQueryParameters(urlQuery url.Values, validDimensions []string) (map[string]string, error)

ExtractQueryParameters creates a map of query parameters (options) by dimension from the provided urlQuery if they exist in the validDimensions list

func GetDimensionOffsetInHeaderRow

func GetDimensionOffsetInHeaderRow(headerRow []string) (int, error)

GetDimensionOffsetInHeaderRow splits the first item of the provided headers by '_', and returns the second item as integer

func GetListOfValidDimensionNames

func GetListOfValidDimensionNames(dimensions []dataset.VersionDimension) []string

GetListOfValidDimensionNames iterates the provided dimensions and returns an array with their names

Types

type API

type API struct {
	Router *mux.Router
	// contains filtered or unexported fields
}

API provides a struct to wrap the api around

func Setup

func Setup(ctx context.Context, r *mux.Router, cfg *config.Config, graphDB IGraph, datasetClient IDatasetClient, cantabularClient CantabularClient, permissions IAuthHandler) *API

Setup creates the API struct and its endpoints with corresponding handlers

func (*API) Close

func (*API) Close(ctx context.Context) error

Close is called during graceful shutdown to give the API an opportunity to perform any required disposal task

type CantabularClient

type CantabularClient interface {
	Checker(context.Context, *healthcheck.CheckState) error
}

type IAuthHandler

type IAuthHandler interface {
	Require(required auth.Permissions, handler http.HandlerFunc) http.HandlerFunc
}

IAuthHandler represents the required methods from authorisation library by Observation API

type IDatasetClient

type IDatasetClient interface {
	GetVersion(ctx context.Context, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version string) (m dataset.Version, err error)
	Get(ctx context.Context, userAuthToken, serviceAuthToken, collectionID, datasetID string) (m dataset.DatasetDetails, err error)
	Checker(ctx context.Context, check *healthcheck.CheckState) error
	GetOptions(ctx context.Context, userAuthToken, serviceAuthToken, collectionID, id, edition, version, dimension string, q *dataset.QueryParams) (m dataset.Options, err error)
}

IDatasetClient represents the required methods from the Dataset Client required by Observation API

type IGraph

type IGraph interface {
	driver.Driver
	StreamCSVRows(ctx context.Context, instanceID, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error)
	ErrorChan() chan error
}

IGraph defines the required methods from GraphDB required by Observation API