marqo

package module
v1.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 4, 2024 License: MIT Imports: 9 Imported by: 1

README

Go

marqo-go (Unofficial Go Client)

Marqo is more than a vector database, it's an end-to-end vector search engine for both text and images. Vector generation, storage and retrieval are handled out of the box through a single API. No need to bring your own embeddings.

Features

  • Easy connection to Marqo with minimal setup
  • Index creation and management
  • Document upsertion and retrieval
  • Advanced search capabilities
  • Index statistics and health monitoring

Getting Started

Prerequisites

  • Docker (Ensure that Docker has at least 8GB memory and 50GB storage)

Installation

  1. Marqo requires docker. To install Docker go to the Docker Official website. Ensure that docker has at least 8GB memory and 50GB storage.

  2. Use docker to run Marqo for Mac users:

## will start marqo-os and marqo service mentioned in docker compose.
docker compose up -d
  1. Get the Marqo Go client:
go get github.com/ganeshdipdumbare/marqo-go@latest

Usage

Example
package main

import (
	"fmt"

	marqo "github.com/ganeshdipdumbare/marqo-go"
)

// Document represents a document

type Document struct {
	ID          string `json:"_id"`
	Title       string `json:"title"`
	Description string `json:"description"`
	Genre       string `json:"genre"`
}

func main() {

	marqoClient, err := marqo.NewClient("http://localhost:8882")
	if err != nil {
		panic(err)
	}
	model, err := marqoClient.GetModels()
	if err != nil {
		panic(err)
	}
	fmt.Printf("Model: %+v\n", model)

	indexes, err := marqoClient.ListIndexes()
	if err != nil {
		panic(err)
	}

	fmt.Printf("Indexes: %+v\n", indexes)

	// create index
	resp, err := marqoClient.CreateIndex(&marqo.CreateIndexRequest{
		IndexName: "test1",
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("CreateIndexResponse: %+v\n", resp)

	// create document
	documents := []Document{
		{
			ID:          "1",
			Title:       "The Great Gatsby",
			Description: "The Great Gatsby is a 1925 novel by American writer F. Scott Fitzgerald.",
			Genre:       "Novel",
		},
		{
			ID:          "2",
			Title:       "The Catcher in the Rye",
			Description: "The Catcher in the Rye is a novel by J. D. Salinger, partially published in serial form in 1945–1946 and as a novel in 1951.",
			Genre:       "Novel",
		},
	}
	docInterface := make([]interface{}, len(documents))
	for i, v := range documents {
		docInterface[i] = v
	}

	upsertResp, err := marqoClient.UpsertDocuments(&marqo.UpsertDocumentsRequest{
		IndexName: "test1",
		Documents: docInterface,
		TensorFields: []string{
			"title",
			"description",
		},
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("UpsertDocumentsResponse: %+v\n", upsertResp)

	// delete documents
	deleteResp, err := marqoClient.DeleteDocuments(&marqo.DeleteDocumentsRequest{
		IndexName:   "test1",
		DocumentIDs: []string{"1", "2"},
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("DeleteDocumentsResponse: %+v\n", deleteResp)

	// Refresh index
	refreshResp, err := marqoClient.RefreshIndex(&marqo.RefreshIndexRequest{
		IndexName: "test1",
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("RefreshIndexResponse: %+v\n", refreshResp)

	// Get index stats
	statsResp, err := marqoClient.GetIndexStats(&marqo.GetIndexStatsRequest{
		IndexName: "test1",
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("GetIndexStatsResponse: %+v\n", statsResp)

	// Get index settings
	settingsResp, err := marqoClient.GetIndexSettings(&marqo.GetIndexSettingsRequest{
		IndexName: "test1",
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("GetIndexSettingsResponse: %+v\n", *settingsResp.IndexDefaults.Model)

	// Get index health
	healthResp, err := marqoClient.GetIndexHealth(&marqo.GetIndexHealthRequest{
		IndexName: "test1",
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("GetIndexHealthResponse: %+v\n", healthResp)

	// Search
	searchQuery := "The Great Gatsby"
	searchResp, err := marqoClient.Search(&marqo.SearchRequest{
		IndexName: "test1",
		Q:         &searchQuery,
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("SearchResponse: %+v\n", searchResp)

	// Bulk search
	//searchQuery := "The Great Gatsby"
	indexName := "test1"
	bulkSearchResp, err := marqoClient.BulkSearch(&marqo.BulkSearchRequest{
		Queries: []marqo.SearchRequest{
			{
				Index: &indexName,
				Q:     &searchQuery,
			},
		},
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("BulkSearchResponse: %+v\n", bulkSearchResp)

}

Improvements

  • Add unit tests for all the APIs.
  • Add end-to-end examples for search.

Contributing

Contributions are welcome! Please open an issue or submit a pull request for any improvements or bug fixes.

License

This project is licensed under the MIT License - see the LICENSE file for details.

Acknowledgements

  • Marqo for providing the vector search engine.
  • Go for the programming language.

Documentation

Overview

Package marqo provides a client for interacting with the Marqo server.

The simplest way to use Marqo is to create a new client and use its methods to interact with the server: 

package main

import (
  "fmt"
  "log"

  marqo "github.com/ganeshdipdumbare/marqo-go"
)

func main() {
  client, err := marqo.NewClient("http://localhost:8882")
  if err != nil {
    log.Fatalf("Failed to create client: %v", err)
  }

  modelsResponse, err := client.GetModels()
  if err != nil {
    log.Fatalf("Failed to get models: %v", err)
  }
  fmt.Printf("Loaded models: %+v\n", modelsResponse.Models)

  createIndexReq := &marqo.CreateIndexRequest{
    IndexName: "example_index",
  }
  createIndexResp, err := client.CreateIndex(createIndexReq)
  if err != nil {
    log.Fatalf("Failed to create index: %v", err)
  }
  fmt.Printf("CreateIndexResponse: %+v\n", createIndexResp)

  upsertDocumentsReq := &marqo.UpsertDocumentsRequest{
    IndexName: "example_index",
    Documents: []interface{}{
      map[string]interface{}{"_id": "1", "title": "Document 1"},
      map[string]interface{}{"_id": "2", "title": "Document 2"},
    },
  }
  upsertDocumentsResp, err := client.UpsertDocuments(upsertDocumentsReq)
  if err != nil {
    log.Fatalf("Failed to upsert documents: %v", err)
  }
  fmt.Printf("UpsertDocumentsResponse: %+v\n", upsertDocumentsResp)

  searchReq := &marqo.SearchRequest{
    IndexName: "example_index",
    Q:         "Document",
  }
  searchResp, err := client.Search(searchReq)
  if err != nil {
    log.Fatalf("Failed to perform search: %v", err)
  }
  fmt.Printf("SearchResponse: %+v\n", searchResp)
}

For a full guide visit https://github.com/ganeshdipdumbare/marqo-go

License

This project is licensed under the MIT License - see the LICENSE file for details.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func WithLogger 

func WithLogger(logger *slog.Logger) func(*Client)

WithLogger sets the logger for the client

func WithMarqoCloudAuth added in v1.0.1 

func WithMarqoCloudAuth(apiKey string) func(*Client)

WithMarqoCloudAuth sets the API key for authentication if you are using MarqoCloud

Types

type ANNParameters 

type ANNParameters struct {
	// The function used to measure the distance between two points in ANN (l1, l2, linf, or cosinesimil. default: cosinesimil)
	SpaceType *string `json:"space_type,omitempty"`
	// The hyperparameters for the ANN method (which is always hnsw for Marqo).
	Parameters *HSNWMethodParameters `json:"parameters,omitempty"`
}

ANNParameters are the ANN parameters for the index

type BulkSearchRequest 

type BulkSearchRequest struct {
	// Body params
	// Queries is the list of search requests
	Queries []SearchRequest `validate:"required" json:"queries"`

	// Query params
	// Device is the device to run the search on
	Device *string `json:"device,omitempty"`
	// Telemetry is whether to send telemetry
	Telemetry *bool `json:"telemetry,omitempty"`
}

BulkSearchRequest is the request to bulk search

type BulkSearchResponse 

type BulkSearchResponse struct {
	// Result is the list of search responses
	Result           []SearchResponse `json:"result"`
	ProcessingTimeMS float64          `json:"processingTimeMs"`
}

BulkSearchResponse is the response from the server

type CUDADevice 

type CUDADevice struct {
	DeviceName  string `json:"device_name"`
	MemoryUsed  string `json:"memory_used"`
	TotalMemory string `json:"total_memory"`
}

type Client 

type Client struct {
	// contains filtered or unexported fields
}

Client is the client for the Marqo server

func NewClient 

func NewClient(url string, opt ...Options) (*Client, error)

NewClient creates a new client for the Marqo server.

This method initializes a new client with the specified endpoint URL and optional parameters.

Parameters: 

url (string): The endpoint URL of your Marqo instance.
opt (...Options): Optional parameters for the client.

Returns: 

*Client: A new Marqo client instance.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the url parameter. 2. Initializes a new Client instance. 3. Applies the optional parameters to the client. 4. Sets the reqClient if not already set. 5. Returns the new client instance if the operation is successful, otherwise returns an error.

Example usage: 

client, err := marqo.NewClient("http://localhost:8882")
if err != nil {
    log.Fatalf("Failed to create client: %v", err)
}
fmt.Printf("Client: %+v\n", client)

func (*Client) BulkSearch 

func (c *Client) BulkSearch(bulkSearchReq *BulkSearchRequest) (*BulkSearchResponse, error)

BulkSearch performs a bulk search on the server.

This method sends a POST request to the server to perform a bulk search with the specified queries.

Parameters: 

bulkSearchReq (*BulkSearchRequest): The request containing the search queries.

Returns: 

*BulkSearchResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the bulkSearchReq parameter. 2. Sends a POST request to the server with the search queries in the request body. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

bulkSearchReq := &BulkSearchRequest{
    Queries: []SearchRequest{...},
}
resp, err := client.BulkSearch(bulkSearchReq)
if err != nil {
    log.Fatalf("Failed to perform bulk search: %v", err)
}
fmt.Printf("BulkSearchResponse: %+v\n", resp)

func (*Client) CreateIndex 

func (c *Client) CreateIndex(createIndexReq *CreateIndexRequest) (*CreateIndexResponse, error)

CreateIndex creates an index

This method sends a POST request to the server to create the specified index.

Parameters: 

createIndexReq (*CreateIndexRequest): The request containing the index details.

Returns: 

*CreateIndexResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Sets default values for the createIndexReq parameter. 2. Validates the createIndexReq parameter. 3. Sends a POST request to the server with the index details in the request body. 4. Checks the response status code and logs any errors. 5. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

createIndexReq := &CreateIndexRequest{
    IndexName: "example_index",
}
resp, err := client.CreateIndex(createIndexReq)
if err != nil {
    log.Fatalf("Failed to create index: %v", err)
}
fmt.Printf("CreateIndexResponse: %+v\n", resp)

func (*Client) DeleteDocuments 

func (c *Client) DeleteDocuments(deleteDocumentsReq *DeleteDocumentsRequest) (*DeleteDocumentsResponse, error)

DeleteDocuments deletes documents from the server.

This method sends a POST request to the server to delete the specified documents.

Parameters: 

deleteDocumentsReq (*DeleteDocumentsRequest): The request containing the document IDs to be deleted.

Returns: 

*DeleteDocumentsResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the deleteDocumentsReq parameter. 2. Sends a POST request to the server with the document IDs in the request body. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

deleteDocumentsReq := &DeleteDocumentsRequest{
    IndexName:   "example_index",
    DocumentIDs: []string{"doc1", "doc2"},
}
resp, err := client.DeleteDocuments(deleteDocumentsReq)
if err != nil {
    log.Fatalf("Failed to delete documents: %v", err)
}
fmt.Printf("DeleteDocumentsResponse: %+v\n", resp)

func (*Client) DeleteIndex 

func (c *Client) DeleteIndex(deleteIndexRequest *DeleteIndexRequest) (*DeleteIndexResponse, error)

DeleteIndex deletes an index

This method sends a DELETE request to the server to delete the specified index.

Parameters: 

deleteIndexRequest (*DeleteIndexRequest): The request containing the index name.

Returns: 

*DeleteIndexResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the deleteIndexRequest parameter. 2. Sends a DELETE request to the server with the index name as a query parameter. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

deleteIndexRequest := &DeleteIndexRequest{
    IndexName: "example_index",
}
resp, err := client.DeleteIndex(deleteIndexRequest)
if err != nil {
    log.Fatalf("Failed to delete index: %v", err)
}
fmt.Printf("DeleteIndexResponse: %+v\n", resp)

func (*Client) EjectModel 

func (c *Client) EjectModel(ejectModelReq *EjectModelRequest) error

EjectModel ejects the model from the server cache.

This method sends a DELETE request to the server to remove the specified model from the server's cache. The model to be ejected is specified in the EjectModelRequest parameter.

Parameters: 

ejectModelReq (*EjectModelRequest): The request containing the model name and device.

Returns: 

error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the ejectModelReq parameter. 2. Sends a DELETE request to the server with the model name and device as query parameters. 3. Checks the response status code and logs any errors. 4. Returns an error if the operation fails, otherwise returns nil.

Example usage: 

ejectModelReq := &EjectModelRequest{
    ModelName:   "example_model",
    ModelDevice: "cpu",
}
err := client.EjectModel(ejectModelReq)
if err != nil {
    log.Fatalf("Failed to eject model: %v", err)
}

func (*Client) GetCPUInfo 

func (c *Client) GetCPUInfo() (*GetCPUInfoResponse, error)

GetCPUInfo returns the CPU info from the server.

This method sends a GET request to the server to retrieve the CPU information.

Returns: 

*GetCPUInfoResponse: The response containing the CPU information.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Sends a GET request to the server to retrieve the CPU information. 2. Checks the response status code and logs any errors. 3. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

resp, err := client.GetCPUInfo()
if err != nil {
    log.Fatalf("Failed to get CPU info: %v", err)
}
fmt.Printf("GetCPUInfoResponse: %+v\n", resp)

func (*Client) GetCUDAInfo 

func (c *Client) GetCUDAInfo() (*GetCUDAInfoResponse, error)

GetCUDAInfo returns the CUDA info from the server. Returns the response from the server and an error if the operation fails.

func (*Client) GetDocument 

func (c *Client) GetDocument(getDocumentReq *GetDocumentRequest) (*GetDocumentResponse, error)

GetDocument gets a document from the server.

This method sends a GET request to the server to retrieve the specified document.

Parameters: 

getDocumentReq (*GetDocumentRequest): The request containing the document ID to be retrieved.

Returns: 

*GetDocumentResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the getDocumentReq parameter. 2. Sends a GET request to the server with the document ID as a query parameter. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

getDocumentReq := &GetDocumentRequest{
    IndexName:  "example_index",
    DocumentID: "doc1",
}
resp, err := client.GetDocument(getDocumentReq)
if err != nil {
    log.Fatalf("Failed to get document: %v", err)
}
fmt.Printf("GetDocumentResponse: %+v\n", resp)

func (*Client) GetDocuments 

func (c *Client) GetDocuments(getDocumentsReq *GetDocumentsRequest) (*GetDocumentsResponse, error)

GetDocuments gets documents from the server.

This method sends a GET request to the server to retrieve the specified documents.

Parameters: 

getDocumentsReq (*GetDocumentsRequest): The request containing the document IDs to be retrieved.

Returns: 

*GetDocumentsResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the getDocumentsReq parameter. 2. Sends a GET request to the server with the document IDs in the request body. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

getDocumentsReq := &GetDocumentsRequest{
    IndexName:   "example_index",
    DocumentIDs: []string{"doc1", "doc2"},
}
resp, err := client.GetDocuments(getDocumentsReq)
if err != nil {
    log.Fatalf("Failed to get documents: %v", err)
}
fmt.Printf("GetDocumentsResponse: %+v\n", resp)

func (*Client) GetIndexHealth 

func (c *Client) GetIndexHealth(getIndexHealthReq *GetIndexHealthRequest) (*GetIndexHealthResponse, error)

GetIndexHealth gets the index health from the server.

This method sends a GET request to the server to retrieve the health status of the specified index.

Parameters: 

getIndexHealthReq (*GetIndexHealthRequest): The request containing the index name.

Returns: 

*GetIndexHealthResponse: The response containing the index health status.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the getIndexHealthReq parameter. 2. Sends a GET request to the server with the index name as a query parameter. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

getIndexHealthReq := &GetIndexHealthRequest{
    IndexName: "example_index",
}
resp, err := client.GetIndexHealth(getIndexHealthReq)
if err != nil {
    log.Fatalf("Failed to get index health: %v", err)
}
fmt.Printf("GetIndexHealthResponse: %+v\n", resp)

func (*Client) GetIndexSettings 

func (c *Client) GetIndexSettings(getIndexSettingsReq *GetIndexSettingsRequest) (*GetIndexSettingsResponse, error)

GetIndexSettings gets the index settings from the server.

This method sends a GET request to the server to retrieve the settings of the specified index.

Parameters: 

getIndexSettingsReq (*GetIndexSettingsRequest): The request containing the index name.

Returns: 

*GetIndexSettingsResponse: The response containing the index settings.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the getIndexSettingsReq parameter. 2. Sends a GET request to the server with the index name as a query parameter. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

getIndexSettingsReq := &GetIndexSettingsRequest{
    IndexName: "example_index",
}
resp, err := client.GetIndexSettings(getIndexSettingsReq)
if err != nil {
    log.Fatalf("Failed to get index settings: %v", err)
}
fmt.Printf("GetIndexSettingsResponse: %+v\n", resp)

func (*Client) GetIndexStats 

func (c *Client) GetIndexStats(getIndexStatsReq *GetIndexStatsRequest) (*GetIndexStatsResponse, error)

GetIndexStats gets the index stats from the server.

This method sends a GET request to the server to retrieve the statistics of the specified index.

Parameters: 

getIndexStatsReq (*GetIndexStatsRequest): The request containing the index name.

Returns: 

*GetIndexStatsResponse: The response containing the index statistics.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the getIndexStatsReq parameter. 2. Sends a GET request to the server with the index name as a query parameter. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

getIndexStatsReq := &GetIndexStatsRequest{
    IndexName: "example_index",
}
resp, err := client.GetIndexStats(getIndexStatsReq)
if err != nil {
    log.Fatalf("Failed to get index stats: %v", err)
}
fmt.Printf("GetIndexStatsResponse: %+v\n", resp)

func (*Client) GetModels 

func (c *Client) GetModels() (*GetModelsResponse, error)

GetModels returns the loaded models from the server.

This method sends a GET request to the server to retrieve the list of loaded models.

Returns: 

*GetModelsResponse: The response containing the list of models.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Sends a GET request to the server to retrieve the models. 2. Checks the response status code and logs any errors. 3. Returns the list of models if the operation is successful, otherwise returns an error.

Example usage: 

modelsResponse, err := client.GetModels()
if err != nil {
    log.Fatalf("Failed to get models: %v", err)
}
fmt.Printf("Loaded models: %+v\n", modelsResponse.Models)

func (*Client) ListIndexes 

func (c *Client) ListIndexes() (*ListIndexesResponse, error)

ListIndexes lists the indexes

func (*Client) RefreshIndex 

func (c *Client) RefreshIndex(refreshIndexReq *RefreshIndexRequest) (*RefreshIndexResponse, error)

RefreshIndex refreshes the index on the server

This method sends a POST request to the server to refresh the specified index.

Parameters: 

refreshIndexReq (*RefreshIndexRequest): The request containing the index name.

Returns: 

*RefreshIndexResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the refreshIndexReq parameter. 2. Sends a POST request to the server with the index name as a query parameter. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

refreshIndexReq := &RefreshIndexRequest{
    IndexName: "example_index",
}
resp, err := client.RefreshIndex(refreshIndexReq)
if err != nil {
    log.Fatalf("Failed to refresh index: %v", err)
}
fmt.Printf("RefreshIndexResponse: %+v\n", resp)

func (*Client) Search 

func (c *Client) Search(searchReq *SearchRequest) (*SearchResponse, error)

Search performs a search on the server.

This method sends a GET request to the server to perform a search with the specified query.

Parameters: 

searchReq (*SearchRequest): The request containing the search query.

Returns: 

*SearchResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the searchReq parameter. 2. Sends a GET request to the server with the search query as query parameters. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

searchReq := &SearchRequest{
    IndexName: "example_index",
    Q:         "example_query",
}
resp, err := client.Search(searchReq)
if err != nil {
    log.Fatalf("Failed to perform search: %v", err)
}
fmt.Printf("SearchResponse: %+v\n", resp)

func (*Client) UpsertDocuments 

func (c *Client) UpsertDocuments(upsertDocumentsReq *UpsertDocumentsRequest) (*UpsertDocumentsResponse, error)

UpsertDocuments upserts documents to the server.

This method sends a POST request to the server to upsert the specified documents.

Parameters: 

upsertDocumentsReq (*UpsertDocumentsRequest): The request containing the documents to be upserted.

Returns: 

*UpsertDocumentsResponse: The response from the server.
error: An error if the operation fails, otherwise nil.

The function performs the following steps: 1. Validates the upsertDocumentsReq parameter. 2. Sends a POST request to the server with the documents in the request body. 3. Checks the response status code and logs any errors. 4. Returns the response from the server if the operation is successful, otherwise returns an error.

Example usage: 

upsertDocumentsReq := &UpsertDocumentsRequest{
    IndexName: "example_index",
    Documents: []interface{}{...},
}
resp, err := client.UpsertDocuments(upsertDocumentsReq)
if err != nil {
    log.Fatalf("Failed to upsert documents: %v", err)
}
fmt.Printf("UpsertDocumentsResponse: %+v\n", resp)

type Context 

type Context struct {
	Tensor []Tensor `json:"tensor"`
}

Context is the tensor context for the search

type CreateIndexRequest 

type CreateIndexRequest struct {
	IndexName     string         `json:"-" validate:"required"`
	IndexDefaults *IndexDefaults `json:"index_defaults,omitempty"`
	// Number of shards for the index (default: 3)
	NumberOfShards *int `json:"number_of_shards,omitempty"`
	// Number of replicas for the index (default: 0)
	NumberOfReplicas *int `json:"number_of_replicas,omitempty"`
}

CreateIndexRequest is the request to create an index

type CreateIndexResponse 

type CreateIndexResponse struct {
	Acknowledged       bool   `json:"acknowledged"`
	ShardsAcknowledged bool   `json:"shards_acknowledged"`
	Index              string `json:"index"`
}

CreateIndexResponse is the response for creating an index

type DeleteDocumentsRequest 

type DeleteDocumentsRequest struct {
	IndexName   string   `json:"-" validate:"required"`
	DocumentIDs []string `json:"document_ids" validate:"required"`
}

DeleteDocumentsRequest is the request to delete documents

type DeleteDocumentsResponse 

type DeleteDocumentsResponse struct {
	IndexName string `json:"index_name"`
	Status    string `json:"status"`
	Type      string `json:"type"`
	Details   struct {
		ReceivedDocumentIds int `json:"receivedDocumentIds"`
		DeletedDocuments    int `json:"deletedDocuments"`
	} `json:"details"`
	Items      []DeletedItem `json:"items"`
	Duration   string        `json:"duration"`
	StartedAt  string        `json:"startedAt"`
	FinishedAt string        `json:"finishedAt"`
}

DeleteDocumentsResponse is the response from the server

type DeleteIndexRequest 

type DeleteIndexRequest struct {
	IndexName string `validate:"required" json:"-"`
}

DeleteIndexRequest is the request to delete an index

type DeleteIndexResponse 

type DeleteIndexResponse struct {
	Acknowledged bool `json:"acknowledged"`
}

DeleteIndexResponse is the response for deleting an index

type DeletedItem 

type DeletedItem struct {
	ID     string `json:"_id"`
	Shards struct {
		Total      int `json:"total"`
		Successful int `json:"successful"`
		Failed     int `json:"failed"`
	} `json:"_shards"`
	Status int    `json:"status"`
	Result string `json:"result"`
}

DeletedItem is the item which was deleted

type EjectModelRequest 

type EjectModelRequest struct {
	ModelName   string `validate:"required" json:"model_name"`
	ModelDevice string `validate:"oneof=cpu cuda" json:"model_device"`
}

EjectModelRequest is the request to eject a model

type GetCPUInfoResponse 

type GetCPUInfoResponse struct {
	CPUUsagePercent   string `json:"cpu_usage_percent"`
	MemoryUsedPercent string `json:"memory_used_percent"`
	MemoryUsedGB      string `json:"memory_used_gb"`
}

GetCPUInfoResponse is the response from the server containing CPU information.

type GetCUDAInfoResponse 

type GetCUDAInfoResponse struct {
	CUDADevices []CUDADevice `json:"cuda_devices"`
}

GetCUDAInfoResponse is the response from the server containing CUDA information.

type GetDocumentRequest 

type GetDocumentRequest struct {
	IndexName    string `json:"-" validate:"required"`
	DocumentID   string `json:"document_id" validate:"required"`
	ExposeFacets bool   `json:"expose_facets,omitempty"`
}

GetDocumentRequest is the request to get a document

type GetDocumentResponse 

type GetDocumentResponse map[string]interface{}

GetDocumentResponse is the response from the server

type GetDocumentsRequest 

type GetDocumentsRequest struct {
	IndexName    string   `json:"-" validate:"required"`
	DocumentIDs  []string `json:"document_ids" validate:"required"`
	ExposeFacets bool     `json:"expose_facets,omitempty"`
}

GetDocumentsRequest is the request to get documents

type GetDocumentsResponse 

type GetDocumentsResponse struct {
	Results []GetDocumentResponse `json:"results"`
}

GetDocumentsResponse is the response from the server

type GetIndexHealthRequest 

type GetIndexHealthRequest struct {
	IndexName string `validate:"required" json:"-"`
}

GetIndexHealthRequest is the request to get the index health

type GetIndexHealthResponse 

type GetIndexHealthResponse struct {
	Status  string `json:"status"`
	Backend struct {
		Status             string `json:"status"`
		StorageIsAvailable bool   `json:"storage_is_available"`
	} `json:"backend"`
}

GetIndexHealthResponse is the response from the server

type GetIndexSettingsRequest 

type GetIndexSettingsRequest struct {
	IndexName string `validate:"required" json:"-"`
}

GetIndexSettingsRequest is the request to get the index settings

type GetIndexSettingsResponse 

type GetIndexSettingsResponse struct {
	IndexDefaults *IndexDefaults `json:"index_defaults"`
}

GetIndexSettingsResponse is the response from the server

type GetIndexStatsRequest 

type GetIndexStatsRequest struct {
	IndexName string `validate:"required" json:"-"`
}

GetIndexStatsRequest is the request to get the index stats

type GetIndexStatsResponse 

type GetIndexStatsResponse struct {
	NumberOfDocuments int `json:"numberOfDocuments"`
	NumberOfVectors   int `json:"numberOfVectors"`
}

GetIndexStatsResponse is the response from the server

type GetModelsResponse 

type GetModelsResponse struct {
	Models []Model `json:"models"`
}

GetModelsResponse is the response from the server

type HSNWMethodParameters 

type HSNWMethodParameters struct {
	// The size of the dynamic list used during k-NN graph creation.
	// Higher values lead to a more accurate graph but slower indexing
	// speed. It is recommended to keep this between 2 and 800 (maximum is 4096)
	// (default: 128)
	EFConstruction *int `json:"ef_construction,omitempty"`
	// The number of bidirectional links that the plugin creates for each
	// new element. Increasing and decreasing this value can have a
	// large impact on memory consumption. Keep this value between 2 and 100.
	// (default: 16)
	M *int `json:"m,omitempty"`
}

HSNWMethodParameters are the HSNW method parameters for the index

type HybridParameters 

type HybridParameters struct {
	RetrievalMethod             string          `json:"retrievalMethod"`
	RankingMethod               string          `json:"rankingMethod"`
	ScoreModifiersTensor        *ScoreModifiers `json:"scoreModifiers,omitempty"`
	ScoreModifiersLexical       *ScoreModifiers `json:"scoreModifiersLexical,omitempty"`
	SearchableAttributesLexical []string        `json:"searchableAttributesLexical,omitempty"`
	SearchableAttributesTensor  []string        `json:"searchableAttributesTensor,omitempty"`
	Alpha                       *float64        `json:"alpha,omitempty"`
	RrfK                        *int            `json:"rrfK,omitempty"`
}

Hybrid search parameters Example usage: 

hybridParameters := HybridParameters{
    RetrievalMethod: "disjunction",
    RankingMethod:   "rrf",
}

see https://github.com/marqo-ai/marqo/blob/mainline/RELEASE.md#release-2100 and https://docs.marqo.ai/2.10/API-Reference/Search/search/#hybrid-parameters

type ImagePreprocessing 

type ImagePreprocessing struct {
	// The method by which images are chunked (options: "simple" or "frcnn")
	PatchMethod *string `json:"patch_method,omitempty"`
}

ImagePreprocessing is the image preprocessing for the index

type IndexDefaults 

type IndexDefaults struct {
	// Fetch images from points and URLs (default: false)
	TreatURLsAndPointersAsImages *bool `json:"treat_urls_and_pointers_as_images,omitempty"`
	// Model to vectorize doc content (default: hf/all_datasets_v4_MiniLM-L6)
	Model           *string          `json:"model,omitempty"`
	ModelProperties *ModelProperties `json:"model_properties,omitempty"`
	// TODO: add search model support in the future
	// SearchModel to vectorize queries (default: hf/all_datasets_v4_MiniLM-L6)
	// SearchModel           *string          `json:"search_model,omitempty"`
	// SearchModelProperties *ModelProperties `json:"search_model_properties,omitempty"`
	// Normalize embeddings to have unit length (default: true)
	NormalizeEmbeddings *bool               `json:"normalize_embeddings,omitempty"`
	TextPreprocessing   *TextPreprocessing  `json:"text_preprocessing,omitempty"`
	ImagePreprocessing  *ImagePreprocessing `json:"image_preprocessing,omitempty"`
	ANNParameters       *ANNParameters      `json:"ann_parameters,omitempty"`
}

IndexDefaults is the defaults for the index

type Item 

type Item struct {
	ID     string `json:"_id"`
	Result string `json:"result"`
	Status int    `json:"status"`
}

Item is the item from the server

type ListIndexesResponse 

type ListIndexesResponse struct {
	Results []Result `json:"results"`
}

ListIndexesResponse is the response for listing indexes

type Model 

type Model struct {
	ModelName   string `json:"model_name"`
	ModelDevice string `json:"model_device"`
}

Model is the model from the server

type ModelProperties 

type ModelProperties struct {
	Name       *string `json:"name,omitempty"`
	Dimensions *int    `json:"dimensions,omitempty"`
	URL        *string `json:"url,omitempty"`
	Type       *string `json:"type,omitempty"`
}

ModelProperties are the properties for the model

type Options 

type Options func(*Client)

Options for the client

type RefreshIndexRequest 

type RefreshIndexRequest struct {
	IndexName string `validate:"required" json:"-"`
}

RefreshIndexRequest is the request to refresh the index

type RefreshIndexResponse 

type RefreshIndexResponse struct {
	Shards struct {
		Total      int `json:"total"`
		Successful int `json:"successful"`
		Failed     int `json:"failed"`
	} `json:"_shards"`
}

RefreshIndexResponse is the response from the server

type Result 

type Result struct {
	IndexName string `json:"index_name"`
}

Result is the result for listing one index

type ScoreModifier 

type ScoreModifier struct {
	FieldName string `json:"field_name"`
	// Weight is the weighting that should be applied to the numeric value
	// default: 1 for multiply_score_by, 0 for add_to_score
	Weight float64 `json:"weight"`
}

ScoreModifier is the score modifier for the search

type ScoreModifiers 

type ScoreModifiers map[string][]ScoreModifier

ScoreModifiers is the score modifiers for the search options for keys: "multiply_score_by", "add_to_score"

type SearchRequest 

type SearchRequest struct {
	// IndexName is the name of the index
	// Note: only used in search request
	IndexName string `json:"-" validate:"required"`

	// Body params
	// This is used inside bulk search request body params
	// Note: Only use in case of bulk search
	Index *string `json:"index,omitempty"`
	// Q is the query string
	Q *string `json:"q,omitempty"`
	// Limit is the number of results to return (default: 20)
	Limit *int `json:"limit,omitempty"`
	// Offset is the number of results to skip (default: 0)
	Offset *int    `json:"offset,omitempty"`
	Filter *string `json:"filter,omitempty"`
	// SearchableAttributes is the list of
	// attributes to search in (default: ["*"]) --> all attributes
	SearchableAttributes []string `json:"searchableAttributes,omitempty"`
	// ShowHighlights return highlights for the document match.
	// Only applicable for TENSOR search. With LEXICAL search,
	// highlights will always be []. (default: true)
	ShowHighlights *bool `json:"showHighlights,omitempty"`
	// SearchMethod is the search method to use,
	// can be LEXICAL or TENSOR or HYBRID (default: TENSOR)
	SearchMethod *string `json:"searchMethod,omitempty"`
	// AttributesToRetrieve is the list of attributes to retrieve
	// (default: ["*"]) --> all attributes
	AttributesToRetrieve []string `json:"attributesToRetrieve,omitempty"`
	// ReRanker method to use for re-ranking results
	// (default: null)
	// options: null, "owl/ViT-B/32", "owl/ViT-B/16", "owl/ViT-L/14"
	ReRanker *string `json:"reRanker,omitempty"`
	// Boost is the map of attribute
	// (string): 2-Array [weight (float), bias (float)]
	// e.g. {"price": [0.5, 0.5]}
	Boost map[string][2]float64 `json:"boost,omitempty"`
	// ImageDownloadHeaders is for the image download. Can be used to authenticate the images for download.
	ImageDownloadHeaders map[string]interface{} `json:"image_download_headers,omitempty"`
	// Context allows you to use your own vectors as context for your queries.
	// Your vectors will be incorporated into the query using a weighted sum
	// approach, allowing you to reduce the number of inference requests for
	// duplicated content. The dimension of the provided vectors should be
	// consistent with the index dimension.
	Context *Context `json:"context,omitempty"`
	// ScoreModifiers is an object with two optional keys: multiply_score_by
	// and add_to_score. The value of each of these keys is an array of objects
	// that each contain the name of a numeric field in the document as the
	// field_name key and the weighting that should be applied to the numeric
	// value, as the weight key.
	ScoreModifiers *ScoreModifiers `json:"scoreModifiers,omitempty"`
	// ModelAuth is an authorisation details used by Marqo to download
	// non-publicly available models.
	ModelAuth map[string]interface{} `json:"modelAuth,omitempty"`
	// TextChunkPrefix is a string to be added to the start of all text queries
	// before vectorisation. Text itself will not be returned or used for
	// lexical search. Only affects vectors generated.
	TextQueryPrefix *string `json:"textQueryPrefix,omitempty"`

	// Query params
	// Device used to search. If device is not specified and CUDA devices are
	// available to Marqo (see here for more info), Marqo will speed up search
	// by using an available CUDA device. Otherwise, the CPU will be used.
	Device *string `json:"device,omitempty"`
	// Telemetry if true, the telemtry object is returned in the search
	// response body. This includes information like latency metrics.
	Telemetry *bool `json:"telemetry,omitempty"`

	// HybridParameters is the hybrid search parameters
	HybridParameters *HybridParameters `json:"hybridParameters,omitempty"`
}

SearchRequest is the request to search

type SearchResponse 

type SearchResponse struct {
	// Hits is the list of hits
	Hits []map[string]interface{} `json:"hits"`
	// Limit is the number of results to return (default: 20)
	Limit int `json:"limit"`
	// Offset is the number of results to skip (default: 0)
	Offset int `json:"offset"`
	// ProcessingTimeMS is the processing time in milliseconds
	ProcessingTimeMS float64 `json:"processingTimeMs"`
	// Query is the query string
	Query string `json:"query"`
}

SearchResponse is the response from the server

type Tensor 

type Tensor struct {
	Vector []float64 `json:"vector"`
	Weight float64   `json:"weight"`
}

Tensor is the tensor for the search

type TextPreprocessing 

type TextPreprocessing struct {
	// SplitLength is length of chunks after splitting
	// by split method (default: 2)
	SplitLength *int `json:"split_length,omitempty"`
	// SplitOverlap is overlap between adjacent chunks (default: 0)
	SplitOverlap *int `json:"split_overlap,omitempty"`
	// SplitMethod method to split text into chunks (default: "sentence", options: "sentence", "word", "character" or "passage")
	SplitMethod *string `json:"split_method,omitempty"`
}

TextPreprocessing is the text preprocessing for the index

type UpsertDocumentsRequest 

type UpsertDocumentsRequest struct {
	IndexName string `json:"-" validate:"required"`
	// Query params
	Refresh   *bool   `json:"-"`
	Device    *string `json:"-"`
	Telemetry *bool   `json:"-"`
	// Body params
	Documents            []interface{}          `json:"documents" validate:"required"`
	TensorFields         []string               `json:"tensorFields,omitempty"`
	UseExistingTensors   *bool                  `json:"useExistingTensors,omitempty"`
	ImageDownloadHeaders map[string]string      `json:"imageDownloadHeaders,omitempty"`
	Mappings             map[string]interface{} `json:"mappings,omitempty"`
	ModelAuth            map[string]interface{} `json:"modelAuth,omitempty"`
	TextChunkPrefix      *string                `json:"textChunkPrefix,omitempty"`
	ClientBatchSize      *int                   `json:"client_batch_size,omitempty"`
}

UpsertDocumentsRequest is the request to upsert documents

type UpsertDocumentsResponse 

type UpsertDocumentsResponse struct {
	Errors           bool    `json:"errors"`
	Items            []Item  `json:"items"`
	ProcessingTimeMS float64 `json:"processingTimeMs"`
	IndexName        string  `json:"index_name"`
}

UpsertDocumentsResponse is the response from the server

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.