Skip to content

Commit

Permalink
Update ConfigureIndex to merge new and previous IndexTags, update…
Browse files Browse the repository at this point in the history
… `README` examples (#89)

## Problem
@jhamon called out that since `ConfigureIndex` is a PATCH operation, we
should probably be explicitly merging existing tags with incoming tags
when configuring an index as a partial update. See here for a python
example:
https://github.com/pinecone-io/pinecone-python-client/pull/426/files#diff-9e7f01373e4589c7e4ded21935023cdf187ab6ce8b2b44a0d5470a27eaf464eeR303-R314

## Solution
- Update `ConfigureIndex` to call `DescribeIndex` and merge tags
properly on updates.
- Update `README` to add examples of using `IndexTags` when creating and
updating indexes.

## Type of Change
- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [ ] Infrastructure change (CI configs, etc)
- [X] Non-code change (docs, etc)
- [ ] None of the above: (explain here)

## Test Plan
`just test`
Make sure CI is green.
  • Loading branch information
austin-denoble authored Dec 20, 2024
1 parent d406377 commit 5a7fff1
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 14 deletions.
50 changes: 39 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ func main() {
Metric: pinecone.Cosine,
Cloud: pinecone.Aws,
Region: "us-east-1",
Tags: &pinecone.IndexTags{"environment": "development"},
})

if err != nil {
Expand Down Expand Up @@ -209,6 +210,7 @@ func main() {
Environment: "us-west1-gcp",
PodType: "s1",
MetadataConfig: podIndexMetadata,
Tags: &pinecone.IndexTags{"environment": "development"},
})

if err != nil {
Expand Down Expand Up @@ -344,9 +346,7 @@ func main() {

### Configure an index

There are multiple ways to configure Pinecone indexes. You are able to configure Deletion Protection for both
pod-based and Serverless indexes. Additionally, you can configure the size of your pods and the number of replicas
for pod-based indexes. Examples for each of these configurations are provided below.
There are multiple ways to configure Pinecone indexes. You are able to configure Deletion Protection and Tags for both pod-based and Serverless indexes. Additionally, you can configure the size of your pods and the number of replicas for pod-based indexes. Examples for each of these configurations are provided below.

```go
package main
Expand Down Expand Up @@ -374,23 +374,50 @@ func main() {
}

// To scale the size of your pods-based index from "x2" to "x4":
_, err := pc.ConfigureIndex(ctx, "my-pod-index", pinecone.ConfigureIndexParams{PodType: "p1.x4"})
_, err := pc.ConfigureIndex(ctx,
"my-pod-index",
pinecone.ConfigureIndexParams{
PodType: "p1.x4",
},
)
if err != nil {
fmt.Printf("Failed to configure index: %v\n", err)
}

// To scale the number of replicas to 4:
_, err := pc.ConfigureIndex(ctx, "my-pod-index", pinecone.ConfigureIndexParams{Replicas: 4})
_, err := pc.ConfigureIndex(ctx,
"my-pod-index",
pinecone.ConfigureIndexParams{
Replicas: 4,
},
)
if err != nil {
fmt.Printf("Failed to configure index: %v\n", err)
}

// To scale both the size of your pods and the number of replicas:
_, err := pc.ConfigureIndex(ctx, "my-pod-index", pinecone.ConfigureIndexParams{PodType: "p1.x4", Replicas: 4})
_, err := pc.ConfigureIndex(ctx,
"my-pod-index",
pinecone.ConfigureIndexParams{
PodType: "p1.x4",
Replicas: 4,
},
)
if err != nil {
fmt.Printf("Failed to configure index: %v\n", err)
}

// To add or remove IndexTags
_, err := pc.ConfigureIndex(ctx,
"my-pod-index",
pinecone.ConfigureIndexParams{
Tags: pinecone.IndexTags{
"environment": "development",
"source": "",
},
},
)

// To enable deletion protection:
_, err := pc.ConfigureIndex(ctx, "my-index", pinecone.ConfigureIndexParams{DeletionProtection: "enabled"})
if err != nil {
Expand Down Expand Up @@ -581,7 +608,7 @@ func main() {

### Import vectors from object storage

You can now [import vectors en masse](https://docs.pinecone.io/guides/data/understanding-imports) from object
You can now [import vectors en masse](https://docs.pinecone.io/guides/data/understanding-imports) from object
storage. `Import` is a long-running, asynchronous operation that imports large numbers of records into a Pinecone
serverless index.

Expand Down Expand Up @@ -619,7 +646,7 @@ The following example imports vectors from an Amazon S3 bucket into a Pinecone s
}

idx, err = pc.DescribeIndex(ctx, "pinecone-index")

if err != nil {
log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err)
}
Expand All @@ -632,15 +659,16 @@ The following example imports vectors from an Amazon S3 bucket into a Pinecone s
storageURI := "s3://my-bucket/my-directory/"

errorMode := "abort" // Will abort if error encountered; other option: "continue"

importRes, err := idxConnection.StartImport(ctx, storageURI, nil, (*pinecone.ImportErrorMode)(&errorMode))

if err != nil {
log.Fatalf("Failed to start import: %v", err)
}

fmt.Printf("import started with ID: %s", importRes.Id)
```

You can [start, cancel, and check the status](https://docs.pinecone.io/guides/data/import-data) of all or one import operation(s).

### Query an index
Expand Down Expand Up @@ -1460,7 +1488,7 @@ indicating higher relevance.

rerankModel := "bge-reranker-v2-m3"
query := "What are some good Turkey dishes for Thanksgiving?"

documents := []pinecone.Document{
{"title": "Turkey Sandwiches", "body": "Turkey is a classic meat to eat at American Thanksgiving."},
{"title": "Lemon Turkey", "body": "A lemon brined Turkey with apple sausage stuffing is a classic Thanksgiving main course."},
Expand Down
29 changes: 27 additions & 2 deletions pinecone/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -934,6 +934,13 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, in ConfigureIn
replicas := pointerOrNil(in.Replicas)
deletionProtection := pointerOrNil(in.DeletionProtection)

// Describe index in order to merge existing tags with incoming tags
idxDesc, err := c.DescribeIndex(ctx, name)
if err != nil {
return nil, err
}
existingTags := idxDesc.Tags

var request db_control.ConfigureIndexRequest
if podType != nil || replicas != nil {
request.Spec =
Expand All @@ -953,8 +960,7 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, in ConfigureIn
}
}
request.DeletionProtection = (*db_control.DeletionProtection)(deletionProtection)
request.Tags = (*db_control.IndexTags)(&in.Tags)
fmt.Printf("request.Tags: %+v\n", request.Tags)
request.Tags = (*db_control.IndexTags)(mergeIndexTags(existingTags, in.Tags))

res, err := c.restClient.ConfigureIndex(ctx, name, request)
if err != nil {
Expand Down Expand Up @@ -1760,6 +1766,25 @@ func buildSharedProviderHeaders(in NewClientBaseParams) []*provider.CustomHeader
return providers
}

func mergeIndexTags(existingTags *IndexTags, newTags IndexTags) *IndexTags {
if existingTags == nil || *existingTags == nil {
existingTags = &IndexTags{}
}
merged := make(IndexTags)

// Copy existing tags
for key, value := range *existingTags {
merged[key] = value
}

// Merge new tags
for key, value := range newTags {
merged[key] = value
}

return &merged
}

func ensureURLScheme(inputURL string) (string, error) {
parsedURL, err := url.Parse(inputURL)
if err != nil {
Expand Down
4 changes: 4 additions & 0 deletions pinecone/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,8 @@ func (ts *IntegrationTests) TestConfigureIndexScaleUpNoPods() {
isReady, _ := WaitUntilIndexReady(ts, context.Background())
require.True(ts.T(), isReady, "Expected index to be ready")

time.Sleep(500 * time.Millisecond)

err = ts.client.DeleteIndex(context.Background(), name)
require.NoError(ts.T(), err)
}
Expand Down Expand Up @@ -244,6 +246,8 @@ func (ts *IntegrationTests) TestConfigureIndexScaleUpNoReplicas() {
isReady, _ := WaitUntilIndexReady(ts, context.Background())
require.True(ts.T(), isReady, "Expected index to be ready")

time.Sleep(500 * time.Millisecond)

err = ts.client.DeleteIndex(context.Background(), name)
require.NoError(ts.T(), err)
}
Expand Down
1 change: 0 additions & 1 deletion pinecone/index_connection_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@ func (ts *IntegrationTests) TestDeleteVectorsByFilter() {

if ts.indexType == "serverless" {
assert.Error(ts.T(), err)
assert.Containsf(ts.T(), err.Error(), "Serverless and Starter indexes do not support deleting with metadata filtering", "Expected error message to contain 'Serverless and Starter indexes do not support deleting with metadata filtering'")
} else {
assert.NoError(ts.T(), err)
}
Expand Down

0 comments on commit 5a7fff1

Please sign in to comment.