From 5a7fff17984765dfabd9765ee6a9d5e0a11324fc Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Fri, 20 Dec 2024 15:32:11 -0500 Subject: [PATCH] Update `ConfigureIndex` to merge new and previous `IndexTags`, update `README` examples (#89) ## Problem @jhamon called out that since `ConfigureIndex` is a PATCH operation, we should probably be explicitly merging existing tags with incoming tags when configuring an index as a partial update. See here for a python example: https://github.com/pinecone-io/pinecone-python-client/pull/426/files#diff-9e7f01373e4589c7e4ded21935023cdf187ab6ce8b2b44a0d5470a27eaf464eeR303-R314 ## Solution - Update `ConfigureIndex` to call `DescribeIndex` and merge tags properly on updates. - Update `README` to add examples of using `IndexTags` when creating and updating indexes. ## Type of Change - [ ] Bug fix (non-breaking change which fixes an issue) - [X] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update - [ ] Infrastructure change (CI configs, etc) - [X] Non-code change (docs, etc) - [ ] None of the above: (explain here) ## Test Plan `just test` Make sure CI is green. --- README.md | 50 ++++++++++++++++++++++++------- pinecone/client.go | 29 ++++++++++++++++-- pinecone/client_test.go | 4 +++ pinecone/index_connection_test.go | 1 - 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 68d8552..8f7c6c9 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ func main() { Metric: pinecone.Cosine, Cloud: pinecone.Aws, Region: "us-east-1", + Tags: &pinecone.IndexTags{"environment": "development"}, }) if err != nil { @@ -209,6 +210,7 @@ func main() { Environment: "us-west1-gcp", PodType: "s1", MetadataConfig: podIndexMetadata, + Tags: &pinecone.IndexTags{"environment": "development"}, }) if err != nil { @@ -344,9 +346,7 @@ func main() { ### Configure an index -There are multiple ways to configure Pinecone indexes. You are able to configure Deletion Protection for both -pod-based and Serverless indexes. Additionally, you can configure the size of your pods and the number of replicas -for pod-based indexes. Examples for each of these configurations are provided below. +There are multiple ways to configure Pinecone indexes. You are able to configure Deletion Protection and Tags for both pod-based and Serverless indexes. Additionally, you can configure the size of your pods and the number of replicas for pod-based indexes. Examples for each of these configurations are provided below. ```go package main @@ -374,23 +374,50 @@ func main() { } // To scale the size of your pods-based index from "x2" to "x4": - _, err := pc.ConfigureIndex(ctx, "my-pod-index", pinecone.ConfigureIndexParams{PodType: "p1.x4"}) + _, err := pc.ConfigureIndex(ctx, + "my-pod-index", + pinecone.ConfigureIndexParams{ + PodType: "p1.x4", + }, + ) if err != nil { fmt.Printf("Failed to configure index: %v\n", err) } // To scale the number of replicas to 4: - _, err := pc.ConfigureIndex(ctx, "my-pod-index", pinecone.ConfigureIndexParams{Replicas: 4}) + _, err := pc.ConfigureIndex(ctx, + "my-pod-index", + pinecone.ConfigureIndexParams{ + Replicas: 4, + }, + ) if err != nil { fmt.Printf("Failed to configure index: %v\n", err) } // To scale both the size of your pods and the number of replicas: - _, err := pc.ConfigureIndex(ctx, "my-pod-index", pinecone.ConfigureIndexParams{PodType: "p1.x4", Replicas: 4}) + _, err := pc.ConfigureIndex(ctx, + "my-pod-index", + pinecone.ConfigureIndexParams{ + PodType: "p1.x4", + Replicas: 4, + }, + ) if err != nil { fmt.Printf("Failed to configure index: %v\n", err) } + // To add or remove IndexTags + _, err := pc.ConfigureIndex(ctx, + "my-pod-index", + pinecone.ConfigureIndexParams{ + Tags: pinecone.IndexTags{ + "environment": "development", + "source": "", + }, + }, + ) + // To enable deletion protection: _, err := pc.ConfigureIndex(ctx, "my-index", pinecone.ConfigureIndexParams{DeletionProtection: "enabled"}) if err != nil { @@ -581,7 +608,7 @@ func main() { ### Import vectors from object storage -You can now [import vectors en masse](https://docs.pinecone.io/guides/data/understanding-imports) from object +You can now [import vectors en masse](https://docs.pinecone.io/guides/data/understanding-imports) from object storage. `Import` is a long-running, asynchronous operation that imports large numbers of records into a Pinecone serverless index. @@ -619,7 +646,7 @@ The following example imports vectors from an Amazon S3 bucket into a Pinecone s } idx, err = pc.DescribeIndex(ctx, "pinecone-index") - + if err != nil { log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) } @@ -632,15 +659,16 @@ The following example imports vectors from an Amazon S3 bucket into a Pinecone s storageURI := "s3://my-bucket/my-directory/" errorMode := "abort" // Will abort if error encountered; other option: "continue" - + importRes, err := idxConnection.StartImport(ctx, storageURI, nil, (*pinecone.ImportErrorMode)(&errorMode)) if err != nil { log.Fatalf("Failed to start import: %v", err) } - + fmt.Printf("import started with ID: %s", importRes.Id) ``` + You can [start, cancel, and check the status](https://docs.pinecone.io/guides/data/import-data) of all or one import operation(s). ### Query an index @@ -1460,7 +1488,7 @@ indicating higher relevance. rerankModel := "bge-reranker-v2-m3" query := "What are some good Turkey dishes for Thanksgiving?" - + documents := []pinecone.Document{ {"title": "Turkey Sandwiches", "body": "Turkey is a classic meat to eat at American Thanksgiving."}, {"title": "Lemon Turkey", "body": "A lemon brined Turkey with apple sausage stuffing is a classic Thanksgiving main course."}, diff --git a/pinecone/client.go b/pinecone/client.go index fd267be..6083680 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -934,6 +934,13 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, in ConfigureIn replicas := pointerOrNil(in.Replicas) deletionProtection := pointerOrNil(in.DeletionProtection) + // Describe index in order to merge existing tags with incoming tags + idxDesc, err := c.DescribeIndex(ctx, name) + if err != nil { + return nil, err + } + existingTags := idxDesc.Tags + var request db_control.ConfigureIndexRequest if podType != nil || replicas != nil { request.Spec = @@ -953,8 +960,7 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, in ConfigureIn } } request.DeletionProtection = (*db_control.DeletionProtection)(deletionProtection) - request.Tags = (*db_control.IndexTags)(&in.Tags) - fmt.Printf("request.Tags: %+v\n", request.Tags) + request.Tags = (*db_control.IndexTags)(mergeIndexTags(existingTags, in.Tags)) res, err := c.restClient.ConfigureIndex(ctx, name, request) if err != nil { @@ -1760,6 +1766,25 @@ func buildSharedProviderHeaders(in NewClientBaseParams) []*provider.CustomHeader return providers } +func mergeIndexTags(existingTags *IndexTags, newTags IndexTags) *IndexTags { + if existingTags == nil || *existingTags == nil { + existingTags = &IndexTags{} + } + merged := make(IndexTags) + + // Copy existing tags + for key, value := range *existingTags { + merged[key] = value + } + + // Merge new tags + for key, value := range newTags { + merged[key] = value + } + + return &merged +} + func ensureURLScheme(inputURL string) (string, error) { parsedURL, err := url.Parse(inputURL) if err != nil { diff --git a/pinecone/client_test.go b/pinecone/client_test.go index 9867ed6..6bf13a2 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -217,6 +217,8 @@ func (ts *IntegrationTests) TestConfigureIndexScaleUpNoPods() { isReady, _ := WaitUntilIndexReady(ts, context.Background()) require.True(ts.T(), isReady, "Expected index to be ready") + time.Sleep(500 * time.Millisecond) + err = ts.client.DeleteIndex(context.Background(), name) require.NoError(ts.T(), err) } @@ -244,6 +246,8 @@ func (ts *IntegrationTests) TestConfigureIndexScaleUpNoReplicas() { isReady, _ := WaitUntilIndexReady(ts, context.Background()) require.True(ts.T(), isReady, "Expected index to be ready") + time.Sleep(500 * time.Millisecond) + err = ts.client.DeleteIndex(context.Background(), name) require.NoError(ts.T(), err) } diff --git a/pinecone/index_connection_test.go b/pinecone/index_connection_test.go index 6d6e5b9..431e8aa 100644 --- a/pinecone/index_connection_test.go +++ b/pinecone/index_connection_test.go @@ -90,7 +90,6 @@ func (ts *IntegrationTests) TestDeleteVectorsByFilter() { if ts.indexType == "serverless" { assert.Error(ts.T(), err) - assert.Containsf(ts.T(), err.Error(), "Serverless and Starter indexes do not support deleting with metadata filtering", "Expected error message to contain 'Serverless and Starter indexes do not support deleting with metadata filtering'") } else { assert.NoError(ts.T(), err) }