Skip to content

Commit

Permalink
fix vector listing failure causing early exit before indexing
Browse files Browse the repository at this point in the history
  • Loading branch information
Southclaws committed Jan 2, 2025
1 parent 847c3be commit 2ddc4a4
Showing 1 changed file with 9 additions and 5 deletions.
14 changes: 9 additions & 5 deletions app/services/semdex/semdexer/pinecone_semdexer/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,16 +100,20 @@ func (s *pineconeSemdexer) buildIndexOps(ctx context.Context, object datagraph.I

vecids := dt.Map(indexedChunk.VectorIds, func(id *string) string { return *id })

indexedChunkTable, err := s.index.FetchVectors(ctx, vecids)
if err != nil {
return nil, nil, fault.Wrap(err, fctx.With(ctx))
indexedChunkTable := map[string]*pinecone.Vector{}
if len(vecids) > 0 {
resp, err := s.index.FetchVectors(ctx, vecids)
if err != nil {
return nil, nil, fault.Wrap(err, fctx.With(ctx))
}
indexedChunkTable = resp.Vectors
}

pool := pond.NewResultPool[*pinecone.Vector](min(runtime.NumCPU(), len(chunkIDs)))
group := pool.NewGroupContext(ctx)

for id, chunk := range inputChunkTable {
_, exists := indexedChunkTable.Vectors[id]
_, exists := indexedChunkTable[id]
if exists {
continue
}
Expand Down Expand Up @@ -146,7 +150,7 @@ func (s *pineconeSemdexer) buildIndexOps(ctx context.Context, object datagraph.I
// build a list of vectors to delete by yielding items that are indexed but
// not present in the input object chunk table.
deletes := []string{}
for id := range indexedChunkTable.Vectors {
for id := range indexedChunkTable {
_, exists := inputChunkTable[id]
if exists {
continue
Expand Down

0 comments on commit 2ddc4a4

Please sign in to comment.