attempt test fix

marqo-ai · Nov 11, 2024 · 7cc77f8 · 7cc77f8
1 parent e32cfa1
commit 7cc77f8
Show file tree

Hide file tree

Showing 5 changed files with 156 additions and 81 deletions.
diff --git a/docs/data-sources/read_indices.md b/docs/data-sources/read_indices.md
@@ -30,7 +30,10 @@ description: |-
 Optional:
 
 - `all_fields` (Attributes List) (see [below for nested schema](#nestedatt--items--all_fields))
+- `audio_preprocessing` (Attributes) (see [below for nested schema](#nestedatt--items--audio_preprocessing))
+- `model_properties` (Attributes) (see [below for nested schema](#nestedatt--items--model_properties))
 - `tensor_fields` (List of String)
+- `video_preprocessing` (Attributes) (see [below for nested schema](#nestedatt--items--video_preprocessing))
 
 Read-Only:
 
@@ -46,7 +49,6 @@ Read-Only:
 - `marqo_endpoint` (String) The Marqo endpoint used by the index
 - `marqo_version` (String) The version of Marqo used by the index
 - `model` (String) The model used by the index
-- `model_properties` (Attributes) (see [below for nested schema](#nestedatt--items--model_properties))
 - `normalize_embeddings` (Boolean) Indicates if embeddings should be normalized
 - `number_of_inferences` (String) The number of inferences made by the index
 - `number_of_replicas` (String) The number of replicas for the index
@@ -70,30 +72,13 @@ Optional:
 - `type` (String)
 
 
-<a id="nestedatt--items--ann_parameters"></a>
-### Nested Schema for `items.ann_parameters`
-
-Read-Only:
-
-- `parameters` (Attributes) (see [below for nested schema](#nestedatt--items--ann_parameters--parameters))
-- `space_type` (String) The space type for ANN parameters
-
-<a id="nestedatt--items--ann_parameters--parameters"></a>
-### Nested Schema for `items.ann_parameters.parameters`
-
-Read-Only:
-
-- `ef_construction` (String) The efConstruction parameter for ANN
-- `m` (String) The m parameter for ANN
-
-
-
-<a id="nestedatt--items--image_preprocessing"></a>
-### Nested Schema for `items.image_preprocessing`
+<a id="nestedatt--items--audio_preprocessing"></a>
+### Nested Schema for `items.audio_preprocessing`
 
-Read-Only:
+Optional:
 
-- `patch_method` (String) The patch method for image preprocessing
+- `split_length` (String)
+- `split_overlap` (String)
 
 
 <a id="nestedatt--items--model_properties"></a>
@@ -142,6 +127,41 @@ Read-Only:
 
 
 
+<a id="nestedatt--items--video_preprocessing"></a>
+### Nested Schema for `items.video_preprocessing`
+
+Optional:
+
+- `split_length` (String)
+- `split_overlap` (String)
+
+
+<a id="nestedatt--items--ann_parameters"></a>
+### Nested Schema for `items.ann_parameters`
+
+Read-Only:
+
+- `parameters` (Attributes) (see [below for nested schema](#nestedatt--items--ann_parameters--parameters))
+- `space_type` (String) The space type for ANN parameters
+
+<a id="nestedatt--items--ann_parameters--parameters"></a>
+### Nested Schema for `items.ann_parameters.parameters`
+
+Read-Only:
+
+- `ef_construction` (String) The efConstruction parameter for ANN
+- `m` (String) The m parameter for ANN
+
+
+
+<a id="nestedatt--items--image_preprocessing"></a>
+### Nested Schema for `items.image_preprocessing`
+
+Read-Only:
+
+- `patch_method` (String) The patch method for image preprocessing
+
+
 <a id="nestedatt--items--text_preprocessing"></a>
 ### Nested Schema for `items.text_preprocessing`
 

diff --git a/internal/provider/indices_datasource.go b/internal/provider/indices_datasource.go
@@ -33,34 +33,32 @@ type allIndicesResourceModel struct {
 
 // indexModel maps index detail data.
 type indexModel struct {
-	Created                      types.String                  `tfsdk:"created"`
-	IndexName                    types.String                  `tfsdk:"index_name"`
-	NumberOfShards               types.String                  `tfsdk:"number_of_shards"`
-	NumberOfReplicas             types.String                  `tfsdk:"number_of_replicas"`
-	IndexStatus                  types.String                  `tfsdk:"index_status"`
-	AllFields                    []AllFieldInput               `tfsdk:"all_fields"`
-	TensorFields                 []string                      `tfsdk:"tensor_fields"`
-	NumberOfInferences           types.String                  `tfsdk:"number_of_inferences"`
-	StorageClass                 types.String                  `tfsdk:"storage_class"`
-	InferenceType                types.String                  `tfsdk:"inference_type"`
-	DocsCount                    types.String                  `tfsdk:"docs_count"`
-	StoreSize                    types.String                  `tfsdk:"store_size"`
-	DocsDeleted                  types.String                  `tfsdk:"docs_deleted"`
-	SearchQueryTotal             types.String                  `tfsdk:"search_query_total"`
-	TreatUrlsAndPointersAsImages types.Bool                    `tfsdk:"treat_urls_and_pointers_as_images"`
-	MarqoEndpoint                types.String                  `tfsdk:"marqo_endpoint"`
-	Type                         types.String                  `tfsdk:"type"`
-	VectorNumericType            types.String                  `tfsdk:"vector_numeric_type"`
-	Model                        types.String                  `tfsdk:"model"`
-	ModelProperties              ModelPropertiesModel          `tfsdk:"model_properties"`
-	NormalizeEmbeddings          types.Bool                    `tfsdk:"normalize_embeddings"`
-	TextPreprocessing            TextPreprocessingModel        `tfsdk:"text_preprocessing"`
-	ImagePreprocessing           ImagePreprocessingModel       `tfsdk:"image_preprocessing"`
-	VideoPreprocessing           VideoPreprocessingModelCreate `tfsdk:"video_preprocessing"`
-	AudioPreprocessing           AudioPreprocessingModelCreate `tfsdk:"audio_preprocessing"`
-	AnnParameters                AnnParametersModel            `tfsdk:"ann_parameters"`
-	MarqoVersion                 types.String                  `tfsdk:"marqo_version"`
-	FilterStringMaxLength        types.String                  `tfsdk:"filter_string_max_length"`
+	Created                      types.String            `tfsdk:"created"`
+	IndexName                    types.String            `tfsdk:"index_name"`
+	NumberOfShards               types.String            `tfsdk:"number_of_shards"`
+	NumberOfReplicas             types.String            `tfsdk:"number_of_replicas"`
+	IndexStatus                  types.String            `tfsdk:"index_status"`
+	AllFields                    []AllFieldInput         `tfsdk:"all_fields"`
+	TensorFields                 []string                `tfsdk:"tensor_fields"`
+	NumberOfInferences           types.String            `tfsdk:"number_of_inferences"`
+	StorageClass                 types.String            `tfsdk:"storage_class"`
+	InferenceType                types.String            `tfsdk:"inference_type"`
+	DocsCount                    types.String            `tfsdk:"docs_count"`
+	StoreSize                    types.String            `tfsdk:"store_size"`
+	DocsDeleted                  types.String            `tfsdk:"docs_deleted"`
+	SearchQueryTotal             types.String            `tfsdk:"search_query_total"`
+	TreatUrlsAndPointersAsImages types.Bool              `tfsdk:"treat_urls_and_pointers_as_images"`
+	MarqoEndpoint                types.String            `tfsdk:"marqo_endpoint"`
+	Type                         types.String            `tfsdk:"type"`
+	VectorNumericType            types.String            `tfsdk:"vector_numeric_type"`
+	Model                        types.String            `tfsdk:"model"`
+	ModelProperties              ModelPropertiesModel    `tfsdk:"model_properties"`
+	NormalizeEmbeddings          types.Bool              `tfsdk:"normalize_embeddings"`
+	TextPreprocessing            TextPreprocessingModel  `tfsdk:"text_preprocessing"`
+	ImagePreprocessing           ImagePreprocessingModel `tfsdk:"image_preprocessing"`
+	AnnParameters                AnnParametersModel      `tfsdk:"ann_parameters"`
+	MarqoVersion                 types.String            `tfsdk:"marqo_version"`
+	FilterStringMaxLength        types.String            `tfsdk:"filter_string_max_length"`
 }
 
 type ModelPropertiesModel struct {
@@ -500,6 +498,18 @@ func (d *indicesDataSource) Read(ctx context.Context, req datasource.ReadRequest
 			FilterStringMaxLength: types.StringValue(fmt.Sprintf("%d", indexDetail.FilterStringMaxLength)),
 		}
 
+		// Handle model properties
+		if items[i].ModelProperties.Name.IsNull() &&
+			items[i].ModelProperties.Dimensions.IsNull() &&
+			items[i].ModelProperties.Type.IsNull() &&
+			items[i].ModelProperties.Tokens.IsNull() &&
+			items[i].ModelProperties.Url.IsNull() &&
+			items[i].ModelProperties.TrustRemoteCode.IsNull() &&
+			items[i].ModelProperties.IsMarqtunedModel.IsNull() {
+			// Set to zero value if all fields are null/empty
+			items[i].ModelProperties = ModelPropertiesModel{}
+		}
+
 		// Remove null fields
 		if items[i].InferenceType.IsNull() {
 			items[i].InferenceType = types.StringNull()

diff --git a/internal/provider/indices_datasource_test.go b/internal/provider/indices_datasource_test.go
@@ -86,8 +86,6 @@ func testAccDataSourceIndexConfig(name string) string {
 						split_overlap = 0
 					}
 				image_preprocessing = {}
-				video_preprocessing = {}
-				audio_preprocessing = {}
 				ann_parameters = {
 				space_type = "prenormalized-angular"
 					parameters = {

diff --git a/internal/provider/indices_resource.go b/internal/provider/indices_resource.go
@@ -531,6 +531,32 @@ func (r *indicesResource) Read(ctx context.Context, req resource.ReadRequest, re
 			newState.Settings.ImagePreprocessing.PatchMethod = types.StringNull()
 		}
 
+		// Handle video and audio preprocessing fields
+		if newState.Settings.VideoPreprocessing != nil &&
+			newState.Settings.VideoPreprocessing.SplitLength.ValueInt64() == 0 &&
+			newState.Settings.VideoPreprocessing.SplitOverlap.ValueInt64() == 0 {
+			newState.Settings.VideoPreprocessing = nil
+		}
+
+		if newState.Settings.AudioPreprocessing != nil &&
+			newState.Settings.AudioPreprocessing.SplitLength.ValueInt64() == 0 &&
+			newState.Settings.AudioPreprocessing.SplitOverlap.ValueInt64() == 0 {
+			newState.Settings.AudioPreprocessing = nil
+		}
+
+		// Handle model properties
+		if newState.Settings.ModelProperties != nil {
+			if newState.Settings.ModelProperties.Name.ValueString() == "" &&
+				newState.Settings.ModelProperties.Dimensions.ValueInt64() == 0 &&
+				newState.Settings.ModelProperties.Type.ValueString() == "" &&
+				newState.Settings.ModelProperties.Tokens.ValueInt64() == 0 &&
+				newState.Settings.ModelProperties.Url.ValueString() == "" &&
+				!newState.Settings.ModelProperties.TrustRemoteCode.ValueBool() &&
+				!newState.Settings.ModelProperties.IsMarqtunedModel.ValueBool() {
+				newState.Settings.ModelProperties = nil
+			}
+		}
+
 		// Remove null fields
 		if newState.Settings.InferenceType.IsNull() {
 			newState.Settings.InferenceType = types.StringNull()
@@ -659,7 +685,6 @@ func (r *indicesResource) Create(ctx context.Context, req resource.CreateRequest
 			model.Settings.ModelProperties.Dimensions.IsNull() &&
 			model.Settings.ModelProperties.Type.IsNull() &&
 			model.Settings.ModelProperties.Tokens.IsNull() &&
-			model.Settings.ModelProperties.ModelLocation == nil &&
 			model.Settings.ModelProperties.Url.IsNull() &&
 			model.Settings.ModelProperties.TrustRemoteCode.IsNull()) {
 		delete(settings, "modelProperties")

diff --git a/internal/provider/indices_resource_test.go b/internal/provider/indices_resource_test.go
@@ -215,9 +215,10 @@ func testAccResourceIndexConfig(name string) string {
 				type = "unstructured"
 				vector_numeric_type = "float"
 				treat_urls_and_pointers_as_images = true
-				model = "hf/e5-small-v2"
+				treat_urls_and_pointers_as_media = true
+				model = "LanguageBind/Video_V1.5_FT_Audio_FT_Image"
 				normalize_embeddings = true
-				inference_type = "marqo.CPU.small"
+				inference_type = "marqo.GPU"
 				number_of_inferences = 1
 				number_of_replicas = 0
 				number_of_shards = 1
@@ -228,7 +229,17 @@ func testAccResourceIndexConfig(name string) string {
 					split_method = "sentence"
 					split_overlap = 0
 				}
-				image_preprocessing = {}
+				image_preprocessing = {
+					patch_method = "null"
+				}
+				video_preprocessing = {
+					split_length = 5
+					split_overlap = 1
+				}
+				audio_preprocessing = {
+					split_length = 5
+					split_overlap = 1
+				}
 				ann_parameters = {
 					space_type = "prenormalized-angular"
 					parameters = {
@@ -247,33 +258,44 @@ func testAccResourceIndexConfigUpdated(name string) string {
 		resource "marqo_index" "test" {
 		index_name = "%s"
 		settings = {
-			type = "unstructured"
-			vector_numeric_type = "float"
-			treat_urls_and_pointers_as_images = true
-			model = "hf/e5-small-v2"
-			normalize_embeddings = true
-			inference_type = "marqo.CPU.large"
-			number_of_inferences = 2
-			number_of_replicas = 0
-			number_of_shards = 1
-			storage_class = "marqo.basic"
-			all_fields = []
-			text_preprocessing = {
-				split_length = 2
-				split_method = "sentence"
-				split_overlap = 0
-			}
-			image_preprocessing = {}
-			ann_parameters = {
-				space_type = "prenormalized-angular"
-				parameters = {
-					ef_construction = 512
-					m = 16
+				type = "unstructured"
+				vector_numeric_type = "float"
+				treat_urls_and_pointers_as_images = true
+				treat_urls_and_pointers_as_media = true
+				model = "LanguageBind/Video_V1.5_FT_Audio_FT_Image"
+				normalize_embeddings = true
+				inference_type = "marqo.GPU"
+				number_of_inferences = 2
+				number_of_replicas = 0
+				number_of_shards = 1
+				storage_class = "marqo.basic"
+				all_fields = []
+				text_preprocessing = {
+					split_length = 2
+					split_method = "sentence"
+					split_overlap = 0
+				}
+				image_preprocessing = {
+					patch_method = "null"
+				}
+				video_preprocessing = {
+					split_length = 5
+					split_overlap = 1
 				}
+				audio_preprocessing = {
+					split_length = 5
+					split_overlap = 1
+				}
+				ann_parameters = {
+					space_type = "prenormalized-angular"
+					parameters = {
+						ef_construction = 512
+						m = 16
+					}
+				}
+				filter_string_max_length = 20
 			}
-			filter_string_max_length = 20
 		}
-	}
 		`, name)
 }