From 3ecaaf7f41c3cce424d685f20981cd301acbb7dc Mon Sep 17 00:00:00 2001 From: Raynor Chavez Date: Tue, 12 Nov 2024 12:51:51 +1100 Subject: [PATCH] attempt test fix --- docs/data-sources/read_indices.md | 1 + internal/provider/indices_datasource.go | 108 +++++++++++++----------- internal/provider/indices_resource.go | 70 ++++++--------- 3 files changed, 89 insertions(+), 90 deletions(-) diff --git a/docs/data-sources/read_indices.md b/docs/data-sources/read_indices.md index 1d6e924..e2debfe 100644 --- a/docs/data-sources/read_indices.md +++ b/docs/data-sources/read_indices.md @@ -58,6 +58,7 @@ Read-Only: - `store_size` (String) The size of the index storage - `text_preprocessing` (Attributes) (see [below for nested schema](#nestedatt--items--text_preprocessing)) - `treat_urls_and_pointers_as_images` (Boolean) Indicates if URLs and pointers should be treated as images +- `treat_urls_and_pointers_as_media` (Boolean) Indicates if URLs and pointers should be treated as media - `type` (String) The type of the index - `vector_numeric_type` (String) The numeric type of the vector diff --git a/internal/provider/indices_datasource.go b/internal/provider/indices_datasource.go index fff1f1b..9baf290 100644 --- a/internal/provider/indices_datasource.go +++ b/internal/provider/indices_datasource.go @@ -33,43 +33,46 @@ type allIndicesResourceModel struct { // indexModel maps index detail data. type indexModel struct { - Created types.String `tfsdk:"created"` - IndexName types.String `tfsdk:"index_name"` - NumberOfShards types.String `tfsdk:"number_of_shards"` - NumberOfReplicas types.String `tfsdk:"number_of_replicas"` - IndexStatus types.String `tfsdk:"index_status"` - AllFields []AllFieldInput `tfsdk:"all_fields"` - TensorFields []string `tfsdk:"tensor_fields"` - NumberOfInferences types.String `tfsdk:"number_of_inferences"` - StorageClass types.String `tfsdk:"storage_class"` - InferenceType types.String `tfsdk:"inference_type"` - DocsCount types.String `tfsdk:"docs_count"` - StoreSize types.String `tfsdk:"store_size"` - DocsDeleted types.String `tfsdk:"docs_deleted"` - SearchQueryTotal types.String `tfsdk:"search_query_total"` - TreatUrlsAndPointersAsImages types.Bool `tfsdk:"treat_urls_and_pointers_as_images"` - MarqoEndpoint types.String `tfsdk:"marqo_endpoint"` - Type types.String `tfsdk:"type"` - VectorNumericType types.String `tfsdk:"vector_numeric_type"` - Model types.String `tfsdk:"model"` - ModelProperties ModelPropertiesModel `tfsdk:"model_properties"` - NormalizeEmbeddings types.Bool `tfsdk:"normalize_embeddings"` - TextPreprocessing TextPreprocessingModel `tfsdk:"text_preprocessing"` - ImagePreprocessing ImagePreprocessingModel `tfsdk:"image_preprocessing"` - AnnParameters AnnParametersModel `tfsdk:"ann_parameters"` - MarqoVersion types.String `tfsdk:"marqo_version"` - FilterStringMaxLength types.String `tfsdk:"filter_string_max_length"` + Created types.String `tfsdk:"created"` + IndexName types.String `tfsdk:"index_name"` + NumberOfShards types.String `tfsdk:"number_of_shards"` + NumberOfReplicas types.String `tfsdk:"number_of_replicas"` + IndexStatus types.String `tfsdk:"index_status"` + AllFields []AllFieldInput `tfsdk:"all_fields"` + TensorFields []string `tfsdk:"tensor_fields"` + NumberOfInferences types.String `tfsdk:"number_of_inferences"` + StorageClass types.String `tfsdk:"storage_class"` + InferenceType types.String `tfsdk:"inference_type"` + DocsCount types.String `tfsdk:"docs_count"` + StoreSize types.String `tfsdk:"store_size"` + DocsDeleted types.String `tfsdk:"docs_deleted"` + SearchQueryTotal types.String `tfsdk:"search_query_total"` + TreatUrlsAndPointersAsImages types.Bool `tfsdk:"treat_urls_and_pointers_as_images"` + TreatUrlsAndPointersAsMedia types.Bool `tfsdk:"treat_urls_and_pointers_as_media"` + MarqoEndpoint types.String `tfsdk:"marqo_endpoint"` + Type types.String `tfsdk:"type"` + VectorNumericType types.String `tfsdk:"vector_numeric_type"` + Model types.String `tfsdk:"model"` + ModelProperties *ModelPropertiesModel `tfsdk:"model_properties"` + NormalizeEmbeddings types.Bool `tfsdk:"normalize_embeddings"` + TextPreprocessing *TextPreprocessingModel `tfsdk:"text_preprocessing"` + ImagePreprocessing *ImagePreprocessingModel `tfsdk:"image_preprocessing"` + VideoPreprocessing *VideoPreprocessingModel `tfsdk:"video_preprocessing"` + AudioPreprocessing *AudioPreprocessingModel `tfsdk:"audio_preprocessing"` + AnnParameters *AnnParametersModel `tfsdk:"ann_parameters"` + MarqoVersion types.String `tfsdk:"marqo_version"` + FilterStringMaxLength types.String `tfsdk:"filter_string_max_length"` } type ModelPropertiesModel struct { - Name types.String `tfsdk:"name"` - Dimensions types.Int64 `tfsdk:"dimensions"` - Type types.String `tfsdk:"type"` - Tokens types.Int64 `tfsdk:"tokens"` - ModelLocation ModelLocationModel `tfsdk:"model_location"` - Url types.String `tfsdk:"url"` - TrustRemoteCode types.Bool `tfsdk:"trust_remote_code"` - IsMarqtunedModel types.Bool `tfsdk:"is_marqtuned_model"` + Name types.String `tfsdk:"name"` + Dimensions types.Int64 `tfsdk:"dimensions"` + Type types.String `tfsdk:"type"` + Tokens types.Int64 `tfsdk:"tokens"` + ModelLocation *ModelLocationModel `tfsdk:"model_location"` + Url types.String `tfsdk:"url"` + TrustRemoteCode types.Bool `tfsdk:"trust_remote_code"` + IsMarqtunedModel types.Bool `tfsdk:"is_marqtuned_model"` } type ModelLocationModel struct { @@ -94,6 +97,16 @@ type TextPreprocessingModel struct { SplitOverlap types.String `tfsdk:"split_overlap"` } +type VideoPreprocessingModel struct { + SplitLength types.String `tfsdk:"split_length"` + SplitOverlap types.String `tfsdk:"split_overlap"` +} + +type AudioPreprocessingModel struct { + SplitLength types.String `tfsdk:"split_length"` + SplitOverlap types.String `tfsdk:"split_overlap"` +} + type AnnParametersModel struct { SpaceType types.String `tfsdk:"space_type"` Parameters parametersModel `tfsdk:"parameters"` @@ -238,6 +251,10 @@ func (d *indicesDataSource) Schema(_ context.Context, _ datasource.SchemaRequest Computed: true, Description: "Indicates if URLs and pointers should be treated as images", }, + "treat_urls_and_pointers_as_media": schema.BoolAttribute{ + Computed: true, + Description: "Indicates if URLs and pointers should be treated as media", + }, "type": schema.StringAttribute{ Computed: true, Description: "The type of the index", @@ -382,8 +399,8 @@ func ConvertMarqoAllFieldInputs(marqoFields []go_marqo.AllFieldInput) []AllField return allFieldsConverted } -func convertModelLocation(location go_marqo.ModelLocation) ModelLocationModel { - modelLocation := ModelLocationModel{ +func convertModelLocation(location go_marqo.ModelLocation) *ModelLocationModel { + modelLocation := &ModelLocationModel{ AuthRequired: types.BoolValue(location.AuthRequired), } @@ -466,11 +483,12 @@ func (d *indicesDataSource) Read(ctx context.Context, req datasource.ReadRequest DocsDeleted: types.StringValue(indexDetail.DocsDeleted), SearchQueryTotal: types.StringValue(indexDetail.SearchQueryTotal), TreatUrlsAndPointersAsImages: types.BoolValue(indexDetail.TreatUrlsAndPointersAsImages), + TreatUrlsAndPointersAsMedia: types.BoolValue(indexDetail.TreatUrlsAndPointersAsMedia), MarqoEndpoint: types.StringValue(indexDetail.MarqoEndpoint), Type: types.StringValue(indexDetail.Type), VectorNumericType: types.StringValue(indexDetail.VectorNumericType), Model: types.StringValue(indexDetail.Model), - ModelProperties: ModelPropertiesModel{ + ModelProperties: &ModelPropertiesModel{ Name: types.StringValue(indexDetail.ModelProperties.Name), Dimensions: types.Int64Value(indexDetail.ModelProperties.Dimensions), Type: types.StringValue(indexDetail.ModelProperties.Type), @@ -481,13 +499,13 @@ func (d *indicesDataSource) Read(ctx context.Context, req datasource.ReadRequest IsMarqtunedModel: types.BoolValue(indexDetail.ModelProperties.IsMarqtunedModel), }, NormalizeEmbeddings: types.BoolValue(indexDetail.NormalizeEmbeddings), - TextPreprocessing: TextPreprocessingModel{ + TextPreprocessing: &TextPreprocessingModel{ SplitLength: types.StringValue(fmt.Sprintf("%d", indexDetail.TextPreprocessing.SplitLength)), SplitMethod: types.StringValue(indexDetail.TextPreprocessing.SplitMethod), SplitOverlap: types.StringValue(fmt.Sprintf("%d", indexDetail.TextPreprocessing.SplitOverlap)), }, // ImagePreprocessing - AnnParameters: AnnParametersModel{ + AnnParameters: &AnnParametersModel{ SpaceType: types.StringValue(indexDetail.AnnParameters.SpaceType), Parameters: parametersModel{ EfConstruction: types.StringValue(fmt.Sprintf("%d", indexDetail.AnnParameters.Parameters.EfConstruction)), @@ -499,15 +517,9 @@ func (d *indicesDataSource) Read(ctx context.Context, req datasource.ReadRequest } // Handle model properties - if items[i].ModelProperties.Name.IsNull() && - items[i].ModelProperties.Dimensions.IsNull() && - items[i].ModelProperties.Type.IsNull() && - items[i].ModelProperties.Tokens.IsNull() && - items[i].ModelProperties.Url.IsNull() && - items[i].ModelProperties.TrustRemoteCode.IsNull() && - items[i].ModelProperties.IsMarqtunedModel.IsNull() { - // Set to zero value if all fields are null/empty - items[i].ModelProperties = ModelPropertiesModel{} + // In the resource + if items[i].ModelProperties.IsEmpty() { + items[i].ModelProperties = nil } // Remove null fields diff --git a/internal/provider/indices_resource.go b/internal/provider/indices_resource.go index 8fbfb98..dfebf25 100644 --- a/internal/provider/indices_resource.go +++ b/internal/provider/indices_resource.go @@ -48,7 +48,7 @@ type IndexSettingsModel struct { TreatUrlsAndPointersAsImages types.Bool `tfsdk:"treat_urls_and_pointers_as_images"` TreatUrlsAndPointersAsMedia types.Bool `tfsdk:"treat_urls_and_pointers_as_media"` Model types.String `tfsdk:"model"` - ModelProperties *ModelPropertiesModelCreate `tfsdk:"model_properties"` + ModelProperties *ModelPropertiesModel `tfsdk:"model_properties"` NormalizeEmbeddings types.Bool `tfsdk:"normalize_embeddings"` TextPreprocessing *TextPreprocessingModelCreate `tfsdk:"text_preprocessing"` ImagePreprocessing *ImagePreprocessingModel `tfsdk:"image_preprocessing"` @@ -65,17 +65,6 @@ type AllFieldInput struct { DependentFields map[string]types.Float64 `tfsdk:"dependent_fields"` } -type ModelPropertiesModelCreate struct { - Name types.String `tfsdk:"name"` - Dimensions types.Int64 `tfsdk:"dimensions"` - Type types.String `tfsdk:"type"` - Tokens types.Int64 `tfsdk:"tokens"` - ModelLocation *ModelLocationModel `tfsdk:"model_location"` - Url types.String `tfsdk:"url"` - TrustRemoteCode types.Bool `tfsdk:"trust_remote_code"` - IsMarqtunedModel types.Bool `tfsdk:"is_marqtuned_model"` -} - type TextPreprocessingModelCreate struct { SplitLength types.Int64 `tfsdk:"split_length"` SplitMethod types.String `tfsdk:"split_method"` @@ -383,27 +372,32 @@ func convertModelLocationToAPI(modelLocation *ModelLocationModel) map[string]int return result } +func (m *ModelPropertiesModel) IsEmpty() bool { + if m == nil { + return true + } + return m.Name.IsNull() && + m.Dimensions.IsNull() && + m.Type.IsNull() && + m.Tokens.IsNull() && + m.Url.IsNull() && + !m.TrustRemoteCode.ValueBool() && + !m.IsMarqtunedModel.ValueBool() && + (m.ModelLocation == nil || m.ModelLocation.IsEmpty()) +} + +func (m *ModelLocationModel) IsEmpty() bool { + if m == nil { + return true + } + return !m.AuthRequired.ValueBool() && + (m.S3 == nil || (m.S3.Bucket.IsNull() && m.S3.Key.IsNull())) && + (m.Hf == nil || (m.Hf.RepoId.IsNull() && m.Hf.Filename.IsNull())) +} + func (r *indicesResource) findAndCreateState(indices []go_marqo.IndexDetail, indexName string) (*IndexResourceModel, bool) { for _, indexDetail := range indices { if indexDetail.IndexName == indexName { - modelLocation := &ModelLocationModel{ - AuthRequired: types.BoolValue(indexDetail.ModelProperties.ModelLocation.AuthRequired), - } - - if indexDetail.ModelProperties.ModelLocation.S3 != nil { - modelLocation.S3 = &S3LocationModel{ - Bucket: types.StringValue(indexDetail.ModelProperties.ModelLocation.S3.Bucket), - Key: types.StringValue(indexDetail.ModelProperties.ModelLocation.S3.Key), - } - } - - if indexDetail.ModelProperties.ModelLocation.Hf != nil { - modelLocation.Hf = &HfLocationModel{ - RepoId: types.StringValue(indexDetail.ModelProperties.ModelLocation.Hf.RepoId), - Filename: types.StringValue(indexDetail.ModelProperties.ModelLocation.Hf.Filename), - } - } - return &IndexResourceModel{ //ID: types.StringValue(indexDetail.IndexName), IndexName: types.StringValue(indexDetail.IndexName), @@ -413,12 +407,12 @@ func (r *indicesResource) findAndCreateState(indices []go_marqo.IndexDetail, ind TreatUrlsAndPointersAsImages: types.BoolValue(indexDetail.TreatUrlsAndPointersAsImages), TreatUrlsAndPointersAsMedia: types.BoolValue(indexDetail.TreatUrlsAndPointersAsMedia), Model: types.StringValue(indexDetail.Model), - ModelProperties: &ModelPropertiesModelCreate{ + ModelProperties: &ModelPropertiesModel{ Name: types.StringValue(indexDetail.ModelProperties.Name), Dimensions: types.Int64Value(indexDetail.ModelProperties.Dimensions), Type: types.StringValue(indexDetail.ModelProperties.Type), Tokens: types.Int64Value(indexDetail.ModelProperties.Tokens), - ModelLocation: modelLocation, + ModelLocation: convertModelLocation(indexDetail.ModelProperties.ModelLocation), Url: types.StringValue(indexDetail.ModelProperties.Url), TrustRemoteCode: types.BoolValue(indexDetail.ModelProperties.TrustRemoteCode), IsMarqtunedModel: types.BoolValue(indexDetail.ModelProperties.IsMarqtunedModel), @@ -545,16 +539,8 @@ func (r *indicesResource) Read(ctx context.Context, req resource.ReadRequest, re } // Handle model properties - if newState.Settings.ModelProperties != nil { - if newState.Settings.ModelProperties.Name.ValueString() == "" && - newState.Settings.ModelProperties.Dimensions.ValueInt64() == 0 && - newState.Settings.ModelProperties.Type.ValueString() == "" && - newState.Settings.ModelProperties.Tokens.ValueInt64() == 0 && - newState.Settings.ModelProperties.Url.ValueString() == "" && - !newState.Settings.ModelProperties.TrustRemoteCode.ValueBool() && - !newState.Settings.ModelProperties.IsMarqtunedModel.ValueBool() { - newState.Settings.ModelProperties = nil - } + if newState.Settings.ModelProperties.IsEmpty() { + newState.Settings.ModelProperties = nil } // Remove null fields