Skip to content

Commit

Permalink
attempt test fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@RaynorChavez
RaynorChavez committed Nov 12, 2024
1 parent 7cc77f8 commit 3ecaaf7
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 90 deletions.
1 change: 1 addition & 0 deletions docs/data-sources/read_indices.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ Read-Only:
- `store_size` (String) The size of the index storage
- `text_preprocessing` (Attributes) (see [below for nested schema](#nestedatt--items--text_preprocessing))
- `treat_urls_and_pointers_as_images` (Boolean) Indicates if URLs and pointers should be treated as images
- `treat_urls_and_pointers_as_media` (Boolean) Indicates if URLs and pointers should be treated as media
- `type` (String) The type of the index
- `vector_numeric_type` (String) The numeric type of the vector

Expand Down
108 changes: 60 additions & 48 deletions internal/provider/indices_datasource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,43 +33,46 @@ type allIndicesResourceModel struct {

// indexModel maps index detail data.
type indexModel struct {
Created types.String `tfsdk:"created"`
IndexName types.String `tfsdk:"index_name"`
NumberOfShards types.String `tfsdk:"number_of_shards"`
NumberOfReplicas types.String `tfsdk:"number_of_replicas"`
IndexStatus types.String `tfsdk:"index_status"`
AllFields []AllFieldInput `tfsdk:"all_fields"`
TensorFields []string `tfsdk:"tensor_fields"`
NumberOfInferences types.String `tfsdk:"number_of_inferences"`
StorageClass types.String `tfsdk:"storage_class"`
InferenceType types.String `tfsdk:"inference_type"`
DocsCount types.String `tfsdk:"docs_count"`
StoreSize types.String `tfsdk:"store_size"`
DocsDeleted types.String `tfsdk:"docs_deleted"`
SearchQueryTotal types.String `tfsdk:"search_query_total"`
TreatUrlsAndPointersAsImages types.Bool `tfsdk:"treat_urls_and_pointers_as_images"`
MarqoEndpoint types.String `tfsdk:"marqo_endpoint"`
Type types.String `tfsdk:"type"`
VectorNumericType types.String `tfsdk:"vector_numeric_type"`
Model types.String `tfsdk:"model"`
ModelProperties ModelPropertiesModel `tfsdk:"model_properties"`
NormalizeEmbeddings types.Bool `tfsdk:"normalize_embeddings"`
TextPreprocessing TextPreprocessingModel `tfsdk:"text_preprocessing"`
ImagePreprocessing ImagePreprocessingModel `tfsdk:"image_preprocessing"`
AnnParameters AnnParametersModel `tfsdk:"ann_parameters"`
MarqoVersion types.String `tfsdk:"marqo_version"`
FilterStringMaxLength types.String `tfsdk:"filter_string_max_length"`
Created types.String `tfsdk:"created"`
IndexName types.String `tfsdk:"index_name"`
NumberOfShards types.String `tfsdk:"number_of_shards"`
NumberOfReplicas types.String `tfsdk:"number_of_replicas"`
IndexStatus types.String `tfsdk:"index_status"`
AllFields []AllFieldInput `tfsdk:"all_fields"`
TensorFields []string `tfsdk:"tensor_fields"`
NumberOfInferences types.String `tfsdk:"number_of_inferences"`
StorageClass types.String `tfsdk:"storage_class"`
InferenceType types.String `tfsdk:"inference_type"`
DocsCount types.String `tfsdk:"docs_count"`
StoreSize types.String `tfsdk:"store_size"`
DocsDeleted types.String `tfsdk:"docs_deleted"`
SearchQueryTotal types.String `tfsdk:"search_query_total"`
TreatUrlsAndPointersAsImages types.Bool `tfsdk:"treat_urls_and_pointers_as_images"`
TreatUrlsAndPointersAsMedia types.Bool `tfsdk:"treat_urls_and_pointers_as_media"`
MarqoEndpoint types.String `tfsdk:"marqo_endpoint"`
Type types.String `tfsdk:"type"`
VectorNumericType types.String `tfsdk:"vector_numeric_type"`
Model types.String `tfsdk:"model"`
ModelProperties *ModelPropertiesModel `tfsdk:"model_properties"`
NormalizeEmbeddings types.Bool `tfsdk:"normalize_embeddings"`
TextPreprocessing *TextPreprocessingModel `tfsdk:"text_preprocessing"`
ImagePreprocessing *ImagePreprocessingModel `tfsdk:"image_preprocessing"`
VideoPreprocessing *VideoPreprocessingModel `tfsdk:"video_preprocessing"`
AudioPreprocessing *AudioPreprocessingModel `tfsdk:"audio_preprocessing"`
AnnParameters *AnnParametersModel `tfsdk:"ann_parameters"`
MarqoVersion types.String `tfsdk:"marqo_version"`
FilterStringMaxLength types.String `tfsdk:"filter_string_max_length"`
}

type ModelPropertiesModel struct {
Name types.String `tfsdk:"name"`
Dimensions types.Int64 `tfsdk:"dimensions"`
Type types.String `tfsdk:"type"`
Tokens types.Int64 `tfsdk:"tokens"`
ModelLocation ModelLocationModel `tfsdk:"model_location"`
Url types.String `tfsdk:"url"`
TrustRemoteCode types.Bool `tfsdk:"trust_remote_code"`
IsMarqtunedModel types.Bool `tfsdk:"is_marqtuned_model"`
Name types.String `tfsdk:"name"`
Dimensions types.Int64 `tfsdk:"dimensions"`
Type types.String `tfsdk:"type"`
Tokens types.Int64 `tfsdk:"tokens"`
ModelLocation *ModelLocationModel `tfsdk:"model_location"`
Url types.String `tfsdk:"url"`
TrustRemoteCode types.Bool `tfsdk:"trust_remote_code"`
IsMarqtunedModel types.Bool `tfsdk:"is_marqtuned_model"`
}

type ModelLocationModel struct {
Expand All @@ -94,6 +97,16 @@ type TextPreprocessingModel struct {
SplitOverlap types.String `tfsdk:"split_overlap"`
}

type VideoPreprocessingModel struct {
SplitLength types.String `tfsdk:"split_length"`
SplitOverlap types.String `tfsdk:"split_overlap"`
}

type AudioPreprocessingModel struct {
SplitLength types.String `tfsdk:"split_length"`
SplitOverlap types.String `tfsdk:"split_overlap"`
}

type AnnParametersModel struct {
SpaceType types.String `tfsdk:"space_type"`
Parameters parametersModel `tfsdk:"parameters"`
Expand Down Expand Up @@ -238,6 +251,10 @@ func (d *indicesDataSource) Schema(_ context.Context, _ datasource.SchemaRequest
Computed: true,
Description: "Indicates if URLs and pointers should be treated as images",
},
"treat_urls_and_pointers_as_media": schema.BoolAttribute{
Computed: true,
Description: "Indicates if URLs and pointers should be treated as media",
},
"type": schema.StringAttribute{
Computed: true,
Description: "The type of the index",
Expand Down Expand Up @@ -382,8 +399,8 @@ func ConvertMarqoAllFieldInputs(marqoFields []go_marqo.AllFieldInput) []AllField
return allFieldsConverted
}

func convertModelLocation(location go_marqo.ModelLocation) ModelLocationModel {
modelLocation := ModelLocationModel{
func convertModelLocation(location go_marqo.ModelLocation) *ModelLocationModel {
modelLocation := &ModelLocationModel{
AuthRequired: types.BoolValue(location.AuthRequired),
}

Expand Down Expand Up @@ -466,11 +483,12 @@ func (d *indicesDataSource) Read(ctx context.Context, req datasource.ReadRequest
DocsDeleted: types.StringValue(indexDetail.DocsDeleted),
SearchQueryTotal: types.StringValue(indexDetail.SearchQueryTotal),
TreatUrlsAndPointersAsImages: types.BoolValue(indexDetail.TreatUrlsAndPointersAsImages),
TreatUrlsAndPointersAsMedia: types.BoolValue(indexDetail.TreatUrlsAndPointersAsMedia),
MarqoEndpoint: types.StringValue(indexDetail.MarqoEndpoint),
Type: types.StringValue(indexDetail.Type),
VectorNumericType: types.StringValue(indexDetail.VectorNumericType),
Model: types.StringValue(indexDetail.Model),
ModelProperties: ModelPropertiesModel{
ModelProperties: &ModelPropertiesModel{
Name: types.StringValue(indexDetail.ModelProperties.Name),
Dimensions: types.Int64Value(indexDetail.ModelProperties.Dimensions),
Type: types.StringValue(indexDetail.ModelProperties.Type),
Expand All @@ -481,13 +499,13 @@ func (d *indicesDataSource) Read(ctx context.Context, req datasource.ReadRequest
IsMarqtunedModel: types.BoolValue(indexDetail.ModelProperties.IsMarqtunedModel),
},
NormalizeEmbeddings: types.BoolValue(indexDetail.NormalizeEmbeddings),
TextPreprocessing: TextPreprocessingModel{
TextPreprocessing: &TextPreprocessingModel{
SplitLength: types.StringValue(fmt.Sprintf("%d", indexDetail.TextPreprocessing.SplitLength)),
SplitMethod: types.StringValue(indexDetail.TextPreprocessing.SplitMethod),
SplitOverlap: types.StringValue(fmt.Sprintf("%d", indexDetail.TextPreprocessing.SplitOverlap)),
},
// ImagePreprocessing
AnnParameters: AnnParametersModel{
AnnParameters: &AnnParametersModel{
SpaceType: types.StringValue(indexDetail.AnnParameters.SpaceType),
Parameters: parametersModel{
EfConstruction: types.StringValue(fmt.Sprintf("%d", indexDetail.AnnParameters.Parameters.EfConstruction)),
Expand All @@ -499,15 +517,9 @@ func (d *indicesDataSource) Read(ctx context.Context, req datasource.ReadRequest
}

// Handle model properties
if items[i].ModelProperties.Name.IsNull() &&
items[i].ModelProperties.Dimensions.IsNull() &&
items[i].ModelProperties.Type.IsNull() &&
items[i].ModelProperties.Tokens.IsNull() &&
items[i].ModelProperties.Url.IsNull() &&
items[i].ModelProperties.TrustRemoteCode.IsNull() &&
items[i].ModelProperties.IsMarqtunedModel.IsNull() {
// Set to zero value if all fields are null/empty
items[i].ModelProperties = ModelPropertiesModel{}
// In the resource
if items[i].ModelProperties.IsEmpty() {
items[i].ModelProperties = nil
}

// Remove null fields
Expand Down
70 changes: 28 additions & 42 deletions internal/provider/indices_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ type IndexSettingsModel struct {
TreatUrlsAndPointersAsImages types.Bool `tfsdk:"treat_urls_and_pointers_as_images"`
TreatUrlsAndPointersAsMedia types.Bool `tfsdk:"treat_urls_and_pointers_as_media"`
Model types.String `tfsdk:"model"`
ModelProperties *ModelPropertiesModelCreate `tfsdk:"model_properties"`
ModelProperties *ModelPropertiesModel `tfsdk:"model_properties"`
NormalizeEmbeddings types.Bool `tfsdk:"normalize_embeddings"`
TextPreprocessing *TextPreprocessingModelCreate `tfsdk:"text_preprocessing"`
ImagePreprocessing *ImagePreprocessingModel `tfsdk:"image_preprocessing"`
Expand All @@ -65,17 +65,6 @@ type AllFieldInput struct {
DependentFields map[string]types.Float64 `tfsdk:"dependent_fields"`
}

type ModelPropertiesModelCreate struct {
Name types.String `tfsdk:"name"`
Dimensions types.Int64 `tfsdk:"dimensions"`
Type types.String `tfsdk:"type"`
Tokens types.Int64 `tfsdk:"tokens"`
ModelLocation *ModelLocationModel `tfsdk:"model_location"`
Url types.String `tfsdk:"url"`
TrustRemoteCode types.Bool `tfsdk:"trust_remote_code"`
IsMarqtunedModel types.Bool `tfsdk:"is_marqtuned_model"`
}

type TextPreprocessingModelCreate struct {
SplitLength types.Int64 `tfsdk:"split_length"`
SplitMethod types.String `tfsdk:"split_method"`
Expand Down Expand Up @@ -383,27 +372,32 @@ func convertModelLocationToAPI(modelLocation *ModelLocationModel) map[string]int
return result
}

func (m *ModelPropertiesModel) IsEmpty() bool {
if m == nil {
return true
}
return m.Name.IsNull() &&
m.Dimensions.IsNull() &&
m.Type.IsNull() &&
m.Tokens.IsNull() &&
m.Url.IsNull() &&
!m.TrustRemoteCode.ValueBool() &&
!m.IsMarqtunedModel.ValueBool() &&
(m.ModelLocation == nil || m.ModelLocation.IsEmpty())
}

func (m *ModelLocationModel) IsEmpty() bool {
if m == nil {
return true
}
return !m.AuthRequired.ValueBool() &&
(m.S3 == nil || (m.S3.Bucket.IsNull() && m.S3.Key.IsNull())) &&
(m.Hf == nil || (m.Hf.RepoId.IsNull() && m.Hf.Filename.IsNull()))
}

func (r *indicesResource) findAndCreateState(indices []go_marqo.IndexDetail, indexName string) (*IndexResourceModel, bool) {
for _, indexDetail := range indices {
if indexDetail.IndexName == indexName {
modelLocation := &ModelLocationModel{
AuthRequired: types.BoolValue(indexDetail.ModelProperties.ModelLocation.AuthRequired),
}

if indexDetail.ModelProperties.ModelLocation.S3 != nil {
modelLocation.S3 = &S3LocationModel{
Bucket: types.StringValue(indexDetail.ModelProperties.ModelLocation.S3.Bucket),
Key: types.StringValue(indexDetail.ModelProperties.ModelLocation.S3.Key),
}
}

if indexDetail.ModelProperties.ModelLocation.Hf != nil {
modelLocation.Hf = &HfLocationModel{
RepoId: types.StringValue(indexDetail.ModelProperties.ModelLocation.Hf.RepoId),
Filename: types.StringValue(indexDetail.ModelProperties.ModelLocation.Hf.Filename),
}
}

return &IndexResourceModel{
//ID: types.StringValue(indexDetail.IndexName),
IndexName: types.StringValue(indexDetail.IndexName),
Expand All @@ -413,12 +407,12 @@ func (r *indicesResource) findAndCreateState(indices []go_marqo.IndexDetail, ind
TreatUrlsAndPointersAsImages: types.BoolValue(indexDetail.TreatUrlsAndPointersAsImages),
TreatUrlsAndPointersAsMedia: types.BoolValue(indexDetail.TreatUrlsAndPointersAsMedia),
Model: types.StringValue(indexDetail.Model),
ModelProperties: &ModelPropertiesModelCreate{
ModelProperties: &ModelPropertiesModel{
Name: types.StringValue(indexDetail.ModelProperties.Name),
Dimensions: types.Int64Value(indexDetail.ModelProperties.Dimensions),
Type: types.StringValue(indexDetail.ModelProperties.Type),
Tokens: types.Int64Value(indexDetail.ModelProperties.Tokens),
ModelLocation: modelLocation,
ModelLocation: convertModelLocation(indexDetail.ModelProperties.ModelLocation),
Url: types.StringValue(indexDetail.ModelProperties.Url),
TrustRemoteCode: types.BoolValue(indexDetail.ModelProperties.TrustRemoteCode),
IsMarqtunedModel: types.BoolValue(indexDetail.ModelProperties.IsMarqtunedModel),
Expand Down Expand Up @@ -545,16 +539,8 @@ func (r *indicesResource) Read(ctx context.Context, req resource.ReadRequest, re
}

// Handle model properties
if newState.Settings.ModelProperties != nil {
if newState.Settings.ModelProperties.Name.ValueString() == "" &&
newState.Settings.ModelProperties.Dimensions.ValueInt64() == 0 &&
newState.Settings.ModelProperties.Type.ValueString() == "" &&
newState.Settings.ModelProperties.Tokens.ValueInt64() == 0 &&
newState.Settings.ModelProperties.Url.ValueString() == "" &&
!newState.Settings.ModelProperties.TrustRemoteCode.ValueBool() &&
!newState.Settings.ModelProperties.IsMarqtunedModel.ValueBool() {
newState.Settings.ModelProperties = nil
}
if newState.Settings.ModelProperties.IsEmpty() {
newState.Settings.ModelProperties = nil
}

// Remove null fields
Expand Down

0 comments on commit 3ecaaf7

Please sign in to comment.