-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update provider to support marqo 2.13 (#30)
This update adds support for the following fields - treat_urls_and_pointers_as_media - video_preprocessing - audio_preprocessing - model_properties Within model_properties: - name - dimensions - type - tokens - model_location - url - trust_remote_code - filter_string_max_length Within model_location: - s3: Has the keys bucket, and key - hf: Has the keys repo_id, and filename - auth_required
- Loading branch information
1 parent
0c9514d
commit 4e31c20
Showing
15 changed files
with
1,344 additions
and
141 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
examples/create_index_custom_model/create_index_custom_model.tf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| terraform { | ||
| required_providers { | ||
| marqo = { | ||
| source = "registry.terraform.io/marqo/marqo" | ||
| } | ||
| } | ||
| } | ||
|
|
||
| provider "marqo" { | ||
| host = "https://api.marqo.ai/api/v2" | ||
| api_key = var.marqo_api_key | ||
| } | ||
|
|
||
| resource "marqo_index" "example" { | ||
| index_name = "example_index_custom_model" | ||
| settings = { | ||
| type = "unstructured" | ||
| vector_numeric_type = "float" | ||
| treat_urls_and_pointers_as_images = true | ||
| model = "your-own-sentence-transformers-mode" | ||
| model_properties = { | ||
| url = "https://marqo-ecs-50-audio-test-dataset.s3.us-east-1.amazonaws.com/test-hf.zip" | ||
| dimensions = 384 | ||
| type = "hf" | ||
| trust_remote_code = false | ||
| } | ||
| normalize_embeddings = false | ||
| inference_type = "marqo.CPU.small" | ||
| all_fields = [] | ||
| number_of_inferences = 1 | ||
| number_of_replicas = 0 | ||
| number_of_shards = 1 | ||
| storage_class = "marqo.basic" | ||
|
|
||
| text_preprocessing = { | ||
| split_length = 2 | ||
| split_method = "sentence" | ||
| split_overlap = 0 | ||
| } | ||
| image_preprocessing = {} | ||
| ann_parameters = { | ||
| space_type = "prenormalized-angular" | ||
| parameters = { | ||
| ef_construction = 512 | ||
| m = 16 | ||
| } | ||
| } | ||
| filter_string_max_length = 20 | ||
| } | ||
| } | ||
|
|
||
| output "created_index" { | ||
| value = marqo_index.example | ||
| } | ||
|
|
||
| variable "marqo_api_key" { | ||
| type = string | ||
| description = "Marqo API key" | ||
| } |
77 changes: 77 additions & 0 deletions
77
examples/create_index_languagebind/create_index_languagebind.tf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| terraform { | ||
| required_providers { | ||
| marqo = { | ||
| source = "marqo/marqo" | ||
| #version = "1.0.1" | ||
| } | ||
| } | ||
| } | ||
|
|
||
| provider "marqo" { | ||
| host = "https://api.marqo.ai/api/v2" | ||
| api_key = var.marqo_api_key | ||
| } | ||
|
|
||
| resource "marqo_index" "example" { | ||
| index_name = "example_index_languagebind" | ||
| settings = { | ||
| type = "structured" | ||
| vector_numeric_type = "float" | ||
| all_fields = [ | ||
| { "name" : "textField", "type" : "text", "features" : ["lexical_search"] }, | ||
| { "name" : "videoField", "type" : "video_pointer" }, | ||
| { "name" : "audioField", "type" : "audio_pointer" }, | ||
| { "name" : "imageField", "type" : "image_pointer" }, | ||
| { | ||
| "name" : "multimodal_field", | ||
| "type" : "multimodal_combination", | ||
| "dependent_fields" : { | ||
| "imageField" : 0.8, | ||
| "textField" : 0.1, | ||
| "videoField" : 0.1, | ||
| "audioField" : 0.1 | ||
| }, | ||
| }, | ||
| ], | ||
| number_of_inferences = 1 | ||
| storage_class = "marqo.basic" | ||
| number_of_replicas = 0 | ||
| number_of_shards = 1 | ||
| tensor_fields = ["multimodal_field", "textField", "videoField", "audioField", "imageField"], | ||
| model = "LanguageBind/Video_V1.5_FT_Audio_FT_Image" | ||
| normalize_embeddings = true | ||
| inference_type = "marqo.GPU" | ||
| text_preprocessing = { | ||
| split_length = 2 | ||
| split_method = "sentence" | ||
| split_overlap = 0 | ||
| } | ||
| image_preprocessing = { | ||
| patch_method = null | ||
| } | ||
| video_preprocessing = { | ||
| split_length = 5 | ||
| split_overlap = 1 | ||
| } | ||
| audio_preprocessing = { | ||
| split_length = 5 | ||
| split_overlap = 1 | ||
| } | ||
| ann_parameters = { | ||
| space_type = "prenormalized-angular" | ||
| parameters = { | ||
| ef_construction = 512 | ||
| m = 16 | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| output "created_index" { | ||
| value = marqo_index.example | ||
| } | ||
|
|
||
| variable "marqo_api_key" { | ||
| type = string | ||
| description = "Marqo API key" | ||
| } |
3 changes: 2 additions & 1 deletion
3
...eate_structured_index/create_dependent.tf → ...ructured_index/create_structured_index.tf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.