Add new models(FashionCLIP and visheration multilingual clip) from th…

…e release 2.12

src/marqo_commons/model_registry/model_properties_data/languagebind_model_properties.py

@@ -0,0 +1,63 @@
+"""
+OpenCLIP model properties
+
+This file contains properties for OpenCLIP models. It is intended to be used in conjunction with the model registry
+and should not be used in isolation.
+"""
+from typing import Dict, List, Optional
+
+from marqo_commons.model_registry.model_properties_data.onnx_clip_properties import OnnxClipModelProperties
+from marqo_commons.model_registry.model_properties_object import ModelProperties, VectorNumericType, Modality, \
+ ModelType, T
+from marqo_commons.model_registry.utils import convert_model_properties_to_dict
+
+
+class LanguagebindModelProperties(ModelProperties):
+ vector_numeric_type: VectorNumericType = VectorNumericType.float32
+ default_memory_size: float = 8
+ modality: List[Modality]
+ type: ModelType = ModelType.languagebind
+ pretrained: Optional[str]
+ notes: str = ""
+
+ @classmethod
+ def get_all_model_properties_objects(cls) -> Dict[str, T]:
+ # use this link to find all the model_configs
+ # https://github.com/mlfoundations/open_clip/tree/main/src/open_clip/model_configs
+ return {
+ 'LanguageBind/Video_V1.5_FT_Audio_FT_Image': LanguagebindModelProperties(
+ name="LanguageBind/Video_V1.5_FT_Audio_FT_Image",
+ dimensions=768,
+ modality=[Modality.video, Modality.audio, Modality.text, Modality.image],
+ ),
+ 'LanguageBind/Video_V1.5_FT_Audio_FT': LanguagebindModelProperties(
+ name="LanguageBind/Video_V1.5_FT_Audio_FT",
+ dimensions=768,
+ modality=[Modality.video, Modality.audio, Modality.text],
+ ),
+ 'LanguageBind/Video_V1.5_FT_Image': LanguagebindModelProperties(
+ name="LanguageBind/Video_V1.5_FT_Image",
+ dimensions=768,
+ modality=[Modality.video, Modality.text, Modality.image],
+ ),
+ 'LanguageBind/Audio_FT_Image': LanguagebindModelProperties(
+ name="LanguageBind/Audio_FT_Image",
+ dimensions=768,
+ modality=[Modality.audio, Modality.text, Modality.image],
+ ),
+ 'LanguageBind/Audio_FT': LanguagebindModelProperties(
+ name="LanguageBind/Audio_FT",
+ dimensions=768,
+ modality=[Modality.audio, Modality.text],
+ ),
+ 'LanguageBind/Video_V1.5_FT': LanguagebindModelProperties(
+ name="LanguageBind/Video_V1.5_FT",
+ dimensions=768,
+ modality=[Modality.video, Modality.text],
+ ),
+ }
+
+
+@convert_model_properties_to_dict
+def _get_languagebind_properties() -> Dict:
+ return LanguagebindModelProperties.get_all_model_properties_objects()

src/marqo_commons/model_registry/model_properties_data/open_clip_properties.py

@@ -4,7 +4,7 @@
 This file contains properties for OpenCLIP models. It is intended to be used in conjunction with the model registry
 and should not be used in isolation.
 """
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from marqo_commons.model_registry.model_properties_data.onnx_clip_properties import OnnxClipModelProperties
 from marqo_commons.model_registry.model_properties_object import ModelProperties, VectorNumericType, Modality, \
@@ -17,7 +17,7 @@ class OpenClipModelProperties(ModelProperties):
  default_memory_size: float = 1.0
  modality: List[Modality] = [Modality.text, Modality.image]
  type: ModelType = ModelType.open_clip
- pretrained: str
+ pretrained: Optional[str]
  notes: str = ""
 
  @classmethod
@@ -454,8 +454,38 @@ def get_all_model_properties_objects(cls) -> Dict[str, T]:
  dimensions=512,
  notes="open_clip model: ViT-B-32-256/datacomp_s34b_b86k",
  pretrained="datacomp_s34b_b86k"
+ ),
+ "Marqo/marqo-fashionCLIP": OpenClipModelProperties(
+ name = "hf-hub:Marqo/marqo-fashionCLIP",
+ dimensions = 512,
+ note = "Marqo's fashionCLIP model",
+ ),
+ "Marqo/marqo-fashionSigLIP": OpenClipModelProperties(
+ name = "hf-hub:Marqo/marqo-fashionSigLIP",
+ dimensions = 768,
+ note = "Marqo's fashionSigLIP model",
+ ),
+ "visheratin/nllb-clip-base-siglip": OpenClipModelProperties(
+ name="hf-hub:visheratin/nllb-clip-base-siglip",
+ dimensions=768,
+ note="A multilingual CLIP model",
+ ),
+ "visheratin/nllb-siglip-mrl-base": OpenClipModelProperties(
+ name="hf-hub:visheratin/nllb-siglip-mrl-base",
+ dimensions=768,
+ note="A multilingual CLIP model",
+ ),
+ "visheratin/nllb-clip-large-siglip": OpenClipModelProperties(
+ name="hf-hub:visheratin/nllb-clip-large-siglip",
+ dimensions=1152,
+ note="A multilingual CLIP model",
+ ),
+ "visheratin/nllb-siglip-mrl-large": OpenClipModelProperties(
+ name="hf-hub:visheratin/nllb-siglip-mrl-large",
+ dimensions=1152,
+ note="A multilingual CLIP model",
  )
-  }
+ }
 
 
 @convert_model_properties_to_dict

src/marqo_commons/model_registry/model_registry.py

@@ -12,6 +12,7 @@
 from marqo_commons.model_registry.model_properties_data.onnx_clip_properties import _get_onnx_clip_properties
 from marqo_commons.model_registry.model_properties_data.random_properties import _get_random_properties
 from marqo_commons.model_registry.model_properties_data.no_model import _get_no_model_properties
+from marqo_commons.model_registry.model_properties_data.languagebind_model_properties import _get_languagebind_properties
 
 
 # we need to keep track of the embed dim and model load functions/classes
@@ -33,6 +34,7 @@ def get_model_properties_dict() -> Dict:
  multilingual_clip_model_properties = _get_multilingual_clip_properties()
  fp16_clip_model_properties = _get_fp16_clip_properties()
  no_model_properties = _get_no_model_properties()
+ languagebind_model_properties = _get_languagebind_properties()
 
  # combine the above dicts
  model_properties = dict(clip_model_properties.items())
@@ -46,6 +48,7 @@ def get_model_properties_dict() -> Dict:
  model_properties.update(multilingual_clip_model_properties)
  model_properties.update(fp16_clip_model_properties)
  model_properties.update(no_model_properties)
+ model_properties.update(languagebind_model_properties)
 
  return model_properties
 

src/marqo_commons/shared_utils/enums.py

@@ -4,6 +4,8 @@
 class Modality(str, Enum):
  image = "image"
  text = "text"
+ audio = "audio"
+ video = "video"
 
 
 class ModelType(str, Enum):
@@ -18,6 +20,7 @@ class ModelType(str, Enum):
  sbert = "sbert"
  test = "test"
  no_model = "no_model"
+ languagebind = "languagebind"
 
 
 class VectorNumericType(str, Enum):

tests/test_model_properties.py

@@ -12,6 +12,7 @@
 from marqo_commons.model_registry.model_properties_data.sbert_onnx_properties import _get_sbert_onnx_properties
 from marqo_commons.model_registry.model_properties_data.sbert_properties import _get_sbert_properties
 from marqo_commons.model_registry.model_properties_data.test_properties import _get_sbert_test_properties
+from marqo_commons.model_registry.model_properties_data.languagebind_model_properties import _get_languagebind_properties
 
 from marqo_commons.model_registry.model_registry import get_model_properties_dict, get_model_properties_json
 from marqo_commons.shared_utils.enums import ModelType
@@ -38,6 +39,7 @@ def test_models_count(self):
  total_count_from_all_model_properties += len(_get_multilingual_clip_properties())
  total_count_from_all_model_properties += len(_get_fp16_clip_properties())
  total_count_from_all_model_properties += len(_get_no_model_properties())
+ total_count_from_all_model_properties += len(_get_languagebind_properties())
  self.assertEqual(
  total_count_from_all_model_properties, total_count_from_model_registry,
  "Number of models in get_model_properties_dict is not equal to total "