Marqo

/

marqo-ecommerce-embeddings-L

like 38

Follow

Marqo 106

Hi! I'm simply running code from example on m2 pro:

from transformers import AutoModel, AutoProcessor
import torch
from PIL import Image
import requests

model_name= 'Marqo/marqo-ecommerce-embeddings-L'
# model_name = 'Marqo/marqo-ecommerce-embeddings-B'

model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)

img = Image.open(requests.get('https://raw.githubusercontent.com/marqo-ai/marqo-ecommerce-embeddings/refs/heads/main/images/dining-chairs.png', stream=True).raw).convert("RGB")
image = [img]
text = ["dining chairs", "a laptop", "toothbrushes"]
processed = processor(text=text, images=image, padding='max_length', return_tensors="pt")
processor.image_processor.do_rescale = False
with torch.no_grad():
    image_features = model.get_image_features(processed['pixel_values'], normalize=True)
    text_features = model.get_text_features(processed['input_ids'], normalize=True)

    text_probs = (100 * image_features @ text_features.T).softmax(dim=-1)

print(text_probs)
# [1.0000e+00, 8.3131e-12, 5.2173e-12]

which gives me the following output:

config.json:   0%|          | 0.00/310 [00:00<?, ?B/s]
marqo_fashionSigLIP.py:   0%|          | 0.00/10.7k [00:00<?, ?B/s]
A new version of the following files was downloaded from https://huggingface.co/Marqo/marqo-ecommerce-embeddings-L:
- marqo_fashionSigLIP.py
. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.
You are using a model of type siglip to instantiate a model of type . This is not supported for all configurations of models and can yield errors.

model.safetensors:   0%|          | 0.00/2.61G [00:00<?, ?B/s]
open_clip_model.safetensors:   0%|          | 0.00/2.61G [00:00<?, ?B/s]
open_clip_config.json:   0%|          | 0.00/884 [00:00<?, ?B/s]
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
Cell In[1], line 9
      6 model_name= 'Marqo/marqo-ecommerce-embeddings-L'
      7 # model_name = 'Marqo/marqo-ecommerce-embeddings-B'
----> 9 model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
     10 processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
     12 img = Image.open(requests.get('https://raw.githubusercontent.com/marqo-ai/marqo-ecommerce-embeddings/refs/heads/main/images/dining-chairs.png', stream=True).raw).convert("RGB")

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py:564, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    562     cls.register(config.__class__, model_class, exist_ok=True)
    563     model_class = add_generation_mixin_to_remote_model(model_class)
--> 564     return model_class.from_pretrained(
    565         pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
    566     )
    567 elif type(config) in cls._model_mapping.keys():
    568     model_class = _get_model_class(config, cls._model_mapping)

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/transformers/modeling_utils.py:309, in restore_default_torch_dtype.<locals>._wrapper(*args, **kwargs)
    307 old_dtype = torch.get_default_dtype()
    308 try:
--> 309     return func(*args, **kwargs)
    310 finally:
    311     torch.set_default_dtype(old_dtype)

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/transformers/modeling_utils.py:4508, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)
   4499     config = cls._autoset_attn_implementation(
   4500         config,
   4501         use_flash_attention_2=use_flash_attention_2,
   4502         torch_dtype=torch_dtype,
   4503         device_map=device_map,
   4504     )
   4506 with ContextManagers(model_init_context):
   4507     # Let's make sure we don't run the init function of buffer modules
-> 4508     model = cls(config, *model_args, **model_kwargs)
   4510 # Make sure to tie the weights correctly
   4511 model.tie_weights()

File ~/.cache/huggingface/modules/transformers_modules/Marqo/marqo-ecommerce-embeddings-L/92e3b2606e34fe9a91b732229689e9fb9d422929/marqo_fashionSigLIP.py:190, in MarqoFashionSigLIP.__init__(self, config)
    188 super().__init__(config)
    189 self.config = config
--> 190 self.model = create_model(config.open_clip_model_name, output_dict=True)
    191 self.model.eval()
    192 self.model.to(self.device)

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/open_clip/factory.py:376, in create_model(model_name, pretrained, precision, device, jit, force_quick_gelu, force_custom_text, force_patch_dropout, force_image_size, force_preprocess_cfg, pretrained_image, pretrained_hf, cache_dir, output_dict, require_pretrained, load_weights_only, **model_kwargs)
    374     model.to(device=device, dtype=dtype)
    375 else:
--> 376     model.to(device=device)
    378 pretrained_loaded = False
    379 if pretrained:

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:1355, in Module.to(self, *args, **kwargs)
   1352         else:
   1353             raise
-> 1355 return self._apply(convert)

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:915, in Module._apply(self, fn, recurse)
    913 if recurse:
    914     for module in self.children():
--> 915         module._apply(fn)
    917 def compute_should_use_set_data(tensor, tensor_applied):
    918     if torch._has_compatible_shallow_copy_type(tensor, tensor_applied):
    919         # If the new tensor has compatible tensor type as the existing tensor,
    920         # the current behavior is to change the tensor in-place using `.data =`,
   (...)    925         # global flag to let the user control whether they want the future
    926         # behavior of overwriting the existing tensor or not.

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:915, in Module._apply(self, fn, recurse)
    913 if recurse:
    914     for module in self.children():
--> 915         module._apply(fn)
    917 def compute_should_use_set_data(tensor, tensor_applied):
    918     if torch._has_compatible_shallow_copy_type(tensor, tensor_applied):
    919         # If the new tensor has compatible tensor type as the existing tensor,
    920         # the current behavior is to change the tensor in-place using `.data =`,
   (...)    925         # global flag to let the user control whether they want the future
    926         # behavior of overwriting the existing tensor or not.

    [... skipping similar frames: Module._apply at line 915 (1 times)]

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:915, in Module._apply(self, fn, recurse)
    913 if recurse:
    914     for module in self.children():
--> 915         module._apply(fn)
    917 def compute_should_use_set_data(tensor, tensor_applied):
    918     if torch._has_compatible_shallow_copy_type(tensor, tensor_applied):
    919         # If the new tensor has compatible tensor type as the existing tensor,
    920         # the current behavior is to change the tensor in-place using `.data =`,
   (...)    925         # global flag to let the user control whether they want the future
    926         # behavior of overwriting the existing tensor or not.

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:942, in Module._apply(self, fn, recurse)
    938 # Tensors stored in modules are graph leaves, and we don't want to
    939 # track autograd history of `param_applied`, so we have to use
    940 # `with torch.no_grad():`
    941 with torch.no_grad():
--> 942     param_applied = fn(param)
    943 p_should_use_set_data = compute_should_use_set_data(param, param_applied)
    945 # subclasses may have multiple child tensors so we need to use swap_tensors

File ~/Projects/sandbox/baml-example/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:1348, in Module.to.<locals>.convert(t)
   1346 except NotImplementedError as e:
   1347     if str(e) == "Cannot copy out of meta tensor; no data!":
-> 1348         raise NotImplementedError(
   1349             f"{e} Please use torch.nn.Module.to_empty() instead of torch.nn.Module.to() "
   1350             f"when moving module from meta to a different device."
   1351         ) from None
   1352     else:
   1353         raise

NotImplementedError: Cannot copy out of meta tensor; no data! Please use torch.nn.Module.to_empty() instead of torch.nn.Module.to() when moving module from meta to a different device.

I think it used to work some time ago, but not any more. I'm running transformers 4.52.4