Merge branch 'main' of https://huggingface.co/jinaai/jina-embeddings-v2

custom_st.py

@@ -1,13 +1,11 @@
-import base64
 import json
 import os
 from io import BytesIO
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-import requests
 import torch
 from torch import nn
-from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoTokenizer
+from transformers import AutoConfig, AutoModel, AutoTokenizer
 
 
 class Transformer(nn.Module):
@@ -35,11 +33,11 @@ class Transformer(nn.Module):
     def __init__(
         self,
         model_name_or_path: str,
-        max_seq_length: int | None = None,
-        model_args: dict[str, Any] | None = None,
-        tokenizer_args: dict[str, Any] | None = None,
-        config_args: dict[str, Any] | None = None,
-        cache_dir: str | None = None,
+        max_seq_length: int = None,
+        model_args: Dict[str, Any] = None,
+        tokenizer_args: Dict[str, Any] = None,
+        config_args: Dict[str, Any] = None,
+        cache_dir: str = None,
         do_lower_case: bool = False,
         tokenizer_name_or_path: str = None,
     ) -> None:
@@ -121,8 +119,10 @@ class Transformer(nn.Module):
         return self.auto_model.config.hidden_size
 
     def tokenize(
-        self, texts: list[str] | list[dict] | list[tuple[str, str]], padding: str | bool = True
-    ) -> dict[str, torch.Tensor]:
+        self,
+        texts: Union[List[str], List[dict], List[Tuple[str, str]]],
+        padding: Union[str, bool] = True
+    ) -> Dict[str, torch.Tensor]:
         """Tokenizes a text and maps tokens to token-ids"""
         output = {}
         if isinstance(texts[0], str):