upload

README.md

@@ -1,3 +1,60 @@
 ---
-license: mit
+language:
+- en
 ---
+
+This model was trained with [Sparsembed](https://github.com/raphaelsty/sparsembed). You can find details on how to use it in the [Sparsembed](https://github.com/raphaelsty/sparsembed) repository.
+
+```sh
+pip install sparsembed
+```
+
+```python
+from sparsembed import model, retrieve
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+device = "cuda" # cpu
+
+batch_size = 10
+
+# List documents to index:
+documents = [
+ {'id': 0,
+  'title': 'Paris',
+  'url': 'https://en.wikipedia.org/wiki/Paris',
+  'text': 'Paris is the capital and most populous city of France.'},
+ {'id': 1,
+  'title': 'Paris',
+  'url': 'https://en.wikipedia.org/wiki/Paris',
+  'text': "Since the 17th century, Paris has been one of Europe's major centres of science, and arts."},
+ {'id': 2,
+  'title': 'Paris',
+  'url': 'https://en.wikipedia.org/wiki/Paris',
+  'text': 'The City of Paris is the centre and seat of government of the region and province of Île-de-France.'
+}]
+
+model = model.SparsEmbed(
+    model=AutoModelForMaskedLM.from_pretrained("raphaelsty/sparsembed-max").to(device),
+    tokenizer=AutoTokenizer.from_pretrained("raphaelsty/sparsembed-max"),
+    device=device
+)
+
+retriever = retrieve.SpladeRetriever(
+    key="id", # Key identifier of each document.
+    on=["title", "text"], # Fields to search.
+    model=model # Splade retriever.
+)
+
+retriever = retriever.add(
+    documents=documents,
+    batch_size=batch_size,
+    k_tokens=256, # Number of activated tokens.
+)
+
+retriever(
+    ["paris", "Toulouse"], # Queries 
+    k_tokens=20, # Maximum number of activated tokens.
+    k=100, # Number of documents to retrieve.
+    batch_size=batch_size
+)
+```

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "./drive/MyDrive/GPU/data/sparsembed_max_frozen",
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForMaskedLM"
+  ],
+  "attention_dropout": 0.1,
+  "dim": 768,
+  "dropout": 0.1,
+  "hidden_dim": 3072,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "output_hidden_states": true,
+  "pad_token_id": 0,
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.32.0",
+  "vocab_size": 30522
+}

linear.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf052e8af3766d810190f7dbd8e80261236df20aa6bc0cf4972201d7b351b13a
+size 197352

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4410c67ed44e716dce6b11f2c8bbb5a632d7e727635b122ce5d0449f0d002b2
+size 267978033

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "device": "cuda",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "max_length": 256,
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}