Update README.md

README.md

@@ -4,14 +4,18 @@ tags:
 - sentence-transformers
 - feature-extraction
 - sentence-similarity
-
+datasets:
+- sbx/superlim-2
+language:
+- sv
 ---
 
 # jzju/sbert-sv-lim2
 
+This model Is trained from [KBLab/bert-base-swedish-cased-new](https://huggingface.co/KBLab/bert-base-swedish-cased-new) with data from [sbx/superlim-2](https://huggingface.co/datasets/sbx/superlim-2) 
+
 This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 256 dimensional dense vector space and can be used for tasks like clustering or semantic search.
 
-<!--- Describe your model here -->
 
 ## Usage (Sentence-Transformers)
 
@@ -32,57 +36,74 @@ embeddings = model.encode(sentences)
 print(embeddings)
 ```
 
-
-
-## Evaluation Results
-
-<!--- Describe how your model was evaluated -->
-
-For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name=jzju/sbert-sv-lim2)
-
-
-## Training
-The model was trained with the parameters:
-
-**DataLoader**:
-
-`torch.utils.data.dataloader.DataLoader` of length 177 with parameters:
-```
-{'batch_size': 64, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
-```
-
-**Loss**:
-
-`sentence_transformers.losses.CosineSimilarityLoss.CosineSimilarityLoss` 
-
-Parameters of the fit()-Method:
-```
-{
-    "epochs": 10,
-    "evaluation_steps": 0,
-    "evaluator": "NoneType",
-    "max_grad_norm": 1,
-    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
-    "optimizer_params": {
-        "lr": 2e-05
-    },
-    "scheduler": "WarmupLinear",
-    "steps_per_epoch": null,
-    "warmup_steps": 100,
-    "weight_decay": 0.01
-}
-```
-
-
-## Full Model Architecture
-```
-SentenceTransformer(
-  (0): Transformer({'max_seq_length': 256, 'do_lower_case': False}) with Transformer model: BertModel 
-  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
-  (2): Dense({'in_features': 768, 'out_features': 256, 'bias': True, 'activation_function': 'torch.nn.modules.activation.Tanh'})
+## Training Code
+```python
+from datasets import load_dataset, concatenate_datasets
+from sentence_transformers import SentenceTransformer, InputExample, losses, models, util, datasets
+from torch.utils.data import DataLoader
+from torch import nn
+import random
+
+word_embedding_model = models.Transformer("KBLab/bert-base-swedish-cased-new", max_seq_length=256)
+pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
+dense_model = models.Dense(
+    in_features=pooling_model.get_sentence_embedding_dimension(), out_features=256, activation_function=nn.Tanh()
 )
-```
-
-## Citing & Authors
-
-<!--- Describe where people can find more information -->
+model = SentenceTransformer(modules=[word_embedding_model, pooling_model, dense_model])
+
+def pair():
+    def norm(x):
+        x["label"] = x["label"] / m
+        return x
+
+    dd = []
+    for sub in ["swepar", "swesim_relatedness", "swesim_similarity"]:
+        ds = concatenate_datasets([d for d in load_dataset("sbx/superlim-2", sub).values()])
+        if "sentence_1" in ds.features:
+            ds = ds.rename_column("sentence_1", "d1")
+            ds = ds.rename_column("sentence_2", "d2")
+        else:
+            ds = ds.rename_column("word_1", "d1")
+            ds = ds.rename_column("word_2", "d2")
+        m = max([d["label"] for d in ds])
+        dd.append(ds.map(norm))
+    ds = concatenate_datasets(dd)
+
+    train_examples = []
+    for d in ds:
+        train_examples.append(InputExample(texts=[d["d1"], d["d2"]], label=d["label"]))
+    train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=64)
+    train_loss = losses.CosineSimilarityLoss(model)
+    model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10, warmup_steps=100)
+
+def nli():
+    ds = concatenate_datasets([d for d in load_dataset("sbx/superlim-2", "swenli").values()])
+
+    def add_to_samples(sent1, sent2, label):
+        if sent1 not in train_data:
+            train_data[sent1] = {0: set(), 1: set(), 2: set()}
+        train_data[sent1][label].add(sent2)
+
+    train_data = {}
+    for d in ds:
+        add_to_samples(d["premise"], d["hypothesis"], d["label"])
+        add_to_samples(d["hypothesis"], d["premise"], d["label"])
+
+    train_samples = []
+    for sent1, others in train_data.items():
+        if len(others[0]) > 0 and len(others[1]) > 0:
+            train_samples.append(
+                InputExample(texts=[sent1, random.choice(list(others[0])), random.choice(list(others[1]))])
+            )
+            train_samples.append(
+                InputExample(texts=[random.choice(list(others[0])), sent1, random.choice(list(others[1]))])
+            )
+    train_dataloader = datasets.NoDuplicatesDataLoader(train_samples, batch_size=64)
+    train_loss = losses.MultipleNegativesRankingLoss(model)
+    model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=100)
+
+pair()
+nli()
+model.save()
+
+```