---
pipeline_tag: sentence-similarity
tags:
- sentence-transformers
- feature-extraction
- sentence-similarity
datasets:
- sbx/superlim-2
language:
- sv
---

# jzju/sbert-sv-lim2

This model Is trained from [KBLab/bert-base-swedish-cased-new](https://huggingface.co/KBLab/bert-base-swedish-cased-new) with data from [sbx/superlim-2](https://huggingface.co/datasets/sbx/superlim-2) 

This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 256 dimensional dense vector space and can be used for tasks like clustering or semantic search.


## Usage (Sentence-Transformers)

Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:

```
pip install -U sentence-transformers
```

Then you can use the model like this:

```python
from sentence_transformers import SentenceTransformer
sentences = ["This is an example sentence", "Each sentence is converted"]

model = SentenceTransformer('jzju/sbert-sv-lim2')
embeddings = model.encode(sentences)
print(embeddings)
```

## Training Code
```python
from datasets import load_dataset, concatenate_datasets
from sentence_transformers import SentenceTransformer, InputExample, losses, models, util, datasets
from torch.utils.data import DataLoader
from torch import nn
import random

word_embedding_model = models.Transformer("KBLab/bert-base-swedish-cased-new", max_seq_length=256)
pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
dense_model = models.Dense(
    in_features=pooling_model.get_sentence_embedding_dimension(), out_features=256, activation_function=nn.Tanh()
)
model = SentenceTransformer(modules=[word_embedding_model, pooling_model, dense_model])

def pair():
    def norm(x):
        x["label"] = x["label"] / m
        return x

    dd = []
    for sub in ["swepar", "swesim_relatedness", "swesim_similarity"]:
        ds = concatenate_datasets([d for d in load_dataset("sbx/superlim-2", sub).values()])
        if "sentence_1" in ds.features:
            ds = ds.rename_column("sentence_1", "d1")
            ds = ds.rename_column("sentence_2", "d2")
        else:
            ds = ds.rename_column("word_1", "d1")
            ds = ds.rename_column("word_2", "d2")
        m = max([d["label"] for d in ds])
        dd.append(ds.map(norm))
    ds = concatenate_datasets(dd)

    train_examples = []
    for d in ds:
        train_examples.append(InputExample(texts=[d["d1"], d["d2"]], label=d["label"]))
    train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=64)
    train_loss = losses.CosineSimilarityLoss(model)
    model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10, warmup_steps=100)

def nli():
    ds = concatenate_datasets([d for d in load_dataset("sbx/superlim-2", "swenli").values()])

    def add_to_samples(sent1, sent2, label):
        if sent1 not in train_data:
            train_data[sent1] = {0: set(), 1: set(), 2: set()}
        train_data[sent1][label].add(sent2)

    train_data = {}
    for d in ds:
        add_to_samples(d["premise"], d["hypothesis"], d["label"])
        add_to_samples(d["hypothesis"], d["premise"], d["label"])

    train_samples = []
    for sent1, others in train_data.items():
        if len(others[0]) > 0 and len(others[1]) > 0:
            train_samples.append(
                InputExample(texts=[sent1, random.choice(list(others[0])), random.choice(list(others[1]))])
            )
            train_samples.append(
                InputExample(texts=[random.choice(list(others[0])), sent1, random.choice(list(others[1]))])
            )
    train_dataloader = datasets.NoDuplicatesDataLoader(train_samples, batch_size=64)
    train_loss = losses.MultipleNegativesRankingLoss(model)
    model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=100)

pair()
nli()
model.save()

```