tokenization_indictrans.py

import os
import json

from typing import Dict, List, Optional, Union, Tuple

from transformers.utils import logging
from sentencepiece import SentencePieceProcessor
from transformers.tokenization_utils import PreTrainedTokenizer


logger = logging.get_logger(__name__)

SPIECE_UNDERLINE = "▁"
SUPPORTED_LANGUAGES = [
    "asm_Beng",
    "awa_Deva",
    "ben_Beng",
    "bho_Deva",
    "brx_Deva",
    "doi_Deva",
    "eng_Latn",
    "gom_Deva",
    "gon_Deva",
    "guj_Gujr",
    "hin_Deva",
    "hne_Deva",
    "kan_Knda",
    "kas_Arab",
    "kas_Deva",
    "kha_Latn",
    "lus_Latn",
    "mag_Deva",
    "mai_Deva",
    "mal_Mlym",
    "mar_Deva",
    "mni_Beng",
    "mni_Mtei",
    "npi_Deva",
    "ory_Orya",
    "pan_Guru",
    "san_Deva",
    "sat_Olck",
    "snd_Arab",
    "snd_Deva",
    "tam_Taml",
    "tel_Telu",
    "urd_Arab",
    "unr_Deva",
]

VOCAB_FILES_NAMES = {
    "src_vocab_fp": "dict.SRC.json",
    "tgt_vocab_fp": "dict.TGT.json",
    "src_spm_fp": "model.SRC",
    "tgt_spm_fp": "model.TGT",
}


class IndicTransTokenizer(PreTrainedTokenizer):
    _added_tokens_encoder = {}
    _added_tokens_decoder = {}

    vocab_files_names = VOCAB_FILES_NAMES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        src_vocab_fp=None,
        tgt_vocab_fp=None,
        src_spm_fp=None,
        tgt_spm_fp=None,
        unk_token="<unk>",
        bos_token="<s>",
        eos_token="</s>",
        pad_token="<pad>",
        do_lower_case=False,
        **kwargs
    ):

        self.src = True

        self.src_vocab_fp = src_vocab_fp
        self.tgt_vocab_fp = tgt_vocab_fp
        self.src_spm_fp = src_spm_fp
        self.tgt_spm_fp = tgt_spm_fp

        self.unk_token = unk_token
        self.pad_token = pad_token
        self.eos_token = eos_token
        self.bos_token = bos_token

        self.encoder = self._load_json(self.src_vocab_fp)
        if self.unk_token not in self.encoder:
            raise KeyError("<unk> token must be in vocab")
        assert self.pad_token in self.encoder
        self.encoder_rev = {v: k for k, v in self.encoder.items()}

        self.decoder = self._load_json(self.tgt_vocab_fp)
        if self.unk_token not in self.encoder:
            raise KeyError("<unk> token must be in vocab")
        assert self.pad_token in self.encoder
        self.decoder_rev = {v: k for k, v in self.decoder.items()}

        # load SentencePiece model for pre-processing
        self.src_spm = self._load_spm(self.src_spm_fp)
        self.tgt_spm = self._load_spm(self.tgt_spm_fp)

        self.current_spm = self.src_spm
        self.current_encoder = self.encoder
        self.current_encoder_rev = self.encoder_rev

        self.unk_token_id = self.encoder[self.unk_token]
        self.pad_token_id = self.encoder[self.pad_token]
        self.eos_token_id = self.encoder[self.eos_token]
        self.bos_token_id = self.encoder[self.bos_token]

        super().__init__(
            src_vocab_file=self.src_vocab_fp,
            tgt_vocab_file=self.src_vocab_fp,
            do_lower_case=do_lower_case,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            pad_token=pad_token,
            **kwargs,
        )
    
    def _switch_to_input_mode(self):
        self.src = True
        self.padding_side = "left"
        self.current_spm = self.src_spm
        self.current_encoder = self.encoder
        self.current_encoder_rev = self.encoder_rev

    def _switch_to_target_mode(self):
        self.src = False
        self.padding_side = "right"
        self.current_spm = self.tgt_spm
        self.current_encoder = self.decoder
        self.current_encoder_rev = self.decoder_rev

    def _load_spm(self, path: str) -> SentencePieceProcessor:
        return SentencePieceProcessor(model_file=path)

    def _save_json(self, data, path: str) -> None:
        with open(path, "w", encoding="utf-8") as f:
            json.dump(data, f, indent=2)

    def _load_json(self, path: str) -> Union[Dict, List]:
        with open(path, "r", encoding="utf-8") as f:
            return json.load(f)

    @property
    def src_vocab_size(self) -> int:
        return len(self.encoder)

    @property
    def tgt_vocab_size(self) -> int:
        return len(self.decoder)

    def get_src_vocab(self) -> Dict[str, int]:
        return dict(self.encoder, **self.added_tokens_encoder)

    def get_tgt_vocab(self) -> Dict[str, int]:
        return dict(self.decoder, **self.added_tokens_decoder)

    # hack override
    def get_vocab(self) -> Dict[str, int]:
        return self.get_src_vocab()

    # hack override
    @property
    def vocab_size(self) -> int:
        return self.src_vocab_size

    def _convert_token_to_id(self, token: str) -> int:
        """Converts an token (str) into an index (integer) using the source/target vocabulary map."""
        return self.current_encoder.get(token, self.current_encoder[self.unk_token])

    def _convert_id_to_token(self, index: int) -> str:
        """Converts an index (integer) into a token (str) using the source/target vocabulary map."""
        return self.current_encoder_rev.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens: List[str]) -> str:
        """Uses sentencepiece model for detokenization"""
        pad_tokens = [token for token in tokens if token == self.pad_token]
        tokens = [token for token in tokens if token != self.pad_token]
        if self.src:
            return (
                " ".join(pad_tokens)
                + " "
                + " ".join(tokens[:2])
                + " "
                + "".join(tokens[2:]).replace(SPIECE_UNDERLINE, " ").strip()
            )
        return (
            "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
            + " "
            + " ".join(pad_tokens)
        )

    def _tokenize(self, text) -> List[str]:
        if self.src:
            tokens = text.split(" ")
            tags = tokens[:2]
            text = " ".join(tokens[2:])
            tokens = self.current_spm.EncodeAsPieces(text)
            return tags + tokens
        else:
            return self.current_spm.EncodeAsPieces(text)

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        if token_ids_1 is None:
            return token_ids_0 + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_0 + [self.eos_token_id] + token_ids_1 + [self.eos_token_id]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        
        src_spm_fp = os.path.join(save_directory, "model.SRC")
        tgt_spm_fp = os.path.join(save_directory, "model.TGT")
        src_vocab_fp = os.path.join(save_directory, "dict.SRC.json")
        tgt_vocab_fp = os.path.join(save_directory, "dict.TGT.json")
        
        self._save_json(self.encoder, src_vocab_fp)
        self._save_json(self.decoder, tgt_vocab_fp)
        
        with open(src_spm_fp, 'wb') as f:
            f.write(self.src_spm.serialized_model_proto())
        
        with open(tgt_spm_fp, 'wb') as f:
            f.write(self.tgt_spm.serialized_model_proto())

        return src_vocab_fp, tgt_vocab_fp, src_spm_fp, tgt_spm_fp