Update app.py

app.py

@@ -1,19 +1,36 @@
-from transformers import AutoTokenizer, TFBertForSeq2SeqLM  # Assuming TFBert model
-
-# Load tokenizer configurations
-source_tokenizer = AutoTokenizer.from_pretrained("https://huggingface.co/Bajiyo/mal_en_transliteration/tree/main/source_tokenizer_config.json")
-target_tokenizer = AutoTokenizer.from_pretrained("https://huggingface.co/Bajiyo/mal_en_transliteration/tree/main/target_tokenizer_config.json")
 from tensorflow.keras.models import load_model
+from tensorflow.keras.preprocessing.text import Tokenizer
+import json
+from gradio import Interface
 
+# Load model (replace with your actual path)
 model = load_model("https://huggingface.co/Bajiyo/mal_en_transliteration/tree/main/transliteration_model.h5")
-# Load the model (replace with your actual model path)
-#model = TFBertForSeq2SeqLM.from_pretrained("https://huggingface.co/Bajiyo/mal_en_transliteration/tree/main/transliteration_model.h5")
+
+# Load tokenizers from configuration files (replace with your paths)
+with open("https://huggingface.co/Bajiyo/mal_en_transliteration/tree/main/source_tokenizer_config.json", "r") as f:
+    source_tokenizer_config = json.load(f)
+source_tokenizer = Tokenizer(num_words=source_tokenizer_config["num_words"])
+source_tokenizer.fit_on_texts(source_tokenizer_config["texts"])  # Assuming pre-defined texts
+
+with open("https://huggingface.co/Bajiyo/mal_en_transliteration/tree/main/target_tokenizer_config.json", "r") as f:
+    target_tokenizer_config = json.load(f)
+target_tokenizer = Tokenizer(num_words=target_tokenizer_config["num_words"])
+target_tokenizer.fit_on_texts(target_tokenizer_config["texts"])  # Assuming pre-defined texts
 
 def translate(malayalam_text):
-    """Function to perform Malayalam to English transliteration"""
-    source_ids = source_tokenizer(malayalam_text, return_tensors="pt")["input_ids"]
-    translated_tokens = model.generate(**source_ids)
-    english_text = target_tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
+    # Preprocessing (tokenization)
+    source_tokens = source_tokenizer.texts_to_sequences([malayalam_text])[0]
+
+    # Padding (adjust maxlen based on your model's requirements)
+    maxlen = 100  # Example value, adjust as needed
+    padded_text = pad_sequences([source_tokens], maxlen=maxlen, padding="post")
+
+    # Make predictions using the model
+    predictions = model.predict(padded_text)
+
+    # Postprocessing (decoding)
+    english_text = target_tokenizer.sequences_to_texts([predictions[0]])[0]
+
     return english_text
 
 interface = gradio.Interface(