Spaces:
Runtime error
Runtime error
File size: 3,329 Bytes
5b74395 7357b85 5f05d95 810915b 5f05d95 810915b 5f05d95 810915b 5f05d95 810915b 5f05d95 a64e097 5f05d95 510623a 810915b 5f05d95 2cb8221 5f05d95 2cb8221 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
from datasets import load_dataset
import pandas as pd
import numpy as np
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Embedding, LSTM, Dense
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.losses import sparse_categorical_crossentropy
import gradio as gr
# Load dataset
dataset = load_dataset("Cosmos-AI/Cosmos-dataset")
# Convert dataset to pandas DataFrame
dataset_df = pd.DataFrame(dataset['train']) # Assuming 'train' split contains both questions and answers
# Prepare data
questions = dataset_df['Question'].astype(str).tolist()
answers = dataset_df['Answer'].astype(str).tolist()
# Tokenize input data
tokenizer = Tokenizer(lower=False, oov_token="<unk>", filters='\t\n')
tokenizer.fit_on_texts(questions + answers)
word_index = tokenizer.word_index
# Convert text sequences to numerical sequences
question_sequences = tokenizer.texts_to_sequences(questions)
answer_sequences = tokenizer.texts_to_sequences(answers)
# Pad sequences to ensure uniform length
max_sequence_length = max(len(seq) for seq in question_sequences + answer_sequences)
print("MAX SEQUENCE LENGTH: " + str(max_sequence_length))
question_sequences = pad_sequences(question_sequences, maxlen=max_sequence_length, padding='post')
answer_sequences = pad_sequences(answer_sequences, maxlen=max_sequence_length, padding='post')
# Convert target sequences to one-hot encoding
num_words = len(word_index) + 1
one_hot_answers = np.zeros((len(answer_sequences), max_sequence_length, num_words), dtype=np.float32)
for i, sequence in enumerate(answer_sequences):
for t, index in enumerate(sequence):
one_hot_answers[i, t, index] = 1
# Define model
model = Sequential([
Embedding(len(word_index) + 1, 64), # Removed input_length parameter
LSTM(64, return_sequences=True),
Dense(len(word_index) + 1, activation='softmax')
])
# Compile model with correct loss function
model.compile(loss=sparse_categorical_crossentropy, optimizer='adam', metrics=['accuracy'])
# Train model
model.fit(question_sequences, answer_sequences, epochs=1000, batch_size=32, steps_per_epoch=1)
# Function to generate response
def generate_response(input_text):
# Tokenize input text
input_sequence = tokenizer.texts_to_sequences([input_text])
input_sequence = pad_sequences(input_sequence, maxlen=max_sequence_length, padding='post')
# Generate response
predicted_sequence = model.predict(input_sequence)
# Decode predicted sequence
response = ""
for timestep in predicted_sequence[0]:
predicted_word_index = np.argmax(timestep)
if predicted_word_index in word_index.values():
predicted_word = next(word for word, idx in word_index.items() if idx == predicted_word_index)
if predicted_word == 'eos': # 'eos' marks the end of the sequence
break
response += predicted_word + " "
else:
response += ' ' # If predicted index not found in word_index
return response.strip()
# Interface
iface = gr.Interface(fn=generate_response, inputs="textbox", outputs="textbox", title="Cosmos AI - A conversational model",)
iface.launch()
|