Spaces:
Sleeping
Sleeping
| import pandas as pd | |
| from fuzzywuzzy import fuzz | |
| from collections import Counter | |
| from nltk.stem import PorterStemmer | |
| from ast import literal_eval | |
| from typing import Union, List | |
| import streamlit as st | |
| class KBVQAEvaluator: | |
| def __init__(self): | |
| """ | |
| Initialize the VQA Processor with the dataset and configuration settings. | |
| """ | |
| self.data_path = 'Files/evaluation_results_final.xlsx' | |
| self.use_fuzzy = False | |
| self.stemmer = PorterStemmer() | |
| self.scores_df = pd.read_excel(self.data_path, sheet_name="Scores") | |
| self.df = pd.read_excel(self.data_path, sheet_name="Main Data") | |
| self.vqa_scores = {} | |
| self.exact_match_scores = {} | |
| def stem_answers(self, answers: Union[str, List[str]]) -> Union[str, List[str]]: | |
| """ | |
| Apply Porter Stemmer to either a single string or a list of strings. | |
| """ | |
| if isinstance(answers, list): | |
| return [" ".join(self.stemmer.stem(word.strip()) for word in answer.split()) for answer in answers] | |
| else: | |
| words = answers.split() | |
| return " ".join(self.stemmer.stem(word.strip()) for word in words) | |
| def calculate_vqa_score(self, ground_truths, model_answer): | |
| """ | |
| Calculate VQA score based on the number of matching answers, with optional fuzzy matching. | |
| """ | |
| if self.use_fuzzy: | |
| fuzzy_matches = sum(fuzz.partial_ratio(model_answer, gt) >= 80 for gt in ground_truths) | |
| return min(fuzzy_matches / 3, 1) | |
| else: | |
| count = Counter(ground_truths) | |
| return min(count.get(model_answer, 0) / 3, 1) | |
| def calculate_exact_match_score(self, ground_truths, model_answer): | |
| """ | |
| Calculate Exact Match score, with optional fuzzy matching. | |
| """ | |
| if self.use_fuzzy: | |
| return int(any(fuzz.partial_ratio(model_answer, gt) >= 80 for gt in ground_truths)) | |
| else: | |
| return int(model_answer in ground_truths) | |
| def evaluate(self): | |
| """ | |
| Process the DataFrame: stem answers, calculate scores, and store results. | |
| """ | |
| self.df['raw_answers_stemmed'] = self.df['raw_answers'].apply(literal_eval).apply(self.stem_answers) | |
| model_configurations = ['caption+detic', 'caption+yolov5', 'only_caption', 'only_detic', 'only_yolov5'] | |
| model_names = ['13b', '7b'] | |
| for name in model_names: | |
| for config in model_configurations: | |
| full_config = f'{name}_{config}' | |
| self.df[f'{full_config}_stemmed'] = self.df[full_config].apply(self.stem_answers) | |
| self.df[f'vqa_score_{full_config}'] = self.df.apply(lambda x: self.calculate_vqa_score(x['raw_answers_stemmed'], x[f'{full_config}_stemmed']), axis=1) | |
| self.df[f'exact_match_score_{full_config}'] = self.df.apply(lambda x: self.calculate_exact_match_score(x['raw_answers_stemmed'], x[f'{full_config}_stemmed']), axis=1) | |
| self.vqa_scores[full_config] = round(self.df[f'vqa_score_{full_config}'].mean()*100, 2) | |
| self.exact_match_scores[full_config] = round(self.df[f'exact_match_score_{full_config}'].mean()*100, 2) | |
| def save_results(self): | |
| # Create a DataFrame for the scores | |
| scores_data = { | |
| 'Model Configuration': list(self.vqa_scores.keys()), | |
| 'VQA Score': list(self.vqa_scores.values()), | |
| 'Exact Match Score': list(self.exact_match_scores.values()) | |
| } | |
| scores_df = pd.DataFrame(scores_data) | |
| # Saving the scores DataFrame to an Excel file | |
| with pd.ExcelWriter('evaluation_results_final.xlsx', engine='openpyxl', mode='w') as writer: | |
| scores_df.to_excel(writer, sheet_name='Scores', index=False) | |
| def run_evaluator(self): | |
| #evaluator.evaluate() | |
| st.table(self.scores_df) | |
| st.dataframe(self.scores_df, width=900, height=500) | |
| #print(evaluator.exact_match_scores) | |
| #evaluator.save_results() | |