Added additional files and source data

.gitignore

@@ -1,4 +1 @@
 .venv
-scrape.py
-app_test.ipynb
-embed_test.ipynb

app.py

@@ -73,7 +73,6 @@ def get_recommendation(query, number_of_recommendations, genres, themes, rating,
             name = data['japanese']
         else:
             name = data['english']
-        english = data['english']
         description = data['description']
         review = data['reviews'][review_index]['text']
         image = data['image']

app_test.ipynb

@@ -0,0 +1,253 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\Jose\\Desktop\\Nuanced_Recommendation_System\\.venv\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:1617: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be deprecated in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "from sklearn.metrics.pairwise import cosine_similarity\n",
+    "\n",
+    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
+    "\n",
+    "import gradio as gr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_n_weighted_scores(embeddings, query, n, objective_weight, subjective_weight):\n",
+    "    query = [model.encode(query)]\n",
+    "\n",
+    "    weighted_scores = []\n",
+    "\n",
+    "    for key, value in embeddings.items():\n",
+    "        objective_embedding = value['objective_embedding']\n",
+    "        subjective_embeddings = value['subjective_embeddings']\n",
+    "        \n",
+    "        objective_score = cosine_similarity(query, objective_embedding).item()\n",
+    "        subjective_scores = cosine_similarity(query, subjective_embeddings)\n",
+    "\n",
+    "        max_score = 0\n",
+    "        max_review_index = 0\n",
+    "        for idx, score in enumerate(subjective_scores[0].tolist()):\n",
+    "            weighted_score = ((objective_score * objective_weight)+(score * subjective_weight))\n",
+    "            if weighted_score > max_score:\n",
+    "                max_score = weighted_score\n",
+    "                max_review_index = idx\n",
+    "        \n",
+    "        weighted_scores.append((key, max_score, max_review_index))\n",
+    "    \n",
+    "    return sorted(weighted_scores, key=lambda x: x[1], reverse=True)[:n]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def filter_anime(embeddings, genres, themes, rating):\n",
+    "    genres = set(genres)\n",
+    "    themes = set(themes)\n",
+    "    rating = set(rating)\n",
+    "\n",
+    "    filtered_anime = embeddings.copy()\n",
+    "    for key, anime in embeddings.items():\n",
+    "\n",
+    "        anime_genres = set(anime['genres'])\n",
+    "        anime_themes = set(anime['themes'])\n",
+    "        anime_rating = set([anime['rating']])\n",
+    "\n",
+    "        if genres.intersection(anime_genres) or 'ALL' in genres:\n",
+    "            pass\n",
+    "        else:\n",
+    "            filtered_anime.pop(key)\n",
+    "            continue\n",
+    "        if themes.intersection(anime_themes) or 'ALL' in themes:\n",
+    "            pass\n",
+    "        else:\n",
+    "            filtered_anime.pop(key)\n",
+    "            continue\n",
+    "        if rating.intersection(anime_rating) or 'ALL' in rating:\n",
+    "            pass\n",
+    "        else:\n",
+    "            filtered_anime.pop(key)\n",
+    "            continue\n",
+    "        \n",
+    "    return filtered_anime"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('./embeddings/data.json') as f:\n",
+    "        data = json.load(f)\n",
+    "        embeddings = data['embeddings']\n",
+    "        filters = data['filters']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_recommendation(query, number_of_recommendations, genres, themes, rating, objective_weight, subjective_weight):\n",
+    "    filtered_anime = filter_anime(embeddings, genres, themes, rating)\n",
+    "    results = []\n",
+    "    weighted_scores = get_n_weighted_scores(filtered_anime, query, number_of_recommendations, float(objective_weight), float(subjective_weight))\n",
+    "    for idx, (key, score, review_index) in enumerate(weighted_scores, start=1):\n",
+    "        data = embeddings[key]\n",
+    "        if not data['english']:\n",
+    "            name = data['japanese']\n",
+    "        else:\n",
+    "            name = data['english']\n",
+    "        description = data['description']\n",
+    "        review = data['reviews'][review_index]['text']\n",
+    "        image = data['image']\n",
+    "\n",
+    "        results.append(gr.Image(label=f\"Recommendation {idx}: {name}\",value=image, height=435, width=500, visible=True))\n",
+    "        results.append(gr.Textbox(label=f\"Synopsis\", value=description, max_lines=7, visible=True))\n",
+    "        results.append(gr.Textbox(label=f\"Most Relevant User Review\",value=review, max_lines=7, visible=True))\n",
+    "\n",
+    "    for _ in range(10-number_of_recommendations):\n",
+    "        results.append(gr.Image(visible=False))\n",
+    "        results.append(gr.Textbox(visible=False))\n",
+    "        results.append(gr.Textbox(visible=False))\n",
+    "    \n",
+    "    return results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7863\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7863/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\Jose\\Desktop\\Nuanced_Recommendation_System\\.venv\\Lib\\site-packages\\gradio\\analytics.py:106: UserWarning: IMPORTANT: You are using gradio version 4.44.1, however version 5.0.1 is available, please upgrade. \n",
+      "--------\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "with gr.Blocks(theme=gr.themes.Soft(primary_hue='red')) as demo:\n",
+    "    with gr.Row():\n",
+    "        with gr.Column():\n",
+    "            gr.Markdown(\n",
+    "                '''\n",
+    "                # Welcome to the Nuanced Recommendation System!\n",
+    "                ### This system **combines** both objective (synopsis, episode count, themes) and subjective (user reviews) data, in order to recommend the most approprate anime. Feel free to refine using the **optional** filters below! \n",
+    "                '''\n",
+    "            )\n",
+    "        with gr.Column():\n",
+    "            pass\n",
+    "        \n",
+    "\n",
+    "    with gr.Row():\n",
+    "        with gr.Column() as input_col:\n",
+    "            query = gr.Textbox(label=\"What are you looking for?\")\n",
+    "            number_of_recommendations = gr.Slider(label= \"# of Recommendations\", minimum=1, maximum=10, value=3, step=1)\n",
+    "            genres = gr.Dropdown(label='Genres',multiselect=True,choices=filters['genres'], value=['ALL'])\n",
+    "            themes = gr.Dropdown(label='Themes',multiselect=True,choices=filters['themes'], value=['ALL'])\n",
+    "            rating = gr.Dropdown(label='Rating',multiselect=True,choices=filters['rating'], value=['ALL'])\n",
+    "            objective_weight = gr.Slider(label= \"Objective Weight\", minimum=0, maximum=1, value=.5, step=.1)\n",
+    "            subjective_weight = gr.Slider(label= \"Subjective Weight\", minimum=0, maximum=1, value=.5, step=.1)\n",
+    "            submit_btn = gr.Button(\"Submit\")\n",
+    "\n",
+    "            examples = gr.Examples(\n",
+    "                examples=[\n",
+    "                    ['A sci-fi anime set in a future where AI and robots have become self-aware', 3, ['Action', 'Sci-Fi', 'Fantasy'], ['ALL'], ['PG-13 - Teens 13 or older'], .8, .2],\n",
+    "                    ['An anime where a group of students form a band, and the story focuses on their personal growth and struggles with adulthood', 5, ['ALL'], ['Music'], ['PG-13 - Teens 13 or older', 'R - 17+ (violence & profanity)'], .3, .7],\n",
+    "                    ['An anime where the main character starts as a villain but slowly redeems themselves', 3, ['Suspense', 'Action'], ['ALL'], ['PG-13 - Teens 13 or older', 'R - 17+ (violence & profanity)'], .2, .8],\n",
+    "                ],\n",
+    "                inputs=[query, number_of_recommendations, genres, themes, rating, objective_weight, subjective_weight],\n",
+    "            )\n",
+    "\n",
+    "        outputs = []\n",
+    "        with gr.Column():\n",
+    "            for i in range(10):\n",
+    "                with gr.Row():\n",
+    "                    with gr.Column():\n",
+    "                        outputs.append(gr.Image(height=435, width=500, visible=False))\n",
+    "                    with gr.Column():\n",
+    "                        outputs.append(gr.Textbox(max_lines=7, visible=False))\n",
+    "                        outputs.append(gr.Textbox(max_lines=7, visible=False))\n",
+    "                        \n",
+    "\n",
+    "    submit_btn.click(\n",
+    "        get_recommendation,\n",
+    "        [query, number_of_recommendations, genres, themes, rating, objective_weight, subjective_weight],\n",
+    "        outputs\n",
+    "    )\n",
+    "\n",
+    "    demo.launch()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

embed_test.ipynb

@@ -0,0 +1,142 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\Jose\\Desktop\\Nuanced_Recommendation_System\\.venv\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:13: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from tqdm.autonotebook import tqdm, trange\n",
+      "c:\\Users\\Jose\\Desktop\\Nuanced_Recommendation_System\\.venv\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:1617: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be deprecated in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "\n",
+    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def update_filters(filters, data):\n",
+    "    for key, value in filters.items():\n",
+    "        dont_add = ['Boys Love', 'Erotica', 'Girls Love', 'Hentai', 'Ecchi', 'Gore', 'Crossdressing', 'Magical Sex Shift', 'Rx - Hentai', 'R+ - Mild Nudity']\n",
+    "        if data[key] and key == 'rating':\n",
+    "            if data[key] not in dont_add:\n",
+    "                value.add(data[key])\n",
+    "        else:\n",
+    "            for val in data[key]:\n",
+    "                if val and val not in dont_add:\n",
+    "                    value.add(val)\n",
+    "    return filters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clean_filters(filters):\n",
+    "    for key, val in filters.items():\n",
+    "        val.add('ALL')\n",
+    "        filters[key] = list(val)\n",
+    "    return filters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print('Embedding Started')\n",
+    "\n",
+    "filters = {\n",
+    "'genres': set(),\n",
+    "'themes': set(),\n",
+    "'rating': set()\n",
+    "}\n",
+    "\n",
+    "embeddings = {}\n",
+    "for name in os.listdir('./anime'):\n",
+    "    with open(f\"./anime/{name}\", 'r') as file:\n",
+    "        data = json.load(file)\n",
+    "\n",
+    "    if not data:\n",
+    "        continue\n",
+    "\n",
+    "    filters = update_filters(filters, data)\n",
+    "\n",
+    "    name = name.replace('.json', '')\n",
+    "    \n",
+    "    data['image'] = f\"./images/{name}.jpg\"\n",
+    "\n",
+    "    text = f'''Episodes: {data['episodes']} \n",
+    "            Premiered: {data['premiered']} \n",
+    "            Broadcast: {data['broadcast']} \n",
+    "            Producers: {' '.join(data['producers'])} \n",
+    "            Licensors: {' '.join(data['licensors'])} \n",
+    "            Studios: {' '.join(data['studios'])} \n",
+    "            Source: {' '.join(data['source'])}  \n",
+    "            Genres: {' '.join(data['genres'])} \n",
+    "            Themes: {' '.join(data['themes'])} \n",
+    "            Demographic: {data['demographic']} \n",
+    "            Duration: {data['duration']} \n",
+    "            Rating: {data['rating']} \n",
+    "            Description: {data['description']}'''\n",
+    "    \n",
+    "    embeddings[name] = data.copy()\n",
+    "    \n",
+    "    embeddings[name]['objective_embedding'] = [model.encode(text).tolist()]\n",
+    "    subjective_embeddings = []\n",
+    "    for review in embeddings[name]['reviews']:\n",
+    "        text = review['text']\n",
+    "        subjective_embeddings.append(model.encode(text).tolist())\n",
+    "        data['review'] = text\n",
+    "    embeddings[name]['subjective_embeddings'] = subjective_embeddings\n",
+    "\n",
+    "filters = clean_filters(filters)\n",
+    "\n",
+    "with open('./embeddings/data.json', 'w') as f:\n",
+    "    json.dump({'embeddings':embeddings, 'filters': filters}, f)\n",
+    "\n",
+    "print('Embedding Complete')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scrape.py

@@ -0,0 +1,184 @@
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+import urllib
+
+import time
+import os
+import json
+import random
+
+driver = webdriver.Chrome()
+
+existing = set([int(name.replace('.json', '')) for name in os.listdir('./anime')])
+# total = set(range(100000))
+# difference = total.difference(existing)
+difference = range(3210, 10000)
+
+for i in difference:
+    random_num = random.choice([4])
+
+    driver.get(f"https://myanimelist.net/anime/{i}/")
+
+    try:
+        try:
+            wait = WebDriverWait(driver, 5)
+            wait.until(EC.presence_of_element_located((By.XPATH, "//div[@class='leftside']//a/img")))
+            image = driver.find_element(By.XPATH, "//div[@class='leftside']//a/img").get_attribute('src')
+            urllib.request.urlretrieve(image, f"./images/{i}.jpg")
+        except:
+            pass
+        try:
+            body = driver.find_element(By.TAG_NAME, 'body')
+        except:
+            body = ''
+        try:
+            description = driver.find_element(By.XPATH, "//p[@itemprop='description']").text
+        except:
+            description = ''
+        try:
+            synonyms = driver.find_element(By.XPATH, "//span[text() = 'Synonyms:']/..").text.replace('Synonyms:', '').strip()
+        except:
+            synonyms = ''
+        try:
+            japanese = driver.find_element(By.XPATH, "//span[text() = 'Japanese:']/..").text.replace('Japanese:', '').strip()
+        except:
+            japanese = ''
+        try:
+            driver.find_element(By.CLASS_NAME, 'js-anime-toggle-alternative-title-button').click()
+            english = driver.find_element(By.XPATH, "//span[text() = 'English:']/..").text.replace('English:', '').strip()
+        except:
+            english = ''
+        try:
+            type = driver.find_element(By.XPATH, "//span[text() = 'Type:']/..").text.replace('Type:', '').strip()
+        except:
+            type = ''
+
+        try:
+            episodes = driver.find_element(By.XPATH, "//span[text() = 'Episodes:']/..").text.replace('Episodes:', '').strip()
+        except:
+            episodes = ''
+        try:
+            premiered = driver.find_element(By.XPATH, "//span[text() = 'Premiered:']/..").text.replace('Premiered:', '').strip()
+        except:
+            premiered = ''
+        try:
+            broadcast = driver.find_element(By.XPATH, "//span[text() = 'Broadcast:']/..").text.replace('Broadcast:', '').strip()
+        except:
+            broadcast = ''
+        try:
+            producers = driver.find_element(By.XPATH, "//span[text() = 'Producers:']/..").text.replace('Producers:', '').strip()
+        except:
+            producers = ''
+        try:
+            licensors = driver.find_element(By.XPATH, "//span[text() = 'Licensors:']/..").text.replace('Licensors:', '').strip()
+        except:
+            licensors = ''
+        try:
+            studios = driver.find_element(By.XPATH, "//span[text() = 'Studios:']/..").text.replace('Studios:', '').strip()
+        except:
+            studios = ''
+        try:
+            source = driver.find_element(By.XPATH, "//span[text() = 'Source:']/..").text.replace('Source:', '').strip()
+        except:
+            source = ''
+
+        try:
+            genres = driver.find_element(By.XPATH, "//span[text() = 'Genres:']/..").text.replace('Genres:', '').strip()
+        except:
+            genres = ''
+        try:
+            themes = driver.find_element(By.XPATH, "//span[text() = 'Themes:']/..").text.replace('Themes:', '').strip()
+        except:
+            try:
+                themes = driver.find_element(By.XPATH, "//span[text() = 'Theme:']/..").text.replace('Theme:', '').strip()
+            except:
+                themes = ''
+
+        try:
+            demographic = driver.find_element(By.XPATH, "//span[text() = 'Demographic:']/..").text.replace('Demographic:', '').strip()
+        except:
+            demographic = ''
+        try:
+            duration = driver.find_element(By.XPATH, "//span[text() = 'Duration:']/..").text.replace('Duration:', '').strip()
+        except:
+            duration = ''
+        try:
+            rating = driver.find_element(By.XPATH, "//span[text() = 'Rating:']/..").text.replace('Rating:', '').strip()
+        except:
+            rating = ''  
+
+        time.sleep(2)  
+
+        try:
+            wait = WebDriverWait(driver, 10)
+            wait.until(EC.presence_of_element_located((By.XPATH, "//div[@id='horiznav_nav']//a[text() = 'Reviews']")))
+            driver.find_element(By.XPATH, "//div[@id='horiznav_nav']//a[text() = 'Reviews']").click()
+        except Exception as e:
+            print(e)
+            try:
+                driver.find_element(By.CLASS_NAME, 'error404')
+                with open(f"anime/{i}.json", "w") as outfile:
+                    json.dump({}, outfile)
+                continue
+            except Exception as e:
+                print(e)
+                driver.close()
+                time.sleep(150)
+                driver = webdriver.Chrome()
+                continue
+        driver.execute_script("window.scrollTo(0, 0)")
+
+        data = []
+        reviews = driver.find_elements(By.CLASS_NAME, 'js-review-element')
+        for review in reviews:
+            visible = review.find_element(By.CLASS_NAME, 'text')
+            sentiment = review.find_element(By.CLASS_NAME, 'tag')
+            wait = WebDriverWait(driver, 10)
+            wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'js-hidden')))
+            hidden = review.find_element(By.CLASS_NAME, 'js-hidden')
+            text = visible.text + hidden.get_attribute('textContent')
+            text = text.strip().replace('\n', ' ')
+            data.append({
+                'sentiment': sentiment.text, 
+                'text': text
+                })
+            
+        reviews = data
+
+        if not reviews:
+            continue
+
+        information = {
+            'synonyms': synonyms,
+            'japanese': japanese,
+            'english': english,
+            'type': type,
+            'episodes': episodes,
+            'premiered': premiered,
+            'broadcast': broadcast,
+            'producers': [x.strip() for x in producers.split(',')],
+            'licensors': [x.strip() for x in licensors.split(',')],
+            'studios': [x.strip() for x in studios.split(',')],
+            'source': [x.strip() for x in source.split(',')],
+            'genres': [x.strip() for x in genres.split(',')],
+            'themes': [x.strip() for x in themes.split(',')],
+            'demographic': demographic.split(','),
+            'duration': duration,
+            'rating': rating,
+            'description': description,
+            'reviews': reviews
+        }
+
+        with open(f"anime/{i}.json", "w") as outfile:
+            json.dump(information, outfile)
+        time.sleep(random_num)
+    except Exception as e:
+        print(e)
+        time.sleep(random_num)
+    
+
+driver.close()