Spaces:
Sleeping
Sleeping
File size: 8,134 Bytes
edaf37d |
1 2 |
{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-08-15T14:44:00.362360Z","iopub.status.busy":"2024-08-15T14:44:00.361946Z","iopub.status.idle":"2024-08-15T14:44:01.523645Z","shell.execute_reply":"2024-08-15T14:44:01.522395Z","shell.execute_reply.started":"2024-08-15T14:44:00.362321Z"},"trusted":true},"outputs":[],"source":["# This Python 3 environment comes with many helpful analytics libraries installed\n","# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n","# For example, here's several helpful packages to load\n","\n","import numpy as np # linear algebra\n","import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","\n","# Input data files are available in the read-only \"../input/\" directory\n","# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n","\n","import os\n","for dirname, _, filenames in os.walk('/kaggle/input'):\n"," for filename in filenames:\n"," print(os.path.join(dirname, filename))\n","\n","# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n","# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-15T14:44:01.526545Z","iopub.status.busy":"2024-08-15T14:44:01.525934Z","iopub.status.idle":"2024-08-15T14:44:01.846387Z","shell.execute_reply":"2024-08-15T14:44:01.845152Z","shell.execute_reply.started":"2024-08-15T14:44:01.526503Z"},"trusted":true},"outputs":[],"source":["movie_details = pd.read_json('/kaggle/input/movie-details/IMDB_movie_details.json', lines=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-15T14:44:01.848814Z","iopub.status.busy":"2024-08-15T14:44:01.848415Z","iopub.status.idle":"2024-08-15T14:44:17.628604Z","shell.execute_reply":"2024-08-15T14:44:17.627484Z","shell.execute_reply.started":"2024-08-15T14:44:01.848783Z"},"trusted":true},"outputs":[],"source":["reviews = pd.read_json('/kaggle/input/bad-words-flag/better_reviews.json')"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-15T14:44:17.631413Z","iopub.status.busy":"2024-08-15T14:44:17.631039Z","iopub.status.idle":"2024-08-15T14:44:17.652198Z","shell.execute_reply":"2024-08-15T14:44:17.650772Z","shell.execute_reply.started":"2024-08-15T14:44:17.631381Z"},"trusted":true},"outputs":[],"source":["print(movie_details.head())\n","print(reviews.head())"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-15T14:44:17.663776Z","iopub.status.busy":"2024-08-15T14:44:17.663306Z","iopub.status.idle":"2024-08-15T14:44:23.371720Z","shell.execute_reply":"2024-08-15T14:44:23.370184Z","shell.execute_reply.started":"2024-08-15T14:44:17.663734Z"},"trusted":true},"outputs":[],"source":["from sklearn.feature_extraction.text import TfidfVectorizer\n","from sklearn.metrics.pairwise import cosine_similarity\n","from transformers import BertTokenizer\n","import torch\n","from torch.utils.data import Dataset, DataLoader\n","from sklearn.model_selection import train_test_split\n","\n","# Preprocess and merge data\n","movie_details.dropna(subset=['plot_synopsis', 'plot_summary'], inplace=True)\n","reviews.dropna(subset=['review_text'], inplace=True)\n","data = pd.merge(reviews, movie_details, on='movie_id')\n","\n","# data = data.head(10000)\n","data.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-15T14:44:23.373876Z","iopub.status.busy":"2024-08-15T14:44:23.373308Z","iopub.status.idle":"2024-08-15T14:44:23.382136Z","shell.execute_reply":"2024-08-15T14:44:23.380852Z","shell.execute_reply.started":"2024-08-15T14:44:23.373843Z"},"trusted":true},"outputs":[],"source":["# Function to split the synopsis into three parts\n","def split_synopsis(text):\n"," parts = len(text.split()) // 3\n"," return text.split()[:parts], text.split()[parts:2*parts], text.split()[2*parts:]\n","\n","# Calculate the proximity of review text to the end of the plot synopsis\n","def calculate_proximity(review, synopsis):\n"," _, _, end = split_synopsis(synopsis)\n"," vectorizer = TfidfVectorizer()\n"," vectors = vectorizer.fit_transform([review, ' '.join(end)])\n"," return cosine_similarity(vectors)[0, 1]"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-15T14:44:55.099211Z","iopub.status.busy":"2024-08-15T14:44:55.098756Z","iopub.status.idle":"2024-08-15T14:45:03.186728Z","shell.execute_reply":"2024-08-15T14:45:03.184877Z","shell.execute_reply.started":"2024-08-15T14:44:55.099178Z"},"trusted":true},"outputs":[],"source":["from nltk.tokenize import sent_tokenize\n","from nltk.sentiment import SentimentIntensityAnalyzer\n","from sklearn.preprocessing import MinMaxScaler\n","import nltk\n","\n","nltk.download('punkt')\n","nltk.download('vader_lexicon')\n","\n","counter = 0\n","\n","def calculate_sentiment_proximity(review, synopsis):\n"," global counter\n"," counter += 1\n"," if counter % 5000 == 0:\n"," print(counter, \"Records Ended!\")\n"," review_sentences = sent_tokenize(review)\n"," synopsis_sentences = sent_tokenize(synopsis)\n"," vectorizer = TfidfVectorizer()\n"," sentiment_analyzer = SentimentIntensityAnalyzer()\n","\n"," if len(synopsis_sentences) == 0:\n"," return 0\n"," # Vectorize the synopsis\n"," synopsis_vectors = vectorizer.fit_transform(synopsis_sentences)\n"," synopsis_sentiments = [sentiment_analyzer.polarity_scores(sentence)['compound'] for sentence in synopsis_sentences]\n","\n"," proximity_scores = []\n"," for sentence in review_sentences:\n"," sentence_vector = vectorizer.transform([sentence])\n"," sentence_sentiment = sentiment_analyzer.polarity_scores(sentence)['compound']\n"," similarities = cosine_similarity(sentence_vector, synopsis_vectors)[0]\n","\n"," # Weighing similarity by sentiment intensity and position\n"," sentiment_weights = [abs(sentence_sentiment - s_sentiment) for s_sentiment in synopsis_sentiments]\n"," weighted_similarities = similarities * np.array(sentiment_weights)\n"," proximity_scores.append(weighted_similarities.max())\n","\n"," return np.mean(proximity_scores)\n","\n","# Applying the function to calculate proximity based on sentiment\n","data['end_proximity'] = data.apply(lambda x: calculate_sentiment_proximity(x['review_text'], x['plot_synopsis']), axis=1)"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.status.busy":"2024-08-15T14:35:59.289260Z","iopub.status.idle":"2024-08-15T14:35:59.289684Z","shell.execute_reply":"2024-08-15T14:35:59.289489Z","shell.execute_reply.started":"2024-08-15T14:35:59.289473Z"},"trusted":true},"outputs":[],"source":["scaler = MinMaxScaler()\n","data['end_proximity'] = scaler.fit_transform(data[['end_proximity']])"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["data.to_json('/kaggle/input/final_dataset2.json', orient='records', lines=True)"]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"datasetId":5547860,"sourceId":9179201,"sourceType":"datasetVersion"},{"datasetId":5547886,"sourceId":9179232,"sourceType":"datasetVersion"}],"dockerImageVersionId":30746,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"}},"nbformat":4,"nbformat_minor":4}
|