Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| # -*- coding: utf-8 -*- | |
| """ | |
| @author: Hamza Farooq | |
| """ | |
| from string import punctuation | |
| from collections import Counter | |
| from heapq import nlargest | |
| import os | |
| from sentence_transformers import SentenceTransformer, CrossEncoder, util | |
| import datetime | |
| import streamlit as st | |
| import matplotlib.pyplot as plt | |
| from wordcloud import WordCloud | |
| from matplotlib import pyplot as plt | |
| from rank_bm25 import BM25Okapi | |
| from sklearn.feature_extraction import _stop_words | |
| import string | |
| from tqdm.autonotebook import tqdm | |
| import numpy as np | |
| import pandas as pd | |
| from sentence_transformers import SentenceTransformer | |
| import scipy.spatial | |
| import pickle | |
| from sentence_transformers import SentenceTransformer, util | |
| import torch | |
| # import utils as utl | |
| import time | |
| import torch | |
| import transformers | |
| from transformers import BartTokenizer, BartForConditionalGeneration | |
| from string import punctuation | |
| # tr = BartTokenizer.from_pretrained('facebook/bart-large-cnn') | |
| import numpy as np | |
| import pandas as pd | |
| from sentence_transformers import SentenceTransformer | |
| import scipy.spatial | |
| from sentence_transformers import SentenceTransformer, util | |
| import torch | |
| def main(): | |
| # Settings | |
| st.set_page_config(layout="wide", page_title='Paris Hotel Finder', page_icon="🎈" ) | |
| from string import punctuation | |
| punctuation=punctuation+ '\n' | |
| from sentence_transformers import SentenceTransformer, util | |
| import torch | |
| import numpy as np | |
| import pandas as pd | |
| from sentence_transformers import SentenceTransformer | |
| import scipy.spatial | |
| from sentence_transformers import SentenceTransformer, util | |
| import torch | |
| #import os | |
| def load_model(): | |
| return SentenceTransformer('all-MiniLM-L6-v2'),SentenceTransformer('multi-qa-MiniLM-L6-cos-v1'),CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2') | |
| embedder,bi_encoder,cross_encoder = load_model() | |
| #original_title = '<p style="font-family:IBM Mono; color:Blue; font-size: 20px;">Original image</p>' | |
| st.title("travelle - Parisian Hotel Finder") | |
| with st.expander("ℹ️ - About this app", expanded=True): | |
| st.write( | |
| """ | |
| - travelle is a hotel search engine that allows users to enter free text query to make the search result personalized to user preference as opposed to other travel websites where a user has to spend hours going through hotel list. | |
| - We use natural language processing and big data to return results customized for your preferences. | |
| - A user can enter just about anything and we will narrow the results to what closely matches your requirements. | |
| - For e.g. a user can enter a query like "Hotel near the Eiffel and cheaper than $300 per night with free breakfast" and we will find the closest results | |
| """ | |
| ) | |
| punctuation=punctuation+ '\n' | |
| #import os | |
| # embedder = SentenceTransformer('all-MiniLM-L6-v2') | |
| def lower_case(input_str): | |
| input_str = input_str.lower() | |
| return input_str | |
| df_all = pd.read_csv('paris_clean_newer.csv') | |
| df_combined = df_all.sort_values(['Hotel']).groupby('Hotel', sort=False).text.apply(''.join).reset_index(name='all_review') | |
| df_combined_paris_summary = pd.read_csv('df_combined_paris.csv') | |
| df_combined_paris_summary = df_combined_paris_summary[['Hotel','summary']] | |
| import re | |
| # df_combined = pd.read_csv('df_combined.csv') | |
| df_combined['all_review'] = df_combined['all_review'].apply(lambda x: re.sub('[^a-zA-z0-9\s]','',x)) | |
| df_combined['all_review']= df_combined['all_review'].apply(lambda x: lower_case(x)) | |
| df_basic = df_all[['Hotel','description','price_per_night']].drop_duplicates() | |
| df_basic = df_basic.merge(df_combined_paris_summary,how='left') | |
| df_combined_e = df_combined.merge(df_basic) | |
| df_combined_e['all_review'] =df_combined_e['description']+ df_combined_e['all_review'] + df_combined_e['price_per_night'] | |
| df = df_combined_e.copy() | |
| df_sentences = df_combined_e.set_index("all_review") | |
| df_sentences = df_sentences["Hotel"].to_dict() | |
| df_sentences_list = list(df_sentences.keys()) | |
| import pandas as pd | |
| from tqdm import tqdm | |
| from sentence_transformers import SentenceTransformer, util | |
| df_sentences_list = [str(d) for d in tqdm(df_sentences_list)] | |
| # | |
| corpus = df_sentences_list | |
| # corpus_embeddings = embedder.encode(corpus,show_progress_bar=True) | |
| corpus_embeddings = np.load('embeddings.npy') | |
| bi_encoder.max_seq_length = 512 #Truncate long passages to 256 tokens | |
| top_k = 32 #Number of passages we want to retrieve with the bi-encoder | |
| #The bi-encoder will retrieve 100 documents. We use a cross-encoder, to re-rank the results list to improve the quality | |
| # corpus_embeddings_h = np.load('embeddings_h_r.npy') | |
| with open('corpus_embeddings_bi_encoder.pickle', 'rb') as pkl: | |
| doc_embedding = pickle.load(pkl) | |
| with open('tokenized_corpus.pickle', 'rb') as pkl: | |
| tokenized_corpus = pickle.load(pkl) | |
| bm25 = BM25Okapi(tokenized_corpus) | |
| passages = corpus | |
| # We lower case our text and remove stop-words from indexing | |
| def bm25_tokenizer(text): | |
| tokenized_doc = [] | |
| for token in text.lower().split(): | |
| token = token.strip(string.punctuation) | |
| if len(token) > 0: | |
| tokenized_doc.append(token) | |
| return tokenized_doc | |
| def search(query): | |
| # q = [str(userinput)] | |
| # doc = nlp(str(userinput)) | |
| # ent_html = displacy.render(doc, style="ent", jupyter=False) | |
| # Display the entity visualization in the browser: | |
| st.markdown(query, unsafe_allow_html=True) | |
| ##### BM25 search (lexical search) ##### | |
| bm25_scores = bm25.get_scores(bm25_tokenizer(query)) | |
| top_n = np.argpartition(bm25_scores, -5)[-5:] | |
| bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n] | |
| bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True) | |
| bm25list = {} | |
| st.title("Top-5 lexical search (BM25) hits") | |
| for hit in bm25_hits[0:5]: | |
| row_dict = df.loc[df['all_review']== corpus[hit['corpus_id']]] | |
| st.subheader(row_dict['Hotel'].values[0]) | |
| de = df_basic.loc[df_basic.Hotel == row_dict['Hotel'].values[0]] | |
| st.write(f'\tPrice Per night: {de.price_per_night.values[0]}') | |
| st.write(f'Description: {de.description.values[0]}') | |
| st.expander(de.description.values[0],expanded=False) | |
| # try: | |
| # st.write('Summary') | |
| # st.expander(de.summary.values[0],expanded=False) | |
| # except: | |
| # None | |
| # doc = corpus[hit['corpus_id']] | |
| # kp.get_key_phrases(doc) | |
| bm25list[row_dict['Hotel'].values[0]] = de.description.values[0][0:200] | |
| #### Sematic Search ##### | |
| # Encode the query using the bi-encoder and find potentially relevant passages | |
| question_embedding = bi_encoder.encode(query, convert_to_tensor=True) | |
| # question_embedding = question_embedding.cuda() | |
| hits = util.semantic_search(question_embedding, corpus_embeddings, top_k=top_k) | |
| hits = hits[0] # Get the hits for the first query | |
| ##### Re-Ranking ##### | |
| # Now, score all retrieved passages with the cross_encoder | |
| cross_inp = [[query, passages[hit['corpus_id']]] for hit in hits] | |
| cross_scores = cross_encoder.predict(cross_inp) | |
| # Sort results by the cross-encoder scores | |
| for idx in range(len(cross_scores)): | |
| hits[idx]['cross-score'] = cross_scores[idx] | |
| # Output of top-5 hits from bi-encoder | |
| st.write("\n-------------------------\n") | |
| st.title("Top-5 Bi-Encoder Retrieval hits") | |
| hits = sorted(hits, key=lambda x: x['score'], reverse=True) | |
| for hit in hits[0:5]: | |
| # st.write("\t{:.3f}\t{}".format(hit['score'], passages[hit['corpus_id']].replace("\n", " "))) | |
| row_dict = df.loc[df['all_review']== corpus[hit['corpus_id']]] | |
| st.subheader(row_dict['Hotel'].values[0]) | |
| de = df_basic.loc[df_basic.Hotel == row_dict['Hotel'].values[0]] | |
| st.write(f'\tPrice Per night: {de.price_per_night.values[0]}') | |
| st.write(f'Description: {de.description.values[0]}') | |
| st.expander(de.description.values[0]) | |
| # try: | |
| # st.write('Summary') | |
| # st.expander(de.summary.values[0],expanded=False) | |
| # except: | |
| # None | |
| # Output of top-5 hits from re-ranker | |
| st.write("\n-------------------------\n") | |
| st.title("Top-5 Cross-Encoder Re-ranker hits") | |
| hits = sorted(hits, key=lambda x: x['cross-score'], reverse=True) | |
| for hit in hits[0:5]: | |
| # st.write("\t{:.3f}\t{}".format(hit['cross-score'], passages[hit['corpus_id']].replace("\n", " "))) | |
| row_dict = df.loc[df['all_review']== corpus[hit['corpus_id']]] | |
| st.subheader(row_dict['Hotel'].values[0]) | |
| de = df_basic.loc[df_basic.Hotel == row_dict['Hotel'].values[0]] | |
| st.write(f'\tPrice Per night: {de.price_per_night.values[0]}') | |
| st.write(f'Description: {de.description.values[0]}') | |
| st.expander(de.description.values[0]) | |
| # try: | |
| # st.write('Summary') | |
| # st.expander(de.summary.values[0],expanded=False) | |
| # except: | |
| # None | |
| sampletext = 'e.g. Hotel near Eiffel Tower with big rooms' | |
| userinput = st.text_input('Tell us what are you looking in your hotel?','e.g. Hotel near Eiffel Tower with big rooms',autocomplete="on") | |
| # da = st.date_input( | |
| # "Date Check-in", | |
| # datetime.date(2023, 6, 3)) | |
| # dst = st.date_input( | |
| # "Date Check-out", | |
| # datetime.date(2023, 6, 8)) | |
| if not userinput or userinput == sampletext: | |
| st.write("Please enter a query to get results") | |
| else: | |
| query = [str(userinput)] | |
| # doc = nlp(str(userinput)) | |
| search(str(userinput)) | |
| # We use cosine-similarity and torch.topk to find the highest 5 scores | |
| if __name__ == '__main__': | |
| main() | |