Update app.py

app.py

@@ -90,18 +90,35 @@ def query_records(search_term):
         except:
             return ' '
 
-    # Fetch records based on the search term
-    query = Works().search_filter(abstract=search_term)
-
+    # Fetch records based on the search term in the abstract!
+    query = Works().search([search_term])
+    query_length = Works().search([search_term]).count()
+    
     records = []
-    for record in chain(*query.paginate(per_page=200)):
+    total_pages = (query_length + 199) // 200  # Calculate total number of pages
+    
+    for i, record in enumerate(chain(*query.paginate(per_page=200))):
         records.append(record)
+        
+        # Calculate progress from 0 to 0.1
+        progress = min(0.1, (i + 1) / query_length * 0.1)
+        
+        # Update progress bar
+        progress(progress, desc="Getting queried data...")
+
 
+    
     records_df = pd.DataFrame(records)
     records_df['abstract'] = [invert_abstract(t) for t in records_df['abstract_inverted_index']]
+
     records_df['parsed_publication'] = [get_pub(x) for x in records_df['primary_location']]
 
     
+    records_df['parsed_publication'] = records_df['parsed_publication'].fillna(' ')
+    records_df['abstract'] = records_df['abstract'].fillna(' ')
+    records_df['title'] = records_df['title'].fillna(' ')
+
+    
     return records_df
 
 
@@ -170,7 +187,7 @@ def create_embeddings(texts_to_embedd):
 
 def predict(text_input, sample_size_slider, reduce_sample_checkbox, progress=gr.Progress()):
 
-    progress(0.1, desc="Getting queried data...")
+    
     # get data.
     records_df = query_records(text_input)
     if reduce_sample_checkbox: