Update app.py

app.py

@@ -47,6 +47,8 @@ import pandas as pd
 from pyalex import Works, Authors, Sources, Institutions, Concepts, Publishers, Funders
 from itertools import chain
 from compress_pickle import load, dump
+from urllib.parse import urlparse, parse_qs
+import re
 
 
 
@@ -67,62 +69,102 @@ import umap
 
 
 
-
-
-
-
-
-def query_records(search_term):
-    def invert_abstract(inv_index):
-        if inv_index is not None:
-            l_inv = [(w, p) for w, pos in inv_index.items() for p in pos]
-            return " ".join(map(lambda x: x[0], sorted(l_inv, key=lambda x: x[1])))
-        else:
+def openalex_url_to_pyalex_query(url):
+    """
+    Convert an OpenAlex search URL to a pyalex query.
+    
+    Args:
+    url (str): The OpenAlex search URL.
+    
+    Returns:
+    tuple: (Works object, dict of parameters)
+    """
+    parsed_url = urlparse(url)
+    query_params = parse_qs(parsed_url.query)
+    
+    # Initialize the Works object
+    query = Works()
+    
+    # Handle filters
+    if 'filter' in query_params:
+        filters = query_params['filter'][0].split(',')
+        for f in filters:
+            if ':' in f:
+                key, value = f.split(':', 1)
+                if key == 'default.search':
+                    query = query.search(value)
+                else:
+                    query = query.filter(**{key: value})
+    
+    # Handle sort
+    if 'sort' in query_params:
+        sort_params = query_params['sort'][0].split(',')
+        for s in sort_params:
+            if s.startswith('-'):
+                query = query.sort(**{s[1:]: 'desc'})
+            else:
+                query = query.sort(**{s: 'asc'})
+    
+    # Handle other parameters
+    params = {}
+    for key in ['page', 'per-page', 'sample', 'seed']:
+        if key in query_params:
+            params[key] = query_params[key][0]
+    
+    return query, params
+
+
+def invert_abstract(inv_index):
+    if inv_index is not None:
+        l_inv = [(w, p) for w, pos in inv_index.items() for p in pos]
+        return " ".join(map(lambda x: x[0], sorted(l_inv, key=lambda x: x[1])))
+    else:
+        return ' '
+
+def get_pub(x):
+    try: 
+        source = x['source']['display_name']
+        if source not in ['parsed_publication','Deleted Journal']:
+            return source
+        else: 
             return ' '
-
-    def get_pub(x):
-        try: 
-            source = x['source']['display_name']
-            if source not in ['parsed_publication','Deleted Journal']:
-                return source
-            else: 
-                return ' '
-        except:
+    except:
             return ' '
 
-    # Fetch records based on the search term in the abstract!
-    query = Works().search([search_term])
-    query_length = Works().search([search_term]).count()
+#def query_records(search_term):
+
+
+#     # Fetch records based on the search term in the abstract!
+#     query = Works().search([search_term])
+#     query_length = Works().search([search_term]).count()
     
-    records = []
-    #total_pages = (query_length + 199) // 200  # Calculate total number of pages
-    progress=gr.Progress()
+#     records = []
+#     #total_pages = (query_length + 199) // 200  # Calculate total number of pages
+#     progress=gr.Progress()
     
-    for i, record in progress.tqdm(enumerate(chain(*query.paginate(per_page=200)))):
-        records.append(record)
+#     for i, record in progress.tqdm(enumerate(chain(*query.paginate(per_page=200)))):
+#         records.append(record)
         
-        # Calculate progress from 0 to 0.1
-        #achieved_progress = min(0.1, (i + 1) / query_length * 0.1)
+#         # Calculate progress from 0 to 0.1
+#         #achieved_progress = min(0.1, (i + 1) / query_length * 0.1)
         
-        # Update progress bar
-        #progress(achieved_progress, desc="Getting queried data...")
+#         # Update progress bar
+#         #progress(achieved_progress, desc="Getting queried data...")
 
 
     
-    records_df = pd.DataFrame(records)
-    records_df['abstract'] = [invert_abstract(t) for t in records_df['abstract_inverted_index']]
+#     records_df = pd.DataFrame(records)
+#     records_df['abstract'] = [invert_abstract(t) for t in records_df['abstract_inverted_index']]
 
-    records_df['parsed_publication'] = [get_pub(x) for x in records_df['primary_location']]
+#     records_df['parsed_publication'] = [get_pub(x) for x in records_df['primary_location']]
 
     
-    records_df['parsed_publication'] = records_df['parsed_publication'].fillna(' ')
-    records_df['abstract'] = records_df['abstract'].fillna(' ')
-    records_df['title'] = records_df['title'].fillna(' ')
+#     records_df['parsed_publication'] = records_df['parsed_publication'].fillna(' ')
+#     records_df['abstract'] = records_df['abstract'].fillna(' ')
+#     records_df['title'] = records_df['title'].fillna(' ')
 
     
-    return records_df
-
-
+#     return records_df
 
 
 ################# Setting up the model for specter2 embeddings ###################
@@ -193,7 +235,38 @@ def predict(text_input, sample_size_slider, reduce_sample_checkbox, progress=gr.
 
     
     # get data.
-    records_df = query_records(text_input)
+
+    query, params = openalex_url_to_pyalex_query(text_input)
+    query_length = query.count()
+    
+    records = []
+    total_pages = (query_length + 199) // 200  # Calculate total number of pages
+
+    
+    for i, record in progress.tqdm(enumerate(chain(*query.paginate(per_page=200)))):
+        records.append(record)
+        
+        # Calculate progress from 0 to 0.1
+        achieved_progress = min(0., (i + 1) / query_length * 0.1)
+        
+        # Update progress bar
+        progress(achieved_progress, desc="Getting queried data...")
+
+
+    
+    records_df = pd.DataFrame(records)
+    records_df['abstract'] = [invert_abstract(t) for t in records_df['abstract_inverted_index']]
+
+    records_df['parsed_publication'] = [get_pub(x) for x in records_df['primary_location']]
+
+    
+    records_df['parsed_publication'] = records_df['parsed_publication'].fillna(' ')
+    records_df['abstract'] = records_df['abstract'].fillna(' ')
+    records_df['title'] = records_df['title'].fillna(' ')
+
+
+
+    
     if reduce_sample_checkbox:
         records_df = records_df.sample(sample_size_slider)
     print(records_df)