add files

README.md

@@ -1,3 +1,39 @@
 ---
 license: apache-2.0
 ---
+
+# AVeriTeC
+
+
+Data, knowledge store and source code to reproduce the baseline experiments for the [AVeriTeC](https://arxiv.org/abs/2305.13117) dataset, which will be used for the 7th [FEVER](https://fever.ai/) workshop co-hosted at EMNLP 2024.
+
+
+### Set up environment
+
+```
+conda create -n averitec python=3.11
+conda activate averitec
+
+pip install -r requirements.txt
+python -m spacy download en_core_web_lg
+python -m nltk.downloader punkt
+python -m nltk.downloader wordnet
+conda install pytorch pytorch-cuda=11.8 -c pytorch -c nvidia
+```
+
+### Scrape text from the URLs obtained by searching queries with the Google API.
+
+We provide up to 1000 URLs for each claim returned from a Google API search using different queries. This is a courtesy aimed at reducing the cost of using the Google Search API for participants of the shared task. The URL files can be found [here](https://huggingface.co/chenxwh/AVeriTeC/tree/main/data_store/urls).
+
+You can use your own scraping tool to extract sentences from the URLs. Alternatively, we have included a scraping tool for this purpose, which can be executed as follows. The processed files are also provided and can be found [here](https://huggingface.co/chenxwh/AVeriTeC/tree/main/data_store/knowledge_store).
+
+```
+bash script/scraper.sh <split> <start_idx> <end_idx> 
+# e.g., bash script/scraper.sh dev 0 500
+```
+
+### Rank the sentences in the knowledge store with BM25
+See [bm25_sentenes.py](https://huggingface.co/chenxwh/AVeriTeC/tree/main/src/reranking/bm25_sentenes.py) for more args
+```
+ python -m src.reranking.bm25_sentenes
+```

script/scraper.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+for ((i=$2;i<$3;i++))
+do
+  echo $i
+  python -m src.retrieval.scraper_for_knowledge_store -i ../AVeriTeC/data_store/"$1"_store/$i.tsv -o data_store/output_"$1" &
+done
+
+wait

src/reranking/bm25_sentenes.py

@@ -0,0 +1,118 @@
+import argparse
+import json
+import os
+import time
+import numpy as np
+import nltk
+from rank_bm25 import BM25Okapi
+
+
+def combine_all_sentences(knowledge_file):
+    # Get all the unique sentences from the scraped urks for this claim
+    sentences, urls = [], []
+
+    with open(knowledge_file, "r", encoding="utf-8") as json_file:
+        for i, line in enumerate(json_file):
+            data = json.loads(line)
+            sentences.extend(data["url2text"])
+            urls.extend([data["url"] for i in range(len(data["url2text"]))])
+    return sentences, urls, i + 1
+
+
+def retrieve_top_k_sentences(query, document, urls, top_k):
+    tokenized_docs = [nltk.word_tokenize(doc) for doc in document]
+    bm25 = BM25Okapi(tokenized_docs)
+    scores = bm25.get_scores(nltk.word_tokenize(query))
+    top_k_idx = np.argsort(scores)[::-1][:top_k]
+
+    return [document[i] for i in top_k_idx], [urls[i] for i in top_k_idx]
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        description="Get top 100 sentences for sentences in the knowlede store"
+    )
+    parser.add_argument(
+        "-k",
+        "--knowledge_store_dir",
+        type=str,
+        default="data_store/output_dev",
+        help="The path of the knowledge_store_dir containing json files with all the retrieved sentences.",
+    )
+    parser.add_argument(
+        "-c",
+        "--claim_file",
+        type=str,
+        default="data/dev.json",
+        help="The path of the file that stores the claim.",
+    )
+    parser.add_argument(
+        "-o",
+        "--json_output",
+        type=str,
+        default="data_store/dev_top_k.json",
+        help="The output dir for JSON files to save the top 100 sentences for each claim.",
+    )
+    parser.add_argument(
+        "--top_k",
+        default=100,
+        type=int,
+        help="How many documents should we pick out with BM25.",
+    )
+    parser.add_argument(
+        "-s",
+        "--start",
+        type=int,
+        default=0,
+        help="Staring index of the files to process.",
+    )
+    parser.add_argument(
+        "-e", "--end", type=int, default=-1, help="End index of the files to process."
+    )
+
+    args = parser.parse_args()
+
+    with open(args.claim_file, "r", encoding="utf-8") as json_file:
+        target_examples = json.load(json_file)
+
+    if args.end == -1:
+        args.end = len(os.listdir(args.knowledge_store_dir))
+        print(args.end)
+
+    files_to_process = list(range(args.start, args.end))
+    total = len(files_to_process)
+
+    with open(args.json_output, "w", encoding="utf-8") as output_json:
+        done = 0
+        for idx, example in enumerate(target_examples):
+            # Load the knowledge store for this example
+            if idx in files_to_process:
+                print(f"Processing claim {idx}... Progress: {done + 1} / {total}")
+                document_in_sentences, sentence_urls, num_urls_this_claim = (
+                    combine_all_sentences(
+                        os.path.join(args.knowledge_store_dir, f"{idx}.json")
+                    )
+                )
+
+                print(
+                    f"Obtained {len(document_in_sentences)} sentenes from {num_urls_this_claim} urls."
+                )
+
+                # Retrieve top_k sentences with bm25
+                st = time.time()
+                top_k_sentences, top_k_urls = retrieve_top_k_sentences(
+                    example["claim"], document_in_sentences, sentence_urls, args.top_k
+                )
+                print(f"Top {args.top_k} retrieved. Time elapsed: {time.time() - st}.")
+
+                json_data = {
+                    "claim_id": idx,
+                    "claim": example["claim"],
+                    f"top_{args.top_k}": [
+                        {"sentence": sent, "url": url}
+                        for sent, url in zip(top_k_sentences, top_k_urls)
+                    ],
+                }
+                output_json.write(json.dumps(json_data, ensure_ascii=False) + "\n")
+                done += 1

src/retrieval/html2lines.py

@@ -0,0 +1,84 @@
+import sys
+from time import sleep
+import trafilatura
+from trafilatura.meta import reset_caches
+from trafilatura.settings import DEFAULT_CONFIG
+import spacy
+
+
+nlp = spacy.load("en_core_web_lg")
+
+
+DEFAULT_CONFIG.MAX_FILE_SIZE = 50000
+MIN_CHAR = 50
+MAX_CHAR = 5000
+
+
+def get_page(url):
+    page = None
+    for _ in range(3):
+        try:
+            # for website that is "maintaining", trafilatura "respect the retry of the html" and waits for 24 hours
+            page = trafilatura.fetch_url(url, config=DEFAULT_CONFIG)
+            assert page is not None
+            print("Fetched " + url, file=sys.stderr)
+            break
+        except:
+            sleep(3)
+    return page
+
+
+def url2lines(url):
+    page = get_page(url)
+
+    if page is None:
+        return []
+
+    lines = html2lines(page)
+    return lines
+
+
+def line_correction(lines, max_size=100):
+    out_lines = []
+    for line in lines:
+        if len(line) < MIN_CHAR:
+            continue
+
+        if len(line) > max_size:
+            doc = nlp(
+                line[:MAX_CHAR]
+            )  # We split lines into sentences, but for performance we take only the first 5k characters per line
+            stack = ""
+            for sent in doc.sents:
+                if len(stack) > 0:
+                    stack += " "
+                stack += str(sent).strip()
+                if len(stack) > max_size:
+                    out_lines.append(stack)
+                    stack = ""
+
+            if (
+                len(stack) > MIN_CHAR
+            ):  # Enusre every lines in the out_lines suffice the MIN_CHAR restriction
+                out_lines.append(stack)
+        else:
+            out_lines.append(line)
+
+    return out_lines
+
+
+def html2lines(page):
+    out_lines = []
+
+    if len(page.strip()) == 0 or page is None:
+        return out_lines
+
+    text = trafilatura.extract(page, config=DEFAULT_CONFIG)
+    reset_caches()
+
+    if text is None:
+        return out_lines
+
+    return text.split(
+        "\n"
+    )  # We just spit out the entire page, so need to reformat later.

src/retrieval/scraper_for_knowledge_store.py

@@ -0,0 +1,158 @@
+import os
+import argparse
+import csv
+from time import sleep
+import time
+import json
+import numpy as np
+import fitz
+import pandas as pd
+import requests
+from src.retrieval.html2lines import url2lines, line_correction
+
+csv.field_size_limit(100000000)
+
+MAX_RETRIES = 3
+TIMEOUT = 5  # time limit for request
+
+
+def scrape_text_from_url(url, temp_name):
+    response = None
+    for attempt in range(MAX_RETRIES):
+        try:
+            response = requests.get(url, timeout=TIMEOUT)
+        except requests.RequestException as e:
+            if attempt < MAX_RETRIES - 1:
+                sleep(3)  # Wait before retrying
+
+    if (
+        response is None or response.status_code == 503
+    ):  # trafilatura does not handle retry with 503, often waiting 24hours as overwriten by the html
+        return []
+
+    if url.endswith(".pdf"):
+        with open(f"pdf_dir/{temp_name}.pdf", "wb") as f:
+            f.write(response.content)
+
+        extracted_text = ""
+        doc = fitz.open(f"pdf_dir/{temp_name}.pdf")
+        for page in doc:  # iterate the document pages
+            extracted_text += page.get_text() if page.get_text() else ""
+
+        return line_correction(extracted_text.split("\n"))
+
+    return line_correction(url2lines(url))
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description="Scraping text from URL")
+    parser.add_argument(
+        "-i",
+        "--tsv_input_file",
+        type=str,
+        help="The path of the input files containing URLs from Google search.",
+    )
+    parser.add_argument(
+        "-o",
+        "--json_output_dir",
+        type=str,
+        default="output",
+        help="The output JSON file to save the scraped data.",
+    )
+    parser.add_argument(
+        "--overwrite_out_file",
+        action="store_true",
+    )
+
+    args = parser.parse_args()
+
+    assert (
+        os.path.splitext(args.tsv_input_file)[-1] == ".tsv"
+    ), "The input should be a tsv file."
+
+    os.makedirs(args.json_output_dir, exist_ok=True)
+
+    total_scraped, empty, total_failed = 0, 0, 0
+
+    print(f"Processing files {args.tsv_input_file}")
+
+    st = time.time()
+
+    claim_id = os.path.splitext(os.path.basename(args.tsv_input_file))[0]
+    json_output_path = os.path.join(args.json_output_dir, f"{claim_id}.json")
+
+    lines_skipped = 0
+    if os.path.exists(json_output_path):
+        if args.overwrite_out_file:
+            os.remove(json_output_path)
+        else:
+            with open(json_output_path, "r", encoding="utf-8") as json_file:
+                existing_data = json_file.readlines()
+                lines_skipped = len(existing_data)
+                print(f"    Skipping {lines_skipped} lines in {json_output_path}")
+
+    # Some tsv files will fail to be laoded, try all 4 different libs to to load them
+    try:
+        df = pd.read_csv(args.tsv_input_file, sep="\t", header=None)
+        data = df.values
+        print("Data loaded successfully with Pandas.")
+
+    except Exception as e:
+        print("Error loading with csv:", e)
+        try:
+            data = np.genfromtxt(
+                args.tsv_input_file, delimiter="\t", dtype=None, encoding=None
+            )
+            print("Data loaded successfully with NumPy.")
+        except Exception as e:
+            print("Error loading with NumPy:", e)
+            # If NumPy loading fails, attempt to load with Pandas
+            try:
+                data = []
+                with open(args.tsv_input_file, "r", newline="") as tsvfile:
+                    reader = csv.reader(tsvfile, delimiter="\t")
+                    for row in reader:
+                        data.append(row)
+                print("Data loaded successfully with csv.")
+            except Exception as e:
+                print("Error loading with csv:", e)
+                data = None
+
+    if len(data) == lines_skipped:
+        print("    No more lines need to be processed!")
+    else:
+        with open(json_output_path, "a", encoding="utf-8") as json_file:
+            for index, row in enumerate(data):
+                if index < lines_skipped:
+                    continue
+                url = row[2]
+                json_data = {
+                    "claim_id": claim_id,
+                    "type": row[1],
+                    "query": row[3],
+                    "url": url,
+                    "url2text": [],
+                }
+                print(f"Scraping text for url_{index}: {url}!")
+                try:
+                    scrape_result = scrape_text_from_url(url, claim_id)
+                    json_data["url2text"] = scrape_result
+
+                    if len(json_data["url2text"]) > 0:
+                        total_scraped += 1
+                    else:
+                        empty += 1
+
+                except Exception as e:
+                    total_failed += 1
+
+            json_file.write(json.dumps(json_data, ensure_ascii=False) + "\n")
+            json_file.flush()
+
+        print(f"Output for {args.tsv_input_file} saved to {json_output_path}")
+        elapsed_time = time.time() - st
+        elapsed_minutes = int(elapsed_time // 60)
+        elapsed_seconds = int(elapsed_time % 60)
+        print(f"Time elapsed: {elapsed_minutes}min {elapsed_seconds}sec")
+        print(f"{total_scraped} scraped, {empty} empty, {total_failed} failed")