|
import os |
|
import argparse |
|
import json |
|
from tqdm import tqdm |
|
|
|
|
|
def load_url_text_map(knowledge_store_dir, claim_id): |
|
url_text_map = {} |
|
knowledge_file = os.path.join(knowledge_store_dir, f"{claim_id}.json") |
|
|
|
if os.path.exists(knowledge_file): |
|
with open(knowledge_file, "r") as f: |
|
for line in f: |
|
data = json.loads(line) |
|
url = data["url"] |
|
url2text = data["url2text"] |
|
concatenated_text = " ".join(url2text) |
|
url_text_map[url] = concatenated_text |
|
|
|
return url_text_map |
|
|
|
|
|
if __name__ == "__main__": |
|
parser = argparse.ArgumentParser( |
|
description="Add scraped_text field to the prediction file." |
|
) |
|
parser.add_argument( |
|
"-i", |
|
"--veracity_prediction_file", |
|
default="data_store/dev_veracity_prediction.json", |
|
help="Json file with the veracity predictions.", |
|
) |
|
parser.add_argument( |
|
"-o", |
|
"--output_file", |
|
default="data_store/dev_veracity_prediction_for_submission.json", |
|
help="Json file with the veracity predictions and the scraped_text.", |
|
) |
|
parser.add_argument( |
|
"--knowledge_store_dir", |
|
type=str, |
|
help="Directory of json files of the knowledge store containing url2text.", |
|
) |
|
args = parser.parse_args() |
|
|
|
predictions = [] |
|
with open(args.veracity_prediction_file) as f: |
|
predictions = json.load(f) |
|
|
|
for claim in tqdm(predictions, desc="Processing claims"): |
|
claim_id = claim["claim_id"] |
|
url_text_map = load_url_text_map(args.knowledge_store_dir, claim_id) |
|
|
|
|
|
for evidence in claim["evidence"]: |
|
url = evidence["url"] |
|
scraped_text = url_text_map.get(url) |
|
if scraped_text: |
|
evidence["scraped_text"] = scraped_text |
|
else: |
|
print( |
|
f"Warning: No scraped text found for claim_id {claim_id} and url {url}" |
|
) |
|
|
|
with open(args.output_file, "w", encoding="utf-8") as output_file: |
|
json.dump(predictions, output_file, ensure_ascii=False, indent=4) |
|
|
|
print(f"Updated JSON saved to {args.output_file}") |
|
|
