Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
rephrsse
Browse files- data/amazon/category_list.json +1425 -0
- data/amazon/intermediate/.gitkeep +0 -0
- download_emb.py +0 -41
- eval.py +0 -134
- generate_emb.py +0 -83
- interactive/pyvis_graph.py +3 -3
- load_dataset.ipynb +0 -508
- src/benchmarks/get_qa_dataset.py +9 -6
- src/benchmarks/qa_datasets/amazon.py +2 -2
- src/benchmarks/qa_datasets/mag.py +2 -2
- src/benchmarks/qa_datasets/primekg.py +3 -3
- src/benchmarks/qa_datasets/stark_qa.py +10 -2
- src/benchmarks/semistruct/amazon.py +4 -2
- src/benchmarks/semistruct/mag.py +1 -0
- src/models/__init__.py +0 -34
- src/models/llm_reranker.py +0 -100
- src/models/model.py +0 -106
- src/models/multi_vss.py +0 -82
- src/models/vss.py +0 -57
- src/tools/api_lib/huggingface.py +0 -18
data/amazon/category_list.json
ADDED
|
@@ -0,0 +1,1425 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
"jerseys",
|
| 3 |
+
"baseball caps",
|
| 4 |
+
"running",
|
| 5 |
+
"gun holsters",
|
| 6 |
+
"imported",
|
| 7 |
+
"equestrian sports",
|
| 8 |
+
"auto accessories",
|
| 9 |
+
"skiing",
|
| 10 |
+
"dcor",
|
| 11 |
+
"swimming",
|
| 12 |
+
"pants",
|
| 13 |
+
"protective gear",
|
| 14 |
+
"hunting knives",
|
| 15 |
+
"shirts",
|
| 16 |
+
"martial arts",
|
| 17 |
+
"jackets",
|
| 18 |
+
"skateboarding",
|
| 19 |
+
"shorts",
|
| 20 |
+
"yoga",
|
| 21 |
+
"archery",
|
| 22 |
+
"socks",
|
| 23 |
+
"airsoft",
|
| 24 |
+
"cell phone accessories",
|
| 25 |
+
"sports water bottles",
|
| 26 |
+
"camp kitchen",
|
| 27 |
+
"sport watches",
|
| 28 |
+
"optics",
|
| 29 |
+
"gloves",
|
| 30 |
+
"sports souvenirs",
|
| 31 |
+
"soccer",
|
| 32 |
+
"strength training equipment",
|
| 33 |
+
"backpacking packs",
|
| 34 |
+
"swimwear",
|
| 35 |
+
"sports equipment",
|
| 36 |
+
"grips",
|
| 37 |
+
"paintball",
|
| 38 |
+
"tennis",
|
| 39 |
+
"machine wash",
|
| 40 |
+
"skateboard parts",
|
| 41 |
+
"outdoor flags",
|
| 42 |
+
"gun maintenance",
|
| 43 |
+
"tack",
|
| 44 |
+
"weapons",
|
| 45 |
+
"snowboarding",
|
| 46 |
+
"artwork",
|
| 47 |
+
"golf clubs",
|
| 48 |
+
"goggles",
|
| 49 |
+
"boys",
|
| 50 |
+
"training equipment",
|
| 51 |
+
"baby clothing",
|
| 52 |
+
"climbing",
|
| 53 |
+
"tactical knives",
|
| 54 |
+
"gun scopes",
|
| 55 |
+
"terminal tackle",
|
| 56 |
+
"hiking daypacks",
|
| 57 |
+
"dance",
|
| 58 |
+
"girls",
|
| 59 |
+
"bedding",
|
| 60 |
+
"hunting apparel",
|
| 61 |
+
"other team sports",
|
| 62 |
+
"fixed blade hunting knives",
|
| 63 |
+
"fitness technology",
|
| 64 |
+
"winter sports accessories",
|
| 65 |
+
"decals",
|
| 66 |
+
"hard lures",
|
| 67 |
+
"hats",
|
| 68 |
+
"hunting accessories",
|
| 69 |
+
"polo shirts",
|
| 70 |
+
"on-course accessories",
|
| 71 |
+
"coolers",
|
| 72 |
+
"rifle scopes",
|
| 73 |
+
"gunsmithing tools",
|
| 74 |
+
"license plate frames",
|
| 75 |
+
"football",
|
| 76 |
+
"wristbands",
|
| 77 |
+
"camping furniture",
|
| 78 |
+
"tires",
|
| 79 |
+
"swords",
|
| 80 |
+
"diving suits",
|
| 81 |
+
"bikes",
|
| 82 |
+
"toy figures",
|
| 83 |
+
"boxing",
|
| 84 |
+
"wheels",
|
| 85 |
+
"fishing apparel",
|
| 86 |
+
"rods",
|
| 87 |
+
"disc sports",
|
| 88 |
+
"ornaments",
|
| 89 |
+
"basketball",
|
| 90 |
+
"casino equipment",
|
| 91 |
+
"drivetrain components",
|
| 92 |
+
"key chains",
|
| 93 |
+
"pet gear",
|
| 94 |
+
"reels",
|
| 95 |
+
"bats",
|
| 96 |
+
"balls",
|
| 97 |
+
"sunglasses",
|
| 98 |
+
"kayaking",
|
| 99 |
+
"surfing",
|
| 100 |
+
"helmets",
|
| 101 |
+
"folding hunting knives",
|
| 102 |
+
"footwear",
|
| 103 |
+
"folding knives",
|
| 104 |
+
"bowling",
|
| 105 |
+
"electrical equipment",
|
| 106 |
+
"personal defense equipment",
|
| 107 |
+
"throw blankets",
|
| 108 |
+
"mitts",
|
| 109 |
+
"personal care",
|
| 110 |
+
"head covers",
|
| 111 |
+
"polyester",
|
| 112 |
+
"wallets",
|
| 113 |
+
"earrings",
|
| 114 |
+
"player equipment",
|
| 115 |
+
"wetsuits",
|
| 116 |
+
"saddles",
|
| 117 |
+
"tents",
|
| 118 |
+
"boat engine parts",
|
| 119 |
+
"exercise bands",
|
| 120 |
+
"ice hockey",
|
| 121 |
+
"cotton",
|
| 122 |
+
"sleeping bags",
|
| 123 |
+
"golf balls",
|
| 124 |
+
"compression shorts",
|
| 125 |
+
"golf cart accessories",
|
| 126 |
+
"cardio training",
|
| 127 |
+
"adult helmets",
|
| 128 |
+
"fishing line",
|
| 129 |
+
"racquets",
|
| 130 |
+
"disc golf",
|
| 131 |
+
"mats",
|
| 132 |
+
"brake parts",
|
| 133 |
+
"golf equipment",
|
| 134 |
+
"artificial bait",
|
| 135 |
+
"headbands",
|
| 136 |
+
"waist packs",
|
| 137 |
+
"one-piece suits",
|
| 138 |
+
"equipment bags",
|
| 139 |
+
"sports sunglasses",
|
| 140 |
+
"baseball mitts",
|
| 141 |
+
"paracord",
|
| 142 |
+
"decks",
|
| 143 |
+
"golf club bags",
|
| 144 |
+
"lacrosse",
|
| 145 |
+
"standard skateboards",
|
| 146 |
+
"hooks",
|
| 147 |
+
"office products",
|
| 148 |
+
"gun scope mounts",
|
| 149 |
+
"toss games",
|
| 150 |
+
"drivers",
|
| 151 |
+
"gun sights",
|
| 152 |
+
"discs",
|
| 153 |
+
"fitness trackers",
|
| 154 |
+
"travel mugs",
|
| 155 |
+
"hardware",
|
| 156 |
+
"coffee mugs",
|
| 157 |
+
"boat covers",
|
| 158 |
+
"baseball bats",
|
| 159 |
+
"hydration packs",
|
| 160 |
+
"compression",
|
| 161 |
+
"scooters",
|
| 162 |
+
"towels",
|
| 163 |
+
"bracelets",
|
| 164 |
+
"chairs",
|
| 165 |
+
"sports medicine",
|
| 166 |
+
"hand wash",
|
| 167 |
+
"game calls",
|
| 168 |
+
"furniture",
|
| 169 |
+
"exercise videos",
|
| 170 |
+
"electronics",
|
| 171 |
+
"backpacks",
|
| 172 |
+
"officially licensed",
|
| 173 |
+
"handlebars",
|
| 174 |
+
"sweaters",
|
| 175 |
+
"tactical backpacks",
|
| 176 |
+
"bath",
|
| 177 |
+
"riding boots",
|
| 178 |
+
"photos",
|
| 179 |
+
"machine washable",
|
| 180 |
+
"lanterns",
|
| 181 |
+
"pedals",
|
| 182 |
+
"volleyball",
|
| 183 |
+
"leggings",
|
| 184 |
+
"leotards",
|
| 185 |
+
"game handling",
|
| 186 |
+
"rifles",
|
| 187 |
+
"gun belts",
|
| 188 |
+
"table tennis",
|
| 189 |
+
"trading cards",
|
| 190 |
+
"china",
|
| 191 |
+
"wall decals",
|
| 192 |
+
"football equipment",
|
| 193 |
+
"topwater lures",
|
| 194 |
+
"golf club parts",
|
| 195 |
+
"compasses",
|
| 196 |
+
"nets",
|
| 197 |
+
"longboards",
|
| 198 |
+
"putters",
|
| 199 |
+
"bows",
|
| 200 |
+
"tubes",
|
| 201 |
+
"ball markers",
|
| 202 |
+
"headlights",
|
| 203 |
+
"hydration",
|
| 204 |
+
"slippers",
|
| 205 |
+
"pet clothing",
|
| 206 |
+
"jigs",
|
| 207 |
+
"infielder's mitts",
|
| 208 |
+
"quartz movement",
|
| 209 |
+
"boxing gloves",
|
| 210 |
+
"swim caps",
|
| 211 |
+
"racquet accessories",
|
| 212 |
+
"bike locks",
|
| 213 |
+
"poker equipment",
|
| 214 |
+
"freeze-dried food",
|
| 215 |
+
"heart rate monitors",
|
| 216 |
+
"darts",
|
| 217 |
+
"tent accessories",
|
| 218 |
+
"pakistan",
|
| 219 |
+
"jump ropes",
|
| 220 |
+
"insect repellent",
|
| 221 |
+
"boat motors",
|
| 222 |
+
"cornhole",
|
| 223 |
+
"honduras",
|
| 224 |
+
"pistols",
|
| 225 |
+
"rifle cases",
|
| 226 |
+
"street signs",
|
| 227 |
+
"bike pumps",
|
| 228 |
+
"tactical flashlights",
|
| 229 |
+
"tactical pouches",
|
| 230 |
+
"boat cabin products",
|
| 231 |
+
"tackle storage",
|
| 232 |
+
"area rugs",
|
| 233 |
+
"downhill skiing",
|
| 234 |
+
"kids' bikes",
|
| 235 |
+
"fishing hats",
|
| 236 |
+
"gymnastics",
|
| 237 |
+
"boots",
|
| 238 |
+
"tights",
|
| 239 |
+
"ice skating",
|
| 240 |
+
"balance trainers",
|
| 241 |
+
"exercise balls",
|
| 242 |
+
"compression tops",
|
| 243 |
+
"zipper closure",
|
| 244 |
+
"soft lures",
|
| 245 |
+
"hammocks",
|
| 246 |
+
"stove accessories",
|
| 247 |
+
"dry bags",
|
| 248 |
+
"shin guards",
|
| 249 |
+
"reflective gear",
|
| 250 |
+
"training gloves",
|
| 251 |
+
"boat plumbing",
|
| 252 |
+
"cue sticks",
|
| 253 |
+
"pet collars",
|
| 254 |
+
"boat trailer accessories",
|
| 255 |
+
"diving masks",
|
| 256 |
+
"mineral crystal",
|
| 257 |
+
"backpack accessories",
|
| 258 |
+
"gun slings",
|
| 259 |
+
"caps",
|
| 260 |
+
"novelty headwear",
|
| 261 |
+
"ice fishing",
|
| 262 |
+
"camping shelters",
|
| 263 |
+
"trekking poles",
|
| 264 |
+
"pennants",
|
| 265 |
+
"shells",
|
| 266 |
+
"family camping tents",
|
| 267 |
+
"halters",
|
| 268 |
+
"masks",
|
| 269 |
+
"fleece",
|
| 270 |
+
"two-piece suits",
|
| 271 |
+
"cheerleading",
|
| 272 |
+
"safety glasses",
|
| 273 |
+
"saddle pads",
|
| 274 |
+
"stand-up paddleboarding",
|
| 275 |
+
"made in usa",
|
| 276 |
+
"soccer equipment",
|
| 277 |
+
"stems",
|
| 278 |
+
"derailleurs",
|
| 279 |
+
"batting gloves",
|
| 280 |
+
"fly fishing",
|
| 281 |
+
"scarves",
|
| 282 |
+
"floormats",
|
| 283 |
+
"bangladesh",
|
| 284 |
+
"bike bells",
|
| 285 |
+
"air guns",
|
| 286 |
+
"standard balls",
|
| 287 |
+
"officially licensed by the nfl",
|
| 288 |
+
"bits",
|
| 289 |
+
"cart bags",
|
| 290 |
+
"camouflage accessories",
|
| 291 |
+
"complete sets",
|
| 292 |
+
"beach towels",
|
| 293 |
+
"gun cleaning kits",
|
| 294 |
+
"grill accessories",
|
| 295 |
+
"punching bags",
|
| 296 |
+
"snowmobiling",
|
| 297 |
+
"football gear",
|
| 298 |
+
"bike racks",
|
| 299 |
+
"water bottle cages",
|
| 300 |
+
"tees",
|
| 301 |
+
"badminton",
|
| 302 |
+
"maintenance supplies",
|
| 303 |
+
"exercise bikes",
|
| 304 |
+
"plaques",
|
| 305 |
+
"bowling balls",
|
| 306 |
+
"seat posts",
|
| 307 |
+
"trucks",
|
| 308 |
+
"pedometers",
|
| 309 |
+
"umbrellas",
|
| 310 |
+
"wall clocks",
|
| 311 |
+
"broadheads",
|
| 312 |
+
"shafts",
|
| 313 |
+
"internal frame backpacks",
|
| 314 |
+
"toy vehicles",
|
| 315 |
+
"sailing",
|
| 316 |
+
"tumblers",
|
| 317 |
+
"license plate covers",
|
| 318 |
+
"knives",
|
| 319 |
+
"backpacking tents",
|
| 320 |
+
"basketballs",
|
| 321 |
+
"irons",
|
| 322 |
+
"bingo equipment",
|
| 323 |
+
"goalkeeper gloves",
|
| 324 |
+
"knee pads",
|
| 325 |
+
"sun shelters",
|
| 326 |
+
"fixed-blade knives",
|
| 327 |
+
"wall banners",
|
| 328 |
+
"tactical bag accessories",
|
| 329 |
+
"cranksets",
|
| 330 |
+
"snowshoeing",
|
| 331 |
+
"sights",
|
| 332 |
+
"cricket",
|
| 333 |
+
"headlamps",
|
| 334 |
+
"rash guards",
|
| 335 |
+
"bindings",
|
| 336 |
+
"straps",
|
| 337 |
+
"car flags",
|
| 338 |
+
"tables",
|
| 339 |
+
"kayaks",
|
| 340 |
+
"shop tools",
|
| 341 |
+
"home gyms",
|
| 342 |
+
"triathlon",
|
| 343 |
+
"whistles",
|
| 344 |
+
"golf gear",
|
| 345 |
+
"braided line",
|
| 346 |
+
"hybrid clubs",
|
| 347 |
+
"car magnets",
|
| 348 |
+
"bags",
|
| 349 |
+
"footballs",
|
| 350 |
+
"dumbbells",
|
| 351 |
+
"visors",
|
| 352 |
+
"hunting arrows",
|
| 353 |
+
"ties",
|
| 354 |
+
"kick scooters",
|
| 355 |
+
"pepper spray",
|
| 356 |
+
"optics mounts",
|
| 357 |
+
"button-down shirts",
|
| 358 |
+
"precise swiss-quartz movement",
|
| 359 |
+
"flying discs",
|
| 360 |
+
"protective padding",
|
| 361 |
+
"treadmill belts",
|
| 362 |
+
"roller skates",
|
| 363 |
+
"sets",
|
| 364 |
+
"wakeboarding",
|
| 365 |
+
"can coolers",
|
| 366 |
+
"synthetic",
|
| 367 |
+
"clothing sets",
|
| 368 |
+
"snorkeling packages",
|
| 369 |
+
"belts",
|
| 370 |
+
"mountain bikes",
|
| 371 |
+
"basketball equipment",
|
| 372 |
+
"taillights",
|
| 373 |
+
"camping cookware",
|
| 374 |
+
"tools",
|
| 375 |
+
"hand strengtheners",
|
| 376 |
+
"hunting hats",
|
| 377 |
+
"softball mitts",
|
| 378 |
+
"spotting scopes",
|
| 379 |
+
"made in us",
|
| 380 |
+
"sledding",
|
| 381 |
+
"air mattresses",
|
| 382 |
+
"breeches",
|
| 383 |
+
"foam rollers",
|
| 384 |
+
"inline skates",
|
| 385 |
+
"targets",
|
| 386 |
+
"brake pads",
|
| 387 |
+
"indonesia",
|
| 388 |
+
"sleeping bag accessories",
|
| 389 |
+
"exercise mats",
|
| 390 |
+
"flights",
|
| 391 |
+
"baseball equipment",
|
| 392 |
+
"flies",
|
| 393 |
+
"racquet strings",
|
| 394 |
+
"duffle bags",
|
| 395 |
+
"dresses",
|
| 396 |
+
"cornhole sets",
|
| 397 |
+
"beer glasses",
|
| 398 |
+
"baseballs",
|
| 399 |
+
"bibs",
|
| 400 |
+
"skirts",
|
| 401 |
+
"mini helmets",
|
| 402 |
+
"chainrings",
|
| 403 |
+
"acrylic",
|
| 404 |
+
"protective body equipment",
|
| 405 |
+
"sticks",
|
| 406 |
+
"folding chairs",
|
| 407 |
+
"brakes",
|
| 408 |
+
"cable locks",
|
| 409 |
+
"sleeping pads",
|
| 410 |
+
"pins",
|
| 411 |
+
"snowshoes",
|
| 412 |
+
"fly tying materials",
|
| 413 |
+
"cruiser bikes",
|
| 414 |
+
"bobbleheads",
|
| 415 |
+
"vests",
|
| 416 |
+
"air rifles",
|
| 417 |
+
"dry boxes",
|
| 418 |
+
"tactical vests",
|
| 419 |
+
"hunting bags",
|
| 420 |
+
"diving fins",
|
| 421 |
+
"blankets",
|
| 422 |
+
"propellers",
|
| 423 |
+
"mouthguards",
|
| 424 |
+
"kids' helmets",
|
| 425 |
+
"slow-pitch softball bats",
|
| 426 |
+
"brake levers",
|
| 427 |
+
"wrestling",
|
| 428 |
+
"stand bags",
|
| 429 |
+
"classic fit",
|
| 430 |
+
"hand warmers",
|
| 431 |
+
"fairway woods",
|
| 432 |
+
"charms",
|
| 433 |
+
"waist trimmers",
|
| 434 |
+
"monofilament line",
|
| 435 |
+
"decoys",
|
| 436 |
+
"rings",
|
| 437 |
+
"entry mats",
|
| 438 |
+
"deck hardware",
|
| 439 |
+
"poker chips",
|
| 440 |
+
"harnesses",
|
| 441 |
+
"paddles",
|
| 442 |
+
"pistol cases",
|
| 443 |
+
"headgear",
|
| 444 |
+
"cleaners",
|
| 445 |
+
"games",
|
| 446 |
+
"trailer hitch covers",
|
| 447 |
+
"racquetball",
|
| 448 |
+
"tops",
|
| 449 |
+
"fins",
|
| 450 |
+
"lighters",
|
| 451 |
+
"magazines",
|
| 452 |
+
"fire starters",
|
| 453 |
+
"rugby",
|
| 454 |
+
"basketball court equipment",
|
| 455 |
+
"rod holders",
|
| 456 |
+
"board bags",
|
| 457 |
+
"blinds",
|
| 458 |
+
"briefs",
|
| 459 |
+
"jammers",
|
| 460 |
+
"navigation lights",
|
| 461 |
+
"bowstrings",
|
| 462 |
+
"forks",
|
| 463 |
+
"skinsuits",
|
| 464 |
+
"outfielder's mitts",
|
| 465 |
+
"cycling computers",
|
| 466 |
+
"elliptical trainers",
|
| 467 |
+
"kayak accessories",
|
| 468 |
+
"bike baskets",
|
| 469 |
+
"benches",
|
| 470 |
+
"headlight-taillight combinations",
|
| 471 |
+
"handlebar bags",
|
| 472 |
+
"headphones",
|
| 473 |
+
"fletches",
|
| 474 |
+
"treadmills",
|
| 475 |
+
"grip tape",
|
| 476 |
+
"hubs",
|
| 477 |
+
"first aid kits",
|
| 478 |
+
"bed-in-a-bag",
|
| 479 |
+
"stand-up paddleboards",
|
| 480 |
+
"rubber sole",
|
| 481 |
+
"made in the usa and imported",
|
| 482 |
+
"fenders",
|
| 483 |
+
"snowboards",
|
| 484 |
+
"bbs",
|
| 485 |
+
"outboard motors",
|
| 486 |
+
"quivers",
|
| 487 |
+
"floor pumps",
|
| 488 |
+
"outdoor statues",
|
| 489 |
+
"pet jerseys",
|
| 490 |
+
"trampolines",
|
| 491 |
+
"importado",
|
| 492 |
+
"racquet grips",
|
| 493 |
+
"wraps",
|
| 494 |
+
"inline skate parts",
|
| 495 |
+
"one size fits most",
|
| 496 |
+
"tote bags",
|
| 497 |
+
"swing trainers",
|
| 498 |
+
"sports water bottle accessories",
|
| 499 |
+
"rope",
|
| 500 |
+
"cuff links",
|
| 501 |
+
"pet t-shirts",
|
| 502 |
+
"bait rigs",
|
| 503 |
+
"full suits",
|
| 504 |
+
"squash",
|
| 505 |
+
"bars",
|
| 506 |
+
"coasters",
|
| 507 |
+
"hockey equipment",
|
| 508 |
+
"night vision",
|
| 509 |
+
"chains",
|
| 510 |
+
"full sized helmets",
|
| 511 |
+
"kettlebells",
|
| 512 |
+
"sauna suits",
|
| 513 |
+
"towables",
|
| 514 |
+
"seating",
|
| 515 |
+
"trolling motors",
|
| 516 |
+
"hand wraps",
|
| 517 |
+
"cargo racks",
|
| 518 |
+
"polarized",
|
| 519 |
+
"weight belts",
|
| 520 |
+
"nylon",
|
| 521 |
+
"usa",
|
| 522 |
+
"mat bags",
|
| 523 |
+
"kayak hardware",
|
| 524 |
+
"skis",
|
| 525 |
+
"scents",
|
| 526 |
+
"armwarmers",
|
| 527 |
+
"midrange discs",
|
| 528 |
+
"bearings",
|
| 529 |
+
"snow sleds",
|
| 530 |
+
"nocks",
|
| 531 |
+
"display cases",
|
| 532 |
+
"medicine balls",
|
| 533 |
+
"locking carabiners",
|
| 534 |
+
"foosball",
|
| 535 |
+
"tent stakes",
|
| 536 |
+
"lamps",
|
| 537 |
+
"balance boards",
|
| 538 |
+
"anchors",
|
| 539 |
+
"markers",
|
| 540 |
+
"seat packs",
|
| 541 |
+
"goals",
|
| 542 |
+
"foot gear",
|
| 543 |
+
"fly line",
|
| 544 |
+
"rackets",
|
| 545 |
+
"tackle boxes",
|
| 546 |
+
"spoons",
|
| 547 |
+
"sandals",
|
| 548 |
+
"batting helmets",
|
| 549 |
+
"ice skates",
|
| 550 |
+
"pet dresses",
|
| 551 |
+
"cooler accessories",
|
| 552 |
+
"bilge pumps",
|
| 553 |
+
"el salvador",
|
| 554 |
+
"roller skate parts",
|
| 555 |
+
"road bikes",
|
| 556 |
+
"pilates",
|
| 557 |
+
"valve caps",
|
| 558 |
+
"boats",
|
| 559 |
+
"standard flights",
|
| 560 |
+
"soccer field equipment",
|
| 561 |
+
"bike pack accessories",
|
| 562 |
+
"fast-pitch softball bats",
|
| 563 |
+
"rocker switches",
|
| 564 |
+
"gun stocks",
|
| 565 |
+
"lunch boxes",
|
| 566 |
+
"made in usa or imported",
|
| 567 |
+
"hunting dog equipment",
|
| 568 |
+
"handlebar tape",
|
| 569 |
+
"screen printed graphics",
|
| 570 |
+
"catcher's mitts",
|
| 571 |
+
"golf club bag accessories",
|
| 572 |
+
"leather",
|
| 573 |
+
"golf club head covers",
|
| 574 |
+
"complete paintball sets",
|
| 575 |
+
"canoeing",
|
| 576 |
+
"fly tying equipment",
|
| 577 |
+
"poles",
|
| 578 |
+
"open fire cookware",
|
| 579 |
+
"elastic closure",
|
| 580 |
+
"softballs",
|
| 581 |
+
"team logo embroidered on left chest",
|
| 582 |
+
"bottom brackets",
|
| 583 |
+
"stabilizers",
|
| 584 |
+
"magnets",
|
| 585 |
+
"pull-up bars",
|
| 586 |
+
"dry flies",
|
| 587 |
+
"sneakers",
|
| 588 |
+
"court equipment",
|
| 589 |
+
"recurve bows",
|
| 590 |
+
"webbing",
|
| 591 |
+
"tree stands",
|
| 592 |
+
"field equipment",
|
| 593 |
+
"bowling roller bags",
|
| 594 |
+
"field hockey",
|
| 595 |
+
"rigging",
|
| 596 |
+
"trailers",
|
| 597 |
+
"compound bows",
|
| 598 |
+
"pull on closure",
|
| 599 |
+
"girths",
|
| 600 |
+
"diving lights",
|
| 601 |
+
"frame-mounted pumps",
|
| 602 |
+
"diploma frames",
|
| 603 |
+
"earmuffs",
|
| 604 |
+
"turkey",
|
| 605 |
+
"pickleball",
|
| 606 |
+
"fishing kayaks",
|
| 607 |
+
"golf umbrellas",
|
| 608 |
+
"quality japanese-quartz movement",
|
| 609 |
+
"release aids",
|
| 610 |
+
"one size fits all",
|
| 611 |
+
"bmx",
|
| 612 |
+
"climbing holds",
|
| 613 |
+
"snorkels",
|
| 614 |
+
"bodyboards",
|
| 615 |
+
"aprons",
|
| 616 |
+
"reins",
|
| 617 |
+
"stun guns",
|
| 618 |
+
"steering equipment",
|
| 619 |
+
"exercise machine attachments",
|
| 620 |
+
"fencing",
|
| 621 |
+
"earplugs",
|
| 622 |
+
"men's",
|
| 623 |
+
"bait traps",
|
| 624 |
+
"cases",
|
| 625 |
+
"cooking utensils",
|
| 626 |
+
"luggage tags",
|
| 627 |
+
"magazine loaders",
|
| 628 |
+
"basketball gear",
|
| 629 |
+
"medals",
|
| 630 |
+
"throw pillows",
|
| 631 |
+
"sleepwear",
|
| 632 |
+
"tree stand accessories",
|
| 633 |
+
"swivels",
|
| 634 |
+
"bike mirrors",
|
| 635 |
+
"barrels",
|
| 636 |
+
"air gun pellets",
|
| 637 |
+
"air hockey",
|
| 638 |
+
"computer mouse pads",
|
| 639 |
+
"kids' tricycles",
|
| 640 |
+
"golf carts",
|
| 641 |
+
"headsets",
|
| 642 |
+
"photomints",
|
| 643 |
+
"bow maintenance accessories",
|
| 644 |
+
"footprints",
|
| 645 |
+
"bumper stickers",
|
| 646 |
+
"electric bicycles",
|
| 647 |
+
"blocks",
|
| 648 |
+
"skate bags",
|
| 649 |
+
"heavy bags",
|
| 650 |
+
"cornhole bags",
|
| 651 |
+
"ski bags",
|
| 652 |
+
"baseball gear",
|
| 653 |
+
"volleyballs",
|
| 654 |
+
"stopwatches",
|
| 655 |
+
"slingshots",
|
| 656 |
+
"made by reebok",
|
| 657 |
+
"children's inline skates",
|
| 658 |
+
"weights",
|
| 659 |
+
"targeting arrows",
|
| 660 |
+
"short sleeve",
|
| 661 |
+
"stirrups",
|
| 662 |
+
"gun brushes",
|
| 663 |
+
"diving gloves",
|
| 664 |
+
"bath accessories",
|
| 665 |
+
"picture frames",
|
| 666 |
+
"handcuffs",
|
| 667 |
+
"plugs",
|
| 668 |
+
"bow cases",
|
| 669 |
+
"cots",
|
| 670 |
+
"flatware",
|
| 671 |
+
"camping pillows",
|
| 672 |
+
"non-polarized",
|
| 673 |
+
"swiss quartz movement",
|
| 674 |
+
"shot glasses",
|
| 675 |
+
"elbow pads",
|
| 676 |
+
"leashes",
|
| 677 |
+
"tool kits",
|
| 678 |
+
"tank tops",
|
| 679 |
+
"plates",
|
| 680 |
+
"divot tools",
|
| 681 |
+
"skates",
|
| 682 |
+
"dice",
|
| 683 |
+
"resistance trainers",
|
| 684 |
+
"drawstring bags",
|
| 685 |
+
"drawstring closure",
|
| 686 |
+
"poly-cotton",
|
| 687 |
+
"plastic frame",
|
| 688 |
+
"brand new",
|
| 689 |
+
"recoil pads",
|
| 690 |
+
"seat clamps",
|
| 691 |
+
"suspension",
|
| 692 |
+
"shoulder pads",
|
| 693 |
+
"comforters",
|
| 694 |
+
"optics accessories",
|
| 695 |
+
"shifters",
|
| 696 |
+
"weight racks",
|
| 697 |
+
"interior lights",
|
| 698 |
+
"pet bandanas",
|
| 699 |
+
"nonlocking carabiners",
|
| 700 |
+
"figure skates",
|
| 701 |
+
"water filters",
|
| 702 |
+
"hybrid bikes",
|
| 703 |
+
"playground balls",
|
| 704 |
+
"rod racks",
|
| 705 |
+
"inflatable rafts",
|
| 706 |
+
"points",
|
| 707 |
+
"billiard balls",
|
| 708 |
+
"bag gloves",
|
| 709 |
+
"throwing equipment",
|
| 710 |
+
"cross-country skiing",
|
| 711 |
+
"attractants",
|
| 712 |
+
"rims",
|
| 713 |
+
"replacement wheels",
|
| 714 |
+
"bat accessories",
|
| 715 |
+
"fuel filters",
|
| 716 |
+
"ankle weights",
|
| 717 |
+
"crossbows",
|
| 718 |
+
"ninja weapons",
|
| 719 |
+
"stuff sacks",
|
| 720 |
+
"officially licensed by the nba",
|
| 721 |
+
"ice spearing equipment",
|
| 722 |
+
"bike covers",
|
| 723 |
+
"spurs",
|
| 724 |
+
"u-locks",
|
| 725 |
+
"kids' accessories",
|
| 726 |
+
"emergency fire starters",
|
| 727 |
+
"the north face size chart",
|
| 728 |
+
"self-inflating pads",
|
| 729 |
+
"tailgating toss games",
|
| 730 |
+
"golf bag towels",
|
| 731 |
+
"fight gloves",
|
| 732 |
+
"weight vests",
|
| 733 |
+
"shift levers",
|
| 734 |
+
"made in the usa",
|
| 735 |
+
"suspension forks",
|
| 736 |
+
"emblems",
|
| 737 |
+
"plastic lens",
|
| 738 |
+
"adjustable benches",
|
| 739 |
+
"officially licensed by the nhl",
|
| 740 |
+
"moisture wicking",
|
| 741 |
+
"optics rings",
|
| 742 |
+
"folding bikes",
|
| 743 |
+
"hockey gear",
|
| 744 |
+
"regulators",
|
| 745 |
+
"training fins",
|
| 746 |
+
"ladders",
|
| 747 |
+
"roller hockey",
|
| 748 |
+
"inversion equipment",
|
| 749 |
+
"canteens",
|
| 750 |
+
"emergency blankets",
|
| 751 |
+
"car rack accessories",
|
| 752 |
+
"steering wheel covers",
|
| 753 |
+
"bocce",
|
| 754 |
+
"cue chalk",
|
| 755 |
+
"fuel tanks",
|
| 756 |
+
"arm guards",
|
| 757 |
+
"lasers",
|
| 758 |
+
"hand wipe with damp cloth",
|
| 759 |
+
"officially licensed by the ncaa",
|
| 760 |
+
"heads",
|
| 761 |
+
"multifunction tools",
|
| 762 |
+
"batteries",
|
| 763 |
+
"shorty suits",
|
| 764 |
+
"predator",
|
| 765 |
+
"compression socks",
|
| 766 |
+
"front derailleurs",
|
| 767 |
+
"topographic maps",
|
| 768 |
+
"rests",
|
| 769 |
+
"regulator accessories",
|
| 770 |
+
"decoy accessories",
|
| 771 |
+
"shoe covers",
|
| 772 |
+
"singlets",
|
| 773 |
+
"poms",
|
| 774 |
+
"pillowcases",
|
| 775 |
+
"shuttlecocks",
|
| 776 |
+
"riding gloves",
|
| 777 |
+
"shovels",
|
| 778 |
+
"underwear",
|
| 779 |
+
"indoor bike storage",
|
| 780 |
+
"fluorocarbon line",
|
| 781 |
+
"paintballs",
|
| 782 |
+
"battery chargers",
|
| 783 |
+
"badge holders",
|
| 784 |
+
"paddle accessories",
|
| 785 |
+
"laser rangefinders",
|
| 786 |
+
"punching bag accessories",
|
| 787 |
+
"foosball accessories",
|
| 788 |
+
"bingo cards",
|
| 789 |
+
"deer",
|
| 790 |
+
"poker sets",
|
| 791 |
+
"seat covers",
|
| 792 |
+
"boat fenders",
|
| 793 |
+
"distance balls",
|
| 794 |
+
"canopies",
|
| 795 |
+
"officially licensed merchandise",
|
| 796 |
+
"vietnam",
|
| 797 |
+
"dry clean only",
|
| 798 |
+
"carry bags",
|
| 799 |
+
"children's roller skates",
|
| 800 |
+
"swim vests",
|
| 801 |
+
"duck",
|
| 802 |
+
"whips",
|
| 803 |
+
"pens",
|
| 804 |
+
"freestyle boards",
|
| 805 |
+
"open-hem sleeves",
|
| 806 |
+
"safety gear",
|
| 807 |
+
"bow slings",
|
| 808 |
+
"stunt scooters",
|
| 809 |
+
"aquatic fitness equipment",
|
| 810 |
+
"painting supplies",
|
| 811 |
+
"screen print graphics",
|
| 812 |
+
"thailand",
|
| 813 |
+
"tire covers",
|
| 814 |
+
"catcher chest protectors",
|
| 815 |
+
"fitted",
|
| 816 |
+
"fishing gloves",
|
| 817 |
+
"boot bags",
|
| 818 |
+
"made in usa and imported",
|
| 819 |
+
"taiwan",
|
| 820 |
+
"scent eliminators",
|
| 821 |
+
"gun cases",
|
| 822 |
+
"bingo daubers",
|
| 823 |
+
"bike frames",
|
| 824 |
+
"belt displays",
|
| 825 |
+
"wet flies",
|
| 826 |
+
"pet leashes",
|
| 827 |
+
"water bottles",
|
| 828 |
+
"lines",
|
| 829 |
+
"headstalls",
|
| 830 |
+
"luggage",
|
| 831 |
+
"tanks",
|
| 832 |
+
"practice nets",
|
| 833 |
+
"skate accessories",
|
| 834 |
+
"hitting nets",
|
| 835 |
+
"wax",
|
| 836 |
+
"wakeboards",
|
| 837 |
+
"pro boxing gloves",
|
| 838 |
+
"starter sets",
|
| 839 |
+
"flags",
|
| 840 |
+
"gun lubrication",
|
| 841 |
+
"shower curtains",
|
| 842 |
+
"football field equipment",
|
| 843 |
+
"pushup stands",
|
| 844 |
+
"backcountry equipment",
|
| 845 |
+
"tablet accessories",
|
| 846 |
+
"rangefinders",
|
| 847 |
+
"fixed gear bikes",
|
| 848 |
+
"tent tarps",
|
| 849 |
+
"bathroom towels",
|
| 850 |
+
"unitards",
|
| 851 |
+
"chalk bags",
|
| 852 |
+
"youth sizes 8-20",
|
| 853 |
+
"original packaging",
|
| 854 |
+
"korea",
|
| 855 |
+
"dartboards",
|
| 856 |
+
"leaders",
|
| 857 |
+
"seating accessories",
|
| 858 |
+
"foosball tables",
|
| 859 |
+
"cue racks",
|
| 860 |
+
"screenprint decoration",
|
| 861 |
+
"shooter sleeves",
|
| 862 |
+
"plumbing fittings",
|
| 863 |
+
"swaziland",
|
| 864 |
+
"chain locks",
|
| 865 |
+
"line spooling accessories",
|
| 866 |
+
"helmet accessories",
|
| 867 |
+
"alarm clocks",
|
| 868 |
+
"food processing",
|
| 869 |
+
"foot warmers",
|
| 870 |
+
"speed skates",
|
| 871 |
+
"mp3 player accessories",
|
| 872 |
+
"loaders",
|
| 873 |
+
"reflectors",
|
| 874 |
+
"bowls",
|
| 875 |
+
"rib knit collar",
|
| 876 |
+
"muff pocket",
|
| 877 |
+
"hunting knife sharpeners",
|
| 878 |
+
"marker maintenance equipment",
|
| 879 |
+
"bowling totes",
|
| 880 |
+
"embroidered logos",
|
| 881 |
+
"putting mats",
|
| 882 |
+
"lob wedges",
|
| 883 |
+
"shuffleboard",
|
| 884 |
+
"sleeve patches",
|
| 885 |
+
"made by outerstuff",
|
| 886 |
+
"lap counters",
|
| 887 |
+
"durable",
|
| 888 |
+
"quartz movement with digital display",
|
| 889 |
+
"combination tables",
|
| 890 |
+
"ball polishers",
|
| 891 |
+
"nicaragua",
|
| 892 |
+
"game feeders",
|
| 893 |
+
"pong games",
|
| 894 |
+
"air pistols",
|
| 895 |
+
"machine washable, line dry, no bleach",
|
| 896 |
+
"liners",
|
| 897 |
+
"protective flooring",
|
| 898 |
+
"practice balls",
|
| 899 |
+
"replacement parts",
|
| 900 |
+
"stomp pads",
|
| 901 |
+
"embroidered graphics",
|
| 902 |
+
"made by adidas",
|
| 903 |
+
"sand wedges",
|
| 904 |
+
"boresighters",
|
| 905 |
+
"kickboards",
|
| 906 |
+
"scratch resistant, shatter resistant sapphimax crystal",
|
| 907 |
+
"billiard cloth",
|
| 908 |
+
"co2 pumps",
|
| 909 |
+
"flasks",
|
| 910 |
+
"girdles",
|
| 911 |
+
"tumbling mats",
|
| 912 |
+
"polyester lining",
|
| 913 |
+
"brake calipers",
|
| 914 |
+
"bmx bikes",
|
| 915 |
+
"buoyancy compensators",
|
| 916 |
+
"fan packs",
|
| 917 |
+
"rubber",
|
| 918 |
+
"windbreakers",
|
| 919 |
+
"bushings",
|
| 920 |
+
"japanese quartz movement",
|
| 921 |
+
"croquet",
|
| 922 |
+
"cables",
|
| 923 |
+
"sheets",
|
| 924 |
+
"seats",
|
| 925 |
+
"collars",
|
| 926 |
+
"officially licensed by mlb",
|
| 927 |
+
"weapon stands",
|
| 928 |
+
"goalposts",
|
| 929 |
+
"travel covers",
|
| 930 |
+
"leads",
|
| 931 |
+
"comfort bikes",
|
| 932 |
+
"traction cleats",
|
| 933 |
+
"table covers",
|
| 934 |
+
"slow-pitch softballs",
|
| 935 |
+
"swiss chronograph movement",
|
| 936 |
+
"bimini tops",
|
| 937 |
+
"locks",
|
| 938 |
+
"handgun scopes",
|
| 939 |
+
"cue tips",
|
| 940 |
+
"workstands",
|
| 941 |
+
"field dressing accessories",
|
| 942 |
+
"waste baskets",
|
| 943 |
+
"gaiters",
|
| 944 |
+
"vibration dampeners",
|
| 945 |
+
"reservoirs",
|
| 946 |
+
"youth bow sets",
|
| 947 |
+
"tracksuits",
|
| 948 |
+
"groin protectors",
|
| 949 |
+
"wrist weights",
|
| 950 |
+
"child carrier trailers",
|
| 951 |
+
"ball retrievers",
|
| 952 |
+
"date function",
|
| 953 |
+
"ballet equipment",
|
| 954 |
+
"v-neck tee",
|
| 955 |
+
"rowers",
|
| 956 |
+
"surfboards",
|
| 957 |
+
"made in china",
|
| 958 |
+
"ring toss",
|
| 959 |
+
"pack covers",
|
| 960 |
+
"slim fit",
|
| 961 |
+
"backboard components",
|
| 962 |
+
"bait storage",
|
| 963 |
+
"pitching machines",
|
| 964 |
+
"bike lube",
|
| 965 |
+
"whitewater kayaks",
|
| 966 |
+
"snow tubes",
|
| 967 |
+
"gauges",
|
| 968 |
+
"spotlights",
|
| 969 |
+
"bike horns",
|
| 970 |
+
"dive skins",
|
| 971 |
+
"synthetic sole",
|
| 972 |
+
"practice swords",
|
| 973 |
+
"holsters",
|
| 974 |
+
"trophy mounts",
|
| 975 |
+
"disc brake sets",
|
| 976 |
+
"showers",
|
| 977 |
+
"bed pillows",
|
| 978 |
+
"night vision monoculars",
|
| 979 |
+
"officially licensed nfl product",
|
| 980 |
+
"officially licensed by major league baseball",
|
| 981 |
+
"step platforms",
|
| 982 |
+
"cleaning tools",
|
| 983 |
+
"crew neck jersey tee",
|
| 984 |
+
"compression sleeves",
|
| 985 |
+
"batting trainers",
|
| 986 |
+
"tank accessories",
|
| 987 |
+
"buckles",
|
| 988 |
+
"mexico",
|
| 989 |
+
"batting tees",
|
| 990 |
+
"step machines",
|
| 991 |
+
"horns",
|
| 992 |
+
"accessory kits",
|
| 993 |
+
"downriggers",
|
| 994 |
+
"made in the usa or imported",
|
| 995 |
+
"diving hoods",
|
| 996 |
+
"chalk",
|
| 997 |
+
"ultimate",
|
| 998 |
+
"chest protectors",
|
| 999 |
+
"punch mitts",
|
| 1000 |
+
"rotors",
|
| 1001 |
+
"hang dry",
|
| 1002 |
+
"altimeters",
|
| 1003 |
+
"loader accessories",
|
| 1004 |
+
"screen print decoration",
|
| 1005 |
+
"stools",
|
| 1006 |
+
"leg machines",
|
| 1007 |
+
"bridles",
|
| 1008 |
+
"shotguns",
|
| 1009 |
+
"laptop bags",
|
| 1010 |
+
"ball racks",
|
| 1011 |
+
"grip repair kits",
|
| 1012 |
+
"vintage feel",
|
| 1013 |
+
"saws",
|
| 1014 |
+
"hangers",
|
| 1015 |
+
"water purifiers",
|
| 1016 |
+
"bear protection",
|
| 1017 |
+
"kids' protective gear",
|
| 1018 |
+
"ice augers",
|
| 1019 |
+
"diving electronics",
|
| 1020 |
+
"button closure",
|
| 1021 |
+
"reel care accessories",
|
| 1022 |
+
"protective mineral crystal dial window",
|
| 1023 |
+
"barstools",
|
| 1024 |
+
"adult",
|
| 1025 |
+
"headset spacers",
|
| 1026 |
+
"speedometers",
|
| 1027 |
+
"shoe bags",
|
| 1028 |
+
"cabinets",
|
| 1029 |
+
"rubbers",
|
| 1030 |
+
"leader rigging",
|
| 1031 |
+
"hockey pucks",
|
| 1032 |
+
"vibration platform machines",
|
| 1033 |
+
"racing skates",
|
| 1034 |
+
"gun solvents",
|
| 1035 |
+
"camping grills",
|
| 1036 |
+
"desk caddies",
|
| 1037 |
+
"hard hats",
|
| 1038 |
+
"diving boots",
|
| 1039 |
+
"first baseman's mitts",
|
| 1040 |
+
"arm machines",
|
| 1041 |
+
"game table accessories",
|
| 1042 |
+
"slacklines",
|
| 1043 |
+
"battery switches",
|
| 1044 |
+
"screenprint",
|
| 1045 |
+
"bodysuits",
|
| 1046 |
+
"single clubs",
|
| 1047 |
+
"shore power cords",
|
| 1048 |
+
"computer mice",
|
| 1049 |
+
"speed bags",
|
| 1050 |
+
"wool",
|
| 1051 |
+
"control cables",
|
| 1052 |
+
"hand grips",
|
| 1053 |
+
"straight bottom hem",
|
| 1054 |
+
"dumbbell racks",
|
| 1055 |
+
"traction pads",
|
| 1056 |
+
"shuffleboard accessories",
|
| 1057 |
+
"mess kits",
|
| 1058 |
+
"outdoor volleyballs",
|
| 1059 |
+
"goalkeeper equipment",
|
| 1060 |
+
"thailand/vietnam",
|
| 1061 |
+
"thermoses",
|
| 1062 |
+
"elk",
|
| 1063 |
+
"diving packages",
|
| 1064 |
+
"insert tape",
|
| 1065 |
+
"wipe with damp cloth",
|
| 1066 |
+
"80/20 cotton/polyester fleece",
|
| 1067 |
+
"trailer lights",
|
| 1068 |
+
"stainless steel case with stainless-steel band",
|
| 1069 |
+
"portable",
|
| 1070 |
+
"neck gaiters",
|
| 1071 |
+
"meet equipment",
|
| 1072 |
+
"trunks",
|
| 1073 |
+
"floor mats",
|
| 1074 |
+
"trailer winches",
|
| 1075 |
+
"rim brake sets",
|
| 1076 |
+
"paddleboard accessories",
|
| 1077 |
+
"open-hem sleeves; adidas performance",
|
| 1078 |
+
"fish scales",
|
| 1079 |
+
"uniquely designed team slippers",
|
| 1080 |
+
"discuses",
|
| 1081 |
+
"lantern accessories",
|
| 1082 |
+
"progressive fashion with vanguard sports cultures",
|
| 1083 |
+
"reel replacement parts",
|
| 1084 |
+
"dock guards",
|
| 1085 |
+
"fuel-flow instruments",
|
| 1086 |
+
"spokes",
|
| 1087 |
+
"shackles",
|
| 1088 |
+
"t-ball bats",
|
| 1089 |
+
"pack pockets",
|
| 1090 |
+
"made in the usa!",
|
| 1091 |
+
"indoor volleyballs",
|
| 1092 |
+
"expedition tents",
|
| 1093 |
+
"megaphones",
|
| 1094 |
+
"rigid forks",
|
| 1095 |
+
"trainer accessories",
|
| 1096 |
+
"screen printed in the usa",
|
| 1097 |
+
"not water resistant",
|
| 1098 |
+
"nose clips",
|
| 1099 |
+
"full zip hoody",
|
| 1100 |
+
"adult/unisex.",
|
| 1101 |
+
"bocce sets",
|
| 1102 |
+
"marker boards",
|
| 1103 |
+
"skimboards",
|
| 1104 |
+
"freeride boards",
|
| 1105 |
+
"embroidered team logo",
|
| 1106 |
+
"gun rails",
|
| 1107 |
+
"longbows",
|
| 1108 |
+
"sapphire crystal",
|
| 1109 |
+
"poker tables",
|
| 1110 |
+
"screen print logo",
|
| 1111 |
+
"gun racks",
|
| 1112 |
+
"wrist guards",
|
| 1113 |
+
"track jackets",
|
| 1114 |
+
"longeing equipment",
|
| 1115 |
+
"chin straps",
|
| 1116 |
+
"authentic",
|
| 1117 |
+
"valances",
|
| 1118 |
+
"fitness trampolines",
|
| 1119 |
+
"casino prize wheels",
|
| 1120 |
+
"aquatic gloves",
|
| 1121 |
+
"billiards",
|
| 1122 |
+
"unicycles",
|
| 1123 |
+
"stringing kits",
|
| 1124 |
+
"pegs",
|
| 1125 |
+
"set-in sleeves",
|
| 1126 |
+
"made by majestic",
|
| 1127 |
+
"scratch-resistant-sapphire crystal",
|
| 1128 |
+
"quality products",
|
| 1129 |
+
"detailed graphic artwork design",
|
| 1130 |
+
"paddling clothing",
|
| 1131 |
+
"shelters",
|
| 1132 |
+
"laces",
|
| 1133 |
+
"pulleys",
|
| 1134 |
+
"horseshoes",
|
| 1135 |
+
"officially licensed by nfl",
|
| 1136 |
+
"saddle blankets",
|
| 1137 |
+
"lightweight",
|
| 1138 |
+
"chronograph",
|
| 1139 |
+
"bingo sets",
|
| 1140 |
+
"remotes",
|
| 1141 |
+
"machine wash cold",
|
| 1142 |
+
"tire repair kits",
|
| 1143 |
+
"golf bracelets",
|
| 1144 |
+
"waterskis",
|
| 1145 |
+
"mascot costumes",
|
| 1146 |
+
"machine wash cold; tumble dry low",
|
| 1147 |
+
"replacement cleats",
|
| 1148 |
+
"wire terminals",
|
| 1149 |
+
"compression sacks",
|
| 1150 |
+
"card shufflers",
|
| 1151 |
+
"boomerangs",
|
| 1152 |
+
"jumping trainers",
|
| 1153 |
+
"quality embroidery",
|
| 1154 |
+
"trailer accessories",
|
| 1155 |
+
"rib protectors",
|
| 1156 |
+
"full athletic fit",
|
| 1157 |
+
"pellets",
|
| 1158 |
+
"suspension service parts",
|
| 1159 |
+
"parts kits",
|
| 1160 |
+
"officially licensed by the mlb",
|
| 1161 |
+
"adjustable",
|
| 1162 |
+
"boxing pads",
|
| 1163 |
+
"dri-power moisture wicking fabric",
|
| 1164 |
+
"toggle switches",
|
| 1165 |
+
"pogo sticks",
|
| 1166 |
+
"race equipment",
|
| 1167 |
+
"stainless-steel case; black dial; date function",
|
| 1168 |
+
"enclosures",
|
| 1169 |
+
"leg warmers",
|
| 1170 |
+
"licensed collegiate product",
|
| 1171 |
+
"tap dancing equipment",
|
| 1172 |
+
"duffles",
|
| 1173 |
+
"rooftop racks",
|
| 1174 |
+
"water pressure pumps",
|
| 1175 |
+
"japanese quartz movement with analog display",
|
| 1176 |
+
"skorts",
|
| 1177 |
+
"curtains",
|
| 1178 |
+
"in-ground",
|
| 1179 |
+
"card sleeves",
|
| 1180 |
+
"hitting mats",
|
| 1181 |
+
"card albums",
|
| 1182 |
+
"binoculars",
|
| 1183 |
+
"desert dry moisture management polo",
|
| 1184 |
+
"storage",
|
| 1185 |
+
"swim fins",
|
| 1186 |
+
"standard weight benches",
|
| 1187 |
+
"stainless steel case",
|
| 1188 |
+
"attacker shafts",
|
| 1189 |
+
"fitness planners",
|
| 1190 |
+
"scorebooks",
|
| 1191 |
+
"silk eye bags",
|
| 1192 |
+
"wire",
|
| 1193 |
+
"air fresheners",
|
| 1194 |
+
"gun cloths",
|
| 1195 |
+
"dock cleats",
|
| 1196 |
+
"desk lamps",
|
| 1197 |
+
"quartz movement with analog display",
|
| 1198 |
+
"field player gloves",
|
| 1199 |
+
"fast-pitch softballs",
|
| 1200 |
+
"bat racks",
|
| 1201 |
+
"optics covers",
|
| 1202 |
+
"precise and high-quality swiss-quartz movement",
|
| 1203 |
+
"night lights",
|
| 1204 |
+
"quickdraws",
|
| 1205 |
+
"capri pants",
|
| 1206 |
+
"electronic basketball",
|
| 1207 |
+
"foam pads",
|
| 1208 |
+
"personal care products",
|
| 1209 |
+
"mirrors",
|
| 1210 |
+
"machine washable, tumble dry low heat",
|
| 1211 |
+
"regular fit",
|
| 1212 |
+
"spikes",
|
| 1213 |
+
"candles",
|
| 1214 |
+
"facemasks",
|
| 1215 |
+
"pucks",
|
| 1216 |
+
"football helmets",
|
| 1217 |
+
"flag football belts",
|
| 1218 |
+
"treadmill motors",
|
| 1219 |
+
"scent accessories",
|
| 1220 |
+
"handbags",
|
| 1221 |
+
"paperweights",
|
| 1222 |
+
"toasters",
|
| 1223 |
+
"standard bases",
|
| 1224 |
+
"wind socks",
|
| 1225 |
+
"mask defoggers",
|
| 1226 |
+
"caster boards",
|
| 1227 |
+
"finger tabs",
|
| 1228 |
+
"fabric",
|
| 1229 |
+
"spray skirts",
|
| 1230 |
+
"bib shorts",
|
| 1231 |
+
"high quality",
|
| 1232 |
+
"drysuits",
|
| 1233 |
+
"tagless label, eliminates scratching",
|
| 1234 |
+
"double needle stitching",
|
| 1235 |
+
"ball storage",
|
| 1236 |
+
"tank covers",
|
| 1237 |
+
"messenger bags",
|
| 1238 |
+
"precise japanese-quartz movement",
|
| 1239 |
+
"pull-on closure",
|
| 1240 |
+
"spoke decorations",
|
| 1241 |
+
"weapon cases",
|
| 1242 |
+
"billiard lighting",
|
| 1243 |
+
"power cages",
|
| 1244 |
+
"sliding pads",
|
| 1245 |
+
"short sleeve basic crew neck tee",
|
| 1246 |
+
"buckle closure",
|
| 1247 |
+
"slap straps",
|
| 1248 |
+
"risers",
|
| 1249 |
+
"raglan sleeves",
|
| 1250 |
+
"support your favorite nhl team",
|
| 1251 |
+
"parts",
|
| 1252 |
+
"ncaa licensed product",
|
| 1253 |
+
"tie-downs",
|
| 1254 |
+
"fitness circles",
|
| 1255 |
+
"catcher helmets",
|
| 1256 |
+
"table tennis sets",
|
| 1257 |
+
"water sports accessories",
|
| 1258 |
+
"gaffs",
|
| 1259 |
+
"jacks",
|
| 1260 |
+
"exercise ball accessories",
|
| 1261 |
+
"forearm guards",
|
| 1262 |
+
"adjustable plastic snapback cap",
|
| 1263 |
+
"swim belts",
|
| 1264 |
+
"face guards",
|
| 1265 |
+
"poker table tops",
|
| 1266 |
+
"padded shorts",
|
| 1267 |
+
"officially licensed product",
|
| 1268 |
+
"snorkel vests",
|
| 1269 |
+
"hearing enhancement aids",
|
| 1270 |
+
"unstrung",
|
| 1271 |
+
"chain guards",
|
| 1272 |
+
"made by new era",
|
| 1273 |
+
"cotton blend",
|
| 1274 |
+
"polyester-blend",
|
| 1275 |
+
"rear shocks",
|
| 1276 |
+
"by adidas",
|
| 1277 |
+
"quickly",
|
| 1278 |
+
"tetherball",
|
| 1279 |
+
"gun snakes",
|
| 1280 |
+
"goose",
|
| 1281 |
+
"net systems",
|
| 1282 |
+
"climbing protection",
|
| 1283 |
+
"shot puts",
|
| 1284 |
+
"training cones",
|
| 1285 |
+
"foam wedges",
|
| 1286 |
+
"marker buoys",
|
| 1287 |
+
"windsurfing",
|
| 1288 |
+
"agility ladders",
|
| 1289 |
+
"training putters",
|
| 1290 |
+
"polycarbonate lens",
|
| 1291 |
+
"kneeboarding",
|
| 1292 |
+
"cornhole boards",
|
| 1293 |
+
"clips",
|
| 1294 |
+
"short sleeves",
|
| 1295 |
+
"desert dry polo",
|
| 1296 |
+
"basic bows",
|
| 1297 |
+
"fleece lining",
|
| 1298 |
+
"tip-ups",
|
| 1299 |
+
"gun holders",
|
| 1300 |
+
"outdoor skates",
|
| 1301 |
+
"short-sleeve shirt",
|
| 1302 |
+
"fishing belts",
|
| 1303 |
+
"co2 systems",
|
| 1304 |
+
"slim flights",
|
| 1305 |
+
"dri-fit",
|
| 1306 |
+
"sweatpants",
|
| 1307 |
+
"indoor surfboard storage",
|
| 1308 |
+
"water helmets",
|
| 1309 |
+
"elastic waistband",
|
| 1310 |
+
"dip stands",
|
| 1311 |
+
"calorie counters",
|
| 1312 |
+
"horse driving equipment",
|
| 1313 |
+
"ladder ball",
|
| 1314 |
+
"performance wear, loose fit",
|
| 1315 |
+
"swiss quartz movement with analog display",
|
| 1316 |
+
"knee high socks",
|
| 1317 |
+
"digital gauges",
|
| 1318 |
+
"indoor kayak storage",
|
| 1319 |
+
"shoulder-to-shoulder taping",
|
| 1320 |
+
"adjustable closure",
|
| 1321 |
+
"wall-mount",
|
| 1322 |
+
"eggs",
|
| 1323 |
+
"roulette equipment",
|
| 1324 |
+
"dog packs",
|
| 1325 |
+
"olympic weight benches",
|
| 1326 |
+
"surfboard racks",
|
| 1327 |
+
"pet harnesses",
|
| 1328 |
+
"machine wash, tumble dry",
|
| 1329 |
+
"made in pakistan",
|
| 1330 |
+
"hecho en ee. uu. e importado",
|
| 1331 |
+
"side entry pockets",
|
| 1332 |
+
"child seats",
|
| 1333 |
+
"officially licensed.",
|
| 1334 |
+
"computer accessories",
|
| 1335 |
+
"self fabric neck band",
|
| 1336 |
+
"utility cord",
|
| 1337 |
+
"spoke tools",
|
| 1338 |
+
"durable mineral crystal",
|
| 1339 |
+
"nipples",
|
| 1340 |
+
"crops",
|
| 1341 |
+
"made with techno-dry",
|
| 1342 |
+
"ankle guards",
|
| 1343 |
+
"shortboards",
|
| 1344 |
+
"women's balls",
|
| 1345 |
+
"classic style hood with drawstring",
|
| 1346 |
+
"official nba product",
|
| 1347 |
+
"tagless heat-seal neck label",
|
| 1348 |
+
"hockey sticks",
|
| 1349 |
+
"padding supplies",
|
| 1350 |
+
"grease",
|
| 1351 |
+
"weather monitors",
|
| 1352 |
+
"external frame backpacks",
|
| 1353 |
+
"great gift for sport fans!",
|
| 1354 |
+
"automatic movement",
|
| 1355 |
+
"pockets",
|
| 1356 |
+
"coverseamed neck double-needle hem",
|
| 1357 |
+
"cotton-blend",
|
| 1358 |
+
"spike wrenches",
|
| 1359 |
+
"unisex",
|
| 1360 |
+
"composite frame",
|
| 1361 |
+
"kicking targets",
|
| 1362 |
+
"crewneck",
|
| 1363 |
+
"bells",
|
| 1364 |
+
"replacement lenses",
|
| 1365 |
+
"preshrunk",
|
| 1366 |
+
"premium knit cap",
|
| 1367 |
+
"39thirty flex fit cap",
|
| 1368 |
+
"mitt accessories",
|
| 1369 |
+
"water globes",
|
| 1370 |
+
"workout shorts",
|
| 1371 |
+
"analog-quartz movement",
|
| 1372 |
+
"heatgear",
|
| 1373 |
+
"machine wash and dry",
|
| 1374 |
+
"sleeveless suits",
|
| 1375 |
+
"cotton/polyester",
|
| 1376 |
+
"pouch pocket",
|
| 1377 |
+
"scoop neck tee",
|
| 1378 |
+
"rib knit cuffs and waistband",
|
| 1379 |
+
"golf gift sets",
|
| 1380 |
+
"motor flushers",
|
| 1381 |
+
"patented one-fit sizing",
|
| 1382 |
+
"tagless technology for a better fit",
|
| 1383 |
+
"nike",
|
| 1384 |
+
"strung",
|
| 1385 |
+
"magnified date window at 3:00",
|
| 1386 |
+
"outdoor thermometers",
|
| 1387 |
+
"sais",
|
| 1388 |
+
"double-end bags",
|
| 1389 |
+
"fly tying kits",
|
| 1390 |
+
"plastic balls",
|
| 1391 |
+
"bean bag chairs",
|
| 1392 |
+
"moisture-wicking finish",
|
| 1393 |
+
"kicking shields",
|
| 1394 |
+
"embroidered logo",
|
| 1395 |
+
"two front pockets",
|
| 1396 |
+
"reversible",
|
| 1397 |
+
"thailand - vietnam",
|
| 1398 |
+
"bangladesh/pakistan",
|
| 1399 |
+
"material:spandex",
|
| 1400 |
+
"goal targets",
|
| 1401 |
+
"canoes",
|
| 1402 |
+
"steering wheels",
|
| 1403 |
+
"india",
|
| 1404 |
+
"waterproof",
|
| 1405 |
+
"comfortable",
|
| 1406 |
+
"mailboxes",
|
| 1407 |
+
"black",
|
| 1408 |
+
"fixed gear bike frames",
|
| 1409 |
+
"coverstiched seams",
|
| 1410 |
+
"canvas",
|
| 1411 |
+
"paper cups",
|
| 1412 |
+
"snap back",
|
| 1413 |
+
"no fur",
|
| 1414 |
+
"great quality jewelry!",
|
| 1415 |
+
"perfect for every day use",
|
| 1416 |
+
"nylon lining",
|
| 1417 |
+
"chain tension idlers",
|
| 1418 |
+
"alarm",
|
| 1419 |
+
"machine wash cold, air dry",
|
| 1420 |
+
"mounts",
|
| 1421 |
+
"rails",
|
| 1422 |
+
"japanese-quartz movement",
|
| 1423 |
+
"lesotho",
|
| 1424 |
+
"embroidered team graphics"
|
| 1425 |
+
]
|
data/amazon/intermediate/.gitkeep
DELETED
|
File without changes
|
download_emb.py
DELETED
|
@@ -1,41 +0,0 @@
|
|
| 1 |
-
import os.path as osp
|
| 2 |
-
import os
|
| 3 |
-
import argparse
|
| 4 |
-
import numpy as np
|
| 5 |
-
from tqdm import tqdm
|
| 6 |
-
import gdown
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def parse_args():
|
| 11 |
-
parser = argparse.ArgumentParser()
|
| 12 |
-
parser.add_argument("--dataset", default="amazon")
|
| 13 |
-
parser.add_argument("--emb_dir", default="emb", type=str)
|
| 14 |
-
return parser.parse_args()
|
| 15 |
-
|
| 16 |
-
if __name__ == "__main__":
|
| 17 |
-
args = parse_args()
|
| 18 |
-
|
| 19 |
-
emb_model = 'text-embedding-ada-002'
|
| 20 |
-
query_emb_token = {'amazon': '1-zyI84MMh6r66-faOFSc2rWTeIUw3VZW',
|
| 21 |
-
'mag': '1HSfUrSKBa7mJbECFbnKPQgd6HSsI8spT',
|
| 22 |
-
'primekg': '1MshwJttPZsHEM2cKA5T13SIrsLeBEdyU'}
|
| 23 |
-
node_emb_token = {'amazon': '18NU7tw_Tcyp9YobxKubLISBncwLaAiJz',
|
| 24 |
-
'mag': '1oVdScsDRuEpCFXtWQcTAx7ycvOggWF17',
|
| 25 |
-
'primekg': '16EJvCMbgkVrQ0BuIBvLBp-BYPaye-Edy'}
|
| 26 |
-
|
| 27 |
-
dataset = args.dataset
|
| 28 |
-
query_emb_url = 'https://drive.google.com/uc?id=' + query_emb_token[dataset]
|
| 29 |
-
node_emb_url = 'https://drive.google.com/uc?id=' + node_emb_token[dataset]
|
| 30 |
-
|
| 31 |
-
emb_dir = osp.join(args.emb_dir, dataset, emb_model)
|
| 32 |
-
query_emb_dir = osp.join(emb_dir, "query")
|
| 33 |
-
node_emb_dir = osp.join(emb_dir, "doc")
|
| 34 |
-
os.makedirs(query_emb_dir, exist_ok=True)
|
| 35 |
-
os.makedirs(node_emb_dir, exist_ok=True)
|
| 36 |
-
query_emb_path = osp.join(query_emb_dir, "query_emb_dict.pt")
|
| 37 |
-
node_emb_path = osp.join(node_emb_dir, "candidate_emb_dict.pt")
|
| 38 |
-
|
| 39 |
-
gdown.download(query_emb_url, query_emb_path, quiet=False)
|
| 40 |
-
gdown.download(node_emb_url, node_emb_path, quiet=False)
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval.py
DELETED
|
@@ -1,134 +0,0 @@
|
|
| 1 |
-
import os.path as osp
|
| 2 |
-
import json
|
| 3 |
-
import os
|
| 4 |
-
import argparse
|
| 5 |
-
import numpy as np
|
| 6 |
-
import pandas as pd
|
| 7 |
-
from tqdm import tqdm
|
| 8 |
-
import torch
|
| 9 |
-
from src.benchmarks import get_qa_dataset, get_semistructured_data
|
| 10 |
-
from src.models import get_model
|
| 11 |
-
from src.tools.args import merge_args, load_args
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
def parse_args():
|
| 15 |
-
parser = argparse.ArgumentParser()
|
| 16 |
-
parser.add_argument("--dataset", default="amazon", choices=['amazon', 'primekg', 'mag'])
|
| 17 |
-
parser.add_argument(
|
| 18 |
-
"--model", default="VSS", choices=["VSS", "MultiVSS", "LLMReranker"]
|
| 19 |
-
)
|
| 20 |
-
parser.add_argument("--split", default="test")
|
| 21 |
-
|
| 22 |
-
# can eval on a subset only
|
| 23 |
-
parser.add_argument("--test_ratio", type=float, default=1.0)
|
| 24 |
-
|
| 25 |
-
# for multivss
|
| 26 |
-
parser.add_argument("--chunk_size", type=int, default=None)
|
| 27 |
-
parser.add_argument("--multi_vss_topk", type=int, default=None)
|
| 28 |
-
parser.add_argument("--aggregate", type=str, default="max")
|
| 29 |
-
|
| 30 |
-
# for vss, multivss, and llm reranker
|
| 31 |
-
parser.add_argument("--emb_model", type=str, default="text-embedding-ada-002")
|
| 32 |
-
|
| 33 |
-
# for llm reranker
|
| 34 |
-
parser.add_argument("--llm_model", type=str, default="gpt-4-1106-preview",
|
| 35 |
-
help='the LLM to rerank candidates.')
|
| 36 |
-
parser.add_argument("--llm_topk", type=int, default=20)
|
| 37 |
-
parser.add_argument("--max_retry", type=int, default=3)
|
| 38 |
-
|
| 39 |
-
# path
|
| 40 |
-
parser.add_argument("--emb_dir", type=str, required=True)
|
| 41 |
-
parser.add_argument("--output_dir", type=str, required=True)
|
| 42 |
-
|
| 43 |
-
# save prediction
|
| 44 |
-
parser.add_argument("--save_pred", action="store_true")
|
| 45 |
-
return parser.parse_args()
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
if __name__ == "__main__":
|
| 49 |
-
args = parse_args()
|
| 50 |
-
default_args = load_args(
|
| 51 |
-
json.load(open("config/default_args.json", "r"))[args.dataset]
|
| 52 |
-
)
|
| 53 |
-
args = merge_args(args, default_args)
|
| 54 |
-
|
| 55 |
-
args.query_emb_dir = osp.join(args.emb_dir, args.dataset, args.emb_model, "query")
|
| 56 |
-
args.node_emb_dir = osp.join(args.emb_dir, args.dataset, args.emb_model, "doc")
|
| 57 |
-
args.chunk_emb_dir = osp.join(args.emb_dir, args.dataset, args.emb_model, "chunk")
|
| 58 |
-
surfix = args.llm_model if args.model == 'LLMReranker' else args.emb_model
|
| 59 |
-
output_dir = osp.join(args.output_dir, "eval", args.dataset, args.model, surfix)
|
| 60 |
-
|
| 61 |
-
os.makedirs(output_dir, exist_ok=True)
|
| 62 |
-
os.makedirs(args.query_emb_dir, exist_ok=True)
|
| 63 |
-
os.makedirs(args.chunk_emb_dir, exist_ok=True)
|
| 64 |
-
os.makedirs(args.node_emb_dir, exist_ok=True)
|
| 65 |
-
json.dump(vars(args), open(osp.join(output_dir, "args.json"), "w"), indent=4)
|
| 66 |
-
|
| 67 |
-
eval_csv_path = osp.join(output_dir, f"eval_results_{args.split}.csv")
|
| 68 |
-
final_eval_path = (
|
| 69 |
-
osp.join(output_dir, f"eval_metrics_{args.split}.json")
|
| 70 |
-
if args.test_ratio == 1.0
|
| 71 |
-
else osp.join(output_dir, f"eval_metrics_{args.split}_{args.test_ratio}.json")
|
| 72 |
-
)
|
| 73 |
-
|
| 74 |
-
kb = get_semistructured_data(args.dataset)
|
| 75 |
-
qa_dataset = get_qa_dataset(args.dataset)
|
| 76 |
-
model = get_model(args, kb)
|
| 77 |
-
|
| 78 |
-
split_idx = qa_dataset.get_idx_split(test_ratio=args.test_ratio)
|
| 79 |
-
|
| 80 |
-
eval_metrics = [
|
| 81 |
-
"mrr",
|
| 82 |
-
"map",
|
| 83 |
-
"rprecision",
|
| 84 |
-
"recall@5",
|
| 85 |
-
"recall@10",
|
| 86 |
-
"recall@20",
|
| 87 |
-
"recall@50",
|
| 88 |
-
"recall@100",
|
| 89 |
-
"hit@1",
|
| 90 |
-
"hit@3",
|
| 91 |
-
"hit@5",
|
| 92 |
-
"hit@10",
|
| 93 |
-
"hit@20",
|
| 94 |
-
"hit@50",
|
| 95 |
-
]
|
| 96 |
-
eval_csv = pd.DataFrame(columns=["idx", "query_id", "pred_rank"] + eval_metrics)
|
| 97 |
-
|
| 98 |
-
existing_idx = []
|
| 99 |
-
if osp.exists(eval_csv_path):
|
| 100 |
-
eval_csv = pd.read_csv(eval_csv_path)
|
| 101 |
-
existing_idx = eval_csv["idx"].tolist()
|
| 102 |
-
|
| 103 |
-
indices = split_idx[args.split].tolist()
|
| 104 |
-
|
| 105 |
-
for idx in tqdm(indices):
|
| 106 |
-
if idx in existing_idx:
|
| 107 |
-
continue
|
| 108 |
-
query, query_id, answer_ids, meta_info = qa_dataset[idx]
|
| 109 |
-
pred_dict = model.forward(query, query_id)
|
| 110 |
-
|
| 111 |
-
answer_ids = torch.LongTensor(answer_ids)
|
| 112 |
-
result = model.evaluate(pred_dict, answer_ids, metrics=eval_metrics)
|
| 113 |
-
|
| 114 |
-
result["idx"], result["query_id"] = idx, query_id
|
| 115 |
-
result["pred_rank"] = torch.LongTensor(list(pred_dict.keys()))[
|
| 116 |
-
torch.argsort(torch.tensor(list(pred_dict.values())), descending=True)[
|
| 117 |
-
:1000
|
| 118 |
-
]
|
| 119 |
-
].tolist()
|
| 120 |
-
|
| 121 |
-
eval_csv = pd.concat([eval_csv, pd.DataFrame([result])], ignore_index=True)
|
| 122 |
-
|
| 123 |
-
if args.save_pred:
|
| 124 |
-
eval_csv.to_csv(eval_csv_path, index=False)
|
| 125 |
-
for metric in eval_metrics:
|
| 126 |
-
print(
|
| 127 |
-
f"{metric}: {np.mean(eval_csv[eval_csv['idx'].isin(indices)][metric])}"
|
| 128 |
-
)
|
| 129 |
-
if args.save_pred:
|
| 130 |
-
eval_csv.to_csv(eval_csv_path, index=False)
|
| 131 |
-
final_metrics = (
|
| 132 |
-
eval_csv[eval_csv["idx"].isin(indices)][eval_metrics].mean().to_dict()
|
| 133 |
-
)
|
| 134 |
-
json.dump(final_metrics, open(final_eval_path, "w"), indent=4)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
generate_emb.py
DELETED
|
@@ -1,83 +0,0 @@
|
|
| 1 |
-
import sys
|
| 2 |
-
|
| 3 |
-
import os
|
| 4 |
-
import os.path as osp
|
| 5 |
-
|
| 6 |
-
import torch
|
| 7 |
-
import random
|
| 8 |
-
import json
|
| 9 |
-
import time
|
| 10 |
-
import pickle
|
| 11 |
-
import argparse
|
| 12 |
-
import numpy as np
|
| 13 |
-
import pandas as pd
|
| 14 |
-
from tqdm import tqdm
|
| 15 |
-
sys.path.append('.')
|
| 16 |
-
from src.benchmarks import get_semistructured_data, get_qa_dataset
|
| 17 |
-
from src.tools.api import get_openai_embeddings
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def parse_args():
|
| 21 |
-
parser = argparse.ArgumentParser()
|
| 22 |
-
parser.add_argument('--dataset', default='amazon',
|
| 23 |
-
choices=['amazon', 'primekg', 'mag']
|
| 24 |
-
)
|
| 25 |
-
parser.add_argument('--emb_model', default='text-embedding-ada-002',
|
| 26 |
-
choices=[
|
| 27 |
-
'text-embedding-ada-002',
|
| 28 |
-
'text-embedding-3-small',
|
| 29 |
-
'text-embedding-3-large'
|
| 30 |
-
]
|
| 31 |
-
)
|
| 32 |
-
parser.add_argument('--mode', default='doc', choices=['doc', 'query'])
|
| 33 |
-
parser.add_argument("--emb_dir", default="emb/", type=str)
|
| 34 |
-
parser.add_argument('--add_rel', action='store_true', default=False,
|
| 35 |
-
help='add relation to the text')
|
| 36 |
-
parser.add_argument('--compact', action='store_true', default=False,
|
| 37 |
-
help='make the text compact when input to the model')
|
| 38 |
-
return parser.parse_args()
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
if __name__ == '__main__':
|
| 43 |
-
args = parse_args()
|
| 44 |
-
emb_dir = osp.join(args.emb_dir, args.dataset, args.emb_model, args.mode)
|
| 45 |
-
os.makedirs(emb_dir, exist_ok=True)
|
| 46 |
-
|
| 47 |
-
if args.mode == 'doc':
|
| 48 |
-
kb = get_semistructured_data(args.dataset)
|
| 49 |
-
lst = kb.candidate_ids
|
| 50 |
-
emb_path = osp.join(emb_dir, f'candidate_emb_dict.pt')
|
| 51 |
-
if args.mode == 'query':
|
| 52 |
-
qa_dataset = get_qa_dataset(args.dataset)
|
| 53 |
-
lst = [qa_dataset[i][1] for i in range(len(qa_dataset))]
|
| 54 |
-
emb_path = osp.join(emb_dir, f'query_emb_dict.pt')
|
| 55 |
-
random.shuffle(lst)
|
| 56 |
-
|
| 57 |
-
if osp.exists(emb_path):
|
| 58 |
-
emb_dict = torch.load(emb_path)
|
| 59 |
-
exisiting_indices = list(emb_dict.keys())
|
| 60 |
-
print(f'Loaded existing embeddings from {emb_path}. Size: {len(emb_dict)}')
|
| 61 |
-
else:
|
| 62 |
-
emb_dict = {}
|
| 63 |
-
exisiting_indices = []
|
| 64 |
-
|
| 65 |
-
texts, indices = [], []
|
| 66 |
-
for idx in tqdm(lst):
|
| 67 |
-
if idx in exisiting_indices:
|
| 68 |
-
continue
|
| 69 |
-
if args.mode == 'query':
|
| 70 |
-
text = qa_dataset.get_query_by_qid(idx)
|
| 71 |
-
elif args.mode == 'doc':
|
| 72 |
-
text = kb.get_doc_info(idx, add_rel=args.add_rel, compact=args.compact)
|
| 73 |
-
texts.append(text)
|
| 74 |
-
indices.append(idx)
|
| 75 |
-
|
| 76 |
-
print(f'Generating embeddings for {len(texts)} texts...')
|
| 77 |
-
embs = get_openai_embeddings(texts, model=args.emb_model).view(len(texts), -1).cpu()
|
| 78 |
-
print('Embedding size:', embs.size())
|
| 79 |
-
|
| 80 |
-
for idx, emb in zip(indices, embs):
|
| 81 |
-
emb_dict[idx] = emb
|
| 82 |
-
torch.save(emb_dict, emb_path)
|
| 83 |
-
print(f'Saved embeddings to {emb_path}!')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
interactive/pyvis_graph.py
CHANGED
|
@@ -8,7 +8,7 @@ import re
|
|
| 8 |
from src.benchmarks import get_semistructured_data
|
| 9 |
|
| 10 |
CONCURRENCY_LIMIT = 1000
|
| 11 |
-
TITLE = "STaRK
|
| 12 |
BRAND_NAME = {
|
| 13 |
"amazon": "STaRK-Amazon",
|
| 14 |
"mag": "STaRK-MAG",
|
|
@@ -236,9 +236,9 @@ def main():
|
|
| 236 |
["1", "2", "inf"], value="2", label="Number of Hops"
|
| 237 |
)
|
| 238 |
query_btn = gr.Button(
|
| 239 |
-
value="
|
| 240 |
variant="primary",
|
| 241 |
-
elem_id=f"{name}-fetch-btn"
|
| 242 |
)
|
| 243 |
|
| 244 |
with gr.Row():
|
|
|
|
| 8 |
from src.benchmarks import get_semistructured_data
|
| 9 |
|
| 10 |
CONCURRENCY_LIMIT = 1000
|
| 11 |
+
TITLE = "STaRK Semi-structured Knowledge Base Explorer"
|
| 12 |
BRAND_NAME = {
|
| 13 |
"amazon": "STaRK-Amazon",
|
| 14 |
"mag": "STaRK-MAG",
|
|
|
|
| 236 |
["1", "2", "inf"], value="2", label="Number of Hops"
|
| 237 |
)
|
| 238 |
query_btn = gr.Button(
|
| 239 |
+
value="Display Semi-structured Data",
|
| 240 |
variant="primary",
|
| 241 |
+
elem_id=f"{name}-fetch-btn"
|
| 242 |
)
|
| 243 |
|
| 244 |
with gr.Row():
|
load_dataset.ipynb
DELETED
|
@@ -1,508 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cells": [
|
| 3 |
-
{
|
| 4 |
-
"cell_type": "code",
|
| 5 |
-
"execution_count": 1,
|
| 6 |
-
"metadata": {},
|
| 7 |
-
"outputs": [
|
| 8 |
-
{
|
| 9 |
-
"name": "stderr",
|
| 10 |
-
"output_type": "stream",
|
| 11 |
-
"text": [
|
| 12 |
-
"/dfs/scratch0/shirwu/anaconda3/envs/torch2/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
| 13 |
-
" from .autonotebook import tqdm as notebook_tqdm\n"
|
| 14 |
-
]
|
| 15 |
-
},
|
| 16 |
-
{
|
| 17 |
-
"name": "stdout",
|
| 18 |
-
"output_type": "stream",
|
| 19 |
-
"text": [
|
| 20 |
-
"loading dataset from external data\n",
|
| 21 |
-
"Load cached graph with meta link types ['brand']\n"
|
| 22 |
-
]
|
| 23 |
-
}
|
| 24 |
-
],
|
| 25 |
-
"source": [
|
| 26 |
-
"from src.benchmarks.get_qa_dataset import get_qa_dataset\n",
|
| 27 |
-
"from src.benchmarks.get_semistruct import get_semistructured_data\n",
|
| 28 |
-
"\n",
|
| 29 |
-
"dataset_name = 'amazon'\n",
|
| 30 |
-
"\n",
|
| 31 |
-
"qa_dataset = get_qa_dataset(dataset_name)\n",
|
| 32 |
-
"kb = get_semistructured_data(dataset_name)"
|
| 33 |
-
]
|
| 34 |
-
},
|
| 35 |
-
{
|
| 36 |
-
"cell_type": "markdown",
|
| 37 |
-
"metadata": {},
|
| 38 |
-
"source": [
|
| 39 |
-
"### Load QA dataset"
|
| 40 |
-
]
|
| 41 |
-
},
|
| 42 |
-
{
|
| 43 |
-
"cell_type": "code",
|
| 44 |
-
"execution_count": 2,
|
| 45 |
-
"metadata": {},
|
| 46 |
-
"outputs": [
|
| 47 |
-
{
|
| 48 |
-
"name": "stdout",
|
| 49 |
-
"output_type": "stream",
|
| 50 |
-
"text": [
|
| 51 |
-
"Query: Looking for a user-friendly fly fishing knot guide with clear, easy-to-understand illustrations. Ideally, it should be logically organised for easy learning and effective in teaching dependable knot tying techniques. It would be a bonus if it complements the Anglers Accessories Gehrke's Gink that I frequently use. Any recommendations?\n",
|
| 52 |
-
"Query ID: 30\n",
|
| 53 |
-
"Answer:\n",
|
| 54 |
-
" Lake Products THREE-in-One Knot Tying Tool Fly Fishing\n",
|
| 55 |
-
"EZ Tie Blood Knot Tying Tool\n",
|
| 56 |
-
"BenchMaster Pocket Guide - Fly Fishing - Fishing\n"
|
| 57 |
-
]
|
| 58 |
-
}
|
| 59 |
-
],
|
| 60 |
-
"source": [
|
| 61 |
-
"# Get one qa pair, we masked out metadata to avoid answer leaking\n",
|
| 62 |
-
"query, q_id, answer_ids, _ = qa_dataset[1]\n",
|
| 63 |
-
"print('Query:', query)\n",
|
| 64 |
-
"print('Query ID:', q_id)\n",
|
| 65 |
-
"print('Answer:\\n', '\\n'.join([kb[aid].title for aid in answer_ids]))"
|
| 66 |
-
]
|
| 67 |
-
},
|
| 68 |
-
{
|
| 69 |
-
"cell_type": "code",
|
| 70 |
-
"execution_count": 3,
|
| 71 |
-
"metadata": {},
|
| 72 |
-
"outputs": [
|
| 73 |
-
{
|
| 74 |
-
"name": "stdout",
|
| 75 |
-
"output_type": "stream",
|
| 76 |
-
"text": [
|
| 77 |
-
"Number of training examples: 5910\n",
|
| 78 |
-
"Number of validation examples: 1548\n",
|
| 79 |
-
"Number of test examples: 1642\n"
|
| 80 |
-
]
|
| 81 |
-
},
|
| 82 |
-
{
|
| 83 |
-
"data": {
|
| 84 |
-
"text/plain": [
|
| 85 |
-
"{'train': tensor([3885, 4522, 2110, ..., 6839, 3967, 2814]),\n",
|
| 86 |
-
" 'val': tensor([1550, 1486, 6591, ..., 5606, 1204, 3792]),\n",
|
| 87 |
-
" 'test': tensor([2905, 3863, 4651, ..., 3891, 7631, 4472])}"
|
| 88 |
-
]
|
| 89 |
-
},
|
| 90 |
-
"execution_count": 3,
|
| 91 |
-
"metadata": {},
|
| 92 |
-
"output_type": "execute_result"
|
| 93 |
-
}
|
| 94 |
-
],
|
| 95 |
-
"source": [
|
| 96 |
-
"# We provide official random split for training, validation and test\n",
|
| 97 |
-
"print('Number of training examples:', len(qa_dataset.get_subset('train')))\n",
|
| 98 |
-
"print('Number of validation examples:', len(qa_dataset.get_subset('val')))\n",
|
| 99 |
-
"print('Number of test examples:', len(qa_dataset.get_subset('test')))\n",
|
| 100 |
-
"\n",
|
| 101 |
-
"# Alternatively, you can get the split indices\n",
|
| 102 |
-
"qa_dataset.get_idx_split()"
|
| 103 |
-
]
|
| 104 |
-
},
|
| 105 |
-
{
|
| 106 |
-
"cell_type": "markdown",
|
| 107 |
-
"metadata": {},
|
| 108 |
-
"source": [
|
| 109 |
-
"### Load Knowledge Base"
|
| 110 |
-
]
|
| 111 |
-
},
|
| 112 |
-
{
|
| 113 |
-
"cell_type": "code",
|
| 114 |
-
"execution_count": 4,
|
| 115 |
-
"metadata": {},
|
| 116 |
-
"outputs": [
|
| 117 |
-
{
|
| 118 |
-
"data": {
|
| 119 |
-
"text/plain": [
|
| 120 |
-
"[('brand', 'has_brand', 'product'),\n",
|
| 121 |
-
" ('product', 'also_buy', 'product'),\n",
|
| 122 |
-
" ('product', 'also_view', 'product'),\n",
|
| 123 |
-
" ('product', 'has_brand', 'brand')]"
|
| 124 |
-
]
|
| 125 |
-
},
|
| 126 |
-
"execution_count": 4,
|
| 127 |
-
"metadata": {},
|
| 128 |
-
"output_type": "execute_result"
|
| 129 |
-
}
|
| 130 |
-
],
|
| 131 |
-
"source": [
|
| 132 |
-
"# You can see part of the knowledge base schema here\n",
|
| 133 |
-
"kb.get_tuples()"
|
| 134 |
-
]
|
| 135 |
-
},
|
| 136 |
-
{
|
| 137 |
-
"cell_type": "code",
|
| 138 |
-
"execution_count": 5,
|
| 139 |
-
"metadata": {},
|
| 140 |
-
"outputs": [
|
| 141 |
-
{
|
| 142 |
-
"data": {
|
| 143 |
-
"text/plain": [
|
| 144 |
-
"(['product', 'brand'], ['also_buy', 'also_view', 'has_brand'])"
|
| 145 |
-
]
|
| 146 |
-
},
|
| 147 |
-
"execution_count": 5,
|
| 148 |
-
"metadata": {},
|
| 149 |
-
"output_type": "execute_result"
|
| 150 |
-
}
|
| 151 |
-
],
|
| 152 |
-
"source": [
|
| 153 |
-
"# Similarly, you can get the node and relation types \n",
|
| 154 |
-
"kb.node_type_lst(), kb.rel_type_lst()"
|
| 155 |
-
]
|
| 156 |
-
},
|
| 157 |
-
{
|
| 158 |
-
"cell_type": "code",
|
| 159 |
-
"execution_count": 6,
|
| 160 |
-
"metadata": {},
|
| 161 |
-
"outputs": [
|
| 162 |
-
{
|
| 163 |
-
"name": "stdout",
|
| 164 |
-
"output_type": "stream",
|
| 165 |
-
"text": [
|
| 166 |
-
"Number of nodes: 1032407\n",
|
| 167 |
-
"Number of edges: 6455692\n"
|
| 168 |
-
]
|
| 169 |
-
}
|
| 170 |
-
],
|
| 171 |
-
"source": [
|
| 172 |
-
"print('Number of nodes:', kb.num_nodes())\n",
|
| 173 |
-
"print('Number of edges:', kb.num_edges())"
|
| 174 |
-
]
|
| 175 |
-
},
|
| 176 |
-
{
|
| 177 |
-
"cell_type": "code",
|
| 178 |
-
"execution_count": 7,
|
| 179 |
-
"metadata": {},
|
| 180 |
-
"outputs": [
|
| 181 |
-
{
|
| 182 |
-
"data": {
|
| 183 |
-
"text/plain": [
|
| 184 |
-
"{'product': ['title',\n",
|
| 185 |
-
" 'dimensions',\n",
|
| 186 |
-
" 'weight',\n",
|
| 187 |
-
" 'description',\n",
|
| 188 |
-
" 'features',\n",
|
| 189 |
-
" 'reviews',\n",
|
| 190 |
-
" 'Q&A'],\n",
|
| 191 |
-
" 'brand': ['brand_name']}"
|
| 192 |
-
]
|
| 193 |
-
},
|
| 194 |
-
"execution_count": 7,
|
| 195 |
-
"metadata": {},
|
| 196 |
-
"output_type": "execute_result"
|
| 197 |
-
}
|
| 198 |
-
],
|
| 199 |
-
"source": [
|
| 200 |
-
"# We include the attributes in node's textual information as part of the schema\n",
|
| 201 |
-
"# Note that some nodes may not have all attributes while some may have additional attributes\n",
|
| 202 |
-
"kb.node_attr_dict"
|
| 203 |
-
]
|
| 204 |
-
},
|
| 205 |
-
{
|
| 206 |
-
"cell_type": "code",
|
| 207 |
-
"execution_count": 8,
|
| 208 |
-
"metadata": {},
|
| 209 |
-
"outputs": [
|
| 210 |
-
{
|
| 211 |
-
"name": "stdout",
|
| 212 |
-
"output_type": "stream",
|
| 213 |
-
"text": [
|
| 214 |
-
"- product: Lake Products THREE-in-One Knot Tying Tool Fly Fishing\n",
|
| 215 |
-
"- brand: Lake\n",
|
| 216 |
-
"- description: NEW & IMPROVED - Replaces the Two-in-One Knot Tying Tool - still ties many over 14 different knots, but now adds a magnetic hook threader; made of Delron and stainless steel; instruction book included.Precision machined contact firmly grips any fishing line, without causing damageUp and down spring action with stainless steel springThe body is manufactured of strong, lightweight Acetel Delrin for years of reliable serviceStainless Steel Shaft, head and loop will not rust or corrodeAttachment loop to clip onto clothing\n",
|
| 217 |
-
"- features: \n",
|
| 218 |
-
"#1: Precision machined contact firmly grips any fishing line, without causing damage\n",
|
| 219 |
-
"#2: Up and down spring action with stainless steel spring\n",
|
| 220 |
-
"#3: The body is manufactured of strong, lightweight Acetel Delrin for years of reliable service\n",
|
| 221 |
-
"#4: Stainless Steel Shaft, head and loop will not rust or corrode\n",
|
| 222 |
-
"#5: Attachment loop to clip onto clothing\n",
|
| 223 |
-
"- reviews: \n",
|
| 224 |
-
"#9:\n",
|
| 225 |
-
"summary: Works Great\n",
|
| 226 |
-
"text: \"Due to an injury, I have no finger dexterity in either hand and cannot feel the fishing line while holding it between my thumb and finger. It would literally take me about 15 minutes to tie a hook or swivel to my line and was extremely frustrating. This tool allows me to tie the knot in 15 seconds. It really works well and I recommend it.\"\n",
|
| 227 |
-
"#3:\n",
|
| 228 |
-
"summary: It works\n",
|
| 229 |
-
"text: \"I got this last night and decided to give it a try. Size 16 nymph, small tippet, no reading glasses (don't laugh, you'll get there), and light gloves. Threaded the fly and tied a cinch knot with no issues several times. It does a lot more, but this alone made it worth it. Ever tried tying on a fly with cold fingers or low light? The only downside is I wish there was a color difference in the v slot where the line threads in. Not a big enough deal to make a difference in opinion.\"\n",
|
| 230 |
-
"#4:\n",
|
| 231 |
-
"summary: Excellent\n",
|
| 232 |
-
"text: \"There are three curses to advancing years: (1) Fingers shake a bit more, (2) eyes don't see the small stuff as well in low light, and (3) we're still as irresponsible as we ever were--can't seem to stop ourselves from doing something stupid like getting up at 3:30 am to stand freezing in a stream, fingers shaking and eyes blurring even more from the chill.\n",
|
| 233 |
-
"\n",
|
| 234 |
-
"But this nice little gadget lets me tie even #22 flies onto a leader with ease. (And I couldn't really tie those on even when I was 25!) Using this tool to thread the tiniest hook eyes onto a leader may sometimes take a little concentration (must use a leader size appropriate for the hook eye, after all), but threading isn't the main thing we need--invariably our problem is tying the knot itself, without crushing the hackle, burying the hook into our finger, or losing the fly in the river during the attempt to hand-tie the knot. With this tool, in 8 or 10 seconds a secure clinch knot is done. Zero chance of losing the fly, having to start over, or foul language. I now find it easy to make the decision whether to try a different fly, because changing flies costs me almost no fishing time.\n",
|
| 235 |
-
"\n",
|
| 236 |
-
"The tool ties securely to a cord on the vest and is just plain there when (constantly) needed. The knots are as clean & compact and good as a knot can be. One time I did not have this tool with me, and I lost 20 minutes of my last fishing hour changing flies a couple of times.\n",
|
| 237 |
-
"\n",
|
| 238 |
-
"As long as I can get the fly threaded (this thing does often help with that too, although no tool is perfect for the very smallest of hook eyes on a bushy fly), then with this tool the job of tying the knot is a done deal.\n",
|
| 239 |
-
"\n",
|
| 240 |
-
"Also, I've found that Lake Products is very responsive and has a great customer service ethic. They really stand behind their product. I'm very happy with this little gizmo.\n",
|
| 241 |
-
"\n",
|
| 242 |
-
"- Michael Vorhis\n",
|
| 243 |
-
".....Author of OPEN DISTANCE deep sea/aviation thriller\n",
|
| 244 |
-
".....Author of ARCHANGEL suspense thriller\n",
|
| 245 |
-
".....Fly fisherman\"\n",
|
| 246 |
-
"#2:\n",
|
| 247 |
-
"summary: Excellent tool. Does what it says.\n",
|
| 248 |
-
"text: \"Hard to imagine an easier toot to use to tie Surgeon's Knots, Clinch and improved Climnch knots which are alll that I really need on a day to day basis. Its hook threader works like a charm for the size 16 midges i typically use.\"\n",
|
| 249 |
-
"#86:\n",
|
| 250 |
-
"summary: Consider alternatives\n",
|
| 251 |
-
"text: \"This device is supposed to tie a variety of knots and help you thread small hooks. The material quality seems good with the body made from a tough plastic (think cutting board) and the metal parts (including spring) made from stainless. I already thought the price was unreasonably high and was more disappointed when I tried to thread size 18 flies with great difficulty. The threader is no panacea for resolving this problem, but for slightly larger hooks, it seems to work ok. It looks like someone removed the mold marks with a pocket knife and a power sander. Smooth, but crude.\n",
|
| 252 |
-
"\n",
|
| 253 |
-
"While the knot tier works well for tying clinch knots, I found that a pair of forceps, especially ones with a curved tip actually work better, and you probably already have those in your tackle box. There are several good YouTube videos on how to use them.\"\n",
|
| 254 |
-
"#6:\n",
|
| 255 |
-
"summary: it works\n",
|
| 256 |
-
"text: \"I have trouble seeing the sting at twilight due to my need for reading glasses ( which I use sunglass bifocals). THis product helps for threading hooks and tying knots.\"\n",
|
| 257 |
-
"#7:\n",
|
| 258 |
-
"summary: It works\n",
|
| 259 |
-
"text: \"Showed and impressed other fly club members. I got 2, one for rigging and one for changes on the water.\"\n",
|
| 260 |
-
"#1:\n",
|
| 261 |
-
"summary: Makes tying knots much easier.\n",
|
| 262 |
-
"text: \"As a 68 year old my vision isn't as good as it was. I use the improved clinch knot the most and using the tip to pull the tag end through your loop eliminates the need for my reading glasses. Also tying nail or albright knots using the double slotted end is much easier.\"\n",
|
| 263 |
-
"#8:\n",
|
| 264 |
-
"summary: worthless for smaller hooks\n",
|
| 265 |
-
"text: \"Took a chance based on other reviews, but this product is worthless when it comes to threading smaller hooks, particular down-turned eyes in sizes 16 or smaller.\"\n",
|
| 266 |
-
"#26:\n",
|
| 267 |
-
"summary: Great product\n",
|
| 268 |
-
"text: \"Replaced a heavier brass tyer with this one and it works great. Before this product I had the brass tyer and a nail knot tool in my pack now I only have this one freeing up space hanging from a retracter. Don't hesitate to purchase one, you will not be disappointed\"\n",
|
| 269 |
-
"#23:\n",
|
| 270 |
-
"summary: Might be a great product, but there's a good chance you'll never ...\n",
|
| 271 |
-
"text: \"Might be a great product, but there's a good chance you'll never know because of the poor instructions that are provided with the tool. The major issues here are the small sizes of the font and diagrams, which makes it next to impossible to decrypt. I looked for a pdf of the knot typing manual on the seller's website and there is none, only an instructional video for an additional $5 or the same replacement manual for $3. Instruction on the use an application of a new tool is 90% of its value, and the seller must do a better job at this.\"\n",
|
| 272 |
-
"#67:\n",
|
| 273 |
-
"summary: My favorite thing in my fishing kit\n",
|
| 274 |
-
"text: \"Love this thing. I have several of these for various tackle boxes. I got this in the yellow color for when I put it down. The booklet that comes with this is professional and easy to understand. It is a great attention device as you are sitting in the doctor's office and practicing knot tying. Fly fishing to whatever.\n",
|
| 275 |
-
"Note: Fly fishing purists use this when no one is looking.\"\n",
|
| 276 |
-
"#14:\n",
|
| 277 |
-
"summary: Almost ready to give up on fly fishing.\n",
|
| 278 |
-
"text: \"So I was fly-fishing on the river and nothing was biting. So I decided to try a different fly. AftER about five minutes of trying and messing with the fly threader I purchased on a auction site. I pulled the tippet up off of the thread to learn the tippet wasn't in the hook eye, and seeing my fly pop up and drop in the river. I just lost another $1.50 fly and that was my last one. So I decided to try this threader before I quit fly fishing. Not only has this threader work for me everytime. But it's also a knot tyer. So now I can change flies in about a minute, instead of the 20 minutes it did before. This thread has a ring to clip to my lanyard, or to my zinger for easy access. Wish I would have purchased years ago. I might have done more fishing! Yes! I would recommend this to anyone!\"\n",
|
| 279 |
-
"#48:\n",
|
| 280 |
-
"summary: Helpful but not perfect\n",
|
| 281 |
-
"text: \"It's very helpful for 64-year-old hands that struggle with tiny tippet and leader, but the design of the tip could be better. It can get tangled in the tip as line/tippet are drawn through loops. Maybe I just need some more practice, but it's been a challenge.\"\n",
|
| 282 |
-
"#89:\n",
|
| 283 |
-
"summary: Tool for Many Uses; Handy Item (Versatile)\n",
|
| 284 |
-
"text: \"Although its not a solution for all problems encountered with threading lines through the eyes of hooks, flies, & lures, etc., the Lake Products 3-in-1 Knot Tying Tool has proven beneficial (for me) to carry it along each fishing outing. There are some limitations to what it can help to accomplish. For example, you cannot thread a line of large diameter through the eye of a smaller fly. The 6X tippet or smaller diameters (7X, 8X, etc.) are required for the smaller flies. Larger diameter tippets greater than 6X (5X, 4X, 3X, etc.) can be threaded into medium and large flies. I was skeptical at first (and sometimes still don't accomplish all tasks that I would like to) but it has been tremendously helpful with knot tying as well. Tying tippet-knots that create leader at the end of fly line is much simpler and quicker, and so is the knot thats needed once the line is threaded through the eye of the hook/fly. The magnet that holds the fly is relatively strong to secure the fly while threading the eye. Those who struggle with focusing (poor vision) will especially find it helpful. Ive found it a worthwhile purchase and would recommend it to a friend. At times, it has been my only method of achieving certain tasks.\"\n",
|
| 285 |
-
"#88:\n",
|
| 286 |
-
"summary: HANDY DANDY LAKE PRODUCT KNOT TYING TOOL\n",
|
| 287 |
-
"text: \"all i can say about this product is its the best for old eyes and crooked hands. use it at least three times a week, would recommend\"\n",
|
| 288 |
-
"#45:\n",
|
| 289 |
-
"summary: Good to have if you're having trouble seeing small things.\n",
|
| 290 |
-
"text: \"Very easy to use - once you read the instructions. Not intuitive but works as documented. Getting older and eyesight is diminishing so I bought a second one as a spare.\"\n",
|
| 291 |
-
"#5:\n",
|
| 292 |
-
"summary: Three in one tool.\n",
|
| 293 |
-
"text: \"This tool is perfect for my failing eyesight and arthritic fingers. Just what I needed. The fisherman's magic wand for stiff and cold fingers\"\n",
|
| 294 |
-
"#71:\n",
|
| 295 |
-
"summary: Best invention ever.\n",
|
| 296 |
-
"text: \"This tool is great. It speeds up my tying 100% and allows me to tie small flies on tiny tippets without much trouble. It definitely is worth every penny for an older man with arthritis and fading eyesight.\"\n",
|
| 297 |
-
"#70:\n",
|
| 298 |
-
"summary: easy to use tool\n",
|
| 299 |
-
"text: \"Simpe to use and I carry it on all my fishing trips. sure makes it a lot easier to thread flies and tie them on the line\"\n",
|
| 300 |
-
"#69:\n",
|
| 301 |
-
"summary: The Best\n",
|
| 302 |
-
"text: \"Makes it so easy with my shaky hands\"\n",
|
| 303 |
-
"#68:\n",
|
| 304 |
-
"summary: Great tool for all fly knots\n",
|
| 305 |
-
"text: \"Helpful tool. Really helps threading fly for these over 50 eyes\"\n",
|
| 306 |
-
"#73:\n",
|
| 307 |
-
"summary: Five Stars\n",
|
| 308 |
-
"text: \"Great tool!\"\n",
|
| 309 |
-
"#72:\n",
|
| 310 |
-
"summary: Five Stars\n",
|
| 311 |
-
"text: \"As expected\"\n",
|
| 312 |
-
"#64:\n",
|
| 313 |
-
"summary: Still learning to tie knots, and learn all the ...\n",
|
| 314 |
-
"text: \"Still learning to tie knots, and learn all the different knots you can do with this tool. should prove to be very helpful\"\n",
|
| 315 |
-
"#65:\n",
|
| 316 |
-
"summary: Great Tool for all fishermen\n",
|
| 317 |
-
"text: \"Does what it says it does if you follow the instruction booklet which is well written. I tied 5X and 6X tippet together with ease...this is a versatile tool and NOT just for Clinch Knots or improved Clinch Knots which it does with ease.\"\n",
|
| 318 |
-
"#74:\n",
|
| 319 |
-
"summary: Three Stars\n",
|
| 320 |
-
"text: \"GREAT IDEA - BUT STILL TRICKY IF YOUR HANDS SHAKE ANDPOOR EYE SIGHT\"\n",
|
| 321 |
-
"\n"
|
| 322 |
-
]
|
| 323 |
-
}
|
| 324 |
-
],
|
| 325 |
-
"source": [
|
| 326 |
-
"# Each node has textual information\n",
|
| 327 |
-
"print(kb.get_doc_info(answer_ids[0], add_rel=False))"
|
| 328 |
-
]
|
| 329 |
-
},
|
| 330 |
-
{
|
| 331 |
-
"cell_type": "code",
|
| 332 |
-
"execution_count": 9,
|
| 333 |
-
"metadata": {},
|
| 334 |
-
"outputs": [
|
| 335 |
-
{
|
| 336 |
-
"name": "stdout",
|
| 337 |
-
"output_type": "stream",
|
| 338 |
-
"text": [
|
| 339 |
-
"The neighbors of the answer node are: 222\n"
|
| 340 |
-
]
|
| 341 |
-
}
|
| 342 |
-
],
|
| 343 |
-
"source": [
|
| 344 |
-
"# Each node can be linked to other nodes\n",
|
| 345 |
-
"neighbor_lst = kb.get_neighbor_nodes(answer_ids[0], edge_type='*')\n",
|
| 346 |
-
"print('The neighbors of the answer node are:', len(neighbor_lst))"
|
| 347 |
-
]
|
| 348 |
-
},
|
| 349 |
-
{
|
| 350 |
-
"cell_type": "code",
|
| 351 |
-
"execution_count": 10,
|
| 352 |
-
"metadata": {},
|
| 353 |
-
"outputs": [
|
| 354 |
-
{
|
| 355 |
-
"name": "stdout",
|
| 356 |
-
"output_type": "stream",
|
| 357 |
-
"text": [
|
| 358 |
-
"Counter({'product': 221, 'brand': 1})\n"
|
| 359 |
-
]
|
| 360 |
-
}
|
| 361 |
-
],
|
| 362 |
-
"source": [
|
| 363 |
-
"# Count the number of each type\n",
|
| 364 |
-
"from collections import Counter\n",
|
| 365 |
-
"neighbor_types = [kb.get_node_type_by_id(neighbor) for neighbor in neighbor_lst]\n",
|
| 366 |
-
"print(Counter(neighbor_types))"
|
| 367 |
-
]
|
| 368 |
-
},
|
| 369 |
-
{
|
| 370 |
-
"cell_type": "markdown",
|
| 371 |
-
"metadata": {},
|
| 372 |
-
"source": [
|
| 373 |
-
"### Take PrimeKG as another example"
|
| 374 |
-
]
|
| 375 |
-
},
|
| 376 |
-
{
|
| 377 |
-
"cell_type": "code",
|
| 378 |
-
"execution_count": 11,
|
| 379 |
-
"metadata": {},
|
| 380 |
-
"outputs": [
|
| 381 |
-
{
|
| 382 |
-
"name": "stdout",
|
| 383 |
-
"output_type": "stream",
|
| 384 |
-
"text": [
|
| 385 |
-
"loading dataset from external data\n",
|
| 386 |
-
"Loaded from data/primekg/processed!\n"
|
| 387 |
-
]
|
| 388 |
-
}
|
| 389 |
-
],
|
| 390 |
-
"source": [
|
| 391 |
-
"dataset_name = 'primekg'\n",
|
| 392 |
-
"\n",
|
| 393 |
-
"qa_dataset = get_qa_dataset(dataset_name)\n",
|
| 394 |
-
"kb = get_semistructured_data(dataset_name)"
|
| 395 |
-
]
|
| 396 |
-
},
|
| 397 |
-
{
|
| 398 |
-
"cell_type": "code",
|
| 399 |
-
"execution_count": 12,
|
| 400 |
-
"metadata": {},
|
| 401 |
-
"outputs": [
|
| 402 |
-
{
|
| 403 |
-
"data": {
|
| 404 |
-
"text/plain": [
|
| 405 |
-
"('What drugs target the CYP3A4 enzyme and are used to treat strongyloidiasis?',\n",
|
| 406 |
-
" 1,\n",
|
| 407 |
-
" [15450],\n",
|
| 408 |
-
" None)"
|
| 409 |
-
]
|
| 410 |
-
},
|
| 411 |
-
"execution_count": 12,
|
| 412 |
-
"metadata": {},
|
| 413 |
-
"output_type": "execute_result"
|
| 414 |
-
}
|
| 415 |
-
],
|
| 416 |
-
"source": [
|
| 417 |
-
"qa_dataset[1]"
|
| 418 |
-
]
|
| 419 |
-
},
|
| 420 |
-
{
|
| 421 |
-
"cell_type": "code",
|
| 422 |
-
"execution_count": 13,
|
| 423 |
-
"metadata": {},
|
| 424 |
-
"outputs": [
|
| 425 |
-
{
|
| 426 |
-
"name": "stdout",
|
| 427 |
-
"output_type": "stream",
|
| 428 |
-
"text": [
|
| 429 |
-
"- name: Ivermectin\n",
|
| 430 |
-
"- type: drug\n",
|
| 431 |
-
"- source: DrugBank\n",
|
| 432 |
-
"- details:\n",
|
| 433 |
-
" - description: Ivermectin is a broad-spectrum anti-parasite medication. It was first marketed under the name Stromectol® and used against worms (except tapeworms), but, in 2012, it was approved for the topical treatment of head lice infestations in patients 6 months of age and older, and marketed under the name Sklice™ as well. Ivermectin is mainly used in humans in the treatment of onchocerciasis, but is also effective against other worm infestations (such as strongyloidiasis, ascariasis, trichuriasis and enterobiasis).\n",
|
| 434 |
-
" - half_life: 16 hours (also reported at 22-28 hours)\n",
|
| 435 |
-
" - indication: For the treatment of intestinal (i.e., nondisseminated) strongyloidiasis due to the nematode parasite <i>Strongyloides stercoralis</i>. Also for the treatment of onchocerciasis (river blindness) due to the nematode parasite <i>Onchocerca volvulus</i>. Can be used to treat scabies caused by <i>Sarcoptes scabiei</i>.\n",
|
| 436 |
-
" - mechanism_of_action: Ivermectin binds selectively and with high affinity to glutamate-gated chloride ion channels in invertebrate muscle and nerve cells of the microfilaria. This binding causes an increase in the permeability of the cell membrane to chloride ions and results in hyperpolarization of the cell, leading to paralysis and death of the parasite. Ivermectin also is believed to act as an agonist of the neurotransmitter gamma-aminobutyric acid (GABA), thereby disrupting GABA-mediated central nervous system (CNS) neurosynaptic transmission. Ivermectin may also impair normal intrauterine development of O. volvulus microfilariae and may inhibit their release from the uteri of gravid female worms.\n",
|
| 437 |
-
" - protein_binding: 93%\n",
|
| 438 |
-
" - pharmacodynamics: Ivermectin is a semisynthetic, anthelminitic agent. It is an avermectin which a group of pentacyclic sixteen-membered lactone (i.e. a macrocyclic lactone disaccharide) derived from the soil bacterium Streptomyces avermitilis. Avermectins are potent anti-parasitic agents. Ivermectin is the most common avermectin. It is a broad spectrum antiparasitic drug for oral administration. It is sometimes used to treat human onchocerciasis (river blindness). It is the mixture of 22,23-dihydro-avermectin B1a (at least 90%) and 22,23-dihydro-avermectin B1b (less than 10%).\n",
|
| 439 |
-
" - state: Ivermectin is a solid.\n",
|
| 440 |
-
" - atc_1: Ivermectin is anatomically related to antiparasitic products, insecticides and repellents and dermatologicals.\n",
|
| 441 |
-
" - atc_2: Ivermectin is in the therapeutic group of anthelmintics and other dermatological preparations.\n",
|
| 442 |
-
" - atc_3: Ivermectin is pharmacologically related to antinematodal agents and other dermatological preparations.\n",
|
| 443 |
-
" - atc_4: The chemical and functional group of is avermectines and other dermatologicals.\n",
|
| 444 |
-
" - category: Ivermectin is part of Agents Causing Muscle Toxicity ; Agrochemicals ; Anthelmintics ; Anti-Bacterial Agents ; Anti-Infective Agents ; Antinematodal Agents ; Antiparasitic Agents ; Antiparasitic Products, Insecticides and Repellents ; Avermectines ; BCRP/ABCG2 Substrates ; Compounds used in a research, industrial, or household setting ; Cytochrome P-450 CYP3A Inducers ; Cytochrome P-450 CYP3A Inhibitors ; Cytochrome P-450 CYP3A Substrates ; Cytochrome P-450 CYP3A4 Inducers ; Cytochrome P-450 CYP3A4 Inducers (strength unknown) ; Cytochrome P-450 CYP3A4 Substrates ; Cytochrome P-450 Enzyme Inducers ; Cytochrome P-450 Enzyme Inhibitors ; Cytochrome P-450 Substrates ; Dermatologicals ; Insecticides ; Lactones ; OATP1B1/SLCO1B1 Inhibitors ; OATP1B3 inhibitors ; P-glycoprotein inducers ; P-glycoprotein inhibitors ; P-glycoprotein substrates ; Pediculicides ; Pesticides ; Polyketides ; Scabicides and Pediculicides ; Toxic Actions.\n",
|
| 445 |
-
" - group: Ivermectin is approved and investigational and vet_approved.\n",
|
| 446 |
-
" - molecular_weight: The molecular weight is 1736.18.\n",
|
| 447 |
-
"\n"
|
| 448 |
-
]
|
| 449 |
-
}
|
| 450 |
-
],
|
| 451 |
-
"source": [
|
| 452 |
-
"print(kb.get_doc_info(15450, add_rel=False))"
|
| 453 |
-
]
|
| 454 |
-
},
|
| 455 |
-
{
|
| 456 |
-
"cell_type": "code",
|
| 457 |
-
"execution_count": 14,
|
| 458 |
-
"metadata": {},
|
| 459 |
-
"outputs": [
|
| 460 |
-
{
|
| 461 |
-
"name": "stdout",
|
| 462 |
-
"output_type": "stream",
|
| 463 |
-
"text": [
|
| 464 |
-
"- relations:\n",
|
| 465 |
-
"\n",
|
| 466 |
-
" enzyme: {gene/protein: (CYP3A4),}\n",
|
| 467 |
-
" target: {gene/protein: (GABRB3, GLRA3),}\n",
|
| 468 |
-
" transporter: {gene/protein: (ABCC2, ABCG2, ABCC1, ABCB1, SLCO1B1, SLCO1B3),}\n",
|
| 469 |
-
" contraindication: {disease: (filariasis, loiasis),}\n",
|
| 470 |
-
" indication: {disease: (onchocerciasis, strongyloidiasis),}\n",
|
| 471 |
-
" synergistic_interaction: {drug: (Beclomethasone dipropionate, Betamethasone, Triamcinolone, Diethylstilbestrol, Liothyronine, Liotrix, Genistein, Ubidecarenone, Torasemide, Nelfinavir, Lovastatin, Ziprasidone, Phenytoin, Metoprolol, Dicoumarol, Conjugated estrogens, Etonogestrel, Desogestrel, Gefitinib, Meperidine, Duloxetine, Chlorpromazine, Raloxifene, Zidovudine, Ritonavir, Erlotinib, Ciprofloxacin, Nortriptyline, Methotrexate, Cephalexin, Clonidine, Enalapril, Medroxyprogesterone acetate, Chloroquine, Imatinib, Testosterone, Stavudine, Estrone, Tamoxifen, Warfarin, Lamivudine, Norethisterone, Irinotecan, Estradiol, Propofol, Clofazimine, Terbinafine, Tacrolimus, Quinidine, Repaglinide, Salmeterol, Phenprocoumon, Fexofenadine, Isoniazid, Norgestimate, Ethinylestradiol, Isotretinoin, Doxorubicin, Letrozole, Sulfamethoxazole, Fenofibrate, Rifampicin, Benzylpenicillin, Atazanavir, Atorvastatin, Rosuvastatin, Amiodarone, Captopril, Saquinavir, Dexamethasone, Gemfibrozil, Clomipramine, Fosphenytoin, Colchicine, Digitoxin, Acenocoumarol, Topiroxostat, Quercetin, Estrone sulfate, Dronedarone, Vandetanib, Cenobamate, Rufinamide, Simeprevir, Prucalopride, (R)-warfarin, Vismodegib, Pitavastatin, Rilpivirine, Ulipristal, Vemurafenib, Palbociclib, Nintedanib, Tenofovir alafenamide, Grazoprevir, Vinflunine, Pitolisant, Acalabrutinib, Istradefylline, Fostemsavir, Neratinib, Revefenacin, Dacomitinib, Glasdegib, Abemaciclib, Gilteritinib, Copanlisib, Darolutamide, Pexidartinib, Testosterone enanthate, Estradiol acetate, Estradiol benzoate, Estradiol cypionate, Estradiol dienanthate, Estradiol valerate, Tenofovir, Ripretinib, Elexacaftor, Niacin, Clofibrate, Metoclopramide, Cholic Acid, Ethanol, Dronabinol, Montelukast, Zafirlukast, Etoposide, Ifosfamide, Trabectedin, Propylthiouracil, Cannabidiol, Medical Cannabis, Nabiximols, Risedronic acid, Bumetanide, Drospirenone, Progesterone, Bempedoic acid, Mefloquine, Ranitidine, Vitamin D, Tucatinib, Cimetidine, Busulfan, Cobimetinib, Ticlopidine, Cytarabine, Caffeine, Theophylline, Omeprazole, Lansoprazole, Paclitaxel, Docetaxel, Dasatinib, Norelgestromin, Methyldopa, Carbimazole, Cyproterone acetate, Norgestrel, Nizatidine, Procainamide, Thiotepa, Sumatriptan, Safinamide, Procarbazine, Ethyl biscoumacetate, Cyclosporine, Erythromycin, Sildenafil, Indinavir, Terfenadine, Levonorgestrel, Amlodipine, Sorafenib, Cerivastatin, Teniposide, Haloperidol, Lercanidipine, Cyclophosphamide, Vincristine, Carbamazepine, Cisapride, Astemizole, Simvastatin, Mycophenolate mofetil, Mifepristone, Sirolimus, Triazolam, Buprenorphine, Fluvastatin, Pimozide, Sunitinib, Trastuzumab emtansine, Romidepsin, Temsirolimus, Ambrisentan, Midostaurin, Axitinib, Gestodene, Cabazitaxel, Hydroxyprogesterone caproate, Crizotinib, Ponatinib, Idelalisib, Cobicistat, Olaparib, Daclatasvir, Paritaprevir, Asunaprevir, Isavuconazole, Letermovir, Rucaparib, Bortezomib, Venlafaxine, Vinorelbine, Zolpidem, Prochlorperazine, Vinblastine, Doxazosin, Bicalutamide, Rabeprazole, St. John's Wort, Everolimus, Zuclopenthixol, Fusidic acid, Nilotinib, Pazopanib, Panobinostat, Netupitant, Dasabuvir, Rolapitant, Ixazomib, Lasmiditan, Elagolix, Fedratinib, Levosalbutamol, Ipecac, Enasidenib, Remdesivir, Tacrine, Trimethoprim, Albendazole, Norfloxacin, Hesperetin, Leflunomide, Ofloxacin, Aminophylline, Dovitinib, Eltrombopag, Teriflunomide, Pomalidomide, Tasimelteon, Osimertinib, Capmatinib, Abametapir, Voxilaprevir, Lorazepam, Phentermine, Dofetilide, Azithromycin, Pantoprazole, Methysergide, Cabergoline, Vindesine, Dihydroergotamine, Megestrol acetate, Caspofungin, Bosentan, Amphotericin B, Ergotamine, Ethynodiol diacetate, Conivaptan, Ezetimibe, Levacetylmethadol, Mestranol, Bezafibrate, Pranlukast, Roflumilast, Ixabepilone, Tolvaptan, Lacosamide, Bosutinib, Fosaprepitant, Lomitapide, Brentuximab vedotin, Ruxolitinib, Linagliptin, Regorafenib, Dabrafenib, Vorapaxar, Suvorexant, Ceritinib, Dienogest, Sonidegib, Tianeptine, Norethynodrel, Dihydroergocornine, Selexipag, Venetoclax, Velpatasvir, Gestrinone, Nomegestrol, Ribociclib, Ebastine, Baricitinib, Apalutamide, Duvelisib, Entrectinib, Fostamatinib, Alpelisib, Erdafitinib, Brigatinib, Siponimod, Lynestrenol, 9-aminocamptothecin, Lefamulin, Tazemetostat, Methylprednisone, Dihydroergocristine, Diphenadione, Dihydroergocryptine, Chlormadinone, Quingestanol, Demegestone, Etynodiol, Glecaprevir, Nomegestrol acetate, (S)-Warfarin, Ivosidenib, Norethindrone enanthate, Zanubrutinib, Mevastatin, Valsartan, Coumarin, Avatrombopag, Fluindione, Cladribine, Telmisartan, Digoxin, Famciclovir, Naltrexone, Raltegravir, Pibrentasvir, Minocycline, Sulfasalazine, Cholecystokinin, Eprosartan, Fimasartan, Dinoprostone, Iloprost, Ciprofibrate, Fenofibric acid, Somatotropin, Allylestrenol, Naringenin, Daidzin, Leuprolide, Nafarelin, Baclofen, Temocapril, Isosorbide, Tafamidis, Alectinib, Eluxadoline, Afatinib, Atrasentan, Pravastatin, Infliximab, Phenindione, Ouabain, Pamidronic acid, Alendronic acid, Ibandronate, Cholesterol, Levamisole, Elacridar, Lonidamine, Taurocholic acid, Metreleptin, Gossypol, Octylphenoxy polyethoxyethanol, p-Coumaric acid, Novobiocin, Penicillamine, Gimatecan, Gadoxetic acid, Technetium Tc-99m mebrofenin, Ganciclovir, Sincalide, Daptomycin, Lactulose, Acipimox, Mebeverine, 4-hydroxycoumarin, Trestolone, Cloprostenol, Triptolide, Clorindione, Ormeloxifene, Tioclomarol, Norgestrienone, Picosulfuric acid, BCG vaccine, Typhoid vaccine, Vibrio cholerae CVD 103-HgR strain live antigen, Etofibrate, Simfibrate, Ronifibrate, Aluminium clofibrate, Clofibride, Emetine),}\n",
|
| 472 |
-
" side_effect: {effect/phenotype: (Edema, Inflammatory abnormality of the skin, Hyperhidrosis, Keratitis, Abdominal distention, Fever, Pain, Seizure, Headache, Dyspnea, Tremor, Encephalopathy, Vomiting, Abdominal pain, Lymphadenopathy, Hematuria, Back pain, Myalgia, Tachycardia, Hepatitis, Respiratory distress, Arthralgia, Blindness, Vertigo, Lethargy, Fatigue, Palpebral edema, Pruritus, Cough, Confusion, Eosinophilia, Chest pain, Bowel incontinence, Facial edema, Coma, Leukopenia, Nausea, Apathy, Dry skin, Excessive daytime somnolence, Difficulty standing, Poor appetite, Peripheral edema),}\n"
|
| 473 |
-
]
|
| 474 |
-
}
|
| 475 |
-
],
|
| 476 |
-
"source": [
|
| 477 |
-
"print(kb.get_rel_info(15450))"
|
| 478 |
-
]
|
| 479 |
-
},
|
| 480 |
-
{
|
| 481 |
-
"cell_type": "markdown",
|
| 482 |
-
"metadata": {},
|
| 483 |
-
"source": []
|
| 484 |
-
}
|
| 485 |
-
],
|
| 486 |
-
"metadata": {
|
| 487 |
-
"kernelspec": {
|
| 488 |
-
"display_name": "torch2",
|
| 489 |
-
"language": "python",
|
| 490 |
-
"name": "python3"
|
| 491 |
-
},
|
| 492 |
-
"language_info": {
|
| 493 |
-
"codemirror_mode": {
|
| 494 |
-
"name": "ipython",
|
| 495 |
-
"version": 3
|
| 496 |
-
},
|
| 497 |
-
"file_extension": ".py",
|
| 498 |
-
"mimetype": "text/x-python",
|
| 499 |
-
"name": "python",
|
| 500 |
-
"nbconvert_exporter": "python",
|
| 501 |
-
"pygments_lexer": "ipython3",
|
| 502 |
-
"version": "3.8.17"
|
| 503 |
-
},
|
| 504 |
-
"orig_nbformat": 4
|
| 505 |
-
},
|
| 506 |
-
"nbformat": 4,
|
| 507 |
-
"nbformat_minor": 2
|
| 508 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/benchmarks/get_qa_dataset.py
CHANGED
|
@@ -2,20 +2,23 @@ import os.path as osp
|
|
| 2 |
from src.benchmarks.qa_datasets import AmazonSTaRKDataset, PrimeKGSTaRKDataset, MAGSTaRKDataset, STaRKDataset
|
| 3 |
|
| 4 |
|
| 5 |
-
def get_qa_dataset(name, root='data/'):
|
| 6 |
qa_root = osp.join(root, name)
|
| 7 |
if name == 'amazon':
|
| 8 |
split_dir = osp.join(qa_root, 'split')
|
| 9 |
stark_qa_dir = osp.join(qa_root, 'stark_qa')
|
| 10 |
-
dataset = AmazonSTaRKDataset(stark_qa_dir, split_dir
|
| 11 |
-
|
|
|
|
| 12 |
split_dir = osp.join(qa_root, 'split')
|
| 13 |
stark_qa_dir = osp.join(qa_root, 'stark_qa')
|
| 14 |
-
dataset = PrimeKGSTaRKDataset(stark_qa_dir, split_dir
|
| 15 |
-
|
|
|
|
| 16 |
split_dir = osp.join(qa_root, 'split')
|
| 17 |
stark_qa_dir = osp.join(qa_root, 'stark_qa')
|
| 18 |
-
dataset = MAGSTaRKDataset(stark_qa_dir, split_dir
|
|
|
|
| 19 |
else:
|
| 20 |
try:
|
| 21 |
print('loading dataset from external data')
|
|
|
|
| 2 |
from src.benchmarks.qa_datasets import AmazonSTaRKDataset, PrimeKGSTaRKDataset, MAGSTaRKDataset, STaRKDataset
|
| 3 |
|
| 4 |
|
| 5 |
+
def get_qa_dataset(name, root='data/', human_generated_eval=False):
|
| 6 |
qa_root = osp.join(root, name)
|
| 7 |
if name == 'amazon':
|
| 8 |
split_dir = osp.join(qa_root, 'split')
|
| 9 |
stark_qa_dir = osp.join(qa_root, 'stark_qa')
|
| 10 |
+
dataset = AmazonSTaRKDataset(stark_qa_dir, split_dir,
|
| 11 |
+
human_generated_eval=human_generated_eval)
|
| 12 |
+
elif name == 'primekg':
|
| 13 |
split_dir = osp.join(qa_root, 'split')
|
| 14 |
stark_qa_dir = osp.join(qa_root, 'stark_qa')
|
| 15 |
+
dataset = PrimeKGSTaRKDataset(stark_qa_dir, split_dir,
|
| 16 |
+
human_generated_eval=human_generated_eval)
|
| 17 |
+
elif name == 'mag':
|
| 18 |
split_dir = osp.join(qa_root, 'split')
|
| 19 |
stark_qa_dir = osp.join(qa_root, 'stark_qa')
|
| 20 |
+
dataset = MAGSTaRKDataset(stark_qa_dir, split_dir,
|
| 21 |
+
human_generated_eval=human_generated_eval)
|
| 22 |
else:
|
| 23 |
try:
|
| 24 |
print('loading dataset from external data')
|
src/benchmarks/qa_datasets/amazon.py
CHANGED
|
@@ -6,5 +6,5 @@ from src.benchmarks.qa_datasets.stark_qa import STaRKDataset
|
|
| 6 |
|
| 7 |
class AmazonSTaRKDataset(STaRKDataset):
|
| 8 |
|
| 9 |
-
def __init__(self, stark_qa_dir, split_dir):
|
| 10 |
-
super().__init__(stark_qa_dir, split_dir)
|
|
|
|
| 6 |
|
| 7 |
class AmazonSTaRKDataset(STaRKDataset):
|
| 8 |
|
| 9 |
+
def __init__(self, stark_qa_dir, split_dir, human_generated_eval=False):
|
| 10 |
+
super(AmazonSTaRKDataset, self).__init__(stark_qa_dir, split_dir, human_generated_eval)
|
src/benchmarks/qa_datasets/mag.py
CHANGED
|
@@ -6,5 +6,5 @@ from src.benchmarks.qa_datasets.stark_qa import STaRKDataset
|
|
| 6 |
|
| 7 |
class MAGSTaRKDataset(STaRKDataset):
|
| 8 |
|
| 9 |
-
def __init__(self, stark_qa_dir, split_dir):
|
| 10 |
-
super().__init__(stark_qa_dir, split_dir)
|
|
|
|
| 6 |
|
| 7 |
class MAGSTaRKDataset(STaRKDataset):
|
| 8 |
|
| 9 |
+
def __init__(self, stark_qa_dir, split_dir, human_generated_eval=False):
|
| 10 |
+
super(MAGSTaRKDataset, self).__init__(stark_qa_dir, split_dir, human_generated_eval)
|
src/benchmarks/qa_datasets/primekg.py
CHANGED
|
@@ -5,6 +5,6 @@ from src.benchmarks.qa_datasets.stark_qa import STaRKDataset
|
|
| 5 |
|
| 6 |
|
| 7 |
class PrimeKGSTaRKDataset(STaRKDataset):
|
| 8 |
-
|
| 9 |
-
def __init__(self, stark_qa_dir, split_dir):
|
| 10 |
-
super().__init__(stark_qa_dir, split_dir)
|
|
|
|
| 5 |
|
| 6 |
|
| 7 |
class PrimeKGSTaRKDataset(STaRKDataset):
|
| 8 |
+
|
| 9 |
+
def __init__(self, stark_qa_dir, split_dir, human_generated_eval=False):
|
| 10 |
+
super(PrimeKGSTaRKDataset, self).__init__(stark_qa_dir, split_dir, human_generated_eval)
|
src/benchmarks/qa_datasets/stark_qa.py
CHANGED
|
@@ -5,10 +5,15 @@ import pandas as pd
|
|
| 5 |
|
| 6 |
|
| 7 |
class STaRKDataset:
|
| 8 |
-
def __init__(self, query_dir, split_dir):
|
| 9 |
self.query_dir = query_dir
|
| 10 |
self.split_dir = split_dir
|
| 11 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
self.data = pd.read_csv(self.qa_csv_path)
|
| 13 |
|
| 14 |
self.indices = list(self.data['id'])
|
|
@@ -31,6 +36,9 @@ class STaRKDataset:
|
|
| 31 |
'''
|
| 32 |
Return the indices of train/val/test split in a dictionary.
|
| 33 |
'''
|
|
|
|
|
|
|
|
|
|
| 34 |
split_idx = {}
|
| 35 |
for split in ['train', 'val', 'test']:
|
| 36 |
# `{split}.index`stores query ids, not the index in the dataset
|
|
|
|
| 5 |
|
| 6 |
|
| 7 |
class STaRKDataset:
|
| 8 |
+
def __init__(self, query_dir, split_dir, human_generated_eval=False):
|
| 9 |
self.query_dir = query_dir
|
| 10 |
self.split_dir = split_dir
|
| 11 |
+
self.human_generated_eval = human_generated_eval
|
| 12 |
+
if human_generated_eval:
|
| 13 |
+
self.qa_csv_path = osp.join(query_dir, 'stark_qa_human_generated_eval.csv')
|
| 14 |
+
else:
|
| 15 |
+
self.qa_csv_path = osp.join(query_dir, 'stark_qa.csv')
|
| 16 |
+
print('Loading QA dataset from', self.qa_csv_path)
|
| 17 |
self.data = pd.read_csv(self.qa_csv_path)
|
| 18 |
|
| 19 |
self.indices = list(self.data['id'])
|
|
|
|
| 36 |
'''
|
| 37 |
Return the indices of train/val/test split in a dictionary.
|
| 38 |
'''
|
| 39 |
+
if self.human_generated_eval:
|
| 40 |
+
return {'human_generated_eval': torch.LongTensor(self.indices)}
|
| 41 |
+
|
| 42 |
split_idx = {}
|
| 43 |
for split in ['train', 'val', 'test']:
|
| 44 |
# `{split}.index`stores query ids, not the index in the dataset
|
src/benchmarks/semistruct/amazon.py
CHANGED
|
@@ -49,7 +49,7 @@ class AmazonSemiStruct(SemiStructureKB):
|
|
| 49 |
'Office_Products', 'Patio_Lawn_and_Garden', 'Pet_Supplies', 'Sports_and_Outdoors',
|
| 50 |
'Tools_and_Home_Improvement', 'Toys_and_Games', 'Video_Games'])
|
| 51 |
|
| 52 |
-
sub_category = 'data/amazon/
|
| 53 |
SUB_CATEGORIES = set(json.load(open(sub_category, 'r')))
|
| 54 |
link_columns = ['also_buy', 'also_view']
|
| 55 |
review_columns = ['reviewerID', 'summary', 'style', 'reviewText', 'vote', 'overall', 'verified', 'reviewTime']
|
|
@@ -92,7 +92,7 @@ class AmazonSemiStruct(SemiStructureKB):
|
|
| 92 |
cache_path = None if meta_link_types is None else \
|
| 93 |
osp.join(self.processed_data_dir, 'cache', '-'.join(meta_link_types))
|
| 94 |
|
| 95 |
-
if not osp.exists(osp.join(
|
| 96 |
print('Downloading processed data...')
|
| 97 |
processed_path = hf_hub_download(
|
| 98 |
PROCESSED_DATASET["repo"],
|
|
@@ -516,6 +516,8 @@ class AmazonSemiStruct(SemiStructureKB):
|
|
| 516 |
else:
|
| 517 |
node_info[idx][column] = clean_data(df_meta_i[column])
|
| 518 |
|
|
|
|
|
|
|
| 519 |
for name, df in zip(['review', 'qa'], [df_review, df_qa]):
|
| 520 |
for i in tqdm(range(len(df))):
|
| 521 |
df_i = df.iloc[i]
|
|
|
|
| 49 |
'Office_Products', 'Patio_Lawn_and_Garden', 'Pet_Supplies', 'Sports_and_Outdoors',
|
| 50 |
'Tools_and_Home_Improvement', 'Toys_and_Games', 'Video_Games'])
|
| 51 |
|
| 52 |
+
sub_category = 'data/amazon/category_list.json'
|
| 53 |
SUB_CATEGORIES = set(json.load(open(sub_category, 'r')))
|
| 54 |
link_columns = ['also_buy', 'also_view']
|
| 55 |
review_columns = ['reviewerID', 'summary', 'style', 'reviewText', 'vote', 'overall', 'verified', 'reviewTime']
|
|
|
|
| 92 |
cache_path = None if meta_link_types is None else \
|
| 93 |
osp.join(self.processed_data_dir, 'cache', '-'.join(meta_link_types))
|
| 94 |
|
| 95 |
+
if not osp.exists(osp.join(cache_path, 'node_info.pkl')) and download_processed:
|
| 96 |
print('Downloading processed data...')
|
| 97 |
processed_path = hf_hub_download(
|
| 98 |
PROCESSED_DATASET["repo"],
|
|
|
|
| 516 |
else:
|
| 517 |
node_info[idx][column] = clean_data(df_meta_i[column])
|
| 518 |
|
| 519 |
+
review_columns = self.review_columns
|
| 520 |
+
review_columns.remove('style')
|
| 521 |
for name, df in zip(['review', 'qa'], [df_review, df_qa]):
|
| 522 |
for i in tqdm(range(len(df))):
|
| 523 |
df_i = df.iloc[i]
|
src/benchmarks/semistruct/mag.py
CHANGED
|
@@ -7,6 +7,7 @@ import torch
|
|
| 7 |
import pandas as pd
|
| 8 |
import numpy as np
|
| 9 |
from langdetect import detect
|
|
|
|
| 10 |
from huggingface_hub import hf_hub_download
|
| 11 |
import zipfile
|
| 12 |
from ogb.nodeproppred import NodePropPredDataset
|
|
|
|
| 7 |
import pandas as pd
|
| 8 |
import numpy as np
|
| 9 |
from langdetect import detect
|
| 10 |
+
import gdown
|
| 11 |
from huggingface_hub import hf_hub_download
|
| 12 |
import zipfile
|
| 13 |
from ogb.nodeproppred import NodePropPredDataset
|
src/models/__init__.py
DELETED
|
@@ -1,34 +0,0 @@
|
|
| 1 |
-
from .vss import VSS
|
| 2 |
-
from .llm_reranker import LLMReranker
|
| 3 |
-
from .multi_vss import MultiVSS
|
| 4 |
-
|
| 5 |
-
def get_model(args, kb):
|
| 6 |
-
model_name = args.model
|
| 7 |
-
if model_name == 'VSS':
|
| 8 |
-
return VSS(
|
| 9 |
-
kb,
|
| 10 |
-
emb_model=args.emb_model,
|
| 11 |
-
query_emb_dir=args.query_emb_dir,
|
| 12 |
-
candidates_emb_dir=args.node_emb_dir
|
| 13 |
-
)
|
| 14 |
-
if model_name == 'MultiVSS':
|
| 15 |
-
return MultiVSS(
|
| 16 |
-
kb,
|
| 17 |
-
emb_model=args.emb_model,
|
| 18 |
-
query_emb_dir=args.query_emb_dir,
|
| 19 |
-
candidates_emb_dir=args.node_emb_dir,
|
| 20 |
-
chunk_emb_dir=args.chunk_emb_dir,
|
| 21 |
-
aggregate=args.aggregate,
|
| 22 |
-
chunk_size=args.chunk_size,
|
| 23 |
-
max_k=args.multi_vss_topk
|
| 24 |
-
)
|
| 25 |
-
if model_name == 'LLMReranker':
|
| 26 |
-
return LLMReranker(kb,
|
| 27 |
-
emb_model=args.emb_model,
|
| 28 |
-
llm_model=args.llm_model,
|
| 29 |
-
query_emb_dir=args.query_emb_dir,
|
| 30 |
-
candidates_emb_dir=args.node_emb_dir,
|
| 31 |
-
max_cnt = args.max_retry,
|
| 32 |
-
max_k=args.llm_topk
|
| 33 |
-
)
|
| 34 |
-
raise NotImplementedError(f'{model_name} not implemented')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/models/llm_reranker.py
DELETED
|
@@ -1,100 +0,0 @@
|
|
| 1 |
-
import torch
|
| 2 |
-
from typing import Any
|
| 3 |
-
|
| 4 |
-
from src.models.vss import VSS
|
| 5 |
-
from src.models.model import ModelForSemiStructQA
|
| 6 |
-
from src.tools.api import get_llm_output
|
| 7 |
-
import re
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def find_floating_number(text):
|
| 11 |
-
pattern = r'0\.\d+|1\.0'
|
| 12 |
-
matches = re.findall(pattern, text)
|
| 13 |
-
return [round(float(match), 4) for match in matches if float(match) <= 1.1]
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
class LLMReranker(ModelForSemiStructQA):
|
| 17 |
-
|
| 18 |
-
def __init__(self,
|
| 19 |
-
kb,
|
| 20 |
-
llm_model,
|
| 21 |
-
emb_model,
|
| 22 |
-
query_emb_dir,
|
| 23 |
-
candidates_emb_dir,
|
| 24 |
-
sim_weight=0.1,
|
| 25 |
-
max_cnt=3,
|
| 26 |
-
max_k=100
|
| 27 |
-
):
|
| 28 |
-
'''
|
| 29 |
-
Answer the query by GPT model.
|
| 30 |
-
Args:
|
| 31 |
-
kb (src.benchmarks.semistruct.SemiStruct): kb
|
| 32 |
-
llm_model (str): model name
|
| 33 |
-
query_emb_dir (str): directory to query embeddings
|
| 34 |
-
candidates_emb_dir (str): directory to candidate embeddings
|
| 35 |
-
'''
|
| 36 |
-
|
| 37 |
-
super(LLMReranker, self).__init__(kb)
|
| 38 |
-
self.max_k = max_k
|
| 39 |
-
self.emb_model = emb_model
|
| 40 |
-
self.llm_model = llm_model
|
| 41 |
-
self.sim_weight = sim_weight
|
| 42 |
-
self.max_cnt = max_cnt
|
| 43 |
-
|
| 44 |
-
self.query_emb_dir = query_emb_dir
|
| 45 |
-
self.candidates_emb_dir = candidates_emb_dir
|
| 46 |
-
self.parent_vss = VSS(kb, query_emb_dir, candidates_emb_dir, emb_model=emb_model)
|
| 47 |
-
|
| 48 |
-
def forward(self,
|
| 49 |
-
query,
|
| 50 |
-
query_id=None,
|
| 51 |
-
**kwargs: Any):
|
| 52 |
-
|
| 53 |
-
initial_score_dict = self.parent_vss(query, query_id)
|
| 54 |
-
node_ids = list(initial_score_dict.keys())
|
| 55 |
-
node_scores = list(initial_score_dict.values())
|
| 56 |
-
|
| 57 |
-
# get the ids with top k highest scores
|
| 58 |
-
top_k_idx = torch.topk(torch.FloatTensor(node_scores),
|
| 59 |
-
min(self.max_k, len(node_scores)),
|
| 60 |
-
dim=-1).indices.view(-1).tolist()
|
| 61 |
-
top_k_node_ids = [node_ids[i] for i in top_k_idx]
|
| 62 |
-
cand_len = len(top_k_node_ids)
|
| 63 |
-
|
| 64 |
-
pred_dict = {}
|
| 65 |
-
for idx, node_id in enumerate(top_k_node_ids):
|
| 66 |
-
node_type = self.kb.get_node_type_by_id(node_id)
|
| 67 |
-
prompt = (
|
| 68 |
-
f'You are a helpful assistant that examines if a {node_type} satisfies a given query and assign a score from 0.0 to 1.0. If the {node_type} does not satisfy the query, the score should be 0.0. If there exists explicit and strong evidence supporting that {node_type} satisfies the query, the score should be 1.0. If partial evidence or weak evidence exists, the score should be between 0.0 and 1.0.\n'
|
| 69 |
-
f'Here is the query:\n\"{query}\"\n'
|
| 70 |
-
f'Here is the information about the {node_type}:\n' +
|
| 71 |
-
self.kb.get_doc_info(node_id, add_rel=True) + '\n\n' +
|
| 72 |
-
f'Please score the {node_type} based on how well it satisfies the query. ONLY output the floating point score WITHOUT anything else. '
|
| 73 |
-
f'Output: The numeric score of this {node_type} is: '
|
| 74 |
-
)
|
| 75 |
-
|
| 76 |
-
success = False
|
| 77 |
-
for _ in range(self.max_cnt):
|
| 78 |
-
try:
|
| 79 |
-
answer = get_llm_output(prompt,
|
| 80 |
-
self.llm_model,
|
| 81 |
-
max_tokens=5
|
| 82 |
-
)
|
| 83 |
-
answer = find_floating_number(answer)
|
| 84 |
-
if len(answer) == 1:
|
| 85 |
-
answer = answer[0]
|
| 86 |
-
success = True
|
| 87 |
-
break
|
| 88 |
-
except Exception as e:
|
| 89 |
-
print(f'Error: {e}, pass')
|
| 90 |
-
|
| 91 |
-
if success:
|
| 92 |
-
llm_score = float(answer)
|
| 93 |
-
sim_score = (cand_len - idx) / cand_len
|
| 94 |
-
score = llm_score + self.sim_weight * sim_score
|
| 95 |
-
pred_dict[node_id] = score
|
| 96 |
-
else:
|
| 97 |
-
return initial_score_dict
|
| 98 |
-
return pred_dict
|
| 99 |
-
|
| 100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/models/model.py
DELETED
|
@@ -1,106 +0,0 @@
|
|
| 1 |
-
import torch.nn as nn
|
| 2 |
-
from typing import Any, Union
|
| 3 |
-
import torch
|
| 4 |
-
import os
|
| 5 |
-
import os.path as osp
|
| 6 |
-
from torchmetrics.functional.retrieval import retrieval_hit_rate, \
|
| 7 |
-
retrieval_reciprocal_rank, \
|
| 8 |
-
retrieval_recall, retrieval_precision, \
|
| 9 |
-
retrieval_average_precision, \
|
| 10 |
-
retrieval_normalized_dcg, \
|
| 11 |
-
retrieval_r_precision
|
| 12 |
-
from src.tools.api import get_openai_embedding
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
class ModelForSemiStructQA(nn.Module):
|
| 16 |
-
|
| 17 |
-
def __init__(self, kb):
|
| 18 |
-
super(ModelForSemiStructQA, self).__init__()
|
| 19 |
-
self.kb = kb
|
| 20 |
-
self.candidate_ids = kb.candidate_ids
|
| 21 |
-
self.num_candidates = kb.num_candidates
|
| 22 |
-
self.query_emb_dict = {}
|
| 23 |
-
|
| 24 |
-
def forward(self,
|
| 25 |
-
query: Union[str, list],
|
| 26 |
-
candidates=None,
|
| 27 |
-
query_id=None,
|
| 28 |
-
**kwargs: Any):
|
| 29 |
-
'''
|
| 30 |
-
Args:
|
| 31 |
-
query (Union[str, list]): query string or a list of query strings
|
| 32 |
-
candidates (Union[list, None]): a list of candidate ids (optional)
|
| 33 |
-
query_id (Union[int, list, None]): query index (optional)
|
| 34 |
-
|
| 35 |
-
Returns:
|
| 36 |
-
pred_dict (dict): a dictionary of predicted scores or answer ids
|
| 37 |
-
'''
|
| 38 |
-
raise NotImplementedError
|
| 39 |
-
|
| 40 |
-
def _get_query_emb(self, query: str, query_id: int,
|
| 41 |
-
emb_model: str = 'text-embedding-ada-002'):
|
| 42 |
-
if query_id is None:
|
| 43 |
-
query_emb = get_openai_embedding(query, model=emb_model)
|
| 44 |
-
elif len(self.query_emb_dict) > 0:
|
| 45 |
-
query_emb = self.query_emb_dict[query_id]
|
| 46 |
-
else:
|
| 47 |
-
query_emb_dic_path = osp.join(self.query_emb_dir, 'query_emb_dict.pt')
|
| 48 |
-
if os.path.exists(query_emb_dic_path):
|
| 49 |
-
print(f'Load query embeddings from {query_emb_dic_path}')
|
| 50 |
-
self.query_emb_dict = torch.load(query_emb_dic_path)
|
| 51 |
-
query_emb = self.query_emb_dict[query_id]
|
| 52 |
-
else:
|
| 53 |
-
query_emb_dir = osp.join(self.query_emb_dir, 'query_embs')
|
| 54 |
-
if not os.path.exists(query_emb_dir):
|
| 55 |
-
os.makedirs(query_emb_dir)
|
| 56 |
-
query_emb_path = osp.join(query_emb_dir, f'query_{query_id}.pt')
|
| 57 |
-
query_emb = get_openai_embedding(query, model=emb_model)
|
| 58 |
-
torch.save(query_emb, query_emb_path)
|
| 59 |
-
return query_emb
|
| 60 |
-
|
| 61 |
-
def evaluate(self,
|
| 62 |
-
pred_dict: dict,
|
| 63 |
-
answer_ids: torch.LongTensor,
|
| 64 |
-
metrics=['mrr', 'hit@3', 'recall@20'],
|
| 65 |
-
**kwargs: Any):
|
| 66 |
-
'''
|
| 67 |
-
Args:
|
| 68 |
-
pred_dict (torch.Tensor): predicted answer ids or scores
|
| 69 |
-
answer_ids (torch.LongTensor): ground truth answer ids
|
| 70 |
-
metrics (list): a list of metrics to be evaluated,
|
| 71 |
-
including 'mrr', 'hit@k', 'recall@k', 'precision@k', 'map@k', 'ndcg@k'
|
| 72 |
-
Returns:
|
| 73 |
-
eval_metrics (dict): a dictionary of evaluation metrics
|
| 74 |
-
'''
|
| 75 |
-
pred_ids = torch.LongTensor(list(pred_dict.keys())).view(-1)
|
| 76 |
-
pred = torch.FloatTensor(list(pred_dict.values())).view(-1)
|
| 77 |
-
answer_ids = answer_ids.view(-1)
|
| 78 |
-
|
| 79 |
-
all_pred = torch.ones(max(self.candidate_ids) + 1, dtype=torch.float) * min(pred) - 1
|
| 80 |
-
all_pred[pred_ids] = pred
|
| 81 |
-
all_pred = all_pred[self.candidate_ids]
|
| 82 |
-
|
| 83 |
-
bool_gd = torch.zeros(max(self.candidate_ids) + 1, dtype=torch.bool)
|
| 84 |
-
bool_gd[answer_ids] = True
|
| 85 |
-
bool_gd = bool_gd[self.candidate_ids]
|
| 86 |
-
|
| 87 |
-
eval_metrics = {}
|
| 88 |
-
for metric in metrics:
|
| 89 |
-
k = int(metric.split('@')[-1]) if '@' in metric else None
|
| 90 |
-
if 'mrr' == metric:
|
| 91 |
-
result = retrieval_reciprocal_rank(all_pred, bool_gd)
|
| 92 |
-
elif 'rprecision' == metric:
|
| 93 |
-
result = retrieval_r_precision(all_pred, bool_gd)
|
| 94 |
-
elif 'hit' in metric:
|
| 95 |
-
result = retrieval_hit_rate(all_pred, bool_gd, top_k=k)
|
| 96 |
-
elif 'recall' in metric:
|
| 97 |
-
result = retrieval_recall(all_pred, bool_gd, top_k=k)
|
| 98 |
-
elif 'precision' in metric:
|
| 99 |
-
result = retrieval_precision(all_pred, bool_gd, top_k=k)
|
| 100 |
-
elif 'map' in metric:
|
| 101 |
-
result = retrieval_average_precision(all_pred, bool_gd, top_k=k)
|
| 102 |
-
elif 'ndcg' in metric:
|
| 103 |
-
result = retrieval_normalized_dcg(all_pred, bool_gd, top_k=k)
|
| 104 |
-
eval_metrics[metric] = float(result)
|
| 105 |
-
|
| 106 |
-
return eval_metrics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/models/multi_vss.py
DELETED
|
@@ -1,82 +0,0 @@
|
|
| 1 |
-
import os.path as osp
|
| 2 |
-
import torch
|
| 3 |
-
from typing import Any
|
| 4 |
-
from src.models.model import ModelForSemiStructQA
|
| 5 |
-
from src.models.vss import VSS
|
| 6 |
-
from src.tools.api import get_openai_embeddings
|
| 7 |
-
from src.tools.process_text import chunk_text
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
class MultiVSS(ModelForSemiStructQA):
|
| 11 |
-
|
| 12 |
-
def __init__(self,
|
| 13 |
-
kb,
|
| 14 |
-
query_emb_dir,
|
| 15 |
-
candidates_emb_dir,
|
| 16 |
-
chunk_emb_dir,
|
| 17 |
-
emb_model='text-embedding-ada-002',
|
| 18 |
-
aggregate='top3_avg',
|
| 19 |
-
max_k=50,
|
| 20 |
-
chunk_size=256):
|
| 21 |
-
'''
|
| 22 |
-
Multivector Vector Similarity Search
|
| 23 |
-
Args:
|
| 24 |
-
kb (src.benchmarks.semistruct.SemiStruct): kb
|
| 25 |
-
query_emb_dir (str): directory to query embeddings
|
| 26 |
-
candidates_emb_dir (str): directory to candidate embeddings
|
| 27 |
-
chunk_emb_dir (str): directory to chunk embeddings
|
| 28 |
-
'''
|
| 29 |
-
|
| 30 |
-
super().__init__(kb)
|
| 31 |
-
self.kb = kb
|
| 32 |
-
self.aggregate = aggregate # 'max', 'avg', 'top{k}_avg'
|
| 33 |
-
|
| 34 |
-
self.max_k = max_k
|
| 35 |
-
self.chunk_size = chunk_size
|
| 36 |
-
self.emb_model = emb_model
|
| 37 |
-
self.query_emb_dir = query_emb_dir
|
| 38 |
-
self.chunk_emb_dir = chunk_emb_dir
|
| 39 |
-
self.candidates_emb_dir = candidates_emb_dir
|
| 40 |
-
self.parent_vss = VSS(kb, query_emb_dir, candidates_emb_dir, emb_model=emb_model)
|
| 41 |
-
|
| 42 |
-
def forward(self,
|
| 43 |
-
query,
|
| 44 |
-
query_id,
|
| 45 |
-
**kwargs: Any):
|
| 46 |
-
|
| 47 |
-
query_emb = self._get_query_emb(query, query_id)
|
| 48 |
-
|
| 49 |
-
initial_score_dict = self.parent_vss(query, query_id)
|
| 50 |
-
node_ids = list(initial_score_dict.keys())
|
| 51 |
-
node_scores = list(initial_score_dict.values())
|
| 52 |
-
|
| 53 |
-
# get the ids with top k highest scores
|
| 54 |
-
top_k_idx = torch.topk(torch.FloatTensor(node_scores),
|
| 55 |
-
min(self.max_k, len(node_scores)),
|
| 56 |
-
dim=-1
|
| 57 |
-
).indices.view(-1).tolist()
|
| 58 |
-
top_k_node_ids = [node_ids[i] for i in top_k_idx]
|
| 59 |
-
|
| 60 |
-
pred_dict = {}
|
| 61 |
-
for node_id in top_k_node_ids:
|
| 62 |
-
doc = self.kb.get_doc_info(node_id, add_rel=True, compact=True)
|
| 63 |
-
chunks = chunk_text(doc, chunk_size=self.chunk_size)
|
| 64 |
-
chunk_path = osp.join(self.chunk_emb_dir, f'{node_id}_size={self.chunk_size}.pt')
|
| 65 |
-
if osp.exists(chunk_path):
|
| 66 |
-
chunk_embs = torch.load(chunk_path)
|
| 67 |
-
else:
|
| 68 |
-
chunk_embs = get_openai_embeddings(chunks,
|
| 69 |
-
model=self.emb_model)
|
| 70 |
-
torch.save(chunk_embs, chunk_path)
|
| 71 |
-
print(f'chunk_embs.shape: {chunk_embs.shape}')
|
| 72 |
-
|
| 73 |
-
similarity = torch.matmul(query_emb.cuda(), chunk_embs.cuda().T).cpu().view(-1)
|
| 74 |
-
if self.aggregate == 'max':
|
| 75 |
-
pred_dict[node_id] = torch.max(similarity).item()
|
| 76 |
-
elif self.aggregate == 'avg':
|
| 77 |
-
pred_dict[node_id] = torch.mean(similarity).item()
|
| 78 |
-
elif 'top' in self.aggregate:
|
| 79 |
-
k = int(self.aggregate.split('_')[0][len('top'):])
|
| 80 |
-
pred_dict[node_id] = torch.mean(torch.topk(similarity, k=min(k, len(chunks)), dim=-1).values).item()
|
| 81 |
-
|
| 82 |
-
return pred_dict
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/models/vss.py
DELETED
|
@@ -1,57 +0,0 @@
|
|
| 1 |
-
import os.path as osp
|
| 2 |
-
import torch
|
| 3 |
-
from typing import Any
|
| 4 |
-
from src.models.model import ModelForSemiStructQA
|
| 5 |
-
from tqdm import tqdm
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
class VSS(ModelForSemiStructQA):
|
| 9 |
-
|
| 10 |
-
def __init__(self,
|
| 11 |
-
kb,
|
| 12 |
-
query_emb_dir,
|
| 13 |
-
candidates_emb_dir,
|
| 14 |
-
emb_model='text-embedding-ada-002'):
|
| 15 |
-
'''
|
| 16 |
-
Vector Similarity Search
|
| 17 |
-
Args:
|
| 18 |
-
kb (src.benchmarks.semistruct.SemiStruct): kb
|
| 19 |
-
query_emb_dir (str): directory to query embeddings
|
| 20 |
-
candidates_emb_dir (str): directory to candidate embeddings
|
| 21 |
-
'''
|
| 22 |
-
|
| 23 |
-
super(VSS, self).__init__(kb)
|
| 24 |
-
self.emb_model = emb_model
|
| 25 |
-
self.query_emb_dir = query_emb_dir
|
| 26 |
-
self.candidates_emb_dir = candidates_emb_dir
|
| 27 |
-
|
| 28 |
-
candidate_emb_path = osp.join(candidates_emb_dir, 'candidate_emb_dict.pt')
|
| 29 |
-
if osp.exists(candidate_emb_path):
|
| 30 |
-
candidate_emb_dict = torch.load(candidate_emb_path)
|
| 31 |
-
print(f'Loaded candidate_emb_dict from {candidate_emb_path}!')
|
| 32 |
-
else:
|
| 33 |
-
print('Loading candidate embeddings...')
|
| 34 |
-
candidate_emb_dict = {}
|
| 35 |
-
for idx in tqdm(self.candidate_ids):
|
| 36 |
-
candidate_emb_dict[idx] = torch.load(osp.join(candidates_emb_dir, f'{idx}.pt'))
|
| 37 |
-
torch.save(candidate_emb_dict, candidate_emb_path)
|
| 38 |
-
print(f'Saved candidate_emb_dict to {candidate_emb_path}!')
|
| 39 |
-
|
| 40 |
-
assert len(candidate_emb_dict) == len(self.candidate_ids)
|
| 41 |
-
candidate_embs = [candidate_emb_dict[idx] for idx in self.candidate_ids]
|
| 42 |
-
self.candidate_embs = torch.cat(candidate_embs, dim=0)
|
| 43 |
-
|
| 44 |
-
def forward(self,
|
| 45 |
-
query: str,
|
| 46 |
-
query_id: int,
|
| 47 |
-
**kwargs: Any):
|
| 48 |
-
|
| 49 |
-
query_emb = self._get_query_emb(query,
|
| 50 |
-
query_id,
|
| 51 |
-
emb_model=self.emb_model
|
| 52 |
-
)
|
| 53 |
-
similarity = torch.matmul(query_emb.cuda(),
|
| 54 |
-
self.candidate_embs.cuda().T
|
| 55 |
-
).cpu().view(-1)
|
| 56 |
-
pred_dict = {self.candidate_ids[i]: similarity[i] for i in range(len(self.candidate_ids))}
|
| 57 |
-
return pred_dict
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/tools/api_lib/huggingface.py
CHANGED
|
@@ -1,28 +1,10 @@
|
|
| 1 |
import time
|
| 2 |
import torch
|
| 3 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 4 |
-
from transformers import StoppingCriteria, StoppingCriteriaList
|
| 5 |
|
| 6 |
loaded_hf_models = {}
|
| 7 |
|
| 8 |
|
| 9 |
-
class StopAtSpecificTokenCriteria(StoppingCriteria):
|
| 10 |
-
def __init__(self, stop_sequence):
|
| 11 |
-
super().__init__()
|
| 12 |
-
self.stop_sequence = stop_sequence
|
| 13 |
-
|
| 14 |
-
def __call__(self, input_ids, scores, **kwargs):
|
| 15 |
-
# Create a tensor from the stop_sequence
|
| 16 |
-
stop_sequence_tensor = torch.tensor(self.stop_sequence,
|
| 17 |
-
device=input_ids.device,
|
| 18 |
-
dtype=input_ids.dtype
|
| 19 |
-
)
|
| 20 |
-
|
| 21 |
-
# Check if the current sequence ends with the stop_sequence
|
| 22 |
-
current_sequence = input_ids[:, -len(self.stop_sequence) :]
|
| 23 |
-
return bool(torch.all(current_sequence == stop_sequence_tensor).item())
|
| 24 |
-
|
| 25 |
-
|
| 26 |
def complete_text_hf(message,
|
| 27 |
model="huggingface/codellama/CodeLlama-7b-hf",
|
| 28 |
max_tokens=2000,
|
|
|
|
| 1 |
import time
|
| 2 |
import torch
|
| 3 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
|
|
| 4 |
|
| 5 |
loaded_hf_models = {}
|
| 6 |
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
def complete_text_hf(message,
|
| 9 |
model="huggingface/codellama/CodeLlama-7b-hf",
|
| 10 |
max_tokens=2000,
|