Upload 2 files
Browse files
generation_test_hf_script.py
ADDED
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
|
3 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
4 |
+
import torch
|
5 |
+
|
6 |
+
|
7 |
+
def load_rag_benchmark_tester_ds():
|
8 |
+
|
9 |
+
# pull 200 question rag benchmark test dataset from LLMWare HuggingFace repo
|
10 |
+
from datasets import load_dataset
|
11 |
+
|
12 |
+
ds_name = "llmware/rag_instruct_benchmark_tester"
|
13 |
+
|
14 |
+
dataset = load_dataset(ds_name)
|
15 |
+
|
16 |
+
print("update: loading test dataset - ", dataset)
|
17 |
+
|
18 |
+
test_set = []
|
19 |
+
for i, samples in enumerate(dataset["train"]):
|
20 |
+
test_set.append(samples)
|
21 |
+
|
22 |
+
# to view test set samples
|
23 |
+
# print("rag benchmark dataset test samples: ", i, samples)
|
24 |
+
|
25 |
+
return test_set
|
26 |
+
|
27 |
+
|
28 |
+
def run_test(model_name, test_ds):
|
29 |
+
|
30 |
+
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
|
31 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
32 |
+
|
33 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
34 |
+
|
35 |
+
for i, entries in enumerate(test_ds):
|
36 |
+
|
37 |
+
# prepare prompt packaging used in fine-tuning process
|
38 |
+
new_prompt = "<human>: " + entries["context"] + "\n" + entries["query"] + "\n" + "<bot>:"
|
39 |
+
|
40 |
+
inputs = tokenizer(new_prompt, return_tensors="pt")
|
41 |
+
start_of_output = len(inputs.input_ids[0])
|
42 |
+
|
43 |
+
# temperature: set at 0.3 for consistency of output
|
44 |
+
# max_new_tokens: set at 100 - may prematurely stop a few of the summaries
|
45 |
+
|
46 |
+
outputs = model.generate(
|
47 |
+
inputs.input_ids.to(device),
|
48 |
+
eos_token_id=tokenizer.eos_token_id,
|
49 |
+
pad_token_id=tokenizer.eos_token_id,
|
50 |
+
do_sample=True,
|
51 |
+
temperature=0.3,
|
52 |
+
max_new_tokens=100,
|
53 |
+
)
|
54 |
+
|
55 |
+
output_only = tokenizer.decode(outputs[0][start_of_output:],skip_special_tokens=True)
|
56 |
+
|
57 |
+
# quick/optional post-processing clean-up of potential fine-tuning artifacts
|
58 |
+
|
59 |
+
eot = output_only.find("<|endoftext|>")
|
60 |
+
if eot > -1:
|
61 |
+
output_only = output_only[:eot]
|
62 |
+
|
63 |
+
bot = output_only.find("<bot>:")
|
64 |
+
if bot > -1:
|
65 |
+
output_only = output_only[bot+len("<bot>:"):]
|
66 |
+
|
67 |
+
# end - post-processing
|
68 |
+
|
69 |
+
print("\n")
|
70 |
+
print(i, "llm_response - ", output_only)
|
71 |
+
print(i, "gold_answer - ", entries["answer"])
|
72 |
+
|
73 |
+
return 0
|
74 |
+
|
75 |
+
|
76 |
+
if __name__ == "__main__":
|
77 |
+
|
78 |
+
test_ds = load_rag_benchmark_tester_ds()
|
79 |
+
|
80 |
+
model_name = "llmware/dragon-red-pajama-7b-v0"
|
81 |
+
|
82 |
+
output = run_test(model_name,test_ds)
|
83 |
+
|
84 |
+
|
generation_test_llmware_script.py
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
from llmware.prompts import Prompt
|
3 |
+
|
4 |
+
|
5 |
+
def load_rag_benchmark_tester_ds():
|
6 |
+
|
7 |
+
# pull 200 question rag benchmark test dataset from LLMWare HuggingFace repo
|
8 |
+
from datasets import load_dataset
|
9 |
+
|
10 |
+
ds_name = "llmware/rag_instruct_benchmark_tester"
|
11 |
+
|
12 |
+
dataset = load_dataset(ds_name)
|
13 |
+
|
14 |
+
print("update: loading test dataset - ", dataset)
|
15 |
+
|
16 |
+
test_set = []
|
17 |
+
for i, samples in enumerate(dataset["train"]):
|
18 |
+
test_set.append(samples)
|
19 |
+
|
20 |
+
# to view test set samples
|
21 |
+
# print("rag benchmark dataset test samples: ", i, samples)
|
22 |
+
|
23 |
+
return test_set
|
24 |
+
|
25 |
+
|
26 |
+
def run_test(model_name, prompt_list):
|
27 |
+
|
28 |
+
print("\nupdate: Starting RAG Benchmark Inference Test")
|
29 |
+
|
30 |
+
prompter = Prompt().load_model(model_name,from_hf=True)
|
31 |
+
|
32 |
+
for i, entries in enumerate(prompt_list):
|
33 |
+
|
34 |
+
prompt = entries["query"]
|
35 |
+
context = entries["context"]
|
36 |
+
|
37 |
+
response = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context", temperature=0.3)
|
38 |
+
|
39 |
+
fc = prompter.evidence_check_numbers(response)
|
40 |
+
sc = prompter.evidence_comparison_stats(response)
|
41 |
+
sr = prompter.evidence_check_sources(response)
|
42 |
+
|
43 |
+
print("\nupdate: model inference output - ", i, response["llm_response"])
|
44 |
+
print("update: gold_answer - ", i, entries["answer"])
|
45 |
+
|
46 |
+
for entries in fc:
|
47 |
+
print("update: fact check - ", entries["fact_check"])
|
48 |
+
|
49 |
+
for entries in sc:
|
50 |
+
print("update: comparison stats - ", entries["comparison_stats"])
|
51 |
+
|
52 |
+
for entries in sr:
|
53 |
+
print("update: sources - ", entries["source_review"])
|
54 |
+
|
55 |
+
return 0
|
56 |
+
|
57 |
+
|
58 |
+
if __name__ == "__main__":
|
59 |
+
|
60 |
+
core_test_set = load_rag_benchmark_tester_ds()
|
61 |
+
|
62 |
+
model_name = "llmware/dragon-red-pajama-7b-v0"
|
63 |
+
|
64 |
+
output = run_test(model_name, core_test_set)
|