Update app.py
Browse files
app.py
CHANGED
@@ -32,10 +32,8 @@ def fine_tune_model(base_model_name, dataset_name):
|
|
32 |
|
33 |
print("### Dataset")
|
34 |
print(dataset)
|
35 |
-
print("###
|
36 |
print(dataset["train"][:1])
|
37 |
-
print("### Test example")
|
38 |
-
print(dataset["test"][:1])
|
39 |
print("###")
|
40 |
|
41 |
# Load model
|
@@ -57,10 +55,8 @@ def fine_tune_model(base_model_name, dataset_name):
|
|
57 |
|
58 |
print("### Pre-processed dataset")
|
59 |
print(dataset)
|
60 |
-
print("###
|
61 |
print(dataset["train"][:1])
|
62 |
-
print("### Test example")
|
63 |
-
print(dataset["test"][:1])
|
64 |
print("###")
|
65 |
|
66 |
# Split dataset into training and validation sets
|
@@ -138,8 +134,7 @@ def prompt_model(model_name, system_prompt, user_prompt, sql_schema):
|
|
138 |
def load_model(model_name):
|
139 |
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
|
140 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
141 |
-
tokenizer.
|
142 |
-
#tokenizer.pad_token = tokenizer.eos_token
|
143 |
|
144 |
return model, tokenizer
|
145 |
|
|
|
32 |
|
33 |
print("### Dataset")
|
34 |
print(dataset)
|
35 |
+
print("### Example")
|
36 |
print(dataset["train"][:1])
|
|
|
|
|
37 |
print("###")
|
38 |
|
39 |
# Load model
|
|
|
55 |
|
56 |
print("### Pre-processed dataset")
|
57 |
print(dataset)
|
58 |
+
print("### Example")
|
59 |
print(dataset["train"][:1])
|
|
|
|
|
60 |
print("###")
|
61 |
|
62 |
# Split dataset into training and validation sets
|
|
|
134 |
def load_model(model_name):
|
135 |
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
|
136 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
137 |
+
tokenizer.pad_token = tokenizer.eos_token
|
|
|
138 |
|
139 |
return model, tokenizer
|
140 |
|