enable 8bit

app.py

@@ -59,31 +59,34 @@ except Exception:
 if device == "cuda":
     model = AutoModelForCausalLM.from_pretrained(
         BASE_MODEL,
-        load_in_8bit=False,
-        torch_dtype=torch.float16,
+        load_in_8bit=True,
         device_map="auto",
     )
-    model = PeftModel.from_pretrained(model, LORA_WEIGHTS, torch_dtype=torch.float16)
+    model = PeftModel.from_pretrained(model, LORA_WEIGHTS, load_in_8bit=True)
 elif device == "mps":
     model = AutoModelForCausalLM.from_pretrained(
         BASE_MODEL,
         device_map={"": device},
-        torch_dtype=torch.float16,
+        load_in_8bit=True,
     )
     model = PeftModel.from_pretrained(
         model,
         LORA_WEIGHTS,
         device_map={"": device},
-        torch_dtype=torch.float16,
+        load_in_8bit=True,
     )
 else:
     model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True
+        BASE_MODEL,
+        device_map={"": device},
+        low_cpu_mem_usage=True,
+        load_in_8bit=True,
     )
     model = PeftModel.from_pretrained(
         model,
         LORA_WEIGHTS,
         device_map={"": device},
+        load_in_8bit=True,
     )