ai-sage
/

GigaChat-20B-A3B-instruct-int8

8-bit precision

Model card Files Files and versions Community

vltnmmdv commited on 28 days ago

Commit

05794df

·

verified ·

1 Parent(s): 69c6347

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -26,7 +26,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
 model_name = "ai-sage/GigaChat-20B-A3B-instruct-int8"
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map="auto")
 model.generation_config = GenerationConfig.from_pretrained(model_name)
 messages = [

 model_name = "ai-sage/GigaChat-20B-A3B-instruct-int8"
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, device_map="auto")
 model.generation_config = GenerationConfig.from_pretrained(model_name)
 messages = [