File size: 416 Bytes
6b398e8 b05a046 6b398e8 b05a046 6b398e8 43e183b 6b398e8 b05a046 43e183b b05a046 43e183b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
#!/bin/sh
BASEDIR=$(dirname "$0")
cd $BASEDIR
echo Current Directory:
pwd
uname -a
. env/tgi.conf
export MODEL_ID="meta-llama/Llama-2-7b-chat-hf"
export QUANTIZE="--quantize bitsandbytes-fp4"
echo Running $MODEL_ID with TGI
text-generation-launcher --model-id $MODEL_ID --port $PORT --max-input-length 2048 --max-total-tokens 4096 --ngrok --ngrok-authtoken $NGROK_AUTHTOKEN --ngrok-edge $NGROK_EDGE $QUANTIZE
|