| BASEDIR=$(dirname "$0") | |
| cd $BASEDIR | |
| echo Current Directory: | |
| pwd | |
| uname -a | |
| . env/tgi.conf | |
| export MODEL_ID="meta-llama/Llama-2-7b-chat-hf" | |
| export QUANTIZE="--quantize bitsandbytes-fp4" | |
| echo Running $MODEL_ID with TGI | |
| text-generation-launcher --model-id $MODEL_ID --port $PORT --max-input-length 2048 --max-total-tokens 4096 --ngrok --ngrok-authtoken $NGROK_AUTHTOKEN --ngrok-edge $NGROK_EDGE $QUANTIZE | |