update tgi.sh

tgi.sh

@@ -1,9 +1,18 @@
 #!/bin/sh
 
-export HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-7b-chat-hf"
+BASEDIR=$(dirname "$0")
+cd $BASEDIR
+echo Current Directory:
+pwd
 
-echo Running $HUGGINGFACE_MODEL_NAME_OR_PATH with TGI
+uname -a
 
-volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
+# for $NGROK_AUTHTOKEN and $NGROK_EDGE
+souce env/ngrok.conf 
 
-docker run -e HUGGING_FACE_HUB_TOKEN=$HUGGINGFACE_AUTH_TOKEN --shm-size 1g -p 8081:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 --model-id $HUGGINGFACE_MODEL_NAME_OR_PATH
+export MODEL_ID="meta-llama/Llama-2-7b-chat-hf"
+export QUANTIZE=--quantize bitsandbytes-fp4
+
+echo Running $MODEL_ID with TGI
+
+text-generation-launcher --model-id $MODEL_ID --port $PORT --max-input-length 2048 --max-total-tokens 4096 --ngrok --ngrok-authtoken $NGROK_AUTHTOKEN --ngrok-edge NGROK_EDGE $QUANTIZE