completed tgi.sh

.env.example

@@ -91,3 +91,13 @@ CHUNK_OVERLAP=512
 # telegram bot
 TELEGRAM_API_TOKEN=
 CHAT_API_URL=http://localhost:8080/chat_sync
+
+# template for env/tgi.conf
+export PORT=64300
+
+export NGROK_AUTHTOKEN=
+export NGROK_EDGE=
+
+export HUGGINGFACE_HUB_CACHE=$HOME/.cache/huggingface/hub/
+export HUGGING_FACE_HUB_TOKEN=
+ß

tgi.sh

@@ -7,12 +7,11 @@ pwd
 
 uname -a
 
-# for $NGROK_AUTHTOKEN and $NGROK_EDGE
-souce env/ngrok.conf 
+. env/tgi.conf 
 
 export MODEL_ID="meta-llama/Llama-2-7b-chat-hf"
-export QUANTIZE=--quantize bitsandbytes-fp4
+export QUANTIZE="--quantize bitsandbytes-fp4"
 
 echo Running $MODEL_ID with TGI
 
-text-generation-launcher --model-id $MODEL_ID --port $PORT --max-input-length 2048 --max-total-tokens 4096 --ngrok --ngrok-authtoken $NGROK_AUTHTOKEN --ngrok-edge NGROK_EDGE $QUANTIZE
+text-generation-launcher --model-id $MODEL_ID --port $PORT --max-input-length 2048 --max-total-tokens 4096 --ngrok --ngrok-authtoken $NGROK_AUTHTOKEN --ngrok-edge $NGROK_EDGE $QUANTIZE