More handling of inference endpoints: Delete when done.

main_backend_toxicity.py

@@ -1,7 +1,7 @@
 import logging
 import pprint
 import re
-from huggingface_hub import snapshot_download
+from huggingface_hub import snapshot_download, delete_inference_endpoint
 
 from src.backend.inference_endpoint import create_endpoint
 from src.backend.run_toxicity_eval import main
@@ -81,6 +81,7 @@ def run_auto_eval():
         hf_repo=QUEUE_REPO,
         local_dir=EVAL_REQUESTS_PATH_BACKEND,
     )
+    delete_inference_endpoint(endpoint_name)
 
 
 if __name__ == "__main__":

src/backend/inference_endpoint.py

@@ -15,23 +15,15 @@ def create_endpoint(endpoint_name, repository, framework="pytorch", task="text-g
     logger.info("Creating endpoint %s..." % endpoint_name)
     # TODO(mm): Handle situation where it's paused
     try:
-        endpoint = create_inference_endpoint(endpoint_name, repository=repository, framework=framework, task=task, accelerator=accelerator, vendor=vendor, region=region, type=type, instance_size=instance_size, instance_type=instance_type
-        )
+        endpoint = create_inference_endpoint(endpoint_name, repository=repository, framework=framework, task=task, accelerator=accelerator, vendor=vendor, region=region, type=type, instance_size=instance_size, instance_type=instance_type)
     except huggingface_hub.utils._errors.HfHubHTTPError as e:
-        endpoint = update_endpoint_exception(accelerator, e, endpoint,
-                                             endpoint_name, framework,
-                                             instance_size, instance_type,
-                                             repository, task)
+        endpoint = update_endpoint_exception(e, endpoint_name=endpoint_name, repository=repository, framework=framework, task=task, accelerator=accelerator, vendor=vendor, region=region, type=type, instance_size=instance_size, instance_type=instance_type)
     except requests.exceptions.HTTPError as e:
-        endpoint = update_endpoint_exception(accelerator, e, endpoint,
-                                             endpoint_name, framework,
-                                             instance_size, instance_type,
-                                             repository, task)
+        endpoint = update_endpoint_exception(e, endpoint_name, repository=repository, framework=framework, task=task, accelerator=accelerator, vendor=vendor, region=region, type=type, instance_size=instance_size, instance_type=instance_type)
     except Exception as e:
         logger.debug("Hit error")
         logger.debug(e)
         sys.exit()
-
     endpoint.fetch()
     logger.info("Endpoint status: %s." % (endpoint.status))
     if endpoint.status == "scaledToZero":
@@ -54,9 +46,7 @@ def create_endpoint(endpoint_name, repository, framework="pytorch", task="text-g
     return generation_url
 
 
-def update_endpoint_exception(accelerator, e, endpoint, endpoint_name,
-                              framework, instance_size, instance_type,
-                              repository, task):
+def update_endpoint_exception(e, endpoint_name, repository, framework, task, accelerator, vendor, region, type, instance_size, instance_type):
     logger.debug("Hit the following exception:")
     logger.debug(e)
     logger.debug("Attempting to continue.")