added json output

app.py

@@ -1,12 +1,17 @@
-import os 
+import os
+
 os.system("pip install gradio==2.4.6")
 import sys
 import gradio as gr
 
-os.system('pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html')
+os.system(
+    "pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html"
+)
 
 # clone and install Detic
-os.system("git clone https://github.com/facebookresearch/Detic.git --recurse-submodules")
+os.system(
+    "git clone https://github.com/facebookresearch/Detic.git --recurse-submodules"
+)
 os.chdir("Detic")
 
 # Install detectron2
@@ -16,6 +21,7 @@ import torch
 # Setup detectron2 logger
 import detectron2
 from detectron2.utils.logger import setup_logger
+
 setup_logger()
 
 # import some common libraries
@@ -31,8 +37,8 @@ from detectron2.utils.visualizer import Visualizer
 from detectron2.data import MetadataCatalog, DatasetCatalog
 
 # Detic libraries
-sys.path.insert(0, 'third_party/CenterNet2/projects/CenterNet2/')
-sys.path.insert(0, 'third_party/CenterNet2/')
+sys.path.insert(0, "third_party/CenterNet2/projects/CenterNet2/")
+sys.path.insert(0, "third_party/CenterNet2/")
 from centernet.config import add_centernet_config
 from detic.config import add_detic_config
 from detic.modeling.utils import reset_cls_test
@@ -43,31 +49,33 @@ from PIL import Image
 cfg = get_cfg()
 add_centernet_config(cfg)
 add_detic_config(cfg)
-cfg.MODEL.DEVICE='cpu'
+cfg.MODEL.DEVICE = "cpu"
 cfg.merge_from_file("configs/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml")
-cfg.MODEL.WEIGHTS = 'https://dl.fbaipublicfiles.com/detic/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.pth'
+cfg.MODEL.WEIGHTS = "https://dl.fbaipublicfiles.com/detic/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.pth"
 cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model
-cfg.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = 'rand'
-cfg.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = True # For better visualization purpose. Set to False for all classes.
+cfg.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = "rand"
+cfg.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = (
+    True  # For better visualization purpose. Set to False for all classes.
+)
 predictor = DefaultPredictor(cfg)
 
 # Setup the model's vocabulary using build-in datasets
 
 BUILDIN_CLASSIFIER = {
-    'lvis': 'datasets/metadata/lvis_v1_clip_a+cname.npy',
-    'objects365': 'datasets/metadata/o365_clip_a+cnamefix.npy',
-    'openimages': 'datasets/metadata/oid_clip_a+cname.npy',
-    'coco': 'datasets/metadata/coco_clip_a+cname.npy',
+    "lvis": "datasets/metadata/lvis_v1_clip_a+cname.npy",
+    "objects365": "datasets/metadata/o365_clip_a+cnamefix.npy",
+    "openimages": "datasets/metadata/oid_clip_a+cname.npy",
+    "coco": "datasets/metadata/coco_clip_a+cname.npy",
 }
 
 BUILDIN_METADATA_PATH = {
-    'lvis': 'lvis_v1_val',
-    'objects365': 'objects365_v2_val',
-    'openimages': 'oid_val_expanded',
-    'coco': 'coco_2017_val',
+    "lvis": "lvis_v1_val",
+    "objects365": "objects365_v2_val",
+    "openimages": "oid_val_expanded",
+    "coco": "coco_2017_val",
 }
 
-vocabulary = 'lvis' # change to 'lvis', 'objects365', 'openimages', or 'coco'
+vocabulary = "lvis"  # change to 'lvis', 'objects365', 'openimages', or 'coco'
 metadata = MetadataCatalog.get(BUILDIN_METADATA_PATH[vocabulary])
 classifier = BUILDIN_CLASSIFIER[vocabulary]
 num_classes = len(metadata.thing_classes)
@@ -75,6 +83,7 @@ reset_cls_test(predictor.model, classifier, num_classes)
 
 os.system("wget https://web.eecs.umich.edu/~fouhey/fun/desk/desk.jpg")
 
+
 def inference(img):
 
     im = cv2.imread(img)
@@ -83,17 +92,45 @@ def inference(img):
     v = Visualizer(im[:, :, ::-1], metadata)
     out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
 
-    return Image.fromarray(np.uint8(out.get_image())).convert('RGB')
-    
+    detected_objects = []
+    box_locations = outputs["instances"].pred_boxes
+    box_loc_screen = box_locations.tensor.cpu().numpy()
+
+    for i, box_coord in enumerate(box_loc_screen):
+        x0, y0, x1, y1 = box_coord
+        width = x1 - x0
+        height = y1 - y0
+        predicted_label = metadata.thing_classes[outputs["instances"].pred_classes[i]]
+        detected_objects.append(
+            {
+                "prediction": predicted_label,
+                "x": int(x0),
+                "y": int(y0),
+                "w": int(width),
+                "h": int(height),
+            }
+        )
+
+    return Image.fromarray(np.uint8(out.get_image())).convert("RGB"), detected_objects
+
+
 title = "Detic"
 
 description = "Gradio demo for Detic: Detecting Twenty-thousand Classes using Image-level Supervision. To use it, simply upload your image, or click one of the examples to load them. Read more at the links below."
 
 article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2201.02605' target='_blank'>Detecting Twenty-thousand Classes using Image-level Supervision</a> | <a href='https://github.com/facebookresearch/Detic' target='_blank'>Github Repo</a></p>"
 
-examples = [['desk.jpg']]
-gr.Interface(inference, inputs=gr.inputs.Image(type="filepath"), outputs=gr.outputs.Image(type="pil"),enable_queue=True, title=title,
+examples = [["desk.jpg"]]
+gr.Interface(
+    inference,
+    inputs=gr.inputs.Image(type="filepath"),
+    outputs=[
+        gr.outputs.Image(label="Visualization", type="pil"),
+        gr.outputs.JSON(label="Detected Objects"),
+    ],
+    enable_queue=True,
+    title=title,
     description=description,
     article=article,
-    examples=examples
-    ).launch()
+    examples=examples,
+).launch()