remove secondary requirements (i.e. not things that are explicitly installed by the user), fix normalization problem, and use tqdm for image processing progress bar

.gitignore

@@ -6,3 +6,4 @@
 aws/
 google-cloud-sdk
 __pycache__/
+env/

dvc.lock

@@ -3,14 +3,19 @@ process_data:
     src/data/processed
   deps:
   - path: src/code/make_dataset.py
-    md5: 726bf2bed948f73c5c342a96d017539e
+    md5: fd5076d53909a47ce3b6598c26af6c97
+    size: 3783
   - path: src/data/raw/nyu_depth_v2_labeled.mat
     md5: 520609c519fba3ba5ac58c8fefcc3530
+    size: 2972037809
   - path: src/data/raw/splits.mat
     md5: 08e3c3aea27130ac7c01ffd739a4535f
+    size: 2626
   outs:
   - path: src/data/processed/
-    md5: 77adb8603dbf31f3b272e0f51b6c2c29.dir
+    md5: d98a9647a37ab431bfa35815eb4afda0.dir
+    size: 232903470
+    nfiles: 2898
 train:
   cmd: python3 src/code/training.py src/data/processed
   deps:

requirements.txt

@@ -1,107 +1,8 @@
-appdirs==1.4.4
-atpublic==2.0
-backcall==0.2.0
-blis==0.4.1
-cachetools==4.1.1
-catalogue==1.0.0
-certifi==2020.6.20
-cffi==1.14.2
-chardet==3.0.4
-colorama==0.4.3
-commonmark==0.9.1
-configobj==5.0.6
-cycler==0.10.0
-cymem==2.0.3
-dataclasses==0.6
-decorator==4.4.2
-dictdiffer==0.8.1
-distro==1.5.0
-dpath==2.0.1
-dvc==1.9.1
-fastai==2.0.0
-fastcore==1.0.0
-fastprogress==1.0.0
-flatten-json==0.1.7
-flufl.lock==3.2
-funcy==1.14
-future==0.18.2
-gitdb==4.0.5
-GitPython==3.1.7
-google-api-core==1.22.1
-google-auth==1.20.1
-google-cloud-core==1.4.1
-google-cloud-storage==1.19.0
-google-crc32c==0.1.0
-google-resumable-media==0.7.1
-googleapis-common-protos==1.52.0
-grandalf==0.6
+dvc==1.10.1
+fastai==2.1.5
+torch==1.7.0
 h5py==2.10.0
-idna==2.10
-importlib-metadata==1.7.0
-ipykernel==5.3.4
-ipython==7.17.0
-ipython-genutils==0.2.0
-jedi==0.17.2
-joblib==0.16.0
-jsonpath-ng==1.5.1
-kiwisolver==1.2.0
-matplotlib==3.3.1
-murmurhash==1.0.2
-nanotime==0.5.2
-networkx==2.4
-numpy==1.19.1
-olefile==0.46
 opencv-python==4.4.0.42
-packaging==20.4
-pandas==1.1.1
-parso==0.7.1
-pathspec==0.8.0
-pexpect==4.8.0
-pickleshare==0.7.5
-Pillow==7.2.0
-pip==20.2.2
-plac==1.1.3
-ply==3.11
-preshed==3.0.2
-prompt-toolkit==3.0.6
-protobuf==3.13.0
-ptyprocess==0.6.0
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycparser==2.20
-pydot==1.4.1
-Pygments==2.6.1
-pygtrie==2.3.2
-pyparsing==2.4.7
-python-dateutil==2.8.1
-pytz==2020.1
-PyYAML==5.3.1
-requests==2.24.0
-rich==5.2.1
-rsa==4.6
-ruamel.yaml==0.16.10
-ruamel.yaml.clib==0.2.0
-scikit-learn==0.23.2
-scipy==1.5.2
-shortuuid==1.0.1
-shtab==1.3.1
-six==1.15.0
-smmap==3.0.4
-spacy==2.3.2
-srsly==1.0.2
-tabulate==0.8.7
-thinc==7.4.1
-threadpoolctl==2.1.0
-toml==0.10.1
-torch==1.6.0
-torchvision==0.7.0
-tqdm==4.48.2
-traitlets==4.3.3
-typing-extensions==3.7.4.3
-urllib3==1.25.10
-voluptuous==0.11.7
-wasabi==0.7.1
-wcwidth==0.2.5
-wheel==0.35.1
-zc.lockfile==2.0
-zipp==3.1.0
+tqdm==4.52.0
+numpy==1.19.4
+scikit-learn==0.23.2

src/code/make_dataset.py

@@ -39,13 +39,12 @@ import os
 import scipy.io
 import sys
 import cv2
+from tqdm import tqdm
 
 
 def convert_image(i, scene, depth, image, folder):
-    img_depth = depth * 1000.0
-    img_depth_uint16 = img_depth.astype(np.uint16)
-    normalized_depth = np.zeros(img_depth_uint16.shape)
-    normalized_depth = cv2.normalize(img_depth_uint16,  normalized_depth, 0, 255, cv2.NORM_MINMAX)
+    # depth is given in meters (Kinect has a range of around .5m and 4.5m but can sense also at 8m)
+    normalized_depth = cv2.normalize(depth,  None, 0, 255, cv2.NORM_MINMAX)
     cv2.imwrite("%s/%05d_depth.png" % (folder, i), normalized_depth)
 
     image = image[:, :, ::-1]
@@ -75,12 +74,9 @@ if __name__ == "__main__":
     print("reading", sys.argv[1])
 
     images = h5_file['images']
-    scenes = [u''.join(chr(c) for c in h5_file[obj_ref]) for obj_ref in h5_file['sceneTypes'][0]]
-
-    print("processing images")
-    for i, image in enumerate(images):
-        print("image", i + 1, "/", len(images))
+    scenes = [u''.join(chr(c[0]) for c in h5_file[obj_ref]) for obj_ref in h5_file['sceneTypes'][0]]
 
+    for i, image in tqdm(enumerate(images), desc="processing images", total=len(images)):
         idx = int(i) + 1
         if idx in train_images:
             train_test = "train"
@@ -93,4 +89,4 @@ if __name__ == "__main__":
             os.makedirs(folder)
         convert_image(i, scenes[i], depth[i, :, :].T, image.T, folder)
 
-    print("Finished")
+    print("Finished")

src/code/training.py

@@ -1,8 +1,9 @@
 import torch
 import sys
-from fastai2.vision.all import *
+from fastai.vision.all import *
 from torchvision.utils import save_image
 
+
 class ImageImageDataLoaders(DataLoaders):
     "Basic wrapper around several `DataLoader`s with factory methods for Image to Image problems"
     @classmethod