Finished data import and processing setup, bug in training step

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,4 @@
+[core]
+    analytics = false
+['remote "dvc-remote"']
+    url = s3://dagshub-savta-depth

.dvc/plots/confusion.json

@@ -0,0 +1,30 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": "rect",
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "nominal",
+            "sort": "ascending",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "nominal",
+            "sort": "ascending",
+            "title": "<DVC_METRIC_Y_LABEL>"
+        },
+        "color": {
+            "aggregate": "count",
+            "type": "quantitative"
+        },
+        "facet": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    }
+}

.dvc/plots/default.json

@@ -0,0 +1,29 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": {
+        "type": "line"
+    },
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_Y_LABEL>",
+            "scale": {
+                "zero": false
+            }
+        },
+        "color": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    }
+}

.dvc/plots/scatter.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": "point",
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_Y_LABEL>",
+            "scale": {
+                "zero": false
+            }
+        },
+        "color": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    }
+}

.dvc/plots/smooth.json

@@ -0,0 +1,39 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": {
+        "type": "line"
+    },
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_Y_LABEL>",
+            "scale": {
+                "zero": false
+            }
+        },
+        "color": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    },
+    "transform": [
+        {
+            "loess": "<DVC_METRIC_Y>",
+            "on": "<DVC_METRIC_X>",
+            "groupby": [
+                "rev"
+            ],
+            "bandwidth": 0.3
+        }
+    ]
+}

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

Makefile

@@ -19,7 +19,7 @@ endif
 env:
 ifeq (True,$(HAS_CONDA))
 	@echo ">>> Detected conda, creating conda environment."
-	conda create --name $(PROJECT_NAME) python=3
+	conda create --name $(PROJECT_NAME) python=3.7.6
 	@echo ">>> New conda env created. Activate with:\nconda activate $(PROJECT_NAME)"
 else
 	@echo ">>> No conda detected, creating venv environment."

dvc.lock

@@ -0,0 +1,13 @@
+process_data:
+  cmd: python3 src/code/make_dataset.py src/data/raw/nyu_depth_v2_labeled.mat src/data/raw/splits.mat
+    src/data/processed
+  deps:
+  - path: src/code/make_dataset.py
+    md5: b15a09b30657f303d62215ceb73c6f4f
+  - path: src/data/raw/nyu_depth_v2_labeled.mat
+    md5: 520609c519fba3ba5ac58c8fefcc3530
+  - path: src/data/raw/splits.mat
+    md5: 08e3c3aea27130ac7c01ffd739a4535f
+  outs:
+  - path: src/data/processed/
+    md5: 78d932b016048d527f9e73e1781127cd.dir

dvc.yaml

@@ -0,0 +1,10 @@
+stages:
+  process_data:
+    cmd: python3 src/code/make_dataset.py src/data/raw/nyu_depth_v2_labeled.mat src/data/raw/splits.mat
+      src/data/processed
+    deps:
+    - src/code/make_dataset.py
+    - src/data/raw/nyu_depth_v2_labeled.mat
+    - src/data/raw/splits.mat
+    outs:
+    - src/data/processed/

requirements.txt

@@ -1 +1,148 @@
-certifi==2020.6.20
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: linux-64
+_libgcc_mutex=0.1=main
+appdirs=1.4.4=pypi_0
+atpublic=2.0=pypi_0
+attrs=19.3.0=py_0
+backcall=0.2.0=pypi_0
+beautifulsoup4=4.9.1=py37_0
+blas=1.0=mkl
+bottleneck=1.3.2=py37heb32a55_1
+brotlipy=0.7.0=py37h7b6447c_1000
+ca-certificates=2020.6.24=0
+catalogue=1.0.0=py37_1
+certifi=2020.6.20=py37_0
+cffi=1.14.0=py37h2e261b9_0
+chardet=3.0.4=py37_1003
+colorama=0.4.3=pypi_0
+commonmark=0.9.1=pypi_0
+configobj=5.0.6=pypi_0
+cryptography=2.9.2=py37h1ba5d50_0
+cudatoolkit=10.2.89=hfd86e86_1
+cycler=0.10.0=py37_0
+cymem=2.0.3=py37he6710b0_0
+cython-blis=0.4.1=py37h7b6447c_1
+dataclasses=0.7=py37_0
+decorator=4.4.2=pypi_0
+dictdiffer=0.8.1=pypi_0
+distro=1.5.0=pypi_0
+dpath=2.0.1=pypi_0
+dvc=1.6.0=pypi_0
+fastai=1.0.61=1
+fastcore=1.0.0=pyh39e3cac_0
+fastprogress=1.0.0=pyh39e3cac_0
+flatten-json=0.1.7=pypi_0
+flufl-lock=3.2=pypi_0
+freetype=2.10.2=h5ab3b9f_0
+funcy=1.14=pypi_0
+future=0.18.2=pypi_0
+gitdb=4.0.5=pypi_0
+gitpython=3.1.7=pypi_0
+grandalf=0.6=pypi_0
+h5py=2.10.0=pypi_0
+idna=2.10=py_0
+importlib-metadata=1.7.0=py37_0
+importlib_metadata=1.7.0=0
+intel-openmp=2020.1=217
+ipython=7.17.0=pypi_0
+ipython-genutils=0.2.0=pypi_0
+jedi=0.17.2=pypi_0
+joblib=0.16.0=py_0
+jpeg=9b=h024ee3a_2
+jsonpath-ng=1.5.1=pypi_0
+jsonschema=3.0.2=py37_0
+kiwisolver=1.2.0=py37hfd86e86_0
+lcms2=2.11=h396b838_0
+ld_impl_linux-64=2.33.1=h53a641e_7
+libedit=3.1.20191231=h14c3975_1
+libffi=3.2.1=hd88cf55_4
+libgcc-ng=9.1.0=hdf63c60_0
+libgfortran-ng=7.3.0=hdf63c60_0
+libpng=1.6.37=hbc83047_0
+libstdcxx-ng=9.1.0=hdf63c60_0
+libtiff=4.1.0=h2733197_1
+lz4-c=1.9.2=he6710b0_1
+matplotlib=3.3.1=1
+matplotlib-base=3.3.1=py37h817c723_0
+mkl=2020.1=217
+mkl-service=2.3.0=py37he904b0f_0
+mkl_fft=1.1.0=py37h23d657b_0
+mkl_random=1.1.1=py37h0573a6f_0
+murmurhash=1.0.2=py37he6710b0_0
+nanotime=0.5.2=pypi_0
+ncurses=6.2=he6710b0_1
+networkx=2.4=pypi_0
+ninja=1.10.0=py37hfd86e86_0
+numexpr=2.7.1=py37h423224d_0
+numpy=1.19.1=py37hbc911f0_0
+numpy-base=1.19.1=py37hfa32c7d_0
+nvidia-ml-py3=7.352.0=py_0
+olefile=0.46=py37_0
+opencv-python=4.4.0.42=pypi_0
+openssl=1.1.1g=h7b6447c_0
+packaging=20.4=py_0
+pandas=1.1.0=py37he6710b0_0
+parso=0.7.1=pypi_0
+pathspec=0.8.0=pypi_0
+pexpect=4.8.0=pypi_0
+pickleshare=0.7.5=pypi_0
+pillow=7.2.0=py37hb39fc2d_0
+pip=20.2.2=py37_0
+plac=0.9.6=py37_1
+ply=3.11=pypi_0
+preshed=3.0.2=py37he6710b0_1
+prompt-toolkit=3.0.6=pypi_0
+ptyprocess=0.6.0=pypi_0
+pyasn1=0.4.8=pypi_0
+pycparser=2.20=py_2
+pydot=1.4.1=pypi_0
+pygments=2.6.1=pypi_0
+pygtrie=2.3.2=pypi_0
+pyopenssl=19.1.0=py_1
+pyparsing=2.4.7=py_0
+pyrsistent=0.16.0=py37h7b6447c_0
+pysocks=1.7.1=py37_1
+python=3.7.6=cpython_h8356626_6
+python-dateutil=2.8.1=py_0
+python_abi=3.7=1_cp37m
+pytorch=1.6.0=py3.7_cuda10.2.89_cudnn7.6.5_0
+pytz=2020.1=py_0
+pyyaml=5.3.1=py37h7b6447c_1
+readline=8.0=h7b6447c_0
+requests=2.24.0=py_0
+rich=5.2.1=pypi_0
+ruamel-yaml=0.16.10=pypi_0
+ruamel-yaml-clib=0.2.0=pypi_0
+scikit-learn=0.23.1=py37h423224d_0
+scipy=1.5.2=py37h0b6359f_0
+setuptools=49.6.0=py37_0
+shortuuid=1.0.1=pypi_0
+shtab=1.3.1=pypi_0
+six=1.15.0=py_0
+smmap=3.0.4=pypi_0
+soupsieve=2.0.1=py_0
+spacy=2.3.1=py37hfd86e86_0
+sqlite=3.33.0=h62c20be_0
+srsly=1.0.2=py37he6710b0_0
+tabulate=0.8.7=pypi_0
+thinc=7.4.1=py37hfd86e86_0
+threadpoolctl=2.1.0=pyh5ca1d4c_0
+tk=8.6.10=hbc83047_0
+toml=0.10.1=pypi_0
+torchvision=0.7.0=py37_cu102
+tornado=6.0.4=py37h7b6447c_1
+tqdm=4.48.2=py_0
+traitlets=4.3.3=pypi_0
+typing-extensions=3.7.4.2=pypi_0
+urllib3=1.25.10=py_0
+voluptuous=0.11.7=pypi_0
+wasabi=0.7.1=py_0
+wcwidth=0.2.5=pypi_0
+wheel=0.34.2=py37_0
+xz=5.2.5=h7b6447c_0
+yaml=0.2.5=h7b6447c_0
+zc-lockfile=2.0=pypi_0
+zipp=3.1.0=py_0
+zlib=1.2.11=h7b6447c_3
+zstd=1.4.5=h9ceee32_0

src/code/make_dataset.py

@@ -1 +1,94 @@
-print('hello world')
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#######################################################################################
+# The MIT License
+
+# Copyright (c) 2014       Hannes Schulz, University of Bonn  <schulz@ais.uni-bonn.de>
+# Copyright (c) 2013       Benedikt Waldvogel, University of Bonn <mail@bwaldvogel.de>
+# Copyright (c) 2008-2009  Sebastian Nowozin                       <nowozin@gmail.com>
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#######################################################################################
+#
+# Helper script to convert the NYU Depth v2 dataset Matlab file into a set of
+# PNG and JPEG images.
+#
+# See https://github.com/deeplearningais/curfil/wiki/Training-and-Prediction-with-the-NYU-Depth-v2-Dataset
+
+from __future__ import print_function
+
+import h5py
+import numpy as np
+import os
+import scipy.io
+import sys
+import cv2
+
+
+def convert_image(i, scene, depth, image, folder):
+    img_depth = depth * 1000.0
+    img_depth_uint16 = img_depth.astype(np.uint16)
+    cv2.imwrite("%s/%05d_depth.png" % (folder, i), img_depth_uint16)
+
+    image = image[:, :, ::-1]
+    image_black_boundary = np.zeros((480, 640, 3), dtype=np.uint8)
+    image_black_boundary[7:474, 7:632, :] = image[7:474, 7:632, :]
+    cv2.imwrite("%s/%05d.jpg" % (folder, i), image_black_boundary)
+
+
+if __name__ == "__main__":
+
+    if len(sys.argv) < 4:
+        print("usage: %s <h5_file> <train_test_split> <out_folder>" % sys.argv[0], file=sys.stderr)
+        sys.exit(0)
+
+    h5_file = h5py.File(sys.argv[1], "r")
+    # h5py is not able to open that file. but scipy is
+    train_test = scipy.io.loadmat(sys.argv[2])
+    out_folder = sys.argv[3]
+
+    test_images = set([int(x) for x in train_test["testNdxs"]])
+    train_images = set([int(x) for x in train_test["trainNdxs"]])
+    print("%d training images" % len(train_images))
+    print("%d test images" % len(test_images))
+
+    depth = h5_file['depths']
+
+    print("reading", sys.argv[1])
+
+    images = h5_file['images']
+    scenes = [u''.join(chr(c) for c in h5_file[obj_ref]) for obj_ref in h5_file['sceneTypes'][0]]
+
+    print("processing images")
+    for i, image in enumerate(images):
+        print("image", i + 1, "/", len(images))
+
+        idx = int(i) + 1
+        if idx in train_images:
+            train_test = "train"
+        else:
+            assert idx in test_images, "index %d neither found in training set nor in test set" % idx
+            train_test = "test"
+
+        folder = "%s/%s/%s" % (out_folder, train_test, scenes[i])
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+        convert_image(i, scenes[i], depth[i, :, :].T, image.T, folder)
+
+    print("Finished")

src/code/training.py

@@ -0,0 +1,37 @@
+import torch
+import sys
+from fastai.vision import unet_learner, ImageImageList, models, Path, root_mean_squared_error
+
+
+def get_y_fn(x):
+    y = str(x.absolute()).replace('.jpg', '_depth.png')
+    y = Path(y)
+
+    return y
+
+
+def create_databunch(data_path):
+    data = (ImageImageList.from_folder(data_path)
+            .filter_by_func(lambda fname: fname.suffix == '.jpg')
+            .split_by_folder(train='train', valid='test')
+            .label_from_func(get_y_fn).databunch()).normalize()
+    return data
+
+
+def train(data):
+    learner = unet_learner(data, models.resnet34, metrics=root_mean_squared_error, wd=1e-2, loss_func=torch.nn.SmoothL1Loss())
+    learner.fit_one_cycle(1, 1e-3)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print("usage: %s <data_path> <out_folder>" % sys.argv[0], file=sys.stderr)
+        sys.exit(0)
+
+    data = create_databunch(sys.argv[1])
+    data.batch_size = 1
+    data.num_workers = 0
+    learner = train(data)
+
+    learner.save(sys.argv[2])
+    learner.show_results()

src/data/.gitignore

@@ -0,0 +1 @@
+/processed

src/data/raw/.gitignore

@@ -0,0 +1,2 @@
+/nyu_depth_v2_labeled.mat
+/splits.mat

src/data/raw/nyu_depth_v2_labeled.mat.dvc

@@ -0,0 +1,8 @@
+md5: d27a0ba6c898f981797a3388c26c2d0f
+frozen: true
+deps:
+- etag: '"b125b2b1-5aa5b95864fc7"'
+  path: http://horatio.cs.nyu.edu/mit/silberman/nyu_depth_v2/nyu_depth_v2_labeled.mat
+outs:
+- md5: 520609c519fba3ba5ac58c8fefcc3530
+  path: nyu_depth_v2_labeled.mat

src/data/raw/splits.mat.dvc

@@ -0,0 +1,8 @@
+md5: 26011289311c18b92781de66654223a4
+frozen: true
+deps:
+- etag: '"a42-4cb6a5fad2fc0"'
+  path: http://horatio.cs.nyu.edu/mit/silberman/indoor_seg_sup/splits.mat
+outs:
+- md5: 08e3c3aea27130ac7c01ffd739a4535f
+  path: splits.mat