update

.gitignore

@@ -0,0 +1,106 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+data/

Dockerfile

@@ -3,14 +3,14 @@ FROM python:3.11
 WORKDIR /code
 
 COPY ./requirements.txt /code/requirements.txt
-RUN python3 -m pip install --no-cache-dir --upgrade pip
-RUN python3 -m pip install --no-cache-dir --upgrade -r /code/requirements.txt
+RUN python3 -m pip install uv
+RUN python3 -m uv pip install -r /code/requirements.txt
+RUN python3 download_datasets.py
 
 COPY . .
 
-CMD ["panel", "serve", "/code/app.py", "--address", "0.0.0.0", "--port", "7860",  "--allow-websocket-origin", "*", "--num-procs", "2", "--index", "app"]
+CMD ["panel", "serve", "/code/app.py", "--address", "0.0.0.0", "--port", "7860",  "--allow-websocket-origin", "*", "--num-procs", "4", "--index", "app"]
 
-RUN mkdir data
 RUN chmod 777 data
 
 ENV NUMBA_CACHE_DIR=/code/.cache

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Marc Skov Madsen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -10,8 +10,28 @@ license: mit
 
 See [Major Tom Explorer](https://awesome-panel.org/resources/major_tom_explorer/) by [awesome-panel.org](https://awesome-panel.org) for more info.
 
-## Serve
+## Install
+
+```python
+git clone https://huggingface.co/spaces/MarcSkovMadsen/major-tom-explorer
+```
+
+Create and activate your virtual environment
+
+```bash
+python -m venv .venv
+source .venv/bin/activate # linux
+```
+
+Install the dependencies
+
+```bash
+pip install uv
+uv pip install -r requirements_dev.txt
+```
+
+## Serve App
 
 ```python
 panel serve app.py --autoreload
-```
+```

app.py

@@ -1,54 +1,212 @@
-import numpy as np
-import holoviews as hv
 import dask.dataframe as dd
-
-from holoviews import opts
-from holoviews.operation.datashader import aggregate
+import holoviews as hv
+import numpy as np
 import panel as pn
-from utils import get_meta_data
+import param
+from holoviews.operation.datashader import dynspread, rasterize
+
+from utils import (
+    DATASETS,
+    DATASHADER_LOGO,
+    DATASHADER_URL,
+    DESCRIPTION,
+    MAJOR_TOM_LOGO,
+    MAJOR_TOM_PICTURE,
+    MAJOR_TOM_REF_URL,
+    PANEL_LOGO,
+    PANEL_URL,
+    get_closest_rows,
+    get_image,
+    get_meta_data,
+)
+
+
+class DatasetInput(pn.viewable.Viewer):
+    value = param.Selector(objects=DATASETS, allow_None=False, label="Dataset")
+
+    data = param.DataFrame(allow_None=False)
+
+    def __panel__(self):
+        return pn.widgets.RadioButtonGroup.from_param(
+            self.param.value, button_style="outline"
+        )
+
+    @pn.depends("value", watch=True, on_init=True)
+    def _update_data(self):
+        self.data = pn.cache(get_meta_data)(dataset=self.value)
+
+
+class MapInput(pn.viewable.Viewer):
+    data = param.DataFrame(allow_refs=True, allow_None=False)
+
+    data_in_view = param.DataFrame(allow_None=False)
+    data_selected = param.DataFrame(allow_None=False)
+
+    _plot = param.Parameter(allow_None=False)
+    _pointer_x = param.Parameter(allow_None=False)
+    _pointer_y = param.Parameter(allow_None=False)
+    _range_xy = param.Parameter(allow_None=False)
+    _tap = param.Parameter(allow_None=False)
+
+    updating = param.Boolean()
+
+    def __panel__(self):
+        return pn.pane.HoloViews(
+            self._plot, height=550, width=800, loading=self.param.updating
+        )
+
+    @param.depends("data", watch=True, on_init=True)
+    def _handle_data_dask_change(self):
+        with self.param.update(updating=True):
+            data_dask = dd.from_pandas(self.data).persist()
+            points = hv.Points(
+                data_dask, kdims=["centre_easting", "centre_northing"], vdims=[]
+            )
+
+            mean_easting = np.mean(points.range("centre_easting"))
+            mean_northing = np.mean(points.range("centre_northing"))
+            rangexy = hv.streams.RangeXY(source=points)
+            tap = hv.streams.Tap(source=points, x=mean_easting, y=mean_northing)
+
+            agg = rasterize(
+                points, link_inputs=True, x_sampling=0.0001, y_sampling=0.0001
+            )
+            dyn = dynspread(agg)
+            dyn.opts(cmap="kr_r", colorbar=True)
+
+            pointerx = hv.streams.PointerX(x=mean_easting, source=points)
+            pointery = hv.streams.PointerY(y=mean_northing, source=points)
+            vline = hv.DynamicMap(lambda x: hv.VLine(x), streams=[pointerx])
+            hline = hv.DynamicMap(lambda y: hv.HLine(y), streams=[pointery])
+            tiles = hv.Tiles(
+                "https://tile.openstreetmap.org/{Z}/{X}/{Y}.png", name="OSM"
+            ).opts(width=600, height=550, xlabel="Longitude", ylabel="Latitude")
+
+            self.param.update(
+                _plot=tiles * agg * dyn * hline * vline,
+                _pointer_x=pointerx,
+                _pointer_y=pointery,
+                _range_xy=rangexy,
+                _tap=tap,
+            )
+
+            update_viewed = pn.bind(
+                self._update_data_in_view,
+                rangexy.param.x_range,
+                rangexy.param.y_range,
+                watch=True,
+            )
+            update_viewed()
+
+            update_selected = pn.bind(
+                self._update_data_selected, tap.param.x, tap.param.y, watch=True
+            )
+            update_selected()
+
+    def _update_data_in_view(self, x_range, y_range):
+        if not x_range or not y_range:
+            self.data_in_view = self.data.head(0)
+            return
+
+        data = self.data
+        data = data[
+            (data.centre_easting.between(*x_range))
+            & (data.centre_northing.between(*y_range))
+        ]
+        self.data_in_view = data.head(10).reset_index(drop=True)
+
+    def _update_data_selected(self, tap_x, tap_y):
+        self.data_selected = get_closest_rows(self.data, tap_x, tap_y)
 
-pn.extension()
 
-hv.extension('bokeh')
+class ImageInput(pn.viewable.Viewer):
+    data = param.DataFrame(allow_refs=True, allow_None=False)
 
-renderer = hv.renderer('bokeh')
+    updating = param.Boolean()
+    image = param.Parameter()
+    plot = param.Parameter()
 
-opts.defaults(
-    opts.Curve(xaxis=None, yaxis=None, show_grid=False, show_frame=False,
-               color='orangered', framewise=True, width=100),
-    opts.Image(width=800, height=400, shared_axes=False, logz=True, colorbar=True,
-               xaxis=None, yaxis=None, axiswise=True, bgcolor='black'),
-    opts.HLine(color='white', line_width=1),
-    opts.Layout(shared_axes=False),
-    opts.VLine(color='white', line_width=1))
+    _timestamp = param.Selector(label="Timestamp", objects=[None])
 
-pandas_df = pn.cache(get_meta_data)()
-df = dd.from_pandas(pandas_df).persist()
+    def __panel__(self):
+        return pn.Column(
+            pn.widgets.RadioButtonGroup.from_param(
+                self.param._timestamp, button_style="outline"
+            ),
+            pn.Tabs(
+                pn.pane.HoloViews(
+                    hv.DynamicMap(pn.bind(lambda plot: plot, self.param.plot)),
+                    loading=self.param.updating,
+                    height=800,
+                    width=800,
+                    name="Interactive Image",
+                ),
+                pn.pane.Image(
+                    self.param.image,
+                    name="Static Image",
+                    loading=self.param.updating,
+                    width=800,
+                ),
+                dynamic=True,
+            ),
+        )
 
-points = hv.Points(df, kdims=['centre_easting', 'centre_northing'], vdims=[])
+    @pn.depends("data", watch=True, on_init=True)
+    def _update_timestamp(self):
+        if self.data.empty:
+            default_value = None
+            options = [None]
+            print("empty options")
+        else:
+            options = sorted(self.data["timestamp"].unique())
+            default_value = options[0]
+            print("options", options)
 
-agg = aggregate(points, link_inputs=True, x_sampling=0.0001, y_sampling=0.0001)
+        self.param._timestamp.objects = options
+        if not self._timestamp in options:
+            self._timestamp = default_value
 
-agg.opts(cmap="kr_r")
+    @pn.depends("_timestamp", watch=True, on_init=True)
+    def tap_image(self):
+        if self.data.empty or not self._timestamp:
+            self.image = hv.RGB(np.array([]))
+        else:
+            with self.param.update(updating=True):
+                row = self.data[self.data.timestamp == self._timestamp].iloc[0]
+                self.image = image = pn.cache(get_image)(row)
+                image_array = np.array(image)
+                self.plot = hv.RGB(image_array).opts(xaxis=None, yaxis=None)
 
-pointerx = hv.streams.PointerX(x=np.mean(points.range('centre_easting')), source=points)
-pointery = hv.streams.PointerY(y=np.mean(points.range('centre_northing')), source=points)
-box = hv.streams.BoundsXY(source=points, bounds=(0,0,0,0))
-vline = hv.DynamicMap(lambda x: hv.VLine(x), streams=[pointerx])
-hline = hv.DynamicMap(lambda y: hv.HLine(y), streams=[pointery])
 
-tiles = hv.Tiles('https://tile.openstreetmap.org/{Z}/{X}/{Y}.png', name="OSM").opts(width=600, height=550)
-hvobj = tiles * agg * hline * vline
+pn.extension("tabulator", design="fast")
 
-@pn.depends(pointerx.param.x, pointery.param.y)
-def table(x,y):
-    return f"Easting: {x}, Northing: {y}"
+dataset = DatasetInput()
+app = MapInput(data=dataset.param.data)
+image_input = ImageInput(data=app.param.data_selected)
 
-@pn.depends(box.param.bounds)
-def bbox(value):
-    return str(value)
 
-pn.pane.HoloViews(hvobj).servable()
+main = pn.Column(
+    pn.FlexBox(
+        pn.Column(dataset, app),
+        image_input,
+    ),
+)
+sidebar = pn.Column(
+    pn.pane.Image(
+        MAJOR_TOM_LOGO, link_url=MAJOR_TOM_REF_URL, sizing_mode="stretch_width"
+    ),
+    pn.pane.Image(
+        MAJOR_TOM_PICTURE, link_url=MAJOR_TOM_REF_URL, sizing_mode="stretch_width"
+    ),
+    DESCRIPTION,
+    pn.pane.Image(PANEL_LOGO, link_url=PANEL_URL, width=200, margin=(10, 20)),
+    pn.pane.Image(DATASHADER_LOGO, link_url=DATASHADER_URL, width=200, margin=(10, 20)),
+)
 
-pn.panel(table).servable()
-pn.panel(bbox).servable()
+pn.template.FastListTemplate(
+    title="Major TOM Explorer",
+    main=[main],
+    sidebar=[sidebar],
+    main_layout=None,
+    accent="#003247",  # "#A01346"
+).servable()

download_datasets.py

@@ -0,0 +1,7 @@
+from utils import DATASETS, get_meta_data
+
+for dataset in DATASETS:
+    print(f"downloading {dataset}")
+    get_meta_data(dataset=dataset)
+
+print("finished downloading datasets")

requirements.txt

@@ -1,8 +1,10 @@
+aiohttp
 dask[dataframe]
 datashader
 fsspec
 holoviews
 hvplot
 pandas
-pyarrow
 panel
+pyarrow
+requests

requirements_dev.txt

@@ -0,0 +1,7 @@
+-r requirements.txt
+
+black
+isort
+mypy
+pylint
+watchfiles

utils.py

@@ -1,46 +1,106 @@
-import pandas as pd
+from io import BytesIO
 from pathlib import Path
-from fsspec.parquet import open_parquet_file 
-import pyarrow.parquet as pq
+
+import holoviews as hv
+import numpy as np
 import pandas as pd
-from io import BytesIO
+import pyarrow.parquet as pq
+from fsspec.parquet import open_parquet_file
+from holoviews import opts
 from PIL import Image
-import holoviews as hv
 
+MAJOR_TOM_LOGO = "https://cdn-uploads.huggingface.co/production/uploads/6304c06eeb6d777a838eab63/BJKsLwX0GG4W3-gdf40TJ.png"
+MAJOR_TOM_PICTURE = (
+    "https://upload.wikimedia.org/wikipedia/en/6/6d/Major_tom_space_oddity_video.JPG"
+)
+MAJOR_TOM_REF_URL = "https://huggingface.co/Major-TOM"
+PANEL_LOGO = "https://panel.holoviz.org/_static/logo_horizontal_light_theme.png"
+PANEL_URL = "https://panel.holoviz.org"
+DATASHADER_LOGO = "https://datashader.org/_static/logo_horizontal.svg"
+DATASHADER_URL = "https://datashader.org/"
 REPOSITORY = "Major-TOM"
-DATASETS = ['Core-S2L2A', 'Core-S2L1C']
+DATASETS = ["Core-S2L2A", "Core-S2L1C"]
+
+DATA_PATH = Path(__file__).parent / "data"
+
+DESCRIPTION = f"""\
+## Dataset Explorer
+
+This app provides a way of exploring samples present in the [MajorTOM-Core]({MAJOR_TOM_REF_URL}) dataset. It contains nearly every piece of Earth captured by ESA [Sentinel-2](https://sentinels.copernicus.eu/web/sentinel/missions/sentinel-2) satellite.
+
+## Instructions
+
+To find a sample, navigate on the map to a place of interest. Click the map to find a dataset sample at the location you clicked.
 
-DATA_PATH = Path(__file__).parent/"data"
+## Powered by
 
+"""
 
-def _meta_data_url(dataset='Core-S2L2A', repository=REPOSITORY):
-    return f'https://huggingface.co/datasets/{repository}/{dataset}/resolve/main/metadata.parquet'
+hv.extension("bokeh")
 
-def _meta_data_path(dataset='Core-S2L2A', repository=REPOSITORY):
+opts.defaults(
+    # opts.Curve(xaxis=None, yaxis=None, show_grid=False, show_frame=False,
+    #            color='orangered', framewise=True, width=100),
+    opts.HLine(color="gray", line_width=1),
+    # opts.Layout(shared_axes=False),
+    opts.VLine(color="gray", line_width=1),
+)
+
+
+def _meta_data_url(dataset="Core-S2L2A", repository=REPOSITORY):
+    return f"https://huggingface.co/datasets/{repository}/{dataset}/resolve/main/metadata.parquet"
+
+
+def _meta_data_path(dataset="Core-S2L2A", repository=REPOSITORY):
     DATA_PATH.mkdir(parents=True, exist_ok=True)
-    return DATA_PATH/f"{dataset}_metadata.parquet"
+    return DATA_PATH / f"{dataset}_metadata.parquet"
+
 
-def get_meta_data(dataset='Core-S2L2A', repository=REPOSITORY):
+def get_meta_data(dataset="Core-S2L2A", repository=REPOSITORY):
     path = _meta_data_path(dataset=dataset)
     if not path.exists():
         data = pd.read_parquet(_meta_data_url(dataset=dataset))
         data.to_parquet(path)
     data = pd.read_parquet(path)
-    
-    data["centre_easting"], data["centre_northing"] = hv.util.transform.lon_lat_to_easting_northing(data["centre_lon"], data["centre_lat"])
+
+    data["centre_easting"], data["centre_northing"] = (
+        hv.util.transform.lon_lat_to_easting_northing(
+            data["centre_lon"], data["centre_lat"]
+        )
+    )
 
     return data
 
+
 def get_image(row):
     parquet_url = row["parquet_url"]
     parquet_row = row["parquet_row"]
     print(parquet_url)
     print(parquet_row)
-    with open_parquet_file(parquet_url,columns = ["thumbnail"]) as f:
+    with open_parquet_file(parquet_url, columns=["thumbnail"]) as f:
         with pq.ParquetFile(f) as pf:
-            first_row_group = pf.read_row_group(parquet_row, columns=['thumbnail'])
+            first_row_group = pf.read_row_group(parquet_row, columns=["thumbnail"])
 
-    stream = BytesIO(first_row_group['thumbnail'][0].as_py())
+    stream = BytesIO(first_row_group["thumbnail"][0].as_py())
     image = Image.open(stream)
     return image
 
+
+def euclidean_distance(x, y, target_x, target_y):
+    return np.sqrt((x - target_x) ** 2 + (y - target_y) ** 2)
+
+
+def get_closest_row(df, target_easting, target_northing):
+    distance = euclidean_distance(
+        df["centre_easting"], df["centre_northing"], target_easting, target_northing
+    )
+    closest_row = df.loc[distance.idxmin()]
+    return closest_row
+
+
+def get_closest_rows(df, target_easting, target_northing):
+    distance = euclidean_distance(
+        df["centre_easting"], df["centre_northing"], target_easting, target_northing
+    )
+    closest_rows = df[distance == distance.min()]
+    return closest_rows