from io import BytesIO from pathlib import Path import holoviews as hv import numpy as np import pandas as pd import pyarrow.parquet as pq from fsspec.parquet import open_parquet_file from holoviews import opts from PIL import Image MAJOR_TOM_LOGO = "https://cdn-uploads.huggingface.co/production/uploads/6304c06eeb6d777a838eab63/BJKsLwX0GG4W3-gdf40TJ.png" MAJOR_TOM_PICTURE = ( "https://upload.wikimedia.org/wikipedia/en/6/6d/Major_tom_space_oddity_video.JPG" ) MAJOR_TOM_REF_URL = "https://huggingface.co/Major-TOM" PANEL_LOGO = "https://panel.holoviz.org/_static/logo_horizontal_light_theme.png" PANEL_URL = "https://panel.holoviz.org" DATASHADER_LOGO = "https://datashader.org/_static/logo_horizontal.svg" DATASHADER_URL = "https://datashader.org/" REPOSITORY = "Major-TOM" DATASETS = ["Core-S2L2A", "Core-S2L1C"] DATA_PATH = Path(__file__).parent / "data" DESCRIPTION = f"""\ ## Dataset Explorer This app provides a way of exploring samples present in the [MajorTOM-Core]({MAJOR_TOM_REF_URL}) dataset. It contains nearly every piece of Earth captured by ESA [Sentinel-2](https://sentinels.copernicus.eu/web/sentinel/missions/sentinel-2) satellite. ## Instructions To find a sample, navigate on the map to a place of interest. Click the map to find a dataset sample at the location you clicked. ## Powered by """ hv.extension("bokeh") opts.defaults( # opts.Curve(xaxis=None, yaxis=None, show_grid=False, show_frame=False, # color='orangered', framewise=True, width=100), opts.HLine(color="gray", line_width=1), # opts.Layout(shared_axes=False), opts.VLine(color="gray", line_width=1), ) def _meta_data_url(dataset="Core-S2L2A", repository=REPOSITORY): return f"https://huggingface.co/datasets/{repository}/{dataset}/resolve/main/metadata.parquet" def _meta_data_path(dataset="Core-S2L2A", repository=REPOSITORY): DATA_PATH.mkdir(parents=True, exist_ok=True) return DATA_PATH / f"{dataset}_metadata.parquet" def get_meta_data(dataset="Core-S2L2A", repository=REPOSITORY): path = _meta_data_path(dataset=dataset) if not path.exists(): data = pd.read_parquet(_meta_data_url(dataset=dataset)) data.to_parquet(path) data = pd.read_parquet(path) data["centre_easting"], data["centre_northing"] = ( hv.util.transform.lon_lat_to_easting_northing( data["centre_lon"], data["centre_lat"] ) ) return data def get_image(row): parquet_url = row["parquet_url"] parquet_row = row["parquet_row"] print(parquet_url) print(parquet_row) with open_parquet_file(parquet_url, columns=["thumbnail"]) as f: with pq.ParquetFile(f) as pf: first_row_group = pf.read_row_group(parquet_row, columns=["thumbnail"]) stream = BytesIO(first_row_group["thumbnail"][0].as_py()) image = Image.open(stream) return image def euclidean_distance(x, y, target_x, target_y): return np.sqrt((x - target_x) ** 2 + (y - target_y) ** 2) def get_closest_row(df, target_easting, target_northing): distance = euclidean_distance( df["centre_easting"], df["centre_northing"], target_easting, target_northing ) closest_row = df.loc[distance.idxmin()] return closest_row def get_closest_rows(df, target_easting, target_northing): distance = euclidean_distance( df["centre_easting"], df["centre_northing"], target_easting, target_northing ) closest_rows = df[distance == distance.min()] return closest_rows