Spaces:
Runtime error
Runtime error
File size: 1,518 Bytes
963ecfd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import pandas as pd
from pathlib import Path
from fsspec.parquet import open_parquet_file
import pyarrow.parquet as pq
import pandas as pd
from io import BytesIO
from PIL import Image
import holoviews as hv
REPOSITORY = "Major-TOM"
DATASETS = ['Core-S2L2A', 'Core-S2L1C']
DATA_PATH = Path(__file__).parent/"data"
def _meta_data_url(dataset='Core-S2L2A', repository=REPOSITORY):
return f'https://huggingface.co/datasets/{repository}/{dataset}/resolve/main/metadata.parquet'
def _meta_data_path(dataset='Core-S2L2A', repository=REPOSITORY):
DATA_PATH.mkdir(parents=True, exist_ok=True)
return DATA_PATH/f"{dataset}_metadata.parquet"
def get_meta_data(dataset='Core-S2L2A', repository=REPOSITORY):
path = _meta_data_path(dataset=dataset)
if not path.exists():
data = pd.read_parquet(_meta_data_url(dataset=dataset))
data.to_parquet(path)
data = pd.read_parquet(path)
data["centre_easting"], data["centre_northing"] = hv.util.transform.lon_lat_to_easting_northing(data["centre_lon"], data["centre_lat"])
return data
def get_image(row):
parquet_url = row["parquet_url"]
parquet_row = row["parquet_row"]
print(parquet_url)
print(parquet_row)
with open_parquet_file(parquet_url,columns = ["thumbnail"]) as f:
with pq.ParquetFile(f) as pf:
first_row_group = pf.read_row_group(parquet_row, columns=['thumbnail'])
stream = BytesIO(first_row_group['thumbnail'][0].as_py())
image = Image.open(stream)
return image
|