utils.py

import pandas as pd
from pathlib import Path
from fsspec.parquet import open_parquet_file 
import pyarrow.parquet as pq
import pandas as pd
from io import BytesIO
from PIL import Image
import holoviews as hv

REPOSITORY = "Major-TOM"
DATASETS = ['Core-S2L2A', 'Core-S2L1C']

DATA_PATH = Path(__file__).parent/"data"


def _meta_data_url(dataset='Core-S2L2A', repository=REPOSITORY):
    return f'https://huggingface.co/datasets/{repository}/{dataset}/resolve/main/metadata.parquet'

def _meta_data_path(dataset='Core-S2L2A', repository=REPOSITORY):
    DATA_PATH.mkdir(parents=True, exist_ok=True)
    return DATA_PATH/f"{dataset}_metadata.parquet"

def get_meta_data(dataset='Core-S2L2A', repository=REPOSITORY):
    path = _meta_data_path(dataset=dataset)
    if not path.exists():
        data = pd.read_parquet(_meta_data_url(dataset=dataset))
        data.to_parquet(path)
    data = pd.read_parquet(path)
    
    data["centre_easting"], data["centre_northing"] = hv.util.transform.lon_lat_to_easting_northing(data["centre_lon"], data["centre_lat"])

    return data

def get_image(row):
    parquet_url = row["parquet_url"]
    parquet_row = row["parquet_row"]
    print(parquet_url)
    print(parquet_row)
    with open_parquet_file(parquet_url,columns = ["thumbnail"]) as f:
        with pq.ParquetFile(f) as pf:
            first_row_group = pf.read_row_group(parquet_row, columns=['thumbnail'])

    stream = BytesIO(first_row_group['thumbnail'][0].as_py())
    image = Image.open(stream)
    return image