main.py

import requests
import os
import sys
from pathlib import Path

import gradio as gr
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import geopandas as gpd
from pyproj.transformer import Transformer

sys.path.append(os.path.dirname(os.path.realpath(__file__)))
from MapItAnywhere.mia.bev import get_bev
from MapItAnywhere.mia.fpv import get_fpv
from MapItAnywhere.mia.fpv import filters
from MapItAnywhere.mia import logger

def get_city_boundary(query, fetch_shape=False):
    # Use Nominatim API to get the boundary of the city
    base_url = "https://nominatim.openstreetmap.org/search"
    params = {
        'q': query,
        'format': 'json',
        'limit': 1,
        'polygon_geojson': 1 if fetch_shape else 0
    }

    headers = {
        'User-Agent': f'mapperceptionnet_{query}'
    }
    response = requests.get(base_url, params=params, headers=headers)

    if response.status_code != 200:
        logger.error(f"Nominatim error when fetching boundary data for {query}.\n"
                     f"Status code: {response.status_code}. Content: {response.content}")
        return None
    
    data = response.json()

    if data is None:
        logger.warn(f"No data returned by Nominatim for {query}")
        return None
    
    # Extract bbox data from the API response
    bbox_data = data[0]['boundingbox']
    bbox = {
        'west': float(bbox_data[2]),
        'south': float(bbox_data[0]),
        'east': float(bbox_data[3]),
        'north': float(bbox_data[1])
    }

    if fetch_shape:
        # Extract GeoJSON boundary data from the API response
        boundary_geojson = data[0]['geojson']
        boundary_geojson = {
        "type": "FeatureCollection",
        "features": [
            {"type": "Feature",
            "properties": {},
            "geometry": boundary_geojson}]
        }
        return bbox, boundary_geojson
    else:
        return bbox
    
def split_dataframe(df, chunk_size = 100): 
    chunks = list()
    num_chunks = len(df) // chunk_size + 1
    for i in range(num_chunks):
        chunks.append(df[i*chunk_size:(i+1)*chunk_size])
    return chunks

async def fetch(location, filter_undistort, disable_cam_filter, map_length, mpp):
    N=1
    TOTAL_LOOKED_INTO_LIMIT = 10000

    ################ FPV
    downloader = get_fpv.MapillaryDownloader(os.getenv("MLY_TOKEN"))
    bbox = get_city_boundary(query=location)
    tiles = get_fpv.get_tiles_from_boundary(boundary_info=dict(bound_type="auto_bbox", bbox=bbox), zoom=14)
    np.random.shuffle(tiles)
    total_looked_into = 0
    dfs_meta = list()
    for tile in tiles:
        image_points_response = await downloader.get_tiles_image_points([tile])
        if image_points_response is None:
            continue
        try:
            df = get_fpv.parse_image_points_json_data(image_points_response)
            if len(df) == 0:
                continue
            total_looked_into += len(df)
            df_split = split_dataframe(df, chunk_size=100)

            for df in df_split:
                image_ids = df["id"]
                image_infos, num_fail = await get_fpv.fetch_image_infos(image_ids, downloader, infos_dir)
                df_meta = get_fpv.geojson_feature_list_to_pandas(image_infos.values())

                # Some standardization of the data
                df_meta["model"] = df_meta["model"].str.lower().str.replace(' ', '').str.replace('_', '')
                df_meta["make"] = df_meta["make"].str.lower().str.replace(' ', '').str.replace('_', '')

                if filter_undistort:
                    fp = no_cam_filter_pipeline if disable_cam_filter else filter_pipeline
                    df_meta = fp(df_meta)
                
                dfs_meta.append(df_meta)
                total_rows = sum([len(x) for x in dfs_meta])
                if total_rows > N:
                    break
                elif total_looked_into > TOTAL_LOOKED_INTO_LIMIT:
                    yield (f"Went through {total_looked_into} images and could not find images satisfying the filters."
                           "\nPlease rerun or run the data engine locally for bulk time consuming operations.", None, None)
                    return
            if total_rows > N:
                break
        except:
            pass

    df_meta = pd.concat(dfs_meta)
    df_meta = df_meta.sample(N)

    # Calc derrivative attributes
    df_meta["loc_descrip"] = filters.haversine_np(
        lon1=df_meta["geometry.long"], lat1=df_meta["geometry.lat"],
        lon2=df_meta["computed_geometry.long"], lat2=df_meta["computed_geometry.lat"]
    )

    df_meta["angle_descrip"] = filters.angle_dist(
        df_meta["compass_angle"],
        df_meta["computed_compass_angle"]
    )

    for index, row in df_meta.iterrows():
        desc = list()
        # Display attributes
        keys = ["id", "geometry.long", "geometry.lat", "compass_angle",
                "loc_descrip", "angle_descrip",
                "make", "model", "camera_type", 
                "quality_score"]
        for k in keys:
            v = row[k]
            if isinstance(v, float):
                v = f"{v:.4f}"
            bullet = f"{k}: {v}"
            desc.append(bullet)
        metadata_fmt = "\n".join(desc)
    
    yield metadata_fmt, None, None
    image_urls = list(df_meta.set_index("id")["thumb_2048_url"].items())
    num_fail = await get_fpv.fetch_images_pixels(image_urls, downloader, raw_image_dir)
    if num_fail > 0:
        logger.error(f"Failed to download {num_fail} images.")
    
    seq_to_image_ids = df_meta.groupby('sequence')['id'].agg(list).to_dict()
    lon_center = (bbox['east'] + bbox['west']) / 2
    lat_center = (bbox['north'] + bbox['south']) / 2
    projection = get_fpv.Projection(lat_center, lon_center, max_extent=200e3)

    df_meta.index = df_meta["id"]
    image_infos = df_meta.to_dict(orient="index")
    process_sequence_args = get_fpv.default_cfg

    if filter_undistort:
        for seq_id, seq_image_ids in seq_to_image_ids.items():
            try: 
                d, pi = get_fpv.process_sequence(
                    seq_image_ids,
                    image_infos,
                    projection,
                    process_sequence_args,
                    raw_image_dir,
                    out_image_dir,
                )
                if d is None or pi is None:
                    raise Exception("process_sequence returned None")
            except Exception as e:
                logger.error(f"Failed to process sequence {seq_id} skipping it. Error: {repr(e)}.")
        
        fpv = plt.imread(out_image_dir/ f"{row['id']}_undistorted.jpg")
    else:
        fpv = plt.imread(raw_image_dir/ f"{row['id']}.jpg")
    yield metadata_fmt, fpv, None
    ################ BEV
    df = df_meta
    # convert pandas dataframe to geopandas dataframe
    gdf = gpd.GeoDataFrame(df, 
                        geometry=gpd.points_from_xy(
                            df['computed_geometry.long'], 
                            df['computed_geometry.lat']), 
                            crs=4326)
        
    # convert the geopandas dataframe to UTM
    utm_crs = gdf.estimate_utm_crs()
    gdf_utm = gdf.to_crs(utm_crs)
    transformer = Transformer.from_crs(utm_crs, 4326)
        # load OSM data, if available
    padding = 50
    # calculate the required distance from the center to the edge of the image 
    # so that the image will not be out of bounds when we rotate it
    map_length = map_length
    map_length = np.ceil(np.sqrt(map_length**2 + map_length**2))
    distance = map_length * mpp

    # create bounding boxes for each point
    gdf_utm['bounding_box_utm_p1'] = gdf_utm.apply(lambda row: (
        row.geometry.x - distance - padding,
        row.geometry.y - distance - padding,
    ), axis=1)

    gdf_utm['bounding_box_utm_p2'] = gdf_utm.apply(lambda row: (
        row.geometry.x + distance + padding,
        row.geometry.y + distance + padding,
    ), axis=1)

    # convert the bounding box back to lat, long
    gdf_utm['bounding_box_lat_long_p1'] = gdf_utm.apply(lambda row: transformer.transform(*row['bounding_box_utm_p1']), axis=1)
    gdf_utm['bounding_box_lat_long_p2'] = gdf_utm.apply(lambda row: transformer.transform(*row['bounding_box_utm_p2']), axis=1)
    gdf_utm['bbox_min_lat'] = gdf_utm['bounding_box_lat_long_p1'].apply(lambda x: x[0])
    gdf_utm['bbox_min_long'] = gdf_utm['bounding_box_lat_long_p1'].apply(lambda x: x[1])
    gdf_utm['bbox_max_lat'] = gdf_utm['bounding_box_lat_long_p2'].apply(lambda x: x[0])
    gdf_utm['bbox_max_long'] = gdf_utm['bounding_box_lat_long_p2'].apply(lambda x: x[1])
    gdf_utm['bbox_formatted'] = gdf_utm.apply(lambda row: f"{row['bbox_min_long']},{row['bbox_min_lat']},{row['bbox_max_long']},{row['bbox_max_lat']}", axis=1)

    # iterate over the dataframe and get BEV images
    jobs = gdf_utm[['id', 'bbox_formatted', 'computed_compass_angle']] # only need the id and bbox_formatted columns for the jobs
    jobs = jobs.to_dict(orient='records').copy()
        
    get_bev.get_bev_from_bbox_worker_init(osm_cache_dir, bev_dir, semantic_mask_dir, rendered_mask_dir, 
                                          "MapItAnywhere/mia/bev/styles/mia.yml", map_length, mpp, 
                                          None, True, False, True, True, 1)
    for job_dict in jobs:
        get_bev.get_bev_from_bbox_worker(job_dict)
    
    bev = plt.imread(rendered_mask_dir / f"{row['id']}.png")

    yield metadata_fmt, fpv, bev

filter_pipeline = filters.FilterPipeline.load_from_yaml("MapItAnywhere/mia/fpv/filter_pipelines/mia.yaml")
filter_pipeline.verbose=False
no_cam_filter_pipeline = filters.FilterPipeline.load_from_yaml("MapItAnywhere/mia/fpv/filter_pipelines/mia_rural.yaml")
no_cam_filter_pipeline.verbose=False

loc = Path(".")
infos_dir =loc / "infos_dir"
raw_image_dir = loc / "raw_images"
out_image_dir = loc / "images"
osm_cache_dir = loc / "osm_cache"
bev_dir = loc / "bev_raw"
semantic_mask_dir = loc / "semantic_masks" 
rendered_mask_dir = loc / "rendered_semantic_masks" 

all_dirs = [loc, osm_cache_dir, bev_dir, semantic_mask_dir, rendered_mask_dir, out_image_dir, raw_image_dir]
for d in all_dirs:
    os.makedirs(d, exist_ok=True)

logger.info(f"Current working directory: {os.getcwd()}, listdir: {os.listdir('.')}")

demo = gr.Interface(
    fn=fetch,
    inputs=[gr.Text("Pittsburgh, PA, United States", label="Location"),
            gr.Checkbox(value=False, label="Filter & Undistort"),
            gr.Checkbox(value=False, label="Disable camera model filtering"),
            gr.Slider(minimum=64, maximum=512, step=1, label="BEV Dimension", value=224),
            gr.Slider(minimum=0.1, maximum=2, label="Meters Per Pixel", value=0.5)],
    outputs=[gr.Text(label="METADATA"), gr.Image(label="FPV"), gr.Image(label="BEV")],
    title="MapItAnywhere (Data Engine)",
    description="A demo showcasing samples of MIA's capability to retrieve FPV-BEV pairs worldwide."
                "For bulk download/heavy filtering please visit the github and follow the instructions to run locally"
)

logger.info("Starting server")
demo.launch(server_name="0.0.0.0", server_port=7860,share=False)