Convert NetCDF Point Data To GeoParquet

This notebook converts the ALBATROS tide-gauge NetCDF example into GeoParquet. The important steps are to flatten the NetCDF variables into a table and create point geometries from longitude and latitude.

from pathlib import Path


def find_repo_root(start: Path = Path.cwd()) -> Path:
    """Find the repository root when the kernel starts in a subfolder."""
    start = start.resolve()
    for path in (start, *start.parents):
        if (path / ".git").exists() or (path / "downloaded_data").exists():
            return path
    return start


REPO_ROOT = find_repo_root()
DATA_DIR = REPO_ROOT / "downloaded_data"
DATA_DIR.mkdir(exist_ok=True)

print(f"Repository root: {REPO_ROOT}")
print(f"Data directory: {DATA_DIR}")

import json

import geopandas as gpd
import pandas as pd
import xarray as xr

NC_FILE = DATA_DIR / "datacollection_ALBATROSS_tidal_amplitude_phase_tide_gauges.nc"
PARQUET_FILE = DATA_DIR / "tidal_amplitude_phase_tide_gauges.parquet"
METADATA_FILE = DATA_DIR / "tidal_amplitude_phase_tide_gauges.parquet.metadata.json"

Repository root: /home/krasen/hackathon-repo
Data directory: /home/krasen/hackathon-repo/downloaded_data

ds = xr.open_dataset(NC_FILE)
ds

Flatten The NetCDF Dataset¶



df = ds.to_dataframe().reset_index()
longitude_col = next(col for col in ["LONGITUDE", "longitude", "lon"] if col in df.columns)
latitude_col = next(col for col in ["LATITUDE", "latitude", "lat"] if col in df.columns)

df = df.dropna(subset=[longitude_col, latitude_col])
gdf = gpd.GeoDataFrame(
    df,
    geometry=gpd.points_from_xy(df[longitude_col], df[latitude_col]),
    crs="EPSG:4326",
)

gdf.head()

gdf.explore()

# To get effective geo parquet you have to use atleast schema=1.1.0 + sort the data spatially
# Infact if you have more  than 2gbs worth of data its a good idea to partition it spatially, not just sort it
# more info on best practices here: - https://geoparquet.io/
gdf.sort_values('geometry',  inplace=True, ignore_index=True)

gdf.to_parquet(
    PARQUET_FILE,
    engine='pyarrow',
    compression='zstd',
    write_covering_bbox=True, 
    schema_version='1.1.0'
)