diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8195e8f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +# Keep the build context small — only app.py + geocoder.db are needed in the image. +env/ +__pycache__/ +*.pyc +.git/ +.DS_Store + +# Large source data (not needed at runtime; geocoder.db is pre-built) +out.shp +out.shx +out.dbf +out.prj +output.csv +housing.json +housing_survey.csv +housingout.csv +result.csv +result.json +output.db +*.zip diff --git a/.gitignore b/.gitignore index 67a6715..039dde1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ *.swp *.json *.db + +# Python +__pycache__/ +*.pyc +env/ +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ed20707 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install runtime dependencies first for better layer caching. +COPY requirements-web.txt . +RUN pip install --no-cache-dir -r requirements-web.txt + +# App code and the pre-built lookup index. +COPY app.py . +COPY geocoder.db . + +# Cloud Run injects PORT; default to 8080 for local `docker run`. +ENV PORT=8080 +EXPOSE 8080 + +# Shell form so $PORT expands at runtime. +CMD exec uvicorn app:app --host 0.0.0.0 --port ${PORT} diff --git a/README.md b/README.md index f3f8b85..e50b6ac 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,66 @@ python csv_to_json.py result.csv --output data.json The script will create a JSON file with the same name as the input CSV (but with .json extension) if no output path is specified. +## Web Service + +The geocoder can run as a REST API (FastAPI) that returns parcel coordinates by +parcel ID or address. + +### 1. Build the lookup index + +The API reads from a compact, indexed SQLite database instead of scanning the +226MB `output.csv` on every request. Build it once: + +```bash +python build_index.py # output.csv -> geocoder.db (~29MB) +``` + +### 2. Run locally + +```bash +pip install -r requirements-web.txt +uvicorn app:app --reload +``` + +Interactive docs are served at `http://localhost:8000/docs`. + +### Endpoints + +| Endpoint | Description | +| --- | --- | +| `GET /health` | Liveness check; returns parcel count | +| `GET /geocode?parcel_id=R72 12307 0032` | Exact parcel ID lookup | +| `GET /geocode?address=4060 DELPHOS` | Partial, case-insensitive address lookup | + +Example: + +```bash +curl "http://localhost:8000/geocode?address=4060%20DELPHOS" +# {"count":1,"results":[{"parcel_id":"R72 12307 0032","address":"4060 DELPHOS AVE", +# "zip":"45402","latitude":39.763...,"longitude":-84.251...}]} +``` + +Provide exactly one of `parcel_id` or `address`. Address matches are substrings, +so partial terms may return multiple results (capped at 50). + +### Deploy to Google Cloud Run + +The `geocoder.db` index is baked into the container image (read-only), so the +service is stateless and scales to zero when idle. + +```bash +# Prereqs: gcloud CLI authenticated, a project selected, geocoder.db built. +./deploy.sh +``` + +To run the container locally: + +```bash +docker build -t parcel-geocoder . +docker run -p 8080:8080 parcel-geocoder +curl "http://localhost:8080/health" +``` + ## CSV File Structure The script expects the following columns in the CSV file: diff --git a/add_coordinates.py b/add_coordinates.py new file mode 100644 index 0000000..dc3b7e4 --- /dev/null +++ b/add_coordinates.py @@ -0,0 +1,68 @@ +import pandas as pd +import argparse +from pathlib import Path + +def add_coordinates_to_survey(survey_csv, coordinates_csv, output_csv=None): + """ + Add latitude and longitude to housing survey data by looking up coordinates + from the coordinates database using parcel IDs. + + Args: + survey_csv (str): Path to housing survey CSV + coordinates_csv (str): Path to coordinates CSV (from lookup_coordinates.py) + output_csv (str, optional): Path for output CSV. If None, will add '_with_coords' + to original filename + """ + # Read the CSVs + print(f"Reading survey data from {survey_csv}") + survey_df = pd.read_csv(survey_csv) + + print(f"Reading coordinates from {coordinates_csv}") + coords_df = pd.read_csv(coordinates_csv) + + # Convert coordinates to numeric values + coords_df['latitude'] = pd.to_numeric(coords_df['latitude'], errors='coerce') + coords_df['longitude'] = pd.to_numeric(coords_df['longitude'], errors='coerce') + + # Ensure we have a parcel ID column + if 'TAXPINNO' not in survey_df.columns: + raise KeyError("Survey CSV must have a 'TAXPINNO' column") + + # Merge coordinates into survey data + print("Joining coordinates with survey data...") + merged_df = pd.merge( + survey_df, + coords_df[['TAXPINNO', 'latitude', 'longitude']], + left_on='TAXPINNO', + right_on='TAXPINNO', + how='left' + ) + + # Check for unmatched records + unmatched = merged_df[merged_df['latitude'].isna()] + if not unmatched.empty: + print(f"\nWarning: {len(unmatched)} records could not be matched:") + print(unmatched['TAXPINNO'].tolist()) + + # Determine output path + if output_csv is None: + output_csv = Path(survey_csv).stem + '_with_coords.csv' + + # Save the result with float format for coordinates + merged_df.to_csv(output_csv, index=False, float_format='%.6f') + print(f"\nSaved {len(merged_df)} records to {output_csv}") + print(f"Successfully matched {len(merged_df) - len(unmatched)} records with coordinates") + +def main(): + parser = argparse.ArgumentParser(description='Add coordinates to housing survey data') + parser.add_argument('survey_csv', help='Path to housing survey CSV') + parser.add_argument('--coords', default='output.csv', + help='Path to coordinates CSV (default: output.csv)') + parser.add_argument('--output', help='Output CSV path (optional)') + + args = parser.parse_args() + + add_coordinates_to_survey(args.survey_csv, args.coords, args.output) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..d160b32 --- /dev/null +++ b/app.py @@ -0,0 +1,92 @@ +"""FastAPI geocoding service for Montgomery County, Ohio parcels. + +Serves lat/lon lookups by parcel ID or address from the compact SQLite index +built by build_index.py. Read-only: the database is opened per request in +read-only mode, so the process is stateless and safe to run with many workers. +""" + +import os +import sqlite3 +from contextlib import contextmanager + +from fastapi import FastAPI, HTTPException, Query + +DB_FILE = os.environ.get("GEOCODER_DB", "geocoder.db") +MAX_RESULTS = 50 + +app = FastAPI( + title="Montgomery County Parcel Geocoder", + description="Look up parcel coordinates by parcel ID or address.", + version="1.0.0", +) + + +@contextmanager +def get_db(): + # Open read-only so the shipped index can never be mutated by a request. + conn = sqlite3.connect(f"file:{DB_FILE}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + try: + yield conn + finally: + conn.close() + + +def _row_to_dict(row): + return { + "parcel_id": row["parcel_id"], + "address": row["full_address"], + "zip": row["zip"], + "latitude": row["latitude"], + "longitude": row["longitude"], + } + + +@app.get("/health") +def health(): + """Liveness/readiness check that also confirms the index is reachable.""" + try: + with get_db() as conn: + count = conn.execute("SELECT COUNT(*) FROM parcels").fetchone()[0] + return {"status": "ok", "parcels": count} + except sqlite3.Error as exc: + raise HTTPException(status_code=503, detail=f"database unavailable: {exc}") + + +@app.get("/geocode") +def geocode( + parcel_id: str | None = Query( + None, description="Exact parcel ID, e.g. 'R72 12307 0032'" + ), + address: str | None = Query( + None, description="Full or partial address, e.g. '4060 DELPHOS AVE' or 'DELPHOS'" + ), + limit: int = Query(MAX_RESULTS, ge=1, le=MAX_RESULTS), +): + """Look up parcels by exact parcel ID or by (partial) address. + + Provide exactly one of `parcel_id` or `address`. Address matching is + case-insensitive and matches substrings, so partial names return multiple hits. + """ + if bool(parcel_id) == bool(address): + raise HTTPException( + status_code=400, + detail="Provide exactly one of 'parcel_id' or 'address'.", + ) + + with get_db() as conn: + if parcel_id: + rows = conn.execute( + "SELECT * FROM parcels WHERE parcel_id = ? LIMIT ?", + (parcel_id, limit), + ).fetchall() + else: + rows = conn.execute( + "SELECT * FROM parcels WHERE full_address LIKE ? LIMIT ?", + (f"%{address.upper()}%", limit), + ).fetchall() + + if not rows: + raise HTTPException(status_code=404, detail="No matching parcels found.") + + return {"count": len(rows), "results": [_row_to_dict(r) for r in rows]} diff --git a/build_index.py b/build_index.py new file mode 100644 index 0000000..6afbd7c --- /dev/null +++ b/build_index.py @@ -0,0 +1,114 @@ +"""Build a compact, indexed SQLite lookup database from the geocoded parcel CSV. + +The source CSV (output.csv) is ~226MB with ~120 columns. The web service only +needs a handful of fields to answer geocoding queries, so this script streams the +CSV in chunks and writes a small `parcels` table with indexes on parcel ID and a +normalized address, turning multi-second scans into sub-millisecond queries. +""" + +import argparse +import sqlite3 +import pandas as pd + +# Columns pulled from the source CSV. Everything else is dropped. +SOURCE_COLUMNS = [ + "TAXPINNO", + "LOC_NBR", + "LOC_DIR", + "LOC_STREET", + "LOC_SUFFIX", + "LOC_ZIP", + "latitude", + "longitude", +] + +CHUNK_SIZE = 50_000 + + +def build_full_address(row): + """Combine address components into a single normalized (upper-case) string.""" + components = [] + if pd.notna(row.get("LOC_NBR")): + # Convert to int to drop the trailing ".0" pandas adds to numeric columns. + components.append(str(int(row["LOC_NBR"]))) + if pd.notna(row.get("LOC_DIR")): + components.append(str(row["LOC_DIR"])) + if pd.notna(row.get("LOC_STREET")): + components.append(str(row["LOC_STREET"])) + if pd.notna(row.get("LOC_SUFFIX")): + components.append(str(row["LOC_SUFFIX"])) + return " ".join(components).upper() + + +def build_index(input_csv, db_file): + conn = sqlite3.connect(db_file) + cur = conn.cursor() + cur.execute("DROP TABLE IF EXISTS parcels") + cur.execute( + """ + CREATE TABLE parcels ( + parcel_id TEXT, + full_address TEXT, + zip TEXT, + latitude REAL, + longitude REAL + ) + """ + ) + conn.commit() + + total = 0 + reader = pd.read_csv( + input_csv, + usecols=SOURCE_COLUMNS, + chunksize=CHUNK_SIZE, + low_memory=False, + ) + for chunk in reader: + chunk = chunk.dropna(subset=["latitude", "longitude"]) + chunk["full_address"] = chunk.apply(build_full_address, axis=1) + chunk["zip"] = chunk["LOC_ZIP"].apply( + lambda z: str(int(z)) if pd.notna(z) else None + ) + rows = list( + zip( + chunk["TAXPINNO"].astype(str), + chunk["full_address"], + chunk["zip"], + chunk["latitude"].astype(float), + chunk["longitude"].astype(float), + ) + ) + cur.executemany( + "INSERT INTO parcels (parcel_id, full_address, zip, latitude, longitude) " + "VALUES (?, ?, ?, ?, ?)", + rows, + ) + total += len(rows) + print(f" inserted {total:,} rows...") + + conn.commit() + + print("Creating indexes...") + cur.execute("CREATE INDEX idx_parcel_id ON parcels(parcel_id)") + cur.execute("CREATE INDEX idx_full_address ON parcels(full_address)") + conn.commit() + + cur.execute("VACUUM") + conn.commit() + conn.close() + print(f"Done. Wrote {total:,} rows to {db_file}") + + +def main(): + parser = argparse.ArgumentParser( + description="Build a compact SQLite lookup index from the geocoded parcel CSV" + ) + parser.add_argument("--csv", default="output.csv", help="Input CSV (default: output.csv)") + parser.add_argument("--db", default="geocoder.db", help="Output SQLite DB (default: geocoder.db)") + args = parser.parse_args() + build_index(args.csv, args.db) + + +if __name__ == "__main__": + main() diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..edc1c36 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Deploy the geocoder web service to Google Cloud Run. +# +# Prerequisites: +# - gcloud CLI installed and authenticated: gcloud auth login +# - a GCP project selected: gcloud config set project YOUR_PROJECT +# - geocoder.db built locally: python build_index.py +# +# Cloud Run builds the image from the Dockerfile (via Cloud Build), so the +# ~29MB geocoder.db is baked into the read-only container image. No volumes, +# no external database. The service scales to zero when idle. +# +set -euo pipefail + +SERVICE_NAME="${SERVICE_NAME:-parcel-geocoder}" +REGION="${REGION:-us-central1}" + +if [[ ! -f geocoder.db ]]; then + echo "geocoder.db not found. Run 'python build_index.py' first." >&2 + exit 1 +fi + +echo "Deploying '${SERVICE_NAME}' to Cloud Run in ${REGION}..." +gcloud run deploy "${SERVICE_NAME}" \ + --source . \ + --region "${REGION}" \ + --allow-unauthenticated \ + --memory 256Mi \ + --cpu 1 \ + --max-instances 3 + +echo +echo "Done. Test it:" +echo " URL=\$(gcloud run services describe ${SERVICE_NAME} --region ${REGION} --format='value(status.url)')" +echo " curl \"\$URL/health\"" diff --git a/requirements-web.txt b/requirements-web.txt new file mode 100644 index 0000000..838c2a3 --- /dev/null +++ b/requirements-web.txt @@ -0,0 +1,5 @@ +# Runtime dependencies for the web service only. +# The geocoder.db index is pre-built (see build_index.py), so pandas/geopandas +# are NOT needed at runtime — keeping the container image small. +fastapi==0.139.2 +uvicorn[standard]==0.51.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..abb3f5a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# Web service +fastapi==0.139.2 +uvicorn[standard]==0.51.0 + +# Data processing (build_index.py, process_shapefile.py, lookup_coordinates.py) +pandas==2.2.3 +geopandas==1.0.1 +shapely==2.0.6