diff --git a/CHANGELOG b/CHANGELOG index dd6096e..d0a658b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,6 +12,7 @@ candidates against previously written places. Existing databases are upgraded in place when appending; the new columns are nullable and readers are unaffected. +- Add optional dense-county precision rule (`--county-dense-max-precision` / `--county-dense-max-area-km2`, env `WOF_COUNTY_DENSE_MAX_PRECISION` / `WOF_COUNTY_DENSE_MAX_AREA_KM2`): small county polygons get a finer geohash cap so municipality-sized counties are not swallowed by neighbors in coarse cells; enabling the rule without an explicit county cap (`--county-max-precision` / `WOF_COUNTY_MAX_PRECISION`) defaults the cap to one below the clamped dense precision so the area threshold actually differentiates - Add `sample` and `sweep` subcommands to `scripts/validate_with_locationiq.js`: a quota-aware, resumable world validation sweep against LocationIQ (JSONL response cache keyed by rounded coordinates, persisted per-UTC-day request cap, per-country Markdown mismatch report plus machine-readable mismatch JSONL, `--dry-run` for cache-only evaluation) and a GeoNames-TSV sampler that picks the top-N most populous places per country. Name comparison now preserves non-Latin scripts. - Run the test suite in GitHub Actions on pushes to master and all pull requests (Node 20.x and 22.x); raise Jasmine's per-hook timeout so diff --git a/scripts/generate_boundary_index.js b/scripts/generate_boundary_index.js index 1cbd39a..0737731 100755 --- a/scripts/generate_boundary_index.js +++ b/scripts/generate_boundary_index.js @@ -60,6 +60,8 @@ function parseArgs(argv) { localityMaxPrecision: null, localadminMaxPrecision: null, countyMaxPrecision: null, + countyDenseMaxPrecision: null, + countyDenseMaxAreaKm2: null, regionMaxPrecision: null, regionSparseMaxPrecision: null, regionSparseMinAreaKm2: null, @@ -117,6 +119,12 @@ function parseArgs(argv) { } else if (arg === '--county-max-precision') { var countyMax = Number(argv[++i]) opts.countyMaxPrecision = Number.isFinite(countyMax) ? Math.trunc(countyMax) : null + } else if (arg === '--county-dense-max-precision') { + var countyDenseMax = Number(argv[++i]) + opts.countyDenseMaxPrecision = Number.isFinite(countyDenseMax) ? Math.trunc(countyDenseMax) : null + } else if (arg === '--county-dense-max-area-km2') { + var denseAreaKm2 = Number(argv[++i]) + opts.countyDenseMaxAreaKm2 = Number.isFinite(denseAreaKm2) && denseAreaKm2 > 0 ? denseAreaKm2 : null } else if (arg === '--region-max-precision') { var regionMax = Number(argv[++i]) opts.regionMaxPrecision = Number.isFinite(regionMax) ? Math.trunc(regionMax) : null @@ -175,6 +183,10 @@ function usage() { ' --locality-max-precision Max precision override for locality placetype', ' --localadmin-max-precision Max precision override for localadmin placetype', ' --county-max-precision Max precision override for county placetype', + ' --county-dense-max-precision Optional precision for very small county polygons (for example 5).', + ' When set together with the area threshold and no explicit', + ' --county-max-precision, the county cap defaults to one below this', + ' --county-dense-max-area-km2 Bbox area threshold to apply dense county precision', ' --region-max-precision Max precision override for region placetype', ' --region-sparse-max-precision Optional precision for very large region polygons (for example 3)', ' --region-sparse-min-area-km2 Area threshold to apply sparse region precision', @@ -862,7 +874,16 @@ function placetypeCode(placetype) { function resolveMaxPrecisionForPlacetype(opts, placetype, bbox) { if (placetype === 'locality') return opts.localityMaxPrecision if (placetype === 'localadmin') return opts.localadminMaxPrecision - if (placetype === 'county') return opts.countyMaxPrecision + if (placetype === 'county') { + var countyPrecision = opts.countyMaxPrecision + if (Number.isFinite(opts.countyDenseMaxPrecision) && Number.isFinite(opts.countyDenseMaxAreaKm2)) { + var countyAreaKm2 = bboxAreaKm2(bbox) + if (countyAreaKm2 <= opts.countyDenseMaxAreaKm2) { + countyPrecision = Math.max(countyPrecision, opts.countyDenseMaxPrecision) + } + } + return countyPrecision + } if (placetype === 'region') { var regionPrecision = opts.regionMaxPrecision if (Number.isFinite(opts.regionSparseMaxPrecision) && Number.isFinite(opts.regionSparseMinAreaKm2)) { @@ -1812,6 +1833,26 @@ async function main() { throw new Error('--index-mode must be either compact or full') } + if (options.countyDenseMaxPrecision !== null) { + if (!Number.isFinite(options.countyDenseMaxPrecision) || options.countyDenseMaxPrecision < 1) { + options.countyDenseMaxPrecision = null + } else { + options.countyDenseMaxPrecision = Math.trunc(options.countyDenseMaxPrecision) + if (options.countyDenseMaxPrecision > options.maxPrecision) { + options.countyDenseMaxPrecision = options.maxPrecision + } + } + } + if (options.countyMaxPrecision === null && + Number.isFinite(options.countyDenseMaxPrecision) && + Number.isFinite(options.countyDenseMaxAreaKm2)) { + // The dense rule only differentiates when the regular county cap is + // coarser than the dense precision. Without an explicit + // --county-max-precision, default the cap to one below the (clamped) + // dense precision instead of the global max. + options.countyMaxPrecision = Math.max(options.basePrecision, options.countyDenseMaxPrecision - 1) + } + options.localityMaxPrecision = clampPrecision(options.localityMaxPrecision, options.basePrecision, options.maxPrecision) options.localadminMaxPrecision = clampPrecision(options.localadminMaxPrecision, options.basePrecision, options.maxPrecision) options.countyMaxPrecision = clampPrecision(options.countyMaxPrecision, options.basePrecision, options.maxPrecision) @@ -1842,7 +1883,6 @@ async function main() { } } } - var files = collectInputFiles(options) if (!files.length) { throw new Error('No input files were found after filtering') @@ -1919,6 +1959,9 @@ async function main() { if (Number.isFinite(options.regionSparseMaxPrecision) && Number.isFinite(options.regionSparseMinAreaKm2)) { console.log('Sparse region rule: area_km2>=' + options.regionSparseMinAreaKm2 + ' => max_precision=' + options.regionSparseMaxPrecision) } + if (Number.isFinite(options.countyDenseMaxPrecision) && Number.isFinite(options.countyDenseMaxAreaKm2)) { + console.log('Dense county rule: area_km2<=' + options.countyDenseMaxAreaKm2 + ' => max_precision=' + options.countyDenseMaxPrecision) + } if (options.dominantLocalityPopulation > 0) { console.log('Dominant locality rollup: major_population>=' + options.dominantLocalityPopulation + ', ratio>=' + options.dominantLocalityRatio) } else { diff --git a/scripts/generate_wof_boundary.sh b/scripts/generate_wof_boundary.sh index b5da63b..f98ce3e 100755 --- a/scripts/generate_wof_boundary.sh +++ b/scripts/generate_wof_boundary.sh @@ -16,7 +16,12 @@ set -euo pipefail # WOF_MAX_PRECISION Geohash max precision (default: 5) # WOF_LOCALITY_MAX_PRECISION Locality max precision override (default: WOF_MAX_PRECISION) # WOF_LOCALADMIN_MAX_PRECISION Localadmin max precision override (default: WOF_MAX_PRECISION) -# WOF_COUNTY_MAX_PRECISION County max precision override (default: WOF_MAX_PRECISION) +# WOF_COUNTY_MAX_PRECISION County max precision override. When unset, the node script +# picks the default: WOF_MAX_PRECISION, or one below the dense +# precision (clamped to WOF_MAX_PRECISION) when the dense +# county rule is enabled +# WOF_COUNTY_DENSE_MAX_PRECISION Dense small-county precision (default: empty = rule off) +# WOF_COUNTY_DENSE_MAX_AREA_KM2 Bbox area threshold to apply dense county precision (default: empty = rule off) # WOF_REGION_MAX_PRECISION Region max precision override (default: 4) # WOF_REGION_SPARSE_MAX_PRECISION Sparse large-region precision (default: 3) # WOF_REGION_SPARSE_MIN_AREA_KM2 Area threshold for sparse region precision (default: 80000) @@ -53,7 +58,13 @@ WOF_BASE_PRECISION="${WOF_BASE_PRECISION:-4}" WOF_MAX_PRECISION="${WOF_MAX_PRECISION:-5}" WOF_LOCALITY_MAX_PRECISION="${WOF_LOCALITY_MAX_PRECISION:-${WOF_MAX_PRECISION}}" WOF_LOCALADMIN_MAX_PRECISION="${WOF_LOCALADMIN_MAX_PRECISION:-${WOF_MAX_PRECISION}}" -WOF_COUNTY_MAX_PRECISION="${WOF_COUNTY_MAX_PRECISION:-${WOF_MAX_PRECISION}}" +# WOF_COUNTY_MAX_PRECISION deliberately has no shell-side default: when it is +# unset the flag is omitted and the node script derives the county cap (the +# global max precision, or one below the clamped dense precision when the +# dense county rule is enabled), so both entry points behave identically. +WOF_COUNTY_MAX_PRECISION="${WOF_COUNTY_MAX_PRECISION:-}" +WOF_COUNTY_DENSE_MAX_PRECISION="${WOF_COUNTY_DENSE_MAX_PRECISION:-}" +WOF_COUNTY_DENSE_MAX_AREA_KM2="${WOF_COUNTY_DENSE_MAX_AREA_KM2:-}" WOF_REGION_MAX_PRECISION="${WOF_REGION_MAX_PRECISION:-4}" WOF_REGION_SPARSE_MAX_PRECISION="${WOF_REGION_SPARSE_MAX_PRECISION:-3}" WOF_REGION_SPARSE_MIN_AREA_KM2="${WOF_REGION_SPARSE_MIN_AREA_KM2:-80000}" @@ -133,7 +144,6 @@ COMMON_FLAGS=( --max-precision "${WOF_MAX_PRECISION}" --locality-max-precision "${WOF_LOCALITY_MAX_PRECISION}" --localadmin-max-precision "${WOF_LOCALADMIN_MAX_PRECISION}" - --county-max-precision "${WOF_COUNTY_MAX_PRECISION}" --region-max-precision "${WOF_REGION_MAX_PRECISION}" --region-sparse-max-precision "${WOF_REGION_SPARSE_MAX_PRECISION}" --region-sparse-min-area-km2 "${WOF_REGION_SPARSE_MIN_AREA_KM2}" @@ -151,6 +161,18 @@ COMMON_FLAGS=( --ensure-country-locality "${WOF_ENSURE_COUNTRY_LOCALITY}" ) +if [[ -n "${WOF_COUNTY_MAX_PRECISION}" ]]; then + COMMON_FLAGS+=(--county-max-precision "${WOF_COUNTY_MAX_PRECISION}") +fi + +if [[ -n "${WOF_COUNTY_DENSE_MAX_PRECISION}" ]]; then + COMMON_FLAGS+=(--county-dense-max-precision "${WOF_COUNTY_DENSE_MAX_PRECISION}") +fi + +if [[ -n "${WOF_COUNTY_DENSE_MAX_AREA_KM2}" ]]; then + COMMON_FLAGS+=(--county-dense-max-area-km2 "${WOF_COUNTY_DENSE_MAX_AREA_KM2}") +fi + if [[ -n "${WOF_MAX_PLACES}" ]]; then COMMON_FLAGS+=(--max-places "${WOF_MAX_PLACES}") fi diff --git a/spec/boundary_builder_dense_county_spec.js b/spec/boundary_builder_dense_county_spec.js new file mode 100644 index 0000000..22f68b2 --- /dev/null +++ b/spec/boundary_builder_dense_county_spec.js @@ -0,0 +1,187 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const sqlite3 = require('sqlite3'); +const geohash = require('../src/geohash'); + +function all(db, sql) { + return new Promise((resolve, reject) => { + db.all(sql, [], (err, rows) => (err ? reject(err) : resolve(rows || []))); + }); +} + +function close(db) { + return new Promise((resolve, reject) => { + db.close((err) => (err ? reject(err) : resolve())); + }); +} + +const SMALL_COUNTY_ID = 8001; +const LARGE_COUNTY_ID = 8002; + +// A municipality-sized county tucked inside a single precision-4 cell +// (bbox area well under 300 km2) plus a department-sized county spanning +// roughly 3x3 degrees (bbox area far above any dense threshold). +function writeCountiesFixture(inputPath) { + const smallCell = geohash.decodeBbox('s000'); + const lonSpan = smallCell.maxLon - smallCell.minLon; + const latSpan = smallCell.maxLat - smallCell.minLat; + const smallMinLon = smallCell.minLon + (lonSpan * 0.15); + const smallMaxLon = smallCell.minLon + (lonSpan * 0.45); + const smallMinLat = smallCell.minLat + (latSpan * 0.15); + const smallMaxLat = smallCell.minLat + (latSpan * 0.45); + + fs.writeFileSync(inputPath, JSON.stringify({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: SMALL_COUNTY_ID, + properties: { + name: 'Tiny Municipality', + placetype: 'county', + country_id: 'GT', + admin1_id: 1, + population: 40000, + is_current: 1 + }, + geometry: { + type: 'Polygon', + coordinates: [[ + [smallMinLon, smallMinLat], + [smallMaxLon, smallMinLat], + [smallMaxLon, smallMaxLat], + [smallMinLon, smallMaxLat], + [smallMinLon, smallMinLat] + ]] + } + }, + { + type: 'Feature', + id: LARGE_COUNTY_ID, + properties: { + name: 'Wide County', + placetype: 'county', + country_id: 'US', + admin1_id: 2, + population: 900000, + is_current: 1 + }, + geometry: { + type: 'Polygon', + coordinates: [[[20, 20], [23, 20], [23, 23], [20, 23], [20, 20]]] + } + } + ] + })); +} + +function runBuilder(inputPath, dbPath, extraArgs) { + return spawnSync('node', [ + path.join(__dirname, '..', 'scripts', 'generate_boundary_index.js'), + '--database', dbPath, + '--input', inputPath, + '--index-mode', 'compact', + '--include-county', 'true', + '--base-precision', '4', + '--max-precision', '5' + ].concat(extraArgs), { encoding: 'utf8' }); +} + +function lookupLengths(db, placeId) { + return all(db, `SELECT DISTINCT length(geohash) AS len FROM compact_geohash_lookup WHERE place_id = ${placeId} ORDER BY len`); +} + +describe('boundary builder dense county precision', () => { + it('raises small counties under the area threshold to the dense precision', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'offline-geocoder-dense-county-')); + try { + const inputPath = path.join(dir, 'counties.geojson'); + const dbPath = path.join(dir, 'dense.sqlite'); + writeCountiesFixture(inputPath); + + const result = runBuilder(inputPath, dbPath, [ + '--county-max-precision', '4', + '--county-dense-max-precision', '5', + '--county-dense-max-area-km2', '300' + ]); + + expect(result.status).toEqual(0); + expect(result.stdout).toContain('Dense county rule: area_km2<=300 => max_precision=5'); + + const db = new sqlite3.Database(dbPath); + try { + const smallLengths = await lookupLengths(db, SMALL_COUNTY_ID); + expect(smallLengths).toEqual([{ len: 5 }]); + + const largeLengths = await lookupLengths(db, LARGE_COUNTY_ID); + expect(largeLengths).toEqual([{ len: 4 }]); + } finally { + await close(db); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('derives a coarser county cap when only the dense flags are given', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'offline-geocoder-dense-county-')); + try { + const inputPath = path.join(dir, 'counties.geojson'); + const dbPath = path.join(dir, 'derived.sqlite'); + writeCountiesFixture(inputPath); + + // No --county-max-precision: the cap must default to one below the + // dense precision (4), not the global max, or the area threshold + // would be a no-op. + const result = runBuilder(inputPath, dbPath, [ + '--county-dense-max-precision', '5', + '--county-dense-max-area-km2', '300' + ]); + + expect(result.status).toEqual(0); + expect(result.stdout).toContain('county=4'); + + const db = new sqlite3.Database(dbPath); + try { + const smallLengths = await lookupLengths(db, SMALL_COUNTY_ID); + expect(smallLengths).toEqual([{ len: 5 }]); + + const largeLengths = await lookupLengths(db, LARGE_COUNTY_ID); + expect(largeLengths).toEqual([{ len: 4 }]); + } finally { + await close(db); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('keeps counties at the county precision cap when the dense flags are absent', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'offline-geocoder-dense-county-')); + try { + const inputPath = path.join(dir, 'counties.geojson'); + const dbPath = path.join(dir, 'plain.sqlite'); + writeCountiesFixture(inputPath); + + const result = runBuilder(inputPath, dbPath, ['--county-max-precision', '4']); + + expect(result.status).toEqual(0); + expect(result.stdout).not.toContain('Dense county rule'); + + const db = new sqlite3.Database(dbPath); + try { + const smallLengths = await lookupLengths(db, SMALL_COUNTY_ID); + expect(smallLengths).toEqual([{ len: 4 }]); + + const largeLengths = await lookupLengths(db, LARGE_COUNTY_ID); + expect(largeLengths).toEqual([{ len: 4 }]); + } finally { + await close(db); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +});