v05
This commit is contained in:
nq
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Streaming, disk-backed source IMG manifest vs OSM XML POI audit.
|
||||
|
||||
Requires Python 3 standard library only. Do not mistake internal roundtrip
|
||||
consistency for correctness of Garmin TYP semantics or the resulting OBF.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
SCALE = Decimal('10000000')
|
||||
|
||||
def scaled(s):
|
||||
return int((Decimal(str(s)) * SCALE).to_integral_value(rounding=ROUND_HALF_UP))
|
||||
|
||||
def opencsv(p):
|
||||
return gzip.open(p, "rt", newline="", encoding="utf-8-sig") if str(p).endswith(".gz") else open(p, "r", newline="", encoding="utf-8-sig")
|
||||
|
||||
def openxml(p):
|
||||
return gzip.open(p, 'rb') if str(p).endswith('.gz') else open(p, 'rb')
|
||||
|
||||
def parse_rules(path):
|
||||
if not path:
|
||||
return []
|
||||
with open(path, encoding='utf-8-sig', newline='') as f:
|
||||
rows = []
|
||||
for r in csv.DictReader(x for x in f if not x.startswith('#')):
|
||||
if not r.get('code'):
|
||||
continue
|
||||
tags = dict(t.split('=', 1) for t in r['tags'].split(';') if t)
|
||||
rows.append((int(r['code'], 0), r['name_contains'], r['mode'], tags))
|
||||
return rows
|
||||
|
||||
def matching_rules(rules, code, name):
|
||||
return [r for r in rules if r[0] == int(code, 0) and (not r[1] or r[1] in name)]
|
||||
|
||||
def source_audit(ns):
|
||||
# Aggregate duplicates in SQLite instead of holding 400k+ records in RAM.
|
||||
tmp = tempfile.TemporaryDirectory(prefix='garmin-poi-qa-')
|
||||
db = sqlite3.connect(str(Path(tmp.name) / 'audit.sqlite'))
|
||||
db.executescript('''PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF;
|
||||
CREATE TABLE src(code TEXT, lat INTEGER, lon INTEGER, name TEXT, n INTEGER DEFAULT 0,
|
||||
seen INTEGER DEFAULT 0, PRIMARY KEY(code,lat,lon,name)) WITHOUT ROWID;
|
||||
CREATE TABLE extra(code TEXT,lat INTEGER,lon INTEGER,name TEXT,n INTEGER DEFAULT 0,
|
||||
PRIMARY KEY(code,lat,lon,name)) WITHOUT ROWID;''')
|
||||
rules = parse_rules(ns.rules)
|
||||
accepted_total, dropped_total, input_total, output_total = 0, 0, 0, 0
|
||||
with opencsv(ns.source) as f, db:
|
||||
for r in csv.DictReader(f):
|
||||
input_total += 1
|
||||
matching = matching_rules(rules, r['code'], r['name'])
|
||||
if any(x[2] == 'drop' for x in matching):
|
||||
dropped_total += 1
|
||||
continue
|
||||
accepted_total += 1
|
||||
db.execute('''INSERT INTO src(code,lat,lon,name,n) VALUES(?,?,?,?,1)
|
||||
ON CONFLICT(code,lat,lon,name) DO UPDATE SET n=n+1''',
|
||||
(r['code'].lower(), scaled(r['lat']), scaled(r['lon']), r['name']))
|
||||
# Updating primary-key rows in place avoids immense Python Counter memory.
|
||||
violations = Counter()
|
||||
tag_examples = []
|
||||
with openxml(ns.osm) as f, db:
|
||||
for _, el in ET.iterparse(f, events=('end',)):
|
||||
if el.tag != 'node':
|
||||
if el.tag in ('way', 'relation'):
|
||||
el.clear()
|
||||
continue
|
||||
tags = {t.get('k', ''): t.get('v', '') for t in el.findall('tag')}
|
||||
if tags.get('garmin:feature') != 'poi':
|
||||
el.clear()
|
||||
continue
|
||||
output_total += 1
|
||||
name = tags.get('name', '')
|
||||
code = tags.get('garmin:type', '').lower()
|
||||
key = (code, scaled(el.attrib['lat']), scaled(el.attrib['lon']), name)
|
||||
row = db.execute('SELECT n,seen FROM src WHERE code=? AND lat=? AND lon=? AND name=?', key).fetchone()
|
||||
if row and row[1] < row[0]:
|
||||
db.execute('UPDATE src SET seen=seen+1 WHERE code=? AND lat=? AND lon=? AND name=?', key)
|
||||
else:
|
||||
db.execute('''INSERT INTO extra(code,lat,lon,name,n) VALUES(?,?,?,?,1)
|
||||
ON CONFLICT(code,lat,lon,name) DO UPDATE SET n=n+1''', key)
|
||||
for _, contains, mode, mapped in matching_rules(rules, code, name):
|
||||
if mode == 'drop':
|
||||
violations['forbidden dropped POI emitted'] += 1
|
||||
continue
|
||||
for k, expected in mapped.items():
|
||||
if tags.get(k) != expected:
|
||||
violations[f'{code} {k}: expected {expected!r}, got {tags.get(k)!r}'] += 1
|
||||
if len(tag_examples) < 50:
|
||||
tag_examples.append({'code': code, 'name': name, 'lat': el.attrib['lat'],
|
||||
'lon': el.attrib['lon'], 'tag': k, 'expected': expected,
|
||||
'actual': tags.get(k)})
|
||||
el.clear()
|
||||
missing = db.execute('SELECT COALESCE(SUM(n-seen),0) FROM src').fetchone()[0]
|
||||
extra = db.execute('SELECT COALESCE(SUM(n),0) FROM extra').fetchone()[0]
|
||||
if ns.mismatches:
|
||||
with open(ns.mismatches, 'w', newline='', encoding='utf-8') as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(['status', 'code', 'lat', 'lon', 'name', 'count', 'osm_url'])
|
||||
for status, query in [('missing','SELECT code,lat,lon,name,n-seen FROM src WHERE n>seen'),
|
||||
('extra','SELECT code,lat,lon,name,n FROM extra')]:
|
||||
for code, lat, lon, name, n in db.execute(query):
|
||||
la, lo = lat / 1e7, lon / 1e7
|
||||
w.writerow([status, code, f'{la:.7f}', f'{lo:.7f}', name, n,
|
||||
f'https://www.openstreetmap.org/?mlat={la:.7f}&mlon={lo:.7f}#map=18/{la:.7f}/{lo:.7f}'])
|
||||
summary = dict(source_pois=input_total, expected_after_drops=accepted_total,
|
||||
declared_drops=dropped_total, osm_pois=output_total,
|
||||
missing=missing, extra=extra, rule_tag_violations=dict(violations), examples=tag_examples,
|
||||
status='PASS' if missing == extra == 0 and not violations else 'FAIL',
|
||||
limitation='Same IMG decoder is used for manifest and OSM: this validates export fidelity, not ground truth or OBF persistence.')
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
if ns.report:
|
||||
Path(ns.report).write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
db.close()
|
||||
tmp.cleanup()
|
||||
return 0 if summary['status'] == 'PASS' else 1
|
||||
|
||||
def audit_obf(ns):
|
||||
cmd = [ns.inspector, '-vpoi', '-vpoiobjects']
|
||||
if ns.bbox:
|
||||
w, s, e, n = map(float, ns.bbox.split(','))
|
||||
cmd.append(f'-bbox={w},{n},{e},{s}')
|
||||
cmd.append(ns.obf)
|
||||
with open(ns.report, 'w', encoding='utf-8') as f:
|
||||
run = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT)
|
||||
print(json.dumps({'command': cmd, 'exit_code': run.returncode, 'dump_file': ns.report,
|
||||
'note': 'Inspector output permits manual indexed-POI audit; it does NOT independently prove equality of IMG/OSM/OBF entities.'}, indent=2))
|
||||
return run.returncode
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
sub = p.add_subparsers(dest='command', required=True)
|
||||
c = sub.add_parser('compare', help='Full source manifest vs .osm or .osm.gz POI comparison')
|
||||
c.add_argument('--source', required=True, help='--poi-manifest output from garmin2osm')
|
||||
c.add_argument('--osm', required=True)
|
||||
c.add_argument('--rules', help='same overrides CSV used during conversion')
|
||||
c.add_argument('--report', help='JSON output path')
|
||||
c.add_argument('--mismatches', help='CSV missing/extra records with coordinates and map links')
|
||||
c.set_defaults(fn=source_audit)
|
||||
o = sub.add_parser('obf-inspect', help='Invoke existing OsmAndMapCreator inspector.sh (requires installing it)')
|
||||
o.add_argument('--inspector', required=True, help='/path/to/OsmAndMapCreator/inspector.sh')
|
||||
o.add_argument('--obf', required=True)
|
||||
o.add_argument('--bbox', help='west,south,east,north')
|
||||
o.add_argument('--report', required=True)
|
||||
o.set_defaults(fn=audit_obf)
|
||||
ns = p.parse_args()
|
||||
return ns.fn(ns)
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user