v07
This commit is contained in:
nq
Executable
+303
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-command BGMountains IMG -> reviewed OSM -> custom indexed OsmAnd OBF.
|
||||
|
||||
Only Python 3 stdlib needed for orchestration; CMake/C++/OsmAndMapCreator are external.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
UPSTREAM = {
|
||||
'poi': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/poi/poi_types.xml',
|
||||
'rendering': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/obf_creation/rendering_types.xml'
|
||||
}
|
||||
|
||||
def call(argv, cwd=None, log=None, env=None):
|
||||
argv = [str(s) for s in argv]
|
||||
print('+', ' '.join(shlex_quote(s) for s in argv), flush=True)
|
||||
if log:
|
||||
with Path(log).open('w', encoding='utf-8') as output:
|
||||
r = subprocess.run(argv, cwd=cwd, env=env, stdout=output, stderr=subprocess.STDOUT)
|
||||
if r.returncode:
|
||||
print('Command failed; last log lines:', file=sys.stderr)
|
||||
print('\n'.join(Path(log).read_text(errors='replace').splitlines()[-35:]), file=sys.stderr)
|
||||
raise SystemExit(r.returncode)
|
||||
else:
|
||||
subprocess.run(argv, cwd=cwd, env=env, check=True)
|
||||
|
||||
def shlex_quote(s):
|
||||
import shlex
|
||||
return shlex.quote(s)
|
||||
|
||||
def base_xml(provided, creator, key, cache, offline):
|
||||
if provided:
|
||||
p = Path(provided).expanduser().resolve()
|
||||
if not p.is_file(): raise SystemExit('Upstream XML not found: '+str(p))
|
||||
return p
|
||||
names = ['poi_types.xml'] if key == 'poi' else ['rendering_types.xml']
|
||||
if creator:
|
||||
for prefix in [creator, creator/'resources', creator/'config', creator/'poi', creator/'obf_creation']:
|
||||
for name in names:
|
||||
candidate = prefix/name
|
||||
if candidate.is_file(): return candidate.resolve()
|
||||
target = cache/names[0]
|
||||
if target.is_file(): return target
|
||||
if offline:
|
||||
raise SystemExit('Missing upstream '+names[0]+': pass --'+key+'-base FILE or allow download')
|
||||
target.parent.mkdir(parents=True,exist_ok=True)
|
||||
print('Fetching official OsmAnd '+names[0]+' (first run only) ...', flush=True)
|
||||
try:
|
||||
with urllib.request.urlopen(UPSTREAM[key], timeout=50) as u, target.open('wb') as out:
|
||||
shutil.copyfileobj(u,out)
|
||||
except Exception as ex:
|
||||
target.unlink(missing_ok=True)
|
||||
raise SystemExit('Cannot fetch official OsmAnd '+names[0]+': '+str(ex)+'\nProvide --'+key+'-base FILE manually.')
|
||||
return target
|
||||
|
||||
def catalog_rows(p):
|
||||
with Path(p).open('r',encoding='utf-8-sig',newline='') as f:
|
||||
rows=list(csv.DictReader(f))
|
||||
if not rows or not {'code','count','has_typ_icon'}.issubset(rows[0]):
|
||||
raise SystemExit('Invalid POI catalog (expected code,count,has_typ_icon): '+str(p))
|
||||
out=[]
|
||||
seen=set()
|
||||
for r in rows:
|
||||
code='0x'+format(int(r['code'],0),'x')
|
||||
if code in seen:raise SystemExit('Duplicate code in catalog: '+code)
|
||||
seen.add(code)
|
||||
out.append(dict(r,code=code,count=int(r['count'])))
|
||||
return sorted(out,key=lambda r:(-r['count'],int(r['code'],0)))
|
||||
|
||||
def write_xml(tree,path):
|
||||
ET.indent(tree,space=' ')
|
||||
tree.write(path,encoding='utf-8',xml_declaration=True)
|
||||
ET.parse(path)
|
||||
|
||||
def make_configs(poi_base,render_base,catalog,folder,threshold,frequent_zoom=15,rare_zoom=17):
|
||||
# Keep entire upstream definitions! Replacing with a standalone tiny XML is
|
||||
# precisely what used to make ordinary semantic POIs disappear.
|
||||
poi=ET.parse(poi_base); proot=poi.getroot()
|
||||
if proot.tag!='poi_types':raise SystemExit('Not an OsmAnd poi_types.xml: '+str(poi_base))
|
||||
if any(c.get('name')=='bgmountains_raw' for c in proot.findall('poi_category')):
|
||||
raise SystemExit('Input poi_types.xml already has a bgmountains_raw category')
|
||||
cat=ET.SubElement(proot,'poi_category',name='bgmountains_raw', default_tag='bgm_poi', no_edit='true')
|
||||
for r in catalog:
|
||||
code=r['code']
|
||||
ET.SubElement(cat,'poi_type',name='bgm_'+code.replace('0x',''),tag='bgm_poi',value='type_'+code,no_edit='true')
|
||||
pa=folder/'poi_types.bgmountains.xml'
|
||||
write_xml(poi,pa)
|
||||
|
||||
rendering=ET.parse(render_base); rroot=rendering.getroot()
|
||||
if rroot.tag != 'osmand_types':
|
||||
# OsmAnd resources use the tag 'rendering_types'; fail instead of dropping all built-in types.
|
||||
raise SystemExit('Unexpected OsmAnd rendering_types.xml root: '+rroot.tag)
|
||||
cats=[c for c in rroot.iter('category') if c.get('name')=='user_defined']
|
||||
target=cats[0] if cats else ET.SubElement(rroot,'category',name='user_defined')
|
||||
for r in catalog:
|
||||
code=r['code']
|
||||
ET.SubElement(target,'type',tag='bgm_poi',value='type_'+code,
|
||||
minzoom=str(frequent_zoom) if r['count']>=threshold else str(rare_zoom))
|
||||
rp=folder/'rendering_types.bgmountains.xml'
|
||||
write_xml(rendering,rp)
|
||||
summary={
|
||||
'catalog_types':len(catalog),
|
||||
'poi_count_in_catalog':sum(r.get('sample_count',r['count']) for r in catalog),
|
||||
'types_at_or_above_threshold':sum(r['count']>=threshold for r in catalog),
|
||||
'frequent_index_minzoom':frequent_zoom, 'rare_index_minzoom':rare_zoom,
|
||||
'items_at_or_above_threshold':sum(r.get('sample_count',r['count']) for r in catalog if r['count']>=threshold),
|
||||
'types_with_extracted_icon':sum(r['has_typ_icon']=='yes' for r in catalog),
|
||||
'poi_base_sha256':hashlib.sha256(Path(poi_base).read_bytes()).hexdigest(),
|
||||
'rendering_base_sha256':hashlib.sha256(Path(render_base).read_bytes()).hexdigest(),
|
||||
'note':'All Garmin hex POI types indexed; high-frequency raw types inserted into OBF map index from selected frequent zoom; low-frequency from selected rare zoom. Rendering exact Garmin icon PNGs requires installing compatible icons/style in OsmAnd, not merely compiling an OBF.'
|
||||
}
|
||||
(folder/'type-index-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
return pa,rp,summary
|
||||
|
||||
def make_style(icon_csv, folder, catalog):
|
||||
"""Optional addon for otherwise unclassified types using preinstalled OsmAnd icons.
|
||||
|
||||
This cannot install Garmin bitmap icons and must be enabled in OsmAnd.
|
||||
"""
|
||||
known={r['code'] for r in catalog}
|
||||
with Path(icon_csv).open(encoding='utf-8-sig',newline='') as f:
|
||||
rows=list(csv.DictReader(f))
|
||||
visible=[]
|
||||
for r in rows:
|
||||
code='0x'+format(int(r['code'],0),'x')
|
||||
if code not in known:continue # bbox builds are smaller than national config
|
||||
if r['show_in_custom_style'].strip().lower() not in ('yes','true','1'):continue
|
||||
name=r['osmand_builtin_icon'].strip()
|
||||
if not name or not name.replace('_','').replace('-','').isalnum():
|
||||
raise SystemExit('Invalid OsmAnd icon in '+str(icon_csv)+' for '+code)
|
||||
zoom=int(r['minzoom'])
|
||||
if not 1<=zoom<=21:raise SystemExit('Invalid icon zoom for '+code)
|
||||
visible.append((code,name,zoom))
|
||||
root=ET.Element('renderingStyle',name='BGMountains custom POIs',depends='default',
|
||||
defaultColor='#fff8ee',version='1')
|
||||
order=ET.SubElement(root,'order')
|
||||
points=ET.SubElement(root,'point')
|
||||
for code,name,zoom in visible:
|
||||
ET.SubElement(order,'case',tag='bgm_poi',value='type_'+code,minzoom=str(zoom),
|
||||
point='true',objectType='1',order='128')
|
||||
ET.SubElement(points,'case',tag='bgm_poi',value='type_'+code,minzoom=str(zoom),
|
||||
icon=name,iconOrder='10',iconVisibleSize='12')
|
||||
path=folder/'BGMountains-custom.render.xml'
|
||||
write_xml(ET.ElementTree(root),path)
|
||||
return path,len(visible)
|
||||
|
||||
def obtain_creator(cache, offline=False):
|
||||
"""Opt-in download from official OsmAnd nightly build (large file)."""
|
||||
import zipfile
|
||||
link='https://download.osmand.net/latest-night-build/OsmAndMapCreator-main.zip'
|
||||
target=cache/'OsmAndMapCreator-main.zip'
|
||||
extracted=cache/'unpacked'
|
||||
found=list(extracted.rglob('utilities.sh')) if extracted.is_dir() else []
|
||||
if found:return found[0].parent
|
||||
if offline:raise SystemExit('Cannot download MapCreator with --offline; pass --creator')
|
||||
cache.mkdir(parents=True,exist_ok=True)
|
||||
if not target.is_file():
|
||||
print('Downloading official OsmAndMapCreator (~150 MB) ...',flush=True)
|
||||
try:
|
||||
with urllib.request.urlopen(link,timeout=180) as src,target.open('wb') as dest:
|
||||
shutil.copyfileobj(src,dest,length=2**20)
|
||||
except Exception as ex:
|
||||
target.unlink(missing_ok=True)
|
||||
raise SystemExit('Cannot download OsmAndMapCreator: '+str(ex)+'\nDownload manually: '+link)
|
||||
print('Extracting MapCreator...',flush=True)
|
||||
extracted.mkdir(parents=True,exist_ok=True)
|
||||
with zipfile.ZipFile(target) as z:
|
||||
for m in z.infolist():
|
||||
rel=Path(m.filename)
|
||||
if rel.is_absolute() or '..' in rel.parts:
|
||||
raise SystemExit('Unsafe file in MapCreator archive: '+m.filename)
|
||||
z.extract(m,extracted)
|
||||
found=list(extracted.rglob('utilities.sh'))
|
||||
if not found:raise SystemExit('MapCreator ZIP extracted but utilities.sh not found')
|
||||
for name in ('utilities.sh','inspector.sh'):
|
||||
for f in extracted.rglob(name):f.chmod(f.stat().st_mode|0o111)
|
||||
return found[0].parent
|
||||
|
||||
def resolve_creator(path):
|
||||
name = path or os.environ.get('OSMAND_MAPCREATOR')
|
||||
if not name:return None
|
||||
creator=Path(name).expanduser().resolve()
|
||||
if creator.is_file() and creator.name=='utilities.sh':creator=creator.parent
|
||||
if not (creator/'utilities.sh').is_file():
|
||||
raise SystemExit('Cannot find utilities.sh under '+str(creator))
|
||||
return creator
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
inp=p.add_mutually_exclusive_group(required=True)
|
||||
inp.add_argument('--img',help='Original Garmin IMG; converter is compiled and executed')
|
||||
inp.add_argument('--osm',help='Already converted OSM XML; must contain bgm_poi=type_0x... tags (v0.7+)')
|
||||
p.add_argument('--creator',help='OsmAndMapCreator directory containing utilities.sh (or OSMAND_MAPCREATOR env)')
|
||||
p.add_argument('--download-creator',action='store_true',help='Download official OsmAndMapCreator ZIP automatically (~150 MB) when no --creator is set')
|
||||
p.add_argument('--out',default='build/bgmountains-output',help='Output folder')
|
||||
p.add_argument('--jobs',type=int,default=1,help='C++ IMG reader threads (opt-in)')
|
||||
p.add_argument('--bbox',help='Optional W,S,E,N bounds for small-area test')
|
||||
p.add_argument('--threshold',type=int,default=100,help='Frequent-type occurrence threshold')
|
||||
p.add_argument('--frequent-zoom',type=int,default=15,help='Store frequent raw map icons at zoom N and above')
|
||||
p.add_argument('--rare-zoom',type=int,default=17,help='Store rare raw map icons at zoom N and above')
|
||||
p.add_argument('--icons-config',default=str(ROOT/'config/osmand-icons.csv'),help='OsmAnd built-in icon choices for custom addon style')
|
||||
p.add_argument('--poi-overrides',default=str(ROOT/'config/poi-overrides-all-types.csv'))
|
||||
p.add_argument('--poi-name-fixes',default=str(ROOT/'config/poi-name-fixes.csv'))
|
||||
p.add_argument('--poi-additions',default=str(ROOT/'config/poi-additions.csv'))
|
||||
p.add_argument('--poi-base',help='Official upstream poi_types.xml; if omitted use cache/download')
|
||||
p.add_argument('--rendering-base',help='Official upstream rendering_types.xml; if omitted use cache/download')
|
||||
p.add_argument('--offline',action='store_true',help='Never download upstream XML')
|
||||
p.add_argument('--prepare-only',action='store_true',help='Do not run OsmAndMapCreator, but generate OSM and XMLs')
|
||||
p.add_argument('--skip-qa',action='store_true',help='Skip OSM comparison for faster repeat runs')
|
||||
p.add_argument('--java-opts',default='-Xms512M -Xmx8192M',help='JAVA_OPTS for OsmAndMapCreator')
|
||||
p.add_argument('--skip-cpp-build',action='store_true')
|
||||
p.add_argument('--catalog',help='Preexisting POI catalog when supplying --osm (otherwise use packaged nationwide catalog)')
|
||||
p.add_argument('--frequency-catalog',default=str(ROOT/'data/BGMountains-unique-poi-types.csv'),help='Reference nationwide counts for map index zoom decisions even in --bbox builds')
|
||||
args=p.parse_args()
|
||||
if args.jobs<1 or args.jobs>64 or args.threshold<1 or not 1<=args.frequent_zoom<=21 or not 1<=args.rare_zoom<=21: p.error('--jobs 1..64; threshold >=1; zoom 1..21')
|
||||
dest=Path(args.out).expanduser().resolve();dest.mkdir(parents=True,exist_ok=True)
|
||||
creator=resolve_creator(args.creator)
|
||||
if not creator and args.download_creator and not args.prepare_only:
|
||||
creator=obtain_creator(ROOT/'build'/'mapcreator-download',args.offline)
|
||||
if not creator and not args.prepare_only:p.error('Supply --creator /path/to/OsmAndMapCreator (or OSMAND_MAPCREATOR env) or use --prepare-only')
|
||||
if args.img:
|
||||
if not Path(args.img).is_file():p.error('IMG file missing: '+args.img)
|
||||
binary=ROOT/'build'/'garmin2osm'
|
||||
if not args.skip_cpp_build or not binary.is_file():
|
||||
call(['cmake','-S',ROOT,'-B',ROOT/'build','-DCMAKE_BUILD_TYPE=Release'])
|
||||
call(['cmake','--build',ROOT/'build','--config','Release','-j','2'])
|
||||
osm=dest/'BGMountains.osm'
|
||||
cmd=[binary,'--input',Path(args.img).resolve(),'--output',osm,'--poi-overrides',Path(args.poi_overrides).resolve(),
|
||||
'--poi-name-fixes',Path(args.poi_name_fixes).resolve(),'--poi-additions',Path(args.poi_additions).resolve(),
|
||||
'--poi-manifest',dest/'source-pois.csv','--poi-types-out',dest/'poi-types.csv','--jobs',args.jobs]
|
||||
if args.bbox:cmd+=['--bbox',args.bbox]
|
||||
call(cmd,log=dest/'converter.log')
|
||||
catalog=dest/'poi-types.csv'
|
||||
if not args.skip_qa:
|
||||
call([sys.executable,ROOT/'tools/validate_pois.py','compare','--source',dest/'source-pois.csv','--osm',osm,
|
||||
'--rules',Path(args.poi_overrides).resolve(),'--report',dest/'validation.json','--mismatches',dest/'mismatches.csv'],
|
||||
log=dest/'validator.log')
|
||||
else:
|
||||
osm=Path(args.osm).expanduser().resolve()
|
||||
if not osm.is_file():p.error('OSM file missing: '+str(osm))
|
||||
catalog=Path(args.catalog).expanduser().resolve() if args.catalog else ROOT/'data'/'BGMountains-unique-poi-types.csv'
|
||||
rows=catalog_rows(catalog)
|
||||
nationwide={r['code']:r['count'] for r in catalog_rows(args.frequency_catalog)} if args.frequency_catalog else {}
|
||||
for row in rows:
|
||||
row['sample_count']=row['count']
|
||||
row['count']=nationwide.get(row['code'],row['count'])
|
||||
# A separately supplied legacy OSM might lack our new classifier.
|
||||
# For --img, the v0.7 converter above just generated it by construction.
|
||||
if args.osm:
|
||||
import gzip
|
||||
opener=gzip.open if osm.suffix=='.gz' else open
|
||||
with opener(osm,'rt',encoding='utf-8',errors='replace') as f:
|
||||
found=any('k="bgm_poi"' in line for line in f)
|
||||
if not found:
|
||||
raise SystemExit('Existing OSM lacks bgm_poi; re-extract with v0.7 C++ converter.')
|
||||
folder=dest/'osmand-config';folder.mkdir(exist_ok=True)
|
||||
poi_base=base_xml(args.poi_base,creator,'poi',dest/'upstream',args.offline)
|
||||
rendering_base=base_xml(args.rendering_base,creator,'rendering',dest/'upstream',args.offline)
|
||||
poi_cfg,render_cfg,summary=make_configs(poi_base,rendering_base,rows,folder,args.threshold,args.frequent_zoom,args.rare_zoom)
|
||||
style,nicons=make_style(args.icons_config,folder,rows)
|
||||
print('Indexed POI types:',summary['catalog_types'],'frequent:',summary['types_at_or_above_threshold'],'custom style icons:',nicons,flush=True)
|
||||
if args.prepare_only:
|
||||
print('PREPARED (no OBF generated). Files at:',dest)
|
||||
return 0
|
||||
before={str(i.resolve()) for i in dest.glob('*.obf')}
|
||||
# Save Java logs, and preserve OsmAndMapCreator temp data inside user-selected output dir.
|
||||
env=os.environ.copy();env['JAVA_OPTS']=args.java_opts
|
||||
program=creator/'utilities.sh'
|
||||
cmd=[str(program),'generate-obf-no-address',str(osm),
|
||||
'--poi-types='+str(poi_cfg),'--rendering-types='+str(render_cfg)]
|
||||
call(cmd,cwd=dest,log=dest/'mapcreator.log',env=env)
|
||||
after=[i for i in dest.glob('*.obf') if str(i.resolve()) not in before]
|
||||
if not after:
|
||||
# Creator sometimes writes next to the .osm source, even with cwd changed.
|
||||
alt=list(osm.parent.glob('*.obf'))
|
||||
after=[i for i in alt if i.is_file()]
|
||||
if not after:raise SystemExit('MapCreator returned success but no .obf was found; see '+str(dest/'mapcreator.log'))
|
||||
built=max(after,key=lambda p:p.stat().st_mtime)
|
||||
print('OBF created:',built,'bytes:',built.stat().st_size)
|
||||
if (creator/'inspector.sh').is_file():
|
||||
call([str(creator/'inspector.sh'),'-vpoi',str(built)],cwd=dest,log=dest/'obf-poi-index.txt',env=env)
|
||||
(dest/'build-manifest.json').write_text(json.dumps({
|
||||
'created_at_utc':dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'obf':str(built),'obf_bytes':built.stat().st_size,
|
||||
'input_osm':str(osm),'poi_types':str(poi_cfg),'rendering_types':str(render_cfg),
|
||||
'custom_style':str(style), 'custom_style_icon_rules':nicons,
|
||||
'summary':summary
|
||||
},ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
return 0
|
||||
if __name__=='__main__':
|
||||
sys.exit(main())
|
||||
+13
-16
@@ -1,18 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compatibility shim. No manually supplied POI XML files are required anymore.
|
||||
# Preferred: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out
|
||||
set -euo pipefail
|
||||
# Usage: build_obf.sh /path/OsmAndMapCreator map.osm[.gz] [--poi-types full-poi_types.xml] [--rendering-types full-rendering_types.xml]
|
||||
if (( $# < 2 )); then echo "Usage: $0 CREATOR_DIR MAP.osm[.gz] [--poi-types FILE] [--rendering-types FILE]" >&2;exit 2;fi
|
||||
creator="$1"; input="$2";shift 2
|
||||
[[ -f "$creator/utilities.sh" && -f "$input" ]] || { echo 'Missing OsmAnd utilities.sh or map input' >&2;exit 1; }
|
||||
args=(generate-obf-no-address "$input")
|
||||
while (( $# > 0 )); do
|
||||
case "$1" in
|
||||
--poi-types|--rendering-types)
|
||||
(( $# >= 2 )) || { echo "Missing file after $1" >&2;exit 2; }
|
||||
[[ -f "$2" ]] || { echo "Config not found: $2" >&2;exit 2; }
|
||||
args+=("$1=$2");shift 2;;
|
||||
*) echo "Unknown option: $1" >&2;exit 2;;
|
||||
esac
|
||||
done
|
||||
export JAVA_OPTS="${JAVA_OPTS:--Xms256M -Xmx4096M}"
|
||||
"$creator/utilities.sh" "${args[@]}"
|
||||
if (( $# < 2 )); then
|
||||
echo "Usage: $0 CREATOR_DIR INPUT_v0.7.osm[.gz] [OUTPUT_DIR]" >&2
|
||||
echo "Recommended instead: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out" >&2
|
||||
exit 2
|
||||
fi
|
||||
creator="$1"; osm="$2"; out="${3:-$(dirname "$osm")/rebuilt-obf}"
|
||||
root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
source_dir="$(dirname "$osm")"
|
||||
args=(--osm "$osm" --creator "$creator" --out "$out")
|
||||
if [[ -f "$source_dir/poi-types.csv" ]]; then args+=(--catalog "$source_dir/poi-types.csv"); fi
|
||||
exec python3 "$root/scripts/build_bgmountains.py" "${args[@]}"
|
||||
|
||||
Reference in New Issue
Block a user