Author SHA1 Message Date
nq 3433135790 v096 2026-10-19 08:52:24 +00:00
13 changed files with 2480 additions and 2321 deletions
+7 -7
View File
@@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.16) cmake_minimum_required(VERSION 3.16)
project(garmin2osmand VERSION 0.8.0 LANGUAGES CXX) project(garmin2osmand VERSION 0.95.0 LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD_REQUIRED ON)
find_package(PNG REQUIRED) find_package(PNG REQUIRED)
@@ -15,18 +15,18 @@ else()
target_compile_options(garmin2osmand PRIVATE -Wall -Wextra -Wpedantic -O2) target_compile_options(garmin2osmand PRIVATE -Wall -Wextra -Wpedantic -O2)
endif() endif()
enable_testing() enable_testing()
add_test(NAME help COMMAND garmin2osmand --help) add_test(NAME help COMMAND garmin2osm --help)
find_package(Python3 COMPONENTS Interpreter QUIET) find_package(Python3 COMPONENTS Interpreter QUIET)
if(Python3_Interpreter_FOUND) if(Python3_Interpreter_FOUND)
add_test(NAME classic_and_gmp_end_to_end COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_smoke.py $<TARGET_FILE:garmin2osmand>) add_test(NAME legacy_native_vector_parser COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_vector.py $<TARGET_FILE:garmin2osm>)
add_test(NAME native_vector_export COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_vector.py $<TARGET_FILE:garmin2osm>) add_test(NAME v092_native_osm_translation COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v092.py)
add_test(NAME bgmountains_legend_audit COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_legend_audit.py)
endif() endif()
if(Python3_Interpreter_FOUND) if(Python3_Interpreter_FOUND)
add_test(NAME osmand_obf_pipeline_xml COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_obf_pipeline.py) add_test(NAME v093_memory_and_mapping_regressions COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v093.py)
endif() endif()
if(Python3_Interpreter_FOUND) if(Python3_Interpreter_FOUND)
add_test(NAME bgmountains_v08_display_mappings COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v08_config.py) add_test(NAME v095_cli_config_and_obf_orchestration COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v095.py)
add_test(NAME native_registry_audit COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_registry_audit.py)
endif() endif()
+145 -79
View File
@@ -1,103 +1,169 @@
# BGMountains Garmin IMG → OSM → OsmAnd OBF v0.8 # BGMountains Garmin IMG → native OsmAnd OBF — v0.95
**This release fixes the v0.7 icon-visibility blind spot and anonymous POIs.** **This is the integrated source release.** It replaces the v0.92/v0.93 Python
pipeline and v0.94 standalone CSVs. Do not combine scripts from older ZIPs:
those versions accept different flags and do not apply the 79 name-conditional
rules. This archive has **one POI config and one line config**.
- Every one of the **263 validated retained Garmin POI types** has at least one conventional OSM semantic/display tag in `config/poi-overrides-all-types.csv`, in addition to raw `bgm_poi`/`garmin:*` provenance tags. ## Quick start (Ubuntu/Debian)
- **87 codes** occur at least 100 times in the source catalog (409,154 source POIs). **46** use conventional feature categories recognized by stock OsmAnd in principle; **41** use an explicitly marked, synthetic `tourism=information;information=board` **local-OsmAnd display fallback**. Some additional lower-frequency codes also need fallback. This DOES NOT assert that the physical object is an information board.
- Missing original IMG names become stable Bulgarian descriptive names, e.g. `Вода - Каптаж - 1THF1SDNTKJ`, with `bgm:generated_name=yes`. Original nonempty names are preserved.
- `--jobs N` enables parallel extraction, **default one thread**.
- New `tools/audit_icon_coverage.py` verifies source OSM classification coverage, names and produces per-type coordinate report. It **does not** prove OBF index presence or visual display; use an OBF inspector and OsmAnd for that.
- Map-derived icon art and datasets remain governed by BGMountains' applicable copyright conditions. These synthetic POI tags are **not appropriate for uploading unchanged to the real OpenStreetMap database**.
## Quick build (Ubuntu/Debian) ```bash
```sh
sudo apt update sudo apt update
sudo apt install -y build-essential cmake libpng-dev libsqlite3-dev python3 openjdk-21-jre-headless unzip sudo apt install -y cmake g++ make libpng-dev libsqlite3-dev python3 openjdk-21-jre-headless unzip
unzip BGMountains-garmin2osm-v0.95-integrated.zip
cd BGMountains-garmin2osm-v0.95
```
Use **exactly** this from the release root (where `scripts/` and `config/` are):
```bash
nohup python3 scripts/build_bgmountains.py \
--img "$HOME/gmapsupp.img" \
--poi-config config/BGMountains-native-poi-mapping-v094.csv \
--line-config config/line-mapping.csv \
--out bg-mountains-v095-build \
--download-creator \
--jobs 8 \
--java-opts '-Xms512M -Xmx25G' > bg-v095.log 2>&1 &
```
The `--poi-config` and `--line-config` arguments are **implemented and tested**.
Other supported flags: `--creator /path/to/OsmAndMapCreator`, `--bbox W,S,E,N`,
`--prepare-only`, `--keep-raw`, `--template`, `--attach-routes`,
`--disk-headroom-gb N`, `--ignore-disk-check`, `--jobs N` (default 1).
**Check your actual disk space first.** Full-country MapCreator compilation has
previously required over 25 GiB of temporary disk space in addition to the
source OSM (6+ GiB) on your machine. The script uses a conservative estimate
`max(2 GiB, 5 × native-OSM-size + 1 GiB)`, unless `--disk-headroom-gb`
requires more. These figures are not upper bounds. Choose an output filesystem
with generous free space. `-Xmx25G` controls heap RAM, **not disk usage**.
## Safe regional test, no Java or extra download
```bash
python3 scripts/build_bgmountains.py \ python3 scripts/build_bgmountains.py \
--img /absolute/path/to/gmapsupp.img \ --img "$HOME/gmapsupp.img" \
--out ./BGMountains-out \ --poi-config config/BGMountains-native-poi-mapping-v094.csv \
--download-creator --line-config config/line-mapping.csv \
``` --bbox '25.05,42.43,25.16,42.54' \
--jobs 8 \
`--download-creator` is for the official OsmAndMapCreator, and the pipeline downloads the original official `poi_types.xml` and `rendering_types.xml` the first time as needed. Network access is required for these unless you already supply the files. **This runtime did not have MapCreator, so full OBF compilation was not executed here.** --out ./regional-test \
Already installed OsmAndMapCreator:
```sh
python3 scripts/build_bgmountains.py --img ./gmapsupp.img --out ./BGMountains-out \
--creator /path/to/OsmAndMapCreator
```
Opt-in parallelism: add `--jobs 4`. The output directory must be fresh: the converter refuses to overwrite an existing OSM or staging files.
Prepare and validate the OSM without invoking the Java OBF builder:
```sh
python3 scripts/build_bgmountains.py --img ./gmapsupp.img --out ./BGMountains-preview \
--prepare-only --prepare-only
``` ```
**IMPORTANT: You must REBUILD the OSM and OBF.** This update cannot modify an already compiled `.obf` or give v0.7 output its new tags in place. Inspect `regional-test/BGMountains-native.osm`, `semantic-audit.json`,
`route-report.json`, `extract.log`, `translate.log`, `audit.log`. The tool
must exit nonzero on a failed conversion subprocess. You can instead pass an
existing MapCreator with `--creator ...` (omit `--prepare-only`) for a small
real OBF build.
## What the configuration files mean ## Two editable configuration files
| File | Purpose | ### 1. `config/BGMountains-native-poi-mapping-v094.csv`
|---|---|
| `config/poi-overrides-all-types.csv` | **Active** 1,871-row config, including 263 retained types with revised tags. `mode=replace` replaces old hardcoded semantics, retaining provenance/name. All non-present legacy type rows remain for historical comparison. |
| `config/poi-overrides-all-types-v07-user-upload.csv` | Exact backup of the CSV you supplied. |
| `config/poi-labels-bg.csv` | New 263-row Bulgarian fallback name prefixes, editable. |
| `config/poi-icon-audit.csv` | All 263 types, count, original tags vs new tags, classification confidence and sample coordinates. |
| `config/osmand-icons.csv` | Optional **custom render style** icon names/visibility; never controls the default `tourism=information` fallback injected through the active overrides. |
| `data/BGMountains-frequent-poi-types-100plus-v08.csv` | All 87 frequent types, with reviewed tags, classification mode and coordinates. |
| `tools/audit_icon_coverage.py` | Independent POI OSM field audit; one row per present type. |
**How to edit a type:** find `code=0x6500` in active overrides. Edit the **tags** column, leaving a `key=value;key=value` list. Change `mode=replace` only if you understand how the C++ built-in classifications merge. Set a Bulgarian text prefix in `config/poi-labels-bg.csv`. Run again from scratch to rebuild OSM then OBF. 1,871 base hex-code entries plus 79 **name-specific** exceptions (1,950 data
rows). Columns include: `code`, `source_count`, `legend_bg`, `legend_en`,
`meaning_confidence`, `osm_tags`, `label_bg`, `priority`, `zoom_hint`, `action`,
`note`, examples, and optional `name_contains`. For conditional matching,
set `name_contains` in an additional row with the same `code`. The matcher uses
case-sensitive substring matching on the **original Garmin name**, with rules
applied in file order after the base rule. `replace` changes the converter's
semantic defaults; `append` retains previous tags; `raw` leaves unspecified
features unclassified until private-map fallback; `drop` discards the feature.
### Why 0x6500 is not `amenity=drinking_water` The Python pipeline automatically creates `cpp-poi-override.csv` and
`cpp-name-labels.csv` **inside the build output directory**. These are generated
intermediates, **not third and fourth user configuration files**. There is no
need to manually supply them or an icon pack to OsmAnd.
The original legend labels this a **каптаж** (water intake/catchment), not necessarily a drinking-water source. v0.8 records `man_made=water_intake` but adds an explicit *presentation-only* `tourism=information;information=board;bgm:synthetic_icon=yes` for common OsmAnd rendering. The "information board" is NOT a physical claim. The default Bulgarian name prefix is `Вода - Каптаж`. If you want `Вода - Резервоар`, just edit the label CSV; do not misrepresent a small intake as an artificial lake (`natural=water;water=reservoir`). Important native examples:
`0x6602` already has `tourism=viewpoint` and does not use a synthetic board marker. Its failure to appear in a particular existing OBF could be an old build, incomplete POI indexing, or display suppression—only inspecting that actual OBF can distinguish these. - `0x6414`: `amenity=drinking_water;man_made=water_tap` (legacy v18 dual
classification); named dry sources such as `(Сух.)` remove drinking-water
classification and get `drinking_water=no`.
- `0x6500`: `man_made=water_intake;amenity=water_point;drinking_water=unknown`
for **private-map index compatibility**, not evidence that its water is
potable, available or accessible. Named shaft (`Шах...`) exceptions remove
the compatibility alias.
- `0x6511`: `natural=spring` (potability not asserted).
- `0x6402`: `building=yes` by default, explicitly named `Заслон` or `Хижа`
may become shelter/hut (availability/access not verified).
- `0x661d`: agricultural structures remain agricultural unless specifically
identified otherwise.
- `0x6600`: keep numeric spot height `ele=*`, **no fake settlement**.
- Unknown POIs: `place=locality`, `Неуточнено - <stable hash>` if unnamed,
and `bgm:synthetic_fallback=place_locality`; this private-map fallback **does
not guarantee Nearby POI indexing**.
### Stable anonymous names `priority` and `zoom_hint` are documented classification hints only. They do
**not force stock OsmAnd to show an icon at zoom 12**. Likewise `garmin:type`
remains a provenance tag, not a display rule. Only stock OsmAnd classifications
and rendering apply to the resulting OBF.
For an unnamed POI, `name = Bulgarian_label + " - " + FNV1a_base36_11(mapset|type|quantized_lat|quantized_lon)`. The output is deterministic for an identical IMG and coordinates across threads and builds. A very small collision probability remains; the coordinates/mapset/type remain available. These generated names are for an offline map, **not actual surveyed names**. ### 2. `config/line-mapping.csv`
### Test against your three sample coordinates 50 Garmin line types (20 currently have `route_mode=relation`, 30 preserve
source geometry). Columns include Garmin `code`, legend, color,
`osm_way_tags`, `route_mode`, `osmc_symbol`, `network`, confidence and priority.
Selected source overlay segments are grouped into native
`type=route;route=hiking` relations with `osmc:symbol`. Garmin source color is
preserved as relation `colour=*`. The script does not create duplicate routable
paths from decoration lines. Opt-in `--attach-routes` matches overlays to real
existing path geometries, but can use significant memory; it is off by default.
**Existing physical path routing remains in the data.** Exact phone-side color,
icon and at-zoom visibility still depend on stock OsmAnd.
| Type | Coordinates | Expected tags | v0.8 generated name | ## New or unknown IMG maps: generate configuration templates
|---|---|---|---|
| `0x6602` | `42.0346141,24.6528697` | `tourism=viewpoint` | `Панорама - Гледка - 25IVP49H823` |
| `0x6500` | `42.0347643,24.6522689` | `man_made=water_intake;tourism=information;information=board` | `Вода - Каптаж - 1THF1SDNTKJ` |
| `0x6500` | `42.0340991,24.6520114` | Same | `Вода - Каптаж - ED03D455VTZ` |
These examples were directly extracted from the 422MiB IMG into a Plovdiv test OSM and compared to a source manifest. All 57 POIs in that test passed exact semantic-tag checks, with **zero missing and extra**. Serial vs `--jobs 4` POI identities/tags were equal. ```bash
python3 scripts/build_bgmountains.py --img /path/to/unknown.img \
## Review before copying `.obf` into OsmAnd --out ./template-newmap --bbox '23.0,42.0,24.0,43.0' --template
Pipeline artifacts in `--out`:
- `BGMountains.osm` — exact source XML for the OBF builder
- `source-pois.csv`, `poi-types.csv` — extraction manifest and type counts
- `validation.json`, `mismatches.csv` — original-source to OSM audit
- `poi-coverage.csv`, `poi-coverage.json` — export display mapping coverage
- `osmand-config/poi_types.bgmountains.xml`, `rendering_types.bgmountains.xml` — registrations added to **complete upstream XML**
- `osmand-config/BGMountains-custom.render.xml` — optional style; exact Garmin TYP art is NOT embedded into stock OsmAnd
- `*.obf` plus `mapcreator.log`, `obf-poi-index.txt` (only after a successful Java compilation)
The `tourism=information` fallback should give OsmAnd a native searchable POI category in an appropriate compatible build. **Neither an OSM tag nor an indexed OBF proves a symbol will be drawn at every zoom**: the map style, profile, POI filters, icon availability, density/collision and selected offline map still affect visibility. Test at zoom 17–20 with POI layers enabled. Avoid having an old overlapping BGMountains OBF selected at the same time.
To inspect your built file with OsmAndMapCreator:
```sh
/path/to/OsmAndMapCreator/inspector.sh -vpoi -vpoiobjects \
-bbox=24.64,42.05,24.665,42.02 ./BGMountains-out/*.obf \
> ./BGMountains-out/plovdiv-obf-inspector.txt
``` ```
Inspect output for the three named POIs. **There is no tested generic parser of OsmAndMapCreator's inspector text in this package**, and we have not verified a full-country OBF built from v0.8 in this runtime. If OSM has the POI but the inspector doesn't, focus on indexing or generated XML; if inspector has it but the screen doesn't, focus on style/POI display/filter/zoom. Produces `poi-mapping.TEMPLATE.csv` and `line-mapping.TEMPLATE.csv` with
observed types and sample coordinates. Fill them manually and run with
`--poi-config ./template-newmap/poi-mapping.TEMPLATE.csv` and
`--line-config ./template-newmap/line-mapping.TEMPLATE.csv`. **Do not assume
Garmin codes have the same meaning in unrelated maps.** If you need a complete
catalog for a new national map, generate its templates without `--bbox`, but
this can require significant time and intermediate disk space.
### Local-only licensing warning ## Validation and provenance
Do not upload the generated synthetic display tags to OpenStreetMap or assert they accurately describe physical information boards. The original IMG, derived icons and extracts can have third-party copyright restrictions. The source-only code path should be distributed separately from map-derived sample assets until rights are confirmed. ```bash
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
cmake --build build -j 4
ctest --test-dir build --output-on-failure
```
There is also `tools/audit_osmand_poi_mapping.py`, which takes the official
version-matched OsmAnd `poi_types.xml` and produces a registry compatibility
report, **not a claim of final icon visibility**:
```bash
python3 tools/audit_osmand_poi_mapping.py \
--mapping config/BGMountains-native-poi-mapping-v094.csv \
--poi-types /path/to/poi_types.xml \
--report native-poi-audit.csv --json native-poi-audit.json
```
Source: https://github.com/osmandapp/OsmAnd-resources/blob/master/poi/poi_types.xml
## Tested, and NOT tested
- C++ Release build, 6 CTest test groups, conditional POI CSV generation,
CLI options, and simulated MapCreator orchestration passed.
- Original 422 MiB IMG: regional `--jobs 8` and serial preparation succeeded
with identical 427 POI coordinate/name/tag records and 6 route relations.
- Separate region: `(Сух.)` water-tap conditional correctly produced
`man_made=water_tap;drinking_water=no`, and no `amenity=drinking_water`.
- **A real OsmAndMapCreator binary was not available here**; its host was
unreachable. The OBF orchestration unit test uses a **mock**, so native
nationwide OBF compilation and phone rendering are not verified. Do not
mistake that mock for real OBF validation.
License: the source bundle contains historically BGMountains-derived
classification information. Treat that data as private-map material under
applicable source licenses; the package is **not** authorization to republish
BGMountains geography or to import its features into the public OSM database.
File diff suppressed because it is too large Load Diff
+17
View File
@@ -0,0 +1,17 @@
{
"release": "v0.95",
"code_groups": 6,
"test_groups_passed": 6,
"real_img_source": "gmapsupp(2).img (original supplied, 441942016 bytes)",
"real_regional_bbox": "25.05,42.43,25.16,42.54",
"real_regional_pois": 427,
"real_regional_routes": 6,
"parallel_workers_tested": [
1,
8
],
"serial_parallel_poi_tags_names_equal": true,
"conditional_dry_water_result": "man_made=water_tap;drinking_water=no; amenity=drinking_water absent",
"java_real_obf_build": "NOT TESTED (official host unavailable)",
"obf_mock_orchestration": "PASSED; not a real OBF binary"
}
Binary file not shown.
+144 -326
View File
@@ -1,341 +1,159 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""One-command BGMountains IMG -> reviewed OSM -> custom indexed OsmAnd OBF. """BGMountains v0.95 native OsmAnd pipeline: only TWO editable mapping files.
Only Python 3 stdlib needed for orchestration; CMake/C++/OsmAndMapCreator are external. Never needs extracted Garmin icons or custom styles on Android.
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse,datetime,hashlib,json,os,pathlib,shutil,subprocess,sys,time,urllib.request,zipfile
import csv ROOT=pathlib.Path(__file__).resolve().parents[1]
import datetime as dt TRANSLATOR=ROOT/'scripts/native_translate.py'
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import urllib.request
import xml.etree.ElementTree as ET
ROOT = Path(__file__).resolve().parents[1] def run(*args,log=None,cwd=None,env=None):
UPSTREAM = { cmd=[str(x) for x in args];print('+',' '.join(cmd),flush=True)
'poi': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/poi/poi_types.xml',
'rendering': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/obf_creation/rendering_types.xml'
}
def call(argv, cwd=None, log=None, env=None):
argv = [str(s) for s in argv]
print('+', ' '.join(shlex_quote(s) for s in argv), flush=True)
if log: if log:
with Path(log).open('w', encoding='utf-8') as output: with open(log,'w',encoding='utf-8') as f:
r = subprocess.run(argv, cwd=cwd, env=env, stdout=output, stderr=subprocess.STDOUT) p=subprocess.run(cmd,stdout=f,stderr=subprocess.STDOUT,cwd=cwd,env=env)
if r.returncode: if p.returncode:
print('Command failed; last log lines:', file=sys.stderr) print(''.join(pathlib.Path(log).read_text(errors='replace').splitlines(True)[-40:]),file=sys.stderr)
print('\n'.join(Path(log).read_text(errors='replace').splitlines()[-35:]), file=sys.stderr) raise RuntimeError('Command failed, see '+str(log))
raise SystemExit(r.returncode) else:subprocess.run(cmd,check=True,cwd=cwd,env=env)
else:
subprocess.run(argv, cwd=cwd, env=env, check=True)
def shlex_quote(s): def creator_dir(x):
import shlex if not x:return None
return shlex.quote(s) p=pathlib.Path(x).expanduser().resolve();p=p.parent if p.name=='utilities.sh' else p
if not (p/'utilities.sh').exists():raise FileNotFoundError('MapCreator utilities.sh missing: '+str(p))
return p
def base_xml(provided, creator, key, cache, offline): def fetch_creator(target):
if provided: extracted=target/'unpacked';c=list(extracted.rglob('utilities.sh')) if extracted.exists() else []
p = Path(provided).expanduser().resolve() if c:return c[0].parent
if not p.is_file(): raise SystemExit('Upstream XML not found: '+str(p)) target.mkdir(parents=True,exist_ok=True)
return p url='https://download.osmand.net/latest-night-build/OsmAndMapCreator-main.zip'
names = ['poi_types.xml'] if key == 'poi' else ['rendering_types.xml'] zip_path=target/'OsmAndMapCreator-main.zip'
if creator: if not zip_path.exists():
for prefix in [creator, creator/'resources', creator/'config', creator/'poi', creator/'obf_creation']: print('Fetching official MapCreator:',url)
for name in names: with urllib.request.urlopen(url,timeout=120) as src,open(zip_path,'wb') as dst:shutil.copyfileobj(src,dst)
candidate = prefix/name with zipfile.ZipFile(zip_path) as z:
if candidate.is_file(): return candidate.resolve() for entry in z.infolist():
target = cache/names[0] parts=pathlib.PurePosixPath(entry.filename)
if target.is_file(): return target if parts.is_absolute() or '..' in parts.parts:raise ValueError('Unsafe creator ZIP entry')
if offline: z.extractall(extracted)
raise SystemExit('Missing upstream '+names[0]+': pass --'+key+'-base FILE or allow download') c=list(extracted.rglob('utilities.sh'))
target.parent.mkdir(parents=True,exist_ok=True) if not c:raise FileNotFoundError('Creator ZIP lacks utilities.sh')
print('Fetching official OsmAnd '+names[0]+' (first run only) ...', flush=True) for f in extracted.rglob('*.sh'):f.chmod(f.stat().st_mode|0o111)
try: return c[0].parent
with urllib.request.urlopen(UPSTREAM[key], timeout=50) as u, target.open('wb') as out:
shutil.copyfileobj(u,out)
except Exception as ex:
target.unlink(missing_ok=True)
raise SystemExit('Cannot fetch official OsmAnd '+names[0]+': '+str(ex)+'\nProvide --'+key+'-base FILE manually.')
return target
def catalog_rows(p): def disk_preflight(out, source_bytes, extra_gb=0):
with Path(p).open('r',encoding='utf-8-sig',newline='') as f: """Conservative warning/blocking check before disk-heavy MapCreator stage.
rows=list(csv.DictReader(f)) In experience 6 GiB OSM has required >25 GiB scratch. 5x + 1 GiB
if not rows or not {'code','count','has_typ_icon'}.issubset(rows[0]): is intentionally approximate, not a guarantee of completion.
raise SystemExit('Invalid POI catalog (expected code,count,has_typ_icon): '+str(p))
out=[]
seen=set()
for r in rows:
code='0x'+format(int(r['code'],0),'x')
if code in seen:raise SystemExit('Duplicate code in catalog: '+code)
seen.add(code)
out.append(dict(r,code=code,count=int(r['count'])))
return sorted(out,key=lambda r:(-r['count'],int(r['code'],0)))
def write_xml(tree,path):
ET.indent(tree,space=' ')
tree.write(path,encoding='utf-8',xml_declaration=True)
ET.parse(path)
def make_configs(poi_base,render_base,catalog,folder,threshold,frequent_zoom=15,rare_zoom=17,route_styles=()):
# Keep entire upstream definitions! Replacing with a standalone tiny XML is
# precisely what used to make ordinary semantic POIs disappear.
poi=ET.parse(poi_base); proot=poi.getroot()
if proot.tag!='poi_types':raise SystemExit('Not an OsmAnd poi_types.xml: '+str(poi_base))
if any(c.get('name')=='bgmountains_raw' for c in proot.findall('poi_category')):
raise SystemExit('Input poi_types.xml already has a bgmountains_raw category')
cat=ET.SubElement(proot,'poi_category',name='bgmountains_raw', default_tag='bgm_poi', no_edit='true')
for r in catalog:
code=r['code']
ET.SubElement(cat,'poi_type',name='bgm_'+code.replace('0x',''),tag='bgm_poi',value='type_'+code,no_edit='true')
pa=folder/'poi_types.bgmountains.xml'
write_xml(poi,pa)
rendering=ET.parse(render_base); rroot=rendering.getroot()
if rroot.tag != 'osmand_types':
# OsmAnd resources use the tag 'rendering_types'; fail instead of dropping all built-in types.
raise SystemExit('Unexpected OsmAnd rendering_types.xml root: '+rroot.tag)
cats=[c for c in rroot.iter('category') if c.get('name')=='user_defined']
target=cats[0] if cats else ET.SubElement(rroot,'category',name='user_defined')
for r in catalog:
code=r['code']
ET.SubElement(target,'type',tag='bgm_poi',value='type_'+code,
minzoom=str(frequent_zoom) if r['count']>=threshold else str(rare_zoom))
# Route overlays have their OWN map-index main types, not only an ignored metadata tag.
# Do not map decorative overlays to 'highway=path': that would create bogus routing.
for route in route_styles:
if route.get('show','yes').lower() not in ('yes','1','true'): continue
ET.SubElement(target,'type',tag='bgm_route',value='type_'+route['code'],minzoom=route['display_minzoom'])
rp=folder/'rendering_types.bgmountains.xml'
write_xml(rendering,rp)
summary={
'catalog_types':len(catalog), 'route_overlay_types':len(route_styles),
'poi_count_in_catalog':sum(r.get('sample_count',r['count']) for r in catalog),
'types_at_or_above_threshold':sum(r['count']>=threshold for r in catalog),
'frequent_index_minzoom':frequent_zoom, 'rare_index_minzoom':rare_zoom,
'items_at_or_above_threshold':sum(r.get('sample_count',r['count']) for r in catalog if r['count']>=threshold),
'types_with_extracted_icon':sum(r['has_typ_icon']=='yes' for r in catalog),
'poi_base_sha256':hashlib.sha256(Path(poi_base).read_bytes()).hexdigest(),
'rendering_base_sha256':hashlib.sha256(Path(render_base).read_bytes()).hexdigest(),
'note':'Local-only compat data; synthetic information icons are not truth about an informational board. All Garmin hex POI types indexed; high-frequency raw types inserted into OBF map index from selected frequent zoom; low-frequency from selected rare zoom. Rendering exact Garmin icon PNGs requires installing compatible icons/style in OsmAnd, not merely compiling an OBF.'
}
(folder/'type-index-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
return pa,rp,summary
def read_route_styles(path):
with Path(path).open(encoding='utf-8-sig',newline='') as f: rows=list(csv.DictReader(f))
if not rows or not {'code','color','show','display_minzoom','width'}.issubset(rows[0]):
raise SystemExit('Invalid trail-styles.csv: '+str(path))
seen=set()
for r in rows:
code='0x'+format(int(r['code'],0),'x')
if code in seen: raise SystemExit('Duplicate route style type: '+code)
seen.add(code);r['code']=code
if not (len(r['color'])==7 and r['color'][0]=='#' and all(c in '0123456789abcdefABCDEF' for c in r['color'][1:])):
raise SystemExit('Invalid CSS color in route type '+code)
if not (0.2<=float(r['width'])<=15 and 1<=int(r['display_minzoom'])<=21):
raise SystemExit('Invalid route width/zoom '+code)
return rows
def make_style(icon_csv,folder,catalog,route_styles=()):
"""Selectable OsmAnd map style that maps BGM indexes to built-in icons.
The .obf itself only embeds geometry/tags. Android requires installing and
selecting this renderer; stock app ignores bgm_poi and bgm_route styling.
""" """
known={r['code'] for r in catalog} gib=1024**3
with Path(icon_csv).open(encoding='utf-8-sig',newline='') as f: free=shutil.disk_usage(out).free
rows=list(csv.DictReader(f)) required=max(int(2*gib),int(source_bytes*5 + gib),int(extra_gb*gib))
visible=[] if free<required:
for r in rows: raise RuntimeError(f'Insufficient free space for OBF build: free={free/gib:.2f} GiB; '
code='0x'+format(int(r['code'],0),'x') f'estimated minimum={required/gib:.2f} GiB. '
if code not in known or r['show_in_custom_style'].strip().lower() not in ('yes','true','1'): 'Use --prepare-only or set --ignore-disk-check if scratch resides on another filesystem.')
continue print(f'Disk preflight OK: free={free/gib:.1f} GiB, estimated minimum={required/gib:.1f} GiB',flush=True)
name=r['osmand_builtin_icon'].strip()
if not name or not name.replace('_','').replace('-','').isalnum():
raise SystemExit('Invalid OsmAnd icon: '+repr(name)+' code='+code)
zoom=int(r['minzoom'])
if not 1<=zoom<=21: raise SystemExit('Bad icon zoom: '+code)
visible.append((code,name,zoom))
root=ET.Element('renderingStyle',name='BGMountains outdoor v0.9',
depends='default',defaultColor='#fff8ee',version='1')
order=ET.SubElement(root,'order');points=ET.SubElement(root,'point')
lines=ET.SubElement(root,'line')
for code,name,zoom in visible:
ET.SubElement(order,'case',tag='bgm_poi',value='type_'+code,
minzoom=str(zoom),point='true',objectType='1',order='120')
ET.SubElement(points,'case',tag='bgm_poi',value='type_'+code,minzoom=str(zoom),
icon=name,iconOrder='11',iconVisibleSize='16',intersectionSizeFactor='0.7')
route_count=0
for route in route_styles:
if route.get('show','yes').lower() not in ('yes','true','1'):continue
route_count+=1
code=route['code'];zoom=route['display_minzoom']
ET.SubElement(order,'case',tag='bgm_route',value='type_'+code,
minzoom=zoom,objectType='2',order='125')
# Color and width faithfully recover TYP base line color, but NOT its bitmap
# segment pattern. A precise pattern renderer requires a custom app build.
ET.SubElement(lines,'case',tag='bgm_route',value='type_'+code,
minzoom=zoom,color=route['color'],strokeWidth=route['width'],cap='ROUND')
path=folder/'BGMountains-outdoor.render.xml'
write_xml(ET.ElementTree(root),path)
return path,len(visible),route_count
def obtain_creator(cache, offline=False):
"""Opt-in download from official OsmAnd nightly build (large file)."""
import zipfile
link='https://download.osmand.net/latest-night-build/OsmAndMapCreator-main.zip'
target=cache/'OsmAndMapCreator-main.zip'
extracted=cache/'unpacked'
found=list(extracted.rglob('utilities.sh')) if extracted.is_dir() else []
if found:return found[0].parent
if offline:raise SystemExit('Cannot download MapCreator with --offline; pass --creator')
cache.mkdir(parents=True,exist_ok=True)
if not target.is_file():
print('Downloading official OsmAndMapCreator (~150 MB) ...',flush=True)
try:
with urllib.request.urlopen(link,timeout=180) as src,target.open('wb') as dest:
shutil.copyfileobj(src,dest,length=2**20)
except Exception as ex:
target.unlink(missing_ok=True)
raise SystemExit('Cannot download OsmAndMapCreator: '+str(ex)+'\nDownload manually: '+link)
print('Extracting MapCreator...',flush=True)
extracted.mkdir(parents=True,exist_ok=True)
with zipfile.ZipFile(target) as z:
for m in z.infolist():
rel=Path(m.filename)
if rel.is_absolute() or '..' in rel.parts:
raise SystemExit('Unsafe file in MapCreator archive: '+m.filename)
z.extract(m,extracted)
found=list(extracted.rglob('utilities.sh'))
if not found:raise SystemExit('MapCreator ZIP extracted but utilities.sh not found')
for name in ('utilities.sh','inspector.sh'):
for f in extracted.rglob(name):f.chmod(f.stat().st_mode|0o111)
return found[0].parent
def resolve_creator(path):
name = path or os.environ.get('OSMAND_MAPCREATOR')
if not name:return None
creator=Path(name).expanduser().resolve()
if creator.is_file() and creator.name=='utilities.sh':creator=creator.parent
if not (creator/'utilities.sh').is_file():
raise SystemExit('Cannot find utilities.sh under '+str(creator))
return creator
def main(): def main():
p=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.ArgumentDefaultsHelpFormatter) p=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
inp=p.add_mutually_exclusive_group(required=True) p.add_argument('--img',help='Original Garmin IMG')
inp.add_argument('--img',help='Original Garmin IMG; converter is compiled and executed') p.add_argument('--osm',help='Already generated original Garmin-derived raw OSM')
inp.add_argument('--osm',help='Already converted OSM XML; must contain bgm_poi=type_0x... tags (v0.7+)') p.add_argument('--out',default='build/bgmountains-v095')
p.add_argument('--creator',help='OsmAndMapCreator directory containing utilities.sh (or OSMAND_MAPCREATOR env)') p.add_argument('--poi-config',default=str(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'))
p.add_argument('--download-creator',action='store_true',help='Download official OsmAndMapCreator ZIP automatically (~150 MB) when no --creator is set') p.add_argument('--line-config',default=str(ROOT/'config/line-mapping.csv'))
p.add_argument('--out',default='build/bgmountains-output',help='Output folder') p.add_argument('--jobs',type=int,default=1,help='IMG workers: default 1; explicitly set N to parallelize')
p.add_argument('--jobs',type=int,default=1,help='C++ IMG reader threads (opt-in)') p.add_argument('--bbox',help='W,S,E,N region for smoke tests')
p.add_argument('--bbox',help='Optional W,S,E,N bounds for small-area test') p.add_argument('--template',action='store_true',help='Generate blank new-map mapping templates using this IMG (not BGMountains defaults)')
p.add_argument('--threshold',type=int,default=100,help='Frequent-type occurrence threshold') p.add_argument('--attach-routes',action='store_true',help='Opt in to high-memory matching of trail overlays to physical paths (not recommended on full map)')
p.add_argument('--frequent-zoom',type=int,default=15,help='Store frequent raw map icons at zoom N and above') p.add_argument('--no-route-attachment',action='store_true',help='Deprecated: attachment is disabled by default')
p.add_argument('--rare-zoom',type=int,default=17,help='Store rare raw map icons at zoom N and above') p.add_argument('--prepare-only',action='store_true',help='Extract/translate/validate, do not compile OBF')
p.add_argument('--icons-config',default=str(ROOT/'config/osmand-icons.csv'),help='OsmAnd icon mappings') p.add_argument('--creator',help='Existing OsmAndMapCreator directory containing utilities.sh')
p.add_argument('--trail-styles',default=str(ROOT/'config/trail-styles.csv'),help='Garmin TYP coloured route overlay styles') p.add_argument('--download-creator',action='store_true',help='Obtain official OsmAndMapCreator from download.osmand.net')
p.add_argument('--poi-overrides',default=str(ROOT/'config/poi-overrides-all-types.csv')) p.add_argument('--java-opts',default='-Xms512M -Xmx8192M')
p.add_argument('--poi-name-fixes',default=str(ROOT/'config/poi-name-fixes.csv')) p.add_argument('--keep-raw',action='store_true',help='Keep pre-translation OSM for debugging')
p.add_argument('--poi-labels',default=str(ROOT/'config/poi-labels-bg.csv'),help='Bulgarian prefixes for stable generated names on nameless Garmin POIs') p.add_argument('--ignore-disk-check',action='store_true',help='Override conservative OBF scratch space safety check')
p.add_argument('--poi-additions',default=str(ROOT/'config/poi-additions.csv')) p.add_argument('--disk-headroom-gb',type=float,default=0,help='Minimum free GiB required before Java OBF build (0=auto estimate)')
p.add_argument('--poi-base',help='Official upstream poi_types.xml; if omitted use cache/download') a=p.parse_args()
p.add_argument('--rendering-base',help='Official upstream rendering_types.xml; if omitted use cache/download') if bool(a.img)==bool(a.osm):p.error('Provide exactly one of --img or --osm')
p.add_argument('--offline',action='store_true',help='Never download upstream XML') if a.jobs<1 or a.jobs>64:p.error('--jobs must be 1..64')
p.add_argument('--prepare-only',action='store_true',help='Do not run OsmAndMapCreator, but generate OSM and XMLs') if not a.template:
p.add_argument('--skip-qa',action='store_true',help='Skip OSM comparison for faster repeat runs') for typ,pth in (('POI',a.poi_config),('line',a.line_config)):
p.add_argument('--java-opts',default='-Xms512M -Xmx8192M',help='JAVA_OPTS for OsmAndMapCreator') if not pathlib.Path(pth).is_file():p.error(f'{typ} mapping file not found: {pth}')
p.add_argument('--skip-cpp-build',action='store_true') out=pathlib.Path(a.out).resolve();out.mkdir(parents=True,exist_ok=True)
p.add_argument('--catalog',help='Preexisting POI catalog when supplying --osm (otherwise use packaged nationwide catalog)') src=pathlib.Path(a.osm).resolve() if a.osm else None
p.add_argument('--frequency-catalog',default=str(ROOT/'data/BGMountains-unique-poi-types.csv'),help='Reference nationwide counts for map index zoom decisions even in --bbox builds') poi=pathlib.Path(a.poi_config).resolve();line=pathlib.Path(a.line_config).resolve()
args=p.parse_args() # Resolve MapCreator before running an expensive full-country extraction.
if args.jobs<1 or args.jobs>64 or args.threshold<1 or not 1<=args.frequent_zoom<=21 or not 1<=args.rare_zoom<=21: p.error('--jobs 1..64; threshold >=1; zoom 1..21') # Missing download/network credentials must fail before consuming gigabytes.
dest=Path(args.out).expanduser().resolve();dest.mkdir(parents=True,exist_ok=True) creator=None
creator=resolve_creator(args.creator) if not a.prepare_only and not a.template:
if not creator and args.download_creator and not args.prepare_only: creator=creator_dir(a.creator or os.environ.get('OSMAND_MAPCREATOR'))
creator=obtain_creator(ROOT/'build'/'mapcreator-download',args.offline) if not creator and a.download_creator:
if not creator and not args.prepare_only:p.error('Supply --creator /path/to/OsmAndMapCreator (or OSMAND_MAPCREATOR env) or use --prepare-only') creator=fetch_creator(ROOT/'build'/'mapcreator')
if args.img: if not creator:
if not Path(args.img).is_file():p.error('IMG file missing: '+args.img) p.error('OBF requires --creator DIR or --download-creator; for testing use --prepare-only.')
binary=ROOT/'build'/'garmin2osm' bin_path=ROOT/'build'/'garmin2osm'
if not args.skip_cpp_build or not binary.is_file(): if a.img:
call(['cmake','-S',ROOT,'-B',ROOT/'build','-DCMAKE_BUILD_TYPE=Release']) run('cmake','-S',ROOT,'-B',ROOT/'build','-DCMAKE_BUILD_TYPE=Release')
call(['cmake','--build',ROOT/'build','--config','Release','-j','2']) run('cmake','--build',ROOT/'build','-j','2','--target','garmin2osm')
osm=dest/'BGMountains.osm' src=out/'IMG-raw.osm'
cmd=[binary,'--input',Path(args.img).resolve(),'--output',osm,'--poi-overrides',Path(args.poi_overrides).resolve(), cmd=[str(bin_path),'--input',str(pathlib.Path(a.img).resolve()),'--output',str(src),
'--poi-name-fixes',Path(args.poi_name_fixes).resolve(),'--poi-labels',Path(args.poi_labels).resolve(),'--poi-additions',Path(args.poi_additions).resolve(), '--poi-types-out',str(out/'discovered-pois.csv'),'--poi-manifest',str(out/'source-pois.csv'),
'--poi-manifest',dest/'source-pois.csv','--poi-types-out',dest/'poi-types.csv','--jobs',args.jobs] '--jobs',str(a.jobs)]
if args.bbox:cmd+=['--bbox',args.bbox] if a.bbox:cmd+=['--bbox',a.bbox]
call(cmd,log=dest/'converter.log') if not a.template:
catalog=dest/'poi-types.csv' run(sys.executable,TRANSLATOR,'prepare','--poi',poi,'--lines',line,'--out',out)
if not args.skip_qa: cmd+=['--poi-overrides',str(out/'cpp-poi-override.csv'),'--poi-labels',str(out/'cpp-name-labels.csv')]
call([sys.executable,ROOT/'tools/validate_pois.py','compare','--source',dest/'source-pois.csv','--osm',osm, run(*cmd,log=out/'extract.log')
'--rules',Path(args.poi_overrides).resolve(),'--report',dest/'validation.json','--mismatches',dest/'mismatches.csv'], if a.template:
log=dest/'validator.log') if not a.img:p.error('--template requires --img')
else: run(sys.executable,TRANSLATOR,'template','--poi',out/'poi-mapping.TEMPLATE.csv',
osm=Path(args.osm).expanduser().resolve() '--lines',out/'line-mapping.TEMPLATE.csv','--catalog',out/'discovered-pois.csv','--osm',src)
if not osm.is_file():p.error('OSM file missing: '+str(osm)) print('Templates created. Review both files before using them for native OSM conversion.')
catalog=Path(args.catalog).expanduser().resolve() if args.catalog else ROOT/'data'/'BGMountains-unique-poi-types.csv'
call([sys.executable,ROOT/'tools/audit_icon_coverage.py', '--osm',osm, '--catalog',catalog,
'--csv',dest/'poi-coverage.csv','--json',dest/'poi-coverage.json'],
log=dest/'poi-coverage.log')
rows=catalog_rows(catalog)
nationwide={r['code']:r['count'] for r in catalog_rows(args.frequency_catalog)} if args.frequency_catalog else {}
for row in rows:
row['sample_count']=row['count']
row['count']=nationwide.get(row['code'],row['count'])
# A separately supplied legacy OSM might lack our new classifier.
# For --img, the v0.7 converter above just generated it by construction.
if args.osm:
import gzip
opener=gzip.open if osm.suffix=='.gz' else open
with opener(osm,'rt',encoding='utf-8',errors='replace') as f:
found=any('k="bgm_poi"' in line for line in f)
if not found:
raise SystemExit('Existing OSM lacks bgm_poi; re-extract with v0.7 C++ converter.')
folder=dest/'osmand-config';folder.mkdir(exist_ok=True)
poi_base=base_xml(args.poi_base,creator,'poi',dest/'upstream',args.offline)
rendering_base=base_xml(args.rendering_base,creator,'rendering',dest/'upstream',args.offline)
route_styles=read_route_styles(args.trail_styles)
poi_cfg,render_cfg,summary=make_configs(poi_base,rendering_base,rows,folder,args.threshold,args.frequent_zoom,args.rare_zoom,route_styles)
style,nicons,nlines=make_style(args.icons_config,folder,rows,route_styles)
print('Indexed POI types:',summary['catalog_types'],'frequent:',summary['types_at_or_above_threshold'],'custom icons:',nicons,'colored route types:',nlines,flush=True)
if args.prepare_only:
print('PREPARED (no OBF generated). Files at:',dest)
return 0 return 0
before={str(i.resolve()) for i in dest.glob('*.obf')} dst=out/'BGMountains-native.osm'
# Save Java logs, and preserve OsmAndMapCreator temp data inside user-selected output dir. run(sys.executable,TRANSLATOR,'transform','--poi',poi,'--lines',line,'--osm',src,
env=os.environ.copy();env['JAVA_OPTS']=args.java_opts '--out',dst,'--report',out/'route-report.json',log=out/'translate.log')
program=creator/'utilities.sh' if a.attach_routes and not a.no_route_attachment:
cmd=[str(program),'generate-obf-no-address',str(osm), attached=out/'BGMountains-native-attached.osm'
'--poi-types='+str(poi_cfg),'--rendering-types='+str(render_cfg)] run(sys.executable,ROOT/'scripts/attach_native_routes.py','--osm',dst,'--out',attached,
call(cmd,cwd=dest,log=dest/'mapcreator.log',env=env) '--lines',line,'--report',out/'route-attachment.json',log=out/'route-attachment.log')
after=[i for i in dest.glob('*.obf') if str(i.resolve()) not in before] dst.unlink();attached.rename(dst)
if not after: run(sys.executable,TRANSLATOR,'audit','--poi',poi,'--lines',line,'--osm',dst,
# Creator sometimes writes next to the .osm source, even with cwd changed. '--out',out/'semantic-audit.json',log=out/'audit.log')
alt=list(osm.parent.glob('*.obf')) # Fail closed if the translated file is not well-formed XML, including relation refs.
after=[i for i in alt if i.is_file()] import xml.etree.ElementTree as ET
if not after:raise SystemExit('MapCreator returned success but no .obf was found; see '+str(dest/'mapcreator.log')) from native_translate import top_level_elements
built=max(after,key=lambda p:p.stat().st_mtime) n=0
print('OBF created:',built,'bytes:',built.stat().st_size) for e in top_level_elements(dst):
if (creator/'inspector.sh').is_file(): if e.tag=='relation':n+=1
call([str(creator/'inspector.sh'),'-vpoi',str(built)],cwd=dest,log=dest/'obf-poi-index.txt',env=env) if not a.keep_raw and a.img:src.unlink(missing_ok=True)
(dest/'build-manifest.json').write_text(json.dumps({ if a.prepare_only:
'created_at_utc':dt.datetime.now(dt.timezone.utc).isoformat(), print('PREPARED OSM:',dst,'route relations:',n,'No OBF compiled.')
'obf':str(built),'obf_bytes':built.stat().st_size, return 0
'input_osm':str(osm),'poi_types':str(poi_cfg),'rendering_types':str(render_cfg), if not a.ignore_disk_check:disk_preflight(out,dst.stat().st_size,a.disk_headroom_gb)
'custom_style':str(style), 'custom_style_icon_rules':nicons, 'custom_route_rules':nlines, env=os.environ.copy();env['JAVA_OPTS']=a.java_opts
'summary':summary # utilities.sh versions differ in whether JAVA_OPTS is honored; this flag is
},ensure_ascii=False,indent=2)+'\n',encoding='utf-8') # explicitly recognized by the JVM. It also reaches spawned Java processes.
env['JAVA_TOOL_OPTIONS']=a.java_opts
# Native types are built into OsmAnd: NO --poi-types/--rendering-types override,
# NO custom .render.xml and NO extracted Garmin icons.
started=time.time()
run(creator/'utilities.sh','generate-obf-no-address',dst,log=out/'mapcreator.log',cwd=out,env=env)
obfs=[f for f in out.glob('*.obf') if f.stat().st_mtime >= started-2 and f.stat().st_size>0]
if not obfs:raise RuntimeError('Creator returned no OBF in '+str(out))
obf=max(obfs,key=lambda x:x.stat().st_mtime)
if (creator/'inspector.sh').exists():
run(creator/'inspector.sh','-vpoi',obf,log=out/'obf-inspection.txt',cwd=out,env=env)
(out/'build-manifest.json').write_text(json.dumps({
'version':'v0.95','input':a.img or a.osm,'poi_config':str(poi),'line_config':str(line),
'poi_config_sha256':hashlib.sha256(poi.read_bytes()).hexdigest(),
'line_config_sha256':hashlib.sha256(line.read_bytes()).hexdigest(),
'output_obf':str(obf),'obf_bytes':obf.stat().st_size,
'compiled_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'notice':'Native stock rendering. POI display zoom controlled by OsmAnd; no claim that all tags get icons.'},ensure_ascii=False,indent=2)+'\n')
print('BUILT OBF:',obf)
return 0 return 0
if __name__=='__main__': if __name__=='__main__':sys.exit(main())
sys.exit(main())
+2 -13
View File
@@ -1,15 +1,4 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Compatibility shim. No manually supplied POI XML files are required anymore.
# Preferred: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out
set -euo pipefail set -euo pipefail
if (( $# < 2 )); then cd "$(dirname "$0")/.."
echo "Usage: $0 CREATOR_DIR INPUT_v0.7.osm[.gz] [OUTPUT_DIR]" >&2 exec python3 scripts/build_bgmountains.py "$@"
echo "Recommended instead: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out" >&2
exit 2
fi
creator="$1"; osm="$2"; out="${3:-$(dirname "$osm")/rebuilt-obf}"
root="$(cd "$(dirname "$0")/.." && pwd)"
source_dir="$(dirname "$osm")"
args=(--osm "$osm" --creator "$creator" --out "$out")
if [[ -f "$source_dir/poi-types.csv" ]]; then args+=(--catalog "$source_dir/poi-types.csv"); fi
exec python3 "$root/scripts/build_bgmountains.py" "${args[@]}"
+45 -8
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""v0.93: streaming IMG/OSM native-OsmAnd translation and honest QA. """v0.95: streaming IMG/OSM native-OsmAnd translation and honest QA.
Only user-editable controls: config/poi-mapping.csv and config/line-mapping.csv. Only user-editable controls: config/BGMountains-native-poi-mapping-v094.csv and config/line-mapping.csv.
""" """
import argparse,base64,collections,csv,hashlib,json,pathlib,re,sqlite3,sys,xml.etree.ElementTree as ET import argparse,base64,collections,csv,hashlib,json,pathlib,re,sqlite3,sys,xml.etree.ElementTree as ET
from xml.sax.saxutils import escape,quoteattr from xml.sax.saxutils import escape,quoteattr
@@ -34,14 +34,42 @@ def read_csv(p):
lines=(line for line in f if not line.startswith('#')) lines=(line for line in f if not line.startswith('#'))
yield from csv.DictReader(lines) yield from csv.DictReader(lines)
def code(value):return hex(int(value,0)).lower() def code(value):return hex(int(value,0)).lower()
def read_poi_configuration(path):
"""Read one canonical POI config: unique base rule per code and optional
ordered name_contains rules (a v0.94 feature previously lost in Python).
The v0.93/0.94 15-column files remain backward-compatible.
"""
bases={}; conditional=[]
for line_no,row in enumerate(read_csv(path),start=2):
try:c=code(row['code'])
except Exception as ex:raise ValueError(f'Invalid POI code at {path}:{line_no}: {row}') from ex
row=dict(row);row['code']=c
act=(row.get('action') or 'replace').strip().lower()
if act not in ('replace','append','raw','drop'):
raise ValueError(f'Invalid action at {path}:{line_no}: {act}')
tags_from_string(row.get('osm_tags',''))
if row.get('name_contains',''):
conditional.append(row)
else:
if c in bases:raise ValueError(f'Duplicate POI base mapping {c} at {path}:{line_no}')
bases[c]=row
for row in conditional:
if row['code'] not in bases:raise ValueError(f'Conditional rule has no base: {row["code"]}')
return bases,conditional
def mapping(path): def mapping(path):
"""Base POI map; conditional rules are applied by the C++ override stage."""
return read_poi_configuration(path)[0]
def line_mapping(path):
ret={} ret={}
for row in read_csv(path): for row in read_csv(path):
try:c=code(row['code']) try:c=code(row['code'])
except Exception as ex:raise ValueError('Invalid code in '+str(path)+': '+str(row)) from ex except Exception as ex:raise ValueError('Invalid line code in '+str(path)+': '+str(row)) from ex
if c in ret:raise ValueError(f'Duplicate code {c} in {path}') if c in ret:raise ValueError(f'Duplicate line code {c} in {path}')
ret[c]=row ret[c]=row
return ret return ret
def tags_from_string(v): def tags_from_string(v):
d={} d={}
for x in v.split(';'): for x in v.split(';'):
@@ -86,7 +114,7 @@ def add_native_unknown_fallback(elem,tags,rule):
tags['bgm:generated_name']='yes' tags['bgm:generated_name']='yes'
return True return True
def make_cpp_rules(pois,out,labels): def make_cpp_rules(pois,out,labels,conditional=()):
"""C++ v0.9 uses these internal stage files; they are NOT user control files.""" """C++ v0.9 uses these internal stage files; they are NOT user control files."""
with open(out,'w',encoding='utf-8',newline='') as f,open(labels,'w',encoding='utf-8',newline='') as g: with open(out,'w',encoding='utf-8',newline='') as f,open(labels,'w',encoding='utf-8',newline='') as g:
w=csv.writer(f);w.writerow(['code','name_contains','mode','tags','note']) w=csv.writer(f);w.writerow(['code','name_contains','mode','tags','note'])
@@ -99,6 +127,15 @@ def make_cpp_rules(pois,out,labels):
if action=='drop':vals={} if action=='drop':vals={}
w.writerow([c,'',action,';'.join(k+'='+v for k,v in vals.items()),'v0.92 compiled from poi-mapping.csv']) w.writerow([c,'',action,';'.join(k+'='+v for k,v in vals.items()),'v0.92 compiled from poi-mapping.csv'])
if row.get('label_bg'):l.writerow([c,row['label_bg']]) if row.get('label_bg'):l.writerow([c,row['label_bg']])
# Order matters: base rule first, then the explicit name-specific rules
# in CSV order. C++ evaluates these against ORIGINAL Garmin names.
for row in conditional:
c=row['code'];action=(row.get('action') or 'replace').strip().lower()
vals=tags_from_string(row.get('osm_tags',''))
if action=='raw':action='keep_raw';vals={}
w.writerow([c,row['name_contains'],action,
';'.join(k+'='+v for k,v in vals.items()),
'v0.95 name-specific rule: '+row.get('note','')])
def inspect_counts(osm,pois,lines,out,strict=False): def inspect_counts(osm,pois,lines,out,strict=False):
seen=collections.Counter();unresolved=collections.Counter();annotations=collections.Counter();fallback=collections.Counter();bad=collections.Counter();routes=collections.Counter();examples={};category=collections.Counter();n=0 seen=collections.Counter();unresolved=collections.Counter();annotations=collections.Counter();fallback=collections.Counter();bad=collections.Counter();routes=collections.Counter();examples={};category=collections.Counter();n=0
@@ -143,7 +180,7 @@ def transform(source,dest,pois,lines,report,strict=False):
db.execute('CREATE TABLE edges (code TEXT, way INTEGER, a INTEGER, b INTEGER)') db.execute('CREATE TABLE edges (code TEXT, way INTEGER, a INTEGER, b INTEGER)')
poi_count=0; ways=0; candidate_edges=0; emitted=0; synthetic=0; kinds=collections.Counter();seen=collections.Counter();gaps=collections.Counter() poi_count=0; ways=0; candidate_edges=0; emitted=0; synthetic=0; kinds=collections.Counter();seen=collections.Counter();gaps=collections.Counter()
with open(source,'rb') as inp,open(dest,'w',encoding='utf-8') as out: with open(source,'rb') as inp,open(dest,'w',encoding='utf-8') as out:
out.write('<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="garmin2osm-v0.93" upload="never">\n') out.write('<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="garmin2osm-v0.95" upload="never">\n')
for el in top_level_elements(inp): for el in top_level_elements(inp):
if el.tag=='node': if el.tag=='node':
tags=tagdict(el) tags=tagdict(el)
@@ -268,10 +305,10 @@ def main():
a=p.parse_args() a=p.parse_args()
if a.mode=='template':res=template(a.catalog,a.poi,a.lines,a.osm) if a.mode=='template':res=template(a.catalog,a.poi,a.lines,a.osm)
else: else:
ps=mapping(a.poi);ls=mapping(a.lines) ps,conditional=read_poi_configuration(a.poi);ls=line_mapping(a.lines)
if a.mode=='prepare': if a.mode=='prepare':
out=pathlib.Path(a.out);out.mkdir(parents=True,exist_ok=True) out=pathlib.Path(a.out);out.mkdir(parents=True,exist_ok=True)
make_cpp_rules(ps,out/'cpp-poi-override.csv',out/'cpp-name-labels.csv');res={'poi_types':len(ps),'line_types':len(ls)} make_cpp_rules(ps,out/'cpp-poi-override.csv',out/'cpp-name-labels.csv',conditional);res={'poi_types':len(ps),'name_specific_rules':len(conditional),'line_types':len(ls)}
elif a.mode=='transform':res=transform(a.osm,a.out,ps,ls,a.report) elif a.mode=='transform':res=transform(a.osm,a.out,ps,ls,a.report)
else:res=inspect_counts(a.osm,ps,ls,a.out,a.strict) else:res=inspect_counts(a.osm,ps,ls,a.out,a.strict)
print(json.dumps(res,ensure_ascii=False,indent=2));return 0 print(json.dumps(res,ensure_ascii=False,indent=2));return 0
+16
View File
@@ -0,0 +1,16 @@
import unittest, tempfile, csv, pathlib, sys
sys.path.insert(0,str(pathlib.Path(__file__).resolve().parents[1]/"tools"))
import audit_osmand_poi_mapping as a
class TestPoiRegistry(unittest.TestCase):
def test_registry_and_semantics(self):
with tempfile.TemporaryDirectory() as t:
p=pathlib.Path(t)/'types.xml'
p.write_text('''<poi_types><poi_type name="water" tag="amenity" value="drinking_water"/><poi_type name="locality" tag="place" value="locality" no_indx="true"/><poi_additional name="dwater" tag="drinking_water" value="yes"/></poi_types>''')
reg,ex=a.registry(p)
base={'code':'0x6414','source_count':'32922','priority':'critical','osm_tags':'amenity=drinking_water;drinking_water=yes'}
self.assertEqual(a.interpret(base,reg,ex)['status'],'native_indexable_type')
self.assertEqual(a.interpret({**base,'osm_tags':'place=locality'},reg,ex)['status'],'only_nonindexed_type')
self.assertEqual(a.interpret({**base,'osm_tags':'drinking_water=yes'},reg,ex)['status'],'additional_without_primary_type')
self.assertEqual(a.interpret({**base,'osm_tags':'man_made=water_intake'},reg,ex)['status'],'no_matching_native_type')
self.assertEqual(a.interpret({**base,'code':'0x6600','priority':'terrain','osm_tags':''},reg,ex)['severity'],'terrain_exemption')
if __name__=='__main__':unittest.main()
+8 -8
View File
@@ -4,7 +4,7 @@ import csv,importlib.util,json,pathlib,subprocess,sys,tempfile,unittest,xml.etre
ROOT=pathlib.Path(__file__).resolve().parents[1] ROOT=pathlib.Path(__file__).resolve().parents[1]
spec=importlib.util.spec_from_file_location('translator',ROOT/'scripts/native_translate.py') spec=importlib.util.spec_from_file_location('translator',ROOT/'scripts/native_translate.py')
t=importlib.util.module_from_spec(spec);spec.loader.exec_module(t) t=importlib.util.module_from_spec(spec);spec.loader.exec_module(t)
POI=ROOT/'config/poi-mapping.csv';LINES=ROOT/'config/line-mapping.csv' POI=ROOT/'config/BGMountains-native-poi-mapping-v094.csv';LINES=ROOT/'config/line-mapping.csv'
class Native092(unittest.TestCase): class Native092(unittest.TestCase):
def test_native_source_semantics(self): def test_native_source_semantics(self):
rows=t.mapping(POI) rows=t.mapping(POI)
@@ -15,17 +15,17 @@ class Native092(unittest.TestCase):
self.assertIn(rows['0x6600']['action'],('append','raw')) self.assertIn(rows['0x6600']['action'],('append','raw'))
self.assertFalse(any('leisure=marina' in r['osm_tags'] for r in rows.values())) self.assertFalse(any('leisure=marina' in r['osm_tags'] for r in rows.values()))
self.assertFalse(any('tourism=information;information=board' in r['osm_tags'] for r in rows.values())) self.assertFalse(any('tourism=information;information=board' in r['osm_tags'] for r in rows.values()))
self.assertTrue(all('garmin' not in r['osm_tags'] for r in rows.values())) self.assertTrue(all('garmin:type' not in r['osm_tags'] for r in rows.values()))
self.assertTrue(all(int(rows[k]['zoom_hint'])<=12 for k in ('0x6414','0x6511','0x6500','0x660a'))) self.assertTrue(all(int(rows[k]['zoom_hint'])<=12 for k in ('0x6414','0x6511','0x6500','0x660a')))
def test_config_only_two_and_types(self): def test_config_only_two_and_types(self):
self.assertEqual(set(p.name for p in (ROOT/'config').glob('*.csv')),{'poi-mapping.csv','line-mapping.csv'}) self.assertEqual(set(p.name for p in (ROOT/'config').glob('*.csv')),{'BGMountains-native-poi-mapping-v094.csv','line-mapping.csv'})
self.assertTrue(any(x.get('route_mode')=='relation' for x in t.mapping(LINES).values())) self.assertTrue(any(x.get('route_mode')=='relation' for x in t.line_mapping(LINES).values()))
for line in t.mapping(LINES).values(): for line in t.line_mapping(LINES).values():
if line['route_mode']=='relation':self.assertTrue(line['osmc_symbol']) if line['route_mode']=='relation':self.assertTrue(line['osmc_symbol'])
def test_native_relation_members_and_poi_provenance(self): def test_native_relation_members_and_poi_provenance(self):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
d=pathlib.Path(directory);src=d/'sample.osm';dst=d/'native.osm';report=d/'report.json' d=pathlib.Path(directory);src=d/'sample.osm';dst=d/'native.osm';report=d/'report.json'
line=next(r for r in t.mapping(LINES).values() if r['route_mode']=='relation') line=next(r for r in t.line_mapping(LINES).values() if r['route_mode']=='relation')
typ=line['code'] typ=line['code']
xml=['<?xml version="1.0" encoding="UTF-8"?><osm version="0.6">'] xml=['<?xml version="1.0" encoding="UTF-8"?><osm version="0.6">']
for i in range(1,5):xml.append(f'<node id="{i}" lat="41.{i}" lon="23.{i}"/>') for i in range(1,5):xml.append(f'<node id="{i}" lat="41.{i}" lon="23.{i}"/>')
@@ -33,7 +33,7 @@ class Native092(unittest.TestCase):
for i,refs in [(10,[1,2]),(11,[2,3])]: for i,refs in [(10,[1,2]),(11,[2,3])]:
xml.append(f'<way id="{i}">'+''.join(f'<nd ref="{n}"/>' for n in refs)+f'<tag k="garmin:feature" v="extended_polyline"/><tag k="garmin:type" v="{typ}"/><tag k="bgm_route" v="type_{typ}"/></way>') xml.append(f'<way id="{i}">'+''.join(f'<nd ref="{n}"/>' for n in refs)+f'<tag k="garmin:feature" v="extended_polyline"/><tag k="garmin:type" v="{typ}"/><tag k="bgm_route" v="type_{typ}"/></way>')
xml.append('</osm>');src.write_text('\n'.join(xml)) xml.append('</osm>');src.write_text('\n'.join(xml))
result=t.transform(src,dst,t.mapping(POI),t.mapping(LINES),report) result=t.transform(src,dst,t.mapping(POI),t.line_mapping(LINES),report)
self.assertEqual(result['route_candidate_ways'],2);self.assertEqual(result['route_relations'],1) self.assertEqual(result['route_candidate_ways'],2);self.assertEqual(result['route_relations'],1)
root=ET.parse(dst).getroot();rel=root.find('relation');tags=t.tagdict(rel) root=ET.parse(dst).getroot();rel=root.find('relation');tags=t.tagdict(rel)
self.assertEqual(tags['route'],'hiking');self.assertTrue(tags['osmc:symbol'].endswith('_bar')) self.assertEqual(tags['route'],'hiking');self.assertTrue(tags['osmc:symbol'].endswith('_bar'))
@@ -45,7 +45,7 @@ class Native092(unittest.TestCase):
def test_route_attachment_to_real_way_only(self): def test_route_attachment_to_real_way_only(self):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
d=pathlib.Path(directory);src=d/'src.osm';dst=d/'out.osm';rep=d/'report.json' d=pathlib.Path(directory);src=d/'src.osm';dst=d/'out.osm';rep=d/'report.json'
typ=next(row['code'] for row in t.mapping(LINES).values() if row['route_mode']=='relation') typ=next(row['code'] for row in t.line_mapping(LINES).values() if row['route_mode']=='relation')
parts=['<osm version="0.6">','<node id="1" lat="41" lon="23"/>','<node id="2" lat="41.1" lon="23.1"/>', parts=['<osm version="0.6">','<node id="1" lat="41" lon="23"/>','<node id="2" lat="41.1" lon="23.1"/>',
'<node id="3" lat="41.2" lon="23.2"/>','<way id="22"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="highway" v="path"/></way>', '<node id="3" lat="41.2" lon="23.2"/>','<way id="22"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="highway" v="path"/></way>',
'<way id="33"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="garmin:type" v="'+typ+'"/><tag k="garmin:feature" v="extended_polyline"/></way>', '<way id="33"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="garmin:type" v="'+typ+'"/><tag k="garmin:feature" v="extended_polyline"/></way>',
+8 -8
View File
@@ -10,25 +10,25 @@ class Reliability093(unittest.TestCase):
out=pathlib.Path(d) out=pathlib.Path(d)
src=out/'input.osm';dst=out/'output.osm';report=out/'report.json' src=out/'input.osm';dst=out/'output.osm';report=out/'report.json'
src.write_text('''<osm version="0.6"><node id="5" lat="42.1345678" lon="24.2345678"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x9999"/><tag k="garmin:mapset" v="1234"/><tag k="bgm:generated_name" v="yes"/><tag k="name" v="Garmin обект 0x9999 - Q3ABCDEFG9"/></node><node id="6" lat="42.2" lon="24.2"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="176"/><tag k="name" v="176"/></node></osm>''') src.write_text('''<osm version="0.6"><node id="5" lat="42.1345678" lon="24.2345678"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x9999"/><tag k="garmin:mapset" v="1234"/><tag k="bgm:generated_name" v="yes"/><tag k="name" v="Garmin обект 0x9999 - Q3ABCDEFG9"/></node><node id="6" lat="42.2" lon="24.2"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="176"/><tag k="name" v="176"/></node></osm>''')
t.transform(src,dst,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report) t.transform(src,dst,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
nodes=ET.parse(dst).getroot().findall('node');a=t.tagdict(nodes[0]);b=t.tagdict(nodes[1]) nodes=ET.parse(dst).getroot().findall('node');a=t.tagdict(nodes[0]);b=t.tagdict(nodes[1])
self.assertEqual(a['place'],'locality');self.assertEqual(a['name'],'Неуточнено - Q3ABCDEFG9') self.assertEqual(a['place'],'locality');self.assertEqual(a['name'],'Неуточнено - Q3ABCDEFG9')
self.assertEqual(a['garmin:type'],'0x9999');self.assertEqual(a['bgm:classification'],'unverified') self.assertEqual(a['garmin:type'],'0x9999');self.assertEqual(a['bgm:classification'],'unverified')
self.assertNotIn('place',b);self.assertEqual(b['ele'],'176') self.assertNotIn('place',b);self.assertEqual(b['ele'],'176')
data=t.inspect_counts(dst,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),out/'audit.json') data=t.inspect_counts(dst,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),out/'audit.json')
self.assertEqual(data['synthetic_locality_fallback_total'],1) self.assertEqual(data['synthetic_locality_fallback_total'],1)
def test_known_bgm_types_are_not_invented_localities(self): def test_known_bgm_types_are_not_invented_localities(self):
cfg=t.mapping(ROOT/'config/poi-mapping.csv') cfg=t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv')
self.assertEqual(cfg['0x6600']['osm_tags'],'') self.assertIn('bgm:spot_height=yes',cfg['0x6600']['osm_tags']);self.assertEqual(cfg['0x6600']['action'],'append')
self.assertNotIn('place=locality',cfg['0x6600']['osm_tags']) self.assertNotIn('place=locality',cfg['0x6600']['osm_tags'])
self.assertEqual(cfg['0x650b']['osm_tags'],'tourism=information;information=route_marker;hiking=yes') self.assertIn('information=trail_blaze',cfg['0x650b']['osm_tags'])
self.assertEqual(cfg['0x660b']['osm_tags'],'tourism=information;information=trail_blaze;hiking=yes') self.assertIn('information=trail_blaze',cfg['0x660b']['osm_tags'])
def test_fallback_does_not_overwrite_known_native_types(self): def test_fallback_does_not_overwrite_known_native_types(self):
with tempfile.TemporaryDirectory() as d: with tempfile.TemporaryDirectory() as d:
src=pathlib.Path(d)/'before.osm';out=pathlib.Path(d)/'after.osm';rep=pathlib.Path(d)/'rep.json' src=pathlib.Path(d)/'before.osm';out=pathlib.Path(d)/'after.osm';rep=pathlib.Path(d)/'rep.json'
src.write_text('''<osm><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6400"/><tag k="landuse" v="quarry"/><tag k="name" v="Кариера П"/></node><node id="2" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6406"/><tag k="barrier" v="gate"/></node><node id="3" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x640d"/><tag k="aerialway" v="station"/></node></osm>''') src.write_text('''<osm><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6400"/><tag k="landuse" v="quarry"/><tag k="name" v="Кариера П"/></node><node id="2" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6406"/><tag k="barrier" v="gate"/></node><node id="3" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x640d"/><tag k="aerialway" v="station"/></node></osm>''')
t.transform(src,out,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),rep) t.transform(src,out,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),rep)
nodes=ET.parse(out).getroot().findall('node') nodes=ET.parse(out).getroot().findall('node')
self.assertTrue(all('place' not in t.tagdict(e) for e in nodes)) self.assertTrue(all('place' not in t.tagdict(e) for e in nodes))
self.assertEqual(t.tagdict(nodes[0])['name'],'Кариера П') self.assertEqual(t.tagdict(nodes[0])['name'],'Кариера П')
@@ -53,7 +53,7 @@ class Reliability093(unittest.TestCase):
with tempfile.TemporaryDirectory() as d: with tempfile.TemporaryDirectory() as d:
path=pathlib.Path(d)/'sample.osm';report=pathlib.Path(d)/'report.json' path=pathlib.Path(d)/'sample.osm';report=pathlib.Path(d)/'report.json'
path.write_text('''<osm version="0.6"><node id="1" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="132"/></node><node id="2" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x650b"/><tag k="tourism" v="information"/><tag k="information" v="route_marker"/></node><node id="3" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6502"/><tag k="ford" v="yes"/></node></osm>''') path.write_text('''<osm version="0.6"><node id="1" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="132"/></node><node id="2" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x650b"/><tag k="tourism" v="information"/><tag k="information" v="route_marker"/></node><node id="3" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6502"/><tag k="ford" v="yes"/></node></osm>''')
stats=t.inspect_counts(path,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report) stats=t.inspect_counts(path,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
self.assertEqual(stats['unclassified_total'],0) self.assertEqual(stats['unclassified_total'],0)
self.assertEqual(stats['annotations_not_missing_pois']['spot_height'],1) self.assertEqual(stats['annotations_not_missing_pois']['spot_height'],1)
if __name__=='__main__':unittest.main() if __name__=='__main__':unittest.main()
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Version 0.95 regression checks, including its REAL published CLI contract.
Only mock OsmAndMapCreator is used for the orchestration test, not for proving
OBF binary correctness. The real IMG tests were run separately during release.
"""
import csv,importlib.util,json,pathlib,subprocess,sys,tempfile,unittest,xml.etree.ElementTree as ET
ROOT=pathlib.Path(__file__).resolve().parents[1]
SPEC=importlib.util.spec_from_file_location('nt',ROOT/'scripts/native_translate.py')
t=importlib.util.module_from_spec(SPEC);SPEC.loader.exec_module(t)
POI=ROOT/'config/BGMountains-native-poi-mapping-v094.csv'
LINES=ROOT/'config/line-mapping.csv'
SCRIPT=ROOT/'scripts/build_bgmountains.py'
class V095(unittest.TestCase):
def test_mapping_handoff_has_all_named_exceptions(self):
bases,conditional=t.read_poi_configuration(POI)
self.assertEqual(len(bases),1871)
self.assertEqual(len(conditional),79)
self.assertEqual(len(t.line_mapping(LINES)),50)
self.assertEqual(sum(x['code']=='0x6414' and x['name_contains']=='Сух' for x in conditional),1)
with tempfile.TemporaryDirectory() as dd:
d=pathlib.Path(dd)
t.make_cpp_rules(bases,d/'cpp.csv',d/'labels.csv',conditional)
rows=list(csv.DictReader((d/'cpp.csv').open()))
self.assertEqual(len(rows),1950)
self.assertEqual(next(x for x in rows if x['code']=='0x6414' and x['name_contains']=='')['tags'].split(';')[:2],['amenity=drinking_water','man_made=water_tap'])
self.assertEqual(next(x for x in rows if x['code']=='0x6414' and x['name_contains']=='Сух')['tags'].split(';')[:2],['man_made=water_tap','drinking_water=no'])
def test_argument_contract_with_users_flags(self):
helptext=subprocess.run([sys.executable,SCRIPT,'--help'],capture_output=True,text=True,check=True).stdout
for flag in ('--poi-config','--line-config','--download-creator','--jobs','--java-opts','--bbox','--prepare-only','--creator','--template'):
self.assertIn(flag,helptext)
self.assertNotIn('unrecognized arguments',helptext)
def test_smoke_full_orchestration_with_mock_creator(self):
with tempfile.TemporaryDirectory() as dd:
d=pathlib.Path(dd)
creator=d/'creator';creator.mkdir()
(creator/'utilities.sh').write_text('#!/bin/sh\nset -eu\n[ "$1" = "generate-obf-no-address" ]\n[ -f "$2" ]\n[ "${JAVA_TOOL_OPTIONS:-}" = "-Xms512M -Xmx25G" ]\nprintf MOCK-OBF > "${2%.osm}.obf"\n')
(creator/'utilities.sh').chmod(0o755)
(creator/'inspector.sh').write_text('#!/bin/sh\n[ "$1" = "-vpoi" ]\nprintf MOCK-INSPECTION\n')
(creator/'inspector.sh').chmod(0o755)
raw=d/'minimal.osm';raw.write_text('''<osm version="0.6"><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6414"/><tag k="amenity" v="drinking_water"/><tag k="man_made" v="water_tap"/><tag k="name" v="Чешма"/></node></osm>''',encoding='utf-8')
cmd=[sys.executable,str(SCRIPT),'--osm',str(raw),'--poi-config',str(POI),'--line-config',str(LINES),'--out',str(d/'out'),'--jobs','8','--java-opts','-Xms512M -Xmx25G','--creator',str(creator)]
run=subprocess.run(cmd,cwd=ROOT,capture_output=True,text=True)
self.assertEqual(run.returncode,0,run.stderr+'\n'+run.stdout)
self.assertTrue((d/'out'/'BGMountains-native.obf').is_file())
self.assertEqual(json.loads((d/'out'/'build-manifest.json').read_text())['version'],'v0.95')
self.assertEqual(json.loads((d/'out'/'semantic-audit.json').read_text())['status'],'PASS')
tags=t.tagdict(ET.parse(d/'out'/'BGMountains-native.osm').getroot().find('node'))
self.assertEqual(tags['amenity'],'drinking_water')
self.assertEqual(tags['man_made'],'water_tap')
def test_future_template_does_not_require_existing_configs(self):
with tempfile.TemporaryDirectory() as dd:
d=pathlib.Path(dd)
# No fake success from a malformed configuration: expected exit nonzero.
p=subprocess.run([sys.executable,SCRIPT,'--osm',str(d/'notthere.osm'),'--poi-config',str(d/'missing.csv'),'--line-config',str(LINES),'--prepare-only'],capture_output=True,text=True)
self.assertNotEqual(p.returncode,0)
self.assertIn('mapping file not found',p.stderr)
if __name__=='__main__':unittest.main()
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Audit Garmin -> OSM POI mappings against an exact OsmAnd poi_types.xml.
No OBF creation. Exits 0 for ordinary data regressions unless --strict is requested.
The existence of a poi_type does NOT guarantee map icon visibility at any zoom.
"""
import argparse,csv,json,collections,sys
from pathlib import Path
from xml.etree import ElementTree as ET
def registry(path):
root=ET.parse(path).getroot()
main=collections.defaultdict(list);extras=collections.defaultdict(list)
for node in root.iter():
if node.tag not in ('poi_type','poi_additional'):continue
t=node.attrib.get('tag');v=node.attrib.get('value')
if not t or v is None:continue
d=main if node.tag=='poi_type' else extras
d[f'{t}={v}'].append(node.attrib)
return main,extras
def interpret(row,main,extra):
tags=[v.strip() for v in row['osm_tags'].split(';') if '=' in v]
supported=[];noindex=[];additional=[];missing=[]
for tv in tags:
candidates=main.get(tv,[])
if any(x.get('no_indx')!='true' for x in candidates):supported.append(tv)
elif candidates:noindex.append(tv)
elif tv in extra:additional.append(tv)
else:missing.append(tv)
if supported:kind='native_indexable_type'
elif noindex:kind='only_nonindexed_type'
elif additional:kind='additional_without_primary_type'
else:kind='no_matching_native_type'
is_terrain=(row.get('priority')=='terrain' or row.get('code')=='0x6600')
priority=row.get('priority','')
cnt=int(row.get('source_count') or 0)
if is_terrain:severity='terrain_exemption'
elif kind=='native_indexable_type':severity='ok'
elif priority=='critical':severity='critical'
elif cnt>=100:severity='high'
else:severity='low'
return {'code':row.get('code',''),'count':cnt,'priority':priority,'legend_bg':row.get('legend_bg',''),
'osm_tags':row.get('osm_tags',''), 'status':kind, 'severity':severity,
'indexable_poi_tags':';'.join(supported),'nonindexed_poi_tags':';'.join(noindex),
'additional_only_tags':';'.join(additional),'unrecognized_tags':';'.join(missing),
'sample_lat':row.get('example_lat',''),'sample_lon':row.get('example_lon',''),
'explanation':'Registry compatibility only; does not guarantee rendering zoom or OBF preservation'}
def main(argv=None):
ap=argparse.ArgumentParser(description=__doc__)
ap.add_argument('--mapping',required=True,help='Single canonical POI mapping CSV')
ap.add_argument('--poi-types',required=True,help='OsmAnd official poi_types.xml, preferably pinned version')
ap.add_argument('--report',default='poi-native-index-audit.csv')
ap.add_argument('--json',default='poi-native-index-audit.json')
ap.add_argument('--strict',action='store_true',help='Exit nonzero for unindexable critical/high types')
args=ap.parse_args(argv)
native,extra=registry(args.poi_types)
with open(args.mapping,encoding='utf8-sig',newline='') as f:
allrows=list(csv.DictReader(f))
# Name-conditional rows are not additional Garmin types; counting them as
# independent types would multiply source_count and inflate POI statistics.
rows=[r for r in allrows if not r.get('name_contains','')]
out=[interpret(r,native,extra) for r in rows]
with open(args.report,'w',newline='',encoding='utf8') as f:
w=csv.DictWriter(f,fieldnames=out[0].keys());w.writeheader();w.writerows(out)
stats=collections.Counter(r['status'] for r in out)
weighted=collections.Counter()
for r in out:weighted[r['status']]+=r['count']
risk=[r for r in out if r['severity'] in ('critical','high')]
payload={'type_counts':dict(stats),'weighted_source_counts':dict(weighted),'critical_or_high':len(risk),
'top_regressions':sorted(risk,key=lambda z:-z['count'])[:30],
'warning':'POI type registry check ONLY. Does not guarantee OBF map index presence, symbol visibility, zoom or Nearby settings.'}
Path(args.json).write_text(json.dumps(payload,ensure_ascii=False,indent=2),encoding='utf8')
print(f"Inspected {len(out)} Garmin types: {dict(stats)}, high/critical issues: {len(risk)}")
print(f"Wrote {args.report} and {args.json}")
return 2 if args.strict and risk else 0
if __name__=='__main__':sys.exit(main())