v096
This commit is contained in:
nq
+7
-7
@@ -1,5 +1,5 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
project(garmin2osmand VERSION 0.8.0 LANGUAGES CXX)
|
||||
project(garmin2osmand VERSION 0.95.0 LANGUAGES CXX)
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
find_package(PNG REQUIRED)
|
||||
@@ -15,18 +15,18 @@ else()
|
||||
target_compile_options(garmin2osmand PRIVATE -Wall -Wextra -Wpedantic -O2)
|
||||
endif()
|
||||
enable_testing()
|
||||
add_test(NAME help COMMAND garmin2osmand --help)
|
||||
add_test(NAME help COMMAND garmin2osm --help)
|
||||
|
||||
find_package(Python3 COMPONENTS Interpreter QUIET)
|
||||
if(Python3_Interpreter_FOUND)
|
||||
add_test(NAME classic_and_gmp_end_to_end COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_smoke.py $<TARGET_FILE:garmin2osmand>)
|
||||
add_test(NAME native_vector_export COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_vector.py $<TARGET_FILE:garmin2osm>)
|
||||
add_test(NAME bgmountains_legend_audit COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_legend_audit.py)
|
||||
add_test(NAME legacy_native_vector_parser COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_vector.py $<TARGET_FILE:garmin2osm>)
|
||||
add_test(NAME v092_native_osm_translation COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v092.py)
|
||||
endif()
|
||||
if(Python3_Interpreter_FOUND)
|
||||
add_test(NAME osmand_obf_pipeline_xml COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_obf_pipeline.py)
|
||||
add_test(NAME v093_memory_and_mapping_regressions COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v093.py)
|
||||
endif()
|
||||
|
||||
if(Python3_Interpreter_FOUND)
|
||||
add_test(NAME bgmountains_v08_display_mappings COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v08_config.py)
|
||||
add_test(NAME v095_cli_config_and_obf_orchestration COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v095.py)
|
||||
add_test(NAME native_registry_audit COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_registry_audit.py)
|
||||
endif()
|
||||
|
||||
@@ -1,103 +1,169 @@
|
||||
# BGMountains Garmin IMG → OSM → OsmAnd OBF v0.8
|
||||
# BGMountains Garmin IMG → native OsmAnd OBF — v0.95
|
||||
|
||||
**This release fixes the v0.7 icon-visibility blind spot and anonymous POIs.**
|
||||
**This is the integrated source release.** It replaces the v0.92/v0.93 Python
|
||||
pipeline and v0.94 standalone CSVs. Do not combine scripts from older ZIPs:
|
||||
those versions accept different flags and do not apply the 79 name-conditional
|
||||
rules. This archive has **one POI config and one line config**.
|
||||
|
||||
- Every one of the **263 validated retained Garmin POI types** has at least one conventional OSM semantic/display tag in `config/poi-overrides-all-types.csv`, in addition to raw `bgm_poi`/`garmin:*` provenance tags.
|
||||
- **87 codes** occur at least 100 times in the source catalog (409,154 source POIs). **46** use conventional feature categories recognized by stock OsmAnd in principle; **41** use an explicitly marked, synthetic `tourism=information;information=board` **local-OsmAnd display fallback**. Some additional lower-frequency codes also need fallback. This DOES NOT assert that the physical object is an information board.
|
||||
- Missing original IMG names become stable Bulgarian descriptive names, e.g. `Вода - Каптаж - 1THF1SDNTKJ`, with `bgm:generated_name=yes`. Original nonempty names are preserved.
|
||||
- `--jobs N` enables parallel extraction, **default one thread**.
|
||||
- New `tools/audit_icon_coverage.py` verifies source OSM classification coverage, names and produces per-type coordinate report. It **does not** prove OBF index presence or visual display; use an OBF inspector and OsmAnd for that.
|
||||
- Map-derived icon art and datasets remain governed by BGMountains' applicable copyright conditions. These synthetic POI tags are **not appropriate for uploading unchanged to the real OpenStreetMap database**.
|
||||
## Quick start (Ubuntu/Debian)
|
||||
|
||||
## Quick build (Ubuntu/Debian)
|
||||
|
||||
```sh
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libpng-dev libsqlite3-dev python3 openjdk-21-jre-headless unzip
|
||||
sudo apt install -y cmake g++ make libpng-dev libsqlite3-dev python3 openjdk-21-jre-headless unzip
|
||||
unzip BGMountains-garmin2osm-v0.95-integrated.zip
|
||||
cd BGMountains-garmin2osm-v0.95
|
||||
```
|
||||
|
||||
Use **exactly** this from the release root (where `scripts/` and `config/` are):
|
||||
|
||||
```bash
|
||||
nohup python3 scripts/build_bgmountains.py \
|
||||
--img "$HOME/gmapsupp.img" \
|
||||
--poi-config config/BGMountains-native-poi-mapping-v094.csv \
|
||||
--line-config config/line-mapping.csv \
|
||||
--out bg-mountains-v095-build \
|
||||
--download-creator \
|
||||
--jobs 8 \
|
||||
--java-opts '-Xms512M -Xmx25G' > bg-v095.log 2>&1 &
|
||||
```
|
||||
|
||||
The `--poi-config` and `--line-config` arguments are **implemented and tested**.
|
||||
Other supported flags: `--creator /path/to/OsmAndMapCreator`, `--bbox W,S,E,N`,
|
||||
`--prepare-only`, `--keep-raw`, `--template`, `--attach-routes`,
|
||||
`--disk-headroom-gb N`, `--ignore-disk-check`, `--jobs N` (default 1).
|
||||
|
||||
**Check your actual disk space first.** Full-country MapCreator compilation has
|
||||
previously required over 25 GiB of temporary disk space in addition to the
|
||||
source OSM (6+ GiB) on your machine. The script uses a conservative estimate
|
||||
`max(2 GiB, 5 × native-OSM-size + 1 GiB)`, unless `--disk-headroom-gb`
|
||||
requires more. These figures are not upper bounds. Choose an output filesystem
|
||||
with generous free space. `-Xmx25G` controls heap RAM, **not disk usage**.
|
||||
|
||||
## Safe regional test, no Java or extra download
|
||||
|
||||
```bash
|
||||
python3 scripts/build_bgmountains.py \
|
||||
--img /absolute/path/to/gmapsupp.img \
|
||||
--out ./BGMountains-out \
|
||||
--download-creator
|
||||
```
|
||||
|
||||
`--download-creator` is for the official OsmAndMapCreator, and the pipeline downloads the original official `poi_types.xml` and `rendering_types.xml` the first time as needed. Network access is required for these unless you already supply the files. **This runtime did not have MapCreator, so full OBF compilation was not executed here.**
|
||||
|
||||
Already installed OsmAndMapCreator:
|
||||
|
||||
```sh
|
||||
python3 scripts/build_bgmountains.py --img ./gmapsupp.img --out ./BGMountains-out \
|
||||
--creator /path/to/OsmAndMapCreator
|
||||
```
|
||||
|
||||
Opt-in parallelism: add `--jobs 4`. The output directory must be fresh: the converter refuses to overwrite an existing OSM or staging files.
|
||||
|
||||
Prepare and validate the OSM without invoking the Java OBF builder:
|
||||
|
||||
```sh
|
||||
python3 scripts/build_bgmountains.py --img ./gmapsupp.img --out ./BGMountains-preview \
|
||||
--img "$HOME/gmapsupp.img" \
|
||||
--poi-config config/BGMountains-native-poi-mapping-v094.csv \
|
||||
--line-config config/line-mapping.csv \
|
||||
--bbox '25.05,42.43,25.16,42.54' \
|
||||
--jobs 8 \
|
||||
--out ./regional-test \
|
||||
--prepare-only
|
||||
```
|
||||
|
||||
**IMPORTANT: You must REBUILD the OSM and OBF.** This update cannot modify an already compiled `.obf` or give v0.7 output its new tags in place.
|
||||
Inspect `regional-test/BGMountains-native.osm`, `semantic-audit.json`,
|
||||
`route-report.json`, `extract.log`, `translate.log`, `audit.log`. The tool
|
||||
must exit nonzero on a failed conversion subprocess. You can instead pass an
|
||||
existing MapCreator with `--creator ...` (omit `--prepare-only`) for a small
|
||||
real OBF build.
|
||||
|
||||
## What the configuration files mean
|
||||
## Two editable configuration files
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `config/poi-overrides-all-types.csv` | **Active** 1,871-row config, including 263 retained types with revised tags. `mode=replace` replaces old hardcoded semantics, retaining provenance/name. All non-present legacy type rows remain for historical comparison. |
|
||||
| `config/poi-overrides-all-types-v07-user-upload.csv` | Exact backup of the CSV you supplied. |
|
||||
| `config/poi-labels-bg.csv` | New 263-row Bulgarian fallback name prefixes, editable. |
|
||||
| `config/poi-icon-audit.csv` | All 263 types, count, original tags vs new tags, classification confidence and sample coordinates. |
|
||||
| `config/osmand-icons.csv` | Optional **custom render style** icon names/visibility; never controls the default `tourism=information` fallback injected through the active overrides. |
|
||||
| `data/BGMountains-frequent-poi-types-100plus-v08.csv` | All 87 frequent types, with reviewed tags, classification mode and coordinates. |
|
||||
| `tools/audit_icon_coverage.py` | Independent POI OSM field audit; one row per present type. |
|
||||
### 1. `config/BGMountains-native-poi-mapping-v094.csv`
|
||||
|
||||
**How to edit a type:** find `code=0x6500` in active overrides. Edit the **tags** column, leaving a `key=value;key=value` list. Change `mode=replace` only if you understand how the C++ built-in classifications merge. Set a Bulgarian text prefix in `config/poi-labels-bg.csv`. Run again from scratch to rebuild OSM then OBF.
|
||||
1,871 base hex-code entries plus 79 **name-specific** exceptions (1,950 data
|
||||
rows). Columns include: `code`, `source_count`, `legend_bg`, `legend_en`,
|
||||
`meaning_confidence`, `osm_tags`, `label_bg`, `priority`, `zoom_hint`, `action`,
|
||||
`note`, examples, and optional `name_contains`. For conditional matching,
|
||||
set `name_contains` in an additional row with the same `code`. The matcher uses
|
||||
case-sensitive substring matching on the **original Garmin name**, with rules
|
||||
applied in file order after the base rule. `replace` changes the converter's
|
||||
semantic defaults; `append` retains previous tags; `raw` leaves unspecified
|
||||
features unclassified until private-map fallback; `drop` discards the feature.
|
||||
|
||||
### Why 0x6500 is not `amenity=drinking_water`
|
||||
The Python pipeline automatically creates `cpp-poi-override.csv` and
|
||||
`cpp-name-labels.csv` **inside the build output directory**. These are generated
|
||||
intermediates, **not third and fourth user configuration files**. There is no
|
||||
need to manually supply them or an icon pack to OsmAnd.
|
||||
|
||||
The original legend labels this a **каптаж** (water intake/catchment), not necessarily a drinking-water source. v0.8 records `man_made=water_intake` but adds an explicit *presentation-only* `tourism=information;information=board;bgm:synthetic_icon=yes` for common OsmAnd rendering. The "information board" is NOT a physical claim. The default Bulgarian name prefix is `Вода - Каптаж`. If you want `Вода - Резервоар`, just edit the label CSV; do not misrepresent a small intake as an artificial lake (`natural=water;water=reservoir`).
|
||||
Important native examples:
|
||||
|
||||
`0x6602` already has `tourism=viewpoint` and does not use a synthetic board marker. Its failure to appear in a particular existing OBF could be an old build, incomplete POI indexing, or display suppression—only inspecting that actual OBF can distinguish these.
|
||||
- `0x6414`: `amenity=drinking_water;man_made=water_tap` (legacy v18 dual
|
||||
classification); named dry sources such as `(Сух.)` remove drinking-water
|
||||
classification and get `drinking_water=no`.
|
||||
- `0x6500`: `man_made=water_intake;amenity=water_point;drinking_water=unknown`
|
||||
for **private-map index compatibility**, not evidence that its water is
|
||||
potable, available or accessible. Named shaft (`Шах...`) exceptions remove
|
||||
the compatibility alias.
|
||||
- `0x6511`: `natural=spring` (potability not asserted).
|
||||
- `0x6402`: `building=yes` by default, explicitly named `Заслон` or `Хижа`
|
||||
may become shelter/hut (availability/access not verified).
|
||||
- `0x661d`: agricultural structures remain agricultural unless specifically
|
||||
identified otherwise.
|
||||
- `0x6600`: keep numeric spot height `ele=*`, **no fake settlement**.
|
||||
- Unknown POIs: `place=locality`, `Неуточнено - <stable hash>` if unnamed,
|
||||
and `bgm:synthetic_fallback=place_locality`; this private-map fallback **does
|
||||
not guarantee Nearby POI indexing**.
|
||||
|
||||
### Stable anonymous names
|
||||
`priority` and `zoom_hint` are documented classification hints only. They do
|
||||
**not force stock OsmAnd to show an icon at zoom 12**. Likewise `garmin:type`
|
||||
remains a provenance tag, not a display rule. Only stock OsmAnd classifications
|
||||
and rendering apply to the resulting OBF.
|
||||
|
||||
For an unnamed POI, `name = Bulgarian_label + " - " + FNV1a_base36_11(mapset|type|quantized_lat|quantized_lon)`. The output is deterministic for an identical IMG and coordinates across threads and builds. A very small collision probability remains; the coordinates/mapset/type remain available. These generated names are for an offline map, **not actual surveyed names**.
|
||||
### 2. `config/line-mapping.csv`
|
||||
|
||||
### Test against your three sample coordinates
|
||||
50 Garmin line types (20 currently have `route_mode=relation`, 30 preserve
|
||||
source geometry). Columns include Garmin `code`, legend, color,
|
||||
`osm_way_tags`, `route_mode`, `osmc_symbol`, `network`, confidence and priority.
|
||||
Selected source overlay segments are grouped into native
|
||||
`type=route;route=hiking` relations with `osmc:symbol`. Garmin source color is
|
||||
preserved as relation `colour=*`. The script does not create duplicate routable
|
||||
paths from decoration lines. Opt-in `--attach-routes` matches overlays to real
|
||||
existing path geometries, but can use significant memory; it is off by default.
|
||||
**Existing physical path routing remains in the data.** Exact phone-side color,
|
||||
icon and at-zoom visibility still depend on stock OsmAnd.
|
||||
|
||||
| Type | Coordinates | Expected tags | v0.8 generated name |
|
||||
|---|---|---|---|
|
||||
| `0x6602` | `42.0346141,24.6528697` | `tourism=viewpoint` | `Панорама - Гледка - 25IVP49H823` |
|
||||
| `0x6500` | `42.0347643,24.6522689` | `man_made=water_intake;tourism=information;information=board` | `Вода - Каптаж - 1THF1SDNTKJ` |
|
||||
| `0x6500` | `42.0340991,24.6520114` | Same | `Вода - Каптаж - ED03D455VTZ` |
|
||||
## New or unknown IMG maps: generate configuration templates
|
||||
|
||||
These examples were directly extracted from the 422MiB IMG into a Plovdiv test OSM and compared to a source manifest. All 57 POIs in that test passed exact semantic-tag checks, with **zero missing and extra**. Serial vs `--jobs 4` POI identities/tags were equal.
|
||||
|
||||
## Review before copying `.obf` into OsmAnd
|
||||
|
||||
Pipeline artifacts in `--out`:
|
||||
- `BGMountains.osm` — exact source XML for the OBF builder
|
||||
- `source-pois.csv`, `poi-types.csv` — extraction manifest and type counts
|
||||
- `validation.json`, `mismatches.csv` — original-source to OSM audit
|
||||
- `poi-coverage.csv`, `poi-coverage.json` — export display mapping coverage
|
||||
- `osmand-config/poi_types.bgmountains.xml`, `rendering_types.bgmountains.xml` — registrations added to **complete upstream XML**
|
||||
- `osmand-config/BGMountains-custom.render.xml` — optional style; exact Garmin TYP art is NOT embedded into stock OsmAnd
|
||||
- `*.obf` plus `mapcreator.log`, `obf-poi-index.txt` (only after a successful Java compilation)
|
||||
|
||||
The `tourism=information` fallback should give OsmAnd a native searchable POI category in an appropriate compatible build. **Neither an OSM tag nor an indexed OBF proves a symbol will be drawn at every zoom**: the map style, profile, POI filters, icon availability, density/collision and selected offline map still affect visibility. Test at zoom 17–20 with POI layers enabled. Avoid having an old overlapping BGMountains OBF selected at the same time.
|
||||
|
||||
To inspect your built file with OsmAndMapCreator:
|
||||
|
||||
```sh
|
||||
/path/to/OsmAndMapCreator/inspector.sh -vpoi -vpoiobjects \
|
||||
-bbox=24.64,42.05,24.665,42.02 ./BGMountains-out/*.obf \
|
||||
> ./BGMountains-out/plovdiv-obf-inspector.txt
|
||||
```bash
|
||||
python3 scripts/build_bgmountains.py --img /path/to/unknown.img \
|
||||
--out ./template-newmap --bbox '23.0,42.0,24.0,43.0' --template
|
||||
```
|
||||
|
||||
Inspect output for the three named POIs. **There is no tested generic parser of OsmAndMapCreator's inspector text in this package**, and we have not verified a full-country OBF built from v0.8 in this runtime. If OSM has the POI but the inspector doesn't, focus on indexing or generated XML; if inspector has it but the screen doesn't, focus on style/POI display/filter/zoom.
|
||||
Produces `poi-mapping.TEMPLATE.csv` and `line-mapping.TEMPLATE.csv` with
|
||||
observed types and sample coordinates. Fill them manually and run with
|
||||
`--poi-config ./template-newmap/poi-mapping.TEMPLATE.csv` and
|
||||
`--line-config ./template-newmap/line-mapping.TEMPLATE.csv`. **Do not assume
|
||||
Garmin codes have the same meaning in unrelated maps.** If you need a complete
|
||||
catalog for a new national map, generate its templates without `--bbox`, but
|
||||
this can require significant time and intermediate disk space.
|
||||
|
||||
### Local-only licensing warning
|
||||
## Validation and provenance
|
||||
|
||||
Do not upload the generated synthetic display tags to OpenStreetMap or assert they accurately describe physical information boards. The original IMG, derived icons and extracts can have third-party copyright restrictions. The source-only code path should be distributed separately from map-derived sample assets until rights are confirmed.
|
||||
```bash
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build -j 4
|
||||
ctest --test-dir build --output-on-failure
|
||||
```
|
||||
|
||||
There is also `tools/audit_osmand_poi_mapping.py`, which takes the official
|
||||
version-matched OsmAnd `poi_types.xml` and produces a registry compatibility
|
||||
report, **not a claim of final icon visibility**:
|
||||
|
||||
```bash
|
||||
python3 tools/audit_osmand_poi_mapping.py \
|
||||
--mapping config/BGMountains-native-poi-mapping-v094.csv \
|
||||
--poi-types /path/to/poi_types.xml \
|
||||
--report native-poi-audit.csv --json native-poi-audit.json
|
||||
```
|
||||
|
||||
Source: https://github.com/osmandapp/OsmAnd-resources/blob/master/poi/poi_types.xml
|
||||
|
||||
## Tested, and NOT tested
|
||||
|
||||
- C++ Release build, 6 CTest test groups, conditional POI CSV generation,
|
||||
CLI options, and simulated MapCreator orchestration passed.
|
||||
- Original 422 MiB IMG: regional `--jobs 8` and serial preparation succeeded
|
||||
with identical 427 POI coordinate/name/tag records and 6 route relations.
|
||||
- Separate region: `(Сух.)` water-tap conditional correctly produced
|
||||
`man_made=water_tap;drinking_water=no`, and no `amenity=drinking_water`.
|
||||
- **A real OsmAndMapCreator binary was not available here**; its host was
|
||||
unreachable. The OBF orchestration unit test uses a **mock**, so native
|
||||
nationwide OBF compilation and phone rendering are not verified. Do not
|
||||
mistake that mock for real OBF validation.
|
||||
|
||||
License: the source bundle contains historically BGMountains-derived
|
||||
classification information. Treat that data as private-map material under
|
||||
applicable source licenses; the package is **not** authorization to republish
|
||||
BGMountains geography or to import its features into the public OSM database.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"release": "v0.95",
|
||||
"code_groups": 6,
|
||||
"test_groups_passed": 6,
|
||||
"real_img_source": "gmapsupp(2).img (original supplied, 441942016 bytes)",
|
||||
"real_regional_bbox": "25.05,42.43,25.16,42.54",
|
||||
"real_regional_pois": 427,
|
||||
"real_regional_routes": 6,
|
||||
"parallel_workers_tested": [
|
||||
1,
|
||||
8
|
||||
],
|
||||
"serial_parallel_poi_tags_names_equal": true,
|
||||
"conditional_dry_water_result": "man_made=water_tap;drinking_water=no; amenity=drinking_water absent",
|
||||
"java_real_obf_build": "NOT TESTED (official host unavailable)",
|
||||
"obf_mock_orchestration": "PASSED; not a real OBF binary"
|
||||
}
|
||||
Binary file not shown.
+144
-326
@@ -1,341 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-command BGMountains IMG -> reviewed OSM -> custom indexed OsmAnd OBF.
|
||||
"""BGMountains v0.95 native OsmAnd pipeline: only TWO editable mapping files.
|
||||
|
||||
Only Python 3 stdlib needed for orchestration; CMake/C++/OsmAndMapCreator are external.
|
||||
Never needs extracted Garmin icons or custom styles on Android.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
import argparse,datetime,hashlib,json,os,pathlib,shutil,subprocess,sys,time,urllib.request,zipfile
|
||||
ROOT=pathlib.Path(__file__).resolve().parents[1]
|
||||
TRANSLATOR=ROOT/'scripts/native_translate.py'
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
UPSTREAM = {
|
||||
'poi': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/poi/poi_types.xml',
|
||||
'rendering': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/obf_creation/rendering_types.xml'
|
||||
}
|
||||
|
||||
def call(argv, cwd=None, log=None, env=None):
|
||||
argv = [str(s) for s in argv]
|
||||
print('+', ' '.join(shlex_quote(s) for s in argv), flush=True)
|
||||
def run(*args,log=None,cwd=None,env=None):
|
||||
cmd=[str(x) for x in args];print('+',' '.join(cmd),flush=True)
|
||||
if log:
|
||||
with Path(log).open('w', encoding='utf-8') as output:
|
||||
r = subprocess.run(argv, cwd=cwd, env=env, stdout=output, stderr=subprocess.STDOUT)
|
||||
if r.returncode:
|
||||
print('Command failed; last log lines:', file=sys.stderr)
|
||||
print('\n'.join(Path(log).read_text(errors='replace').splitlines()[-35:]), file=sys.stderr)
|
||||
raise SystemExit(r.returncode)
|
||||
else:
|
||||
subprocess.run(argv, cwd=cwd, env=env, check=True)
|
||||
with open(log,'w',encoding='utf-8') as f:
|
||||
p=subprocess.run(cmd,stdout=f,stderr=subprocess.STDOUT,cwd=cwd,env=env)
|
||||
if p.returncode:
|
||||
print(''.join(pathlib.Path(log).read_text(errors='replace').splitlines(True)[-40:]),file=sys.stderr)
|
||||
raise RuntimeError('Command failed, see '+str(log))
|
||||
else:subprocess.run(cmd,check=True,cwd=cwd,env=env)
|
||||
|
||||
def shlex_quote(s):
|
||||
import shlex
|
||||
return shlex.quote(s)
|
||||
def creator_dir(x):
|
||||
if not x:return None
|
||||
p=pathlib.Path(x).expanduser().resolve();p=p.parent if p.name=='utilities.sh' else p
|
||||
if not (p/'utilities.sh').exists():raise FileNotFoundError('MapCreator utilities.sh missing: '+str(p))
|
||||
return p
|
||||
|
||||
def base_xml(provided, creator, key, cache, offline):
|
||||
if provided:
|
||||
p = Path(provided).expanduser().resolve()
|
||||
if not p.is_file(): raise SystemExit('Upstream XML not found: '+str(p))
|
||||
return p
|
||||
names = ['poi_types.xml'] if key == 'poi' else ['rendering_types.xml']
|
||||
if creator:
|
||||
for prefix in [creator, creator/'resources', creator/'config', creator/'poi', creator/'obf_creation']:
|
||||
for name in names:
|
||||
candidate = prefix/name
|
||||
if candidate.is_file(): return candidate.resolve()
|
||||
target = cache/names[0]
|
||||
if target.is_file(): return target
|
||||
if offline:
|
||||
raise SystemExit('Missing upstream '+names[0]+': pass --'+key+'-base FILE or allow download')
|
||||
target.parent.mkdir(parents=True,exist_ok=True)
|
||||
print('Fetching official OsmAnd '+names[0]+' (first run only) ...', flush=True)
|
||||
try:
|
||||
with urllib.request.urlopen(UPSTREAM[key], timeout=50) as u, target.open('wb') as out:
|
||||
shutil.copyfileobj(u,out)
|
||||
except Exception as ex:
|
||||
target.unlink(missing_ok=True)
|
||||
raise SystemExit('Cannot fetch official OsmAnd '+names[0]+': '+str(ex)+'\nProvide --'+key+'-base FILE manually.')
|
||||
return target
|
||||
def fetch_creator(target):
|
||||
extracted=target/'unpacked';c=list(extracted.rglob('utilities.sh')) if extracted.exists() else []
|
||||
if c:return c[0].parent
|
||||
target.mkdir(parents=True,exist_ok=True)
|
||||
url='https://download.osmand.net/latest-night-build/OsmAndMapCreator-main.zip'
|
||||
zip_path=target/'OsmAndMapCreator-main.zip'
|
||||
if not zip_path.exists():
|
||||
print('Fetching official MapCreator:',url)
|
||||
with urllib.request.urlopen(url,timeout=120) as src,open(zip_path,'wb') as dst:shutil.copyfileobj(src,dst)
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
for entry in z.infolist():
|
||||
parts=pathlib.PurePosixPath(entry.filename)
|
||||
if parts.is_absolute() or '..' in parts.parts:raise ValueError('Unsafe creator ZIP entry')
|
||||
z.extractall(extracted)
|
||||
c=list(extracted.rglob('utilities.sh'))
|
||||
if not c:raise FileNotFoundError('Creator ZIP lacks utilities.sh')
|
||||
for f in extracted.rglob('*.sh'):f.chmod(f.stat().st_mode|0o111)
|
||||
return c[0].parent
|
||||
|
||||
def catalog_rows(p):
|
||||
with Path(p).open('r',encoding='utf-8-sig',newline='') as f:
|
||||
rows=list(csv.DictReader(f))
|
||||
if not rows or not {'code','count','has_typ_icon'}.issubset(rows[0]):
|
||||
raise SystemExit('Invalid POI catalog (expected code,count,has_typ_icon): '+str(p))
|
||||
out=[]
|
||||
seen=set()
|
||||
for r in rows:
|
||||
code='0x'+format(int(r['code'],0),'x')
|
||||
if code in seen:raise SystemExit('Duplicate code in catalog: '+code)
|
||||
seen.add(code)
|
||||
out.append(dict(r,code=code,count=int(r['count'])))
|
||||
return sorted(out,key=lambda r:(-r['count'],int(r['code'],0)))
|
||||
|
||||
def write_xml(tree,path):
|
||||
ET.indent(tree,space=' ')
|
||||
tree.write(path,encoding='utf-8',xml_declaration=True)
|
||||
ET.parse(path)
|
||||
|
||||
def make_configs(poi_base,render_base,catalog,folder,threshold,frequent_zoom=15,rare_zoom=17,route_styles=()):
|
||||
# Keep entire upstream definitions! Replacing with a standalone tiny XML is
|
||||
# precisely what used to make ordinary semantic POIs disappear.
|
||||
poi=ET.parse(poi_base); proot=poi.getroot()
|
||||
if proot.tag!='poi_types':raise SystemExit('Not an OsmAnd poi_types.xml: '+str(poi_base))
|
||||
if any(c.get('name')=='bgmountains_raw' for c in proot.findall('poi_category')):
|
||||
raise SystemExit('Input poi_types.xml already has a bgmountains_raw category')
|
||||
cat=ET.SubElement(proot,'poi_category',name='bgmountains_raw', default_tag='bgm_poi', no_edit='true')
|
||||
for r in catalog:
|
||||
code=r['code']
|
||||
ET.SubElement(cat,'poi_type',name='bgm_'+code.replace('0x',''),tag='bgm_poi',value='type_'+code,no_edit='true')
|
||||
pa=folder/'poi_types.bgmountains.xml'
|
||||
write_xml(poi,pa)
|
||||
|
||||
rendering=ET.parse(render_base); rroot=rendering.getroot()
|
||||
if rroot.tag != 'osmand_types':
|
||||
# OsmAnd resources use the tag 'rendering_types'; fail instead of dropping all built-in types.
|
||||
raise SystemExit('Unexpected OsmAnd rendering_types.xml root: '+rroot.tag)
|
||||
cats=[c for c in rroot.iter('category') if c.get('name')=='user_defined']
|
||||
target=cats[0] if cats else ET.SubElement(rroot,'category',name='user_defined')
|
||||
for r in catalog:
|
||||
code=r['code']
|
||||
ET.SubElement(target,'type',tag='bgm_poi',value='type_'+code,
|
||||
minzoom=str(frequent_zoom) if r['count']>=threshold else str(rare_zoom))
|
||||
# Route overlays have their OWN map-index main types, not only an ignored metadata tag.
|
||||
# Do not map decorative overlays to 'highway=path': that would create bogus routing.
|
||||
for route in route_styles:
|
||||
if route.get('show','yes').lower() not in ('yes','1','true'): continue
|
||||
ET.SubElement(target,'type',tag='bgm_route',value='type_'+route['code'],minzoom=route['display_minzoom'])
|
||||
rp=folder/'rendering_types.bgmountains.xml'
|
||||
write_xml(rendering,rp)
|
||||
summary={
|
||||
'catalog_types':len(catalog), 'route_overlay_types':len(route_styles),
|
||||
'poi_count_in_catalog':sum(r.get('sample_count',r['count']) for r in catalog),
|
||||
'types_at_or_above_threshold':sum(r['count']>=threshold for r in catalog),
|
||||
'frequent_index_minzoom':frequent_zoom, 'rare_index_minzoom':rare_zoom,
|
||||
'items_at_or_above_threshold':sum(r.get('sample_count',r['count']) for r in catalog if r['count']>=threshold),
|
||||
'types_with_extracted_icon':sum(r['has_typ_icon']=='yes' for r in catalog),
|
||||
'poi_base_sha256':hashlib.sha256(Path(poi_base).read_bytes()).hexdigest(),
|
||||
'rendering_base_sha256':hashlib.sha256(Path(render_base).read_bytes()).hexdigest(),
|
||||
'note':'Local-only compat data; synthetic information icons are not truth about an informational board. All Garmin hex POI types indexed; high-frequency raw types inserted into OBF map index from selected frequent zoom; low-frequency from selected rare zoom. Rendering exact Garmin icon PNGs requires installing compatible icons/style in OsmAnd, not merely compiling an OBF.'
|
||||
}
|
||||
(folder/'type-index-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
return pa,rp,summary
|
||||
|
||||
def read_route_styles(path):
|
||||
with Path(path).open(encoding='utf-8-sig',newline='') as f: rows=list(csv.DictReader(f))
|
||||
if not rows or not {'code','color','show','display_minzoom','width'}.issubset(rows[0]):
|
||||
raise SystemExit('Invalid trail-styles.csv: '+str(path))
|
||||
seen=set()
|
||||
for r in rows:
|
||||
code='0x'+format(int(r['code'],0),'x')
|
||||
if code in seen: raise SystemExit('Duplicate route style type: '+code)
|
||||
seen.add(code);r['code']=code
|
||||
if not (len(r['color'])==7 and r['color'][0]=='#' and all(c in '0123456789abcdefABCDEF' for c in r['color'][1:])):
|
||||
raise SystemExit('Invalid CSS color in route type '+code)
|
||||
if not (0.2<=float(r['width'])<=15 and 1<=int(r['display_minzoom'])<=21):
|
||||
raise SystemExit('Invalid route width/zoom '+code)
|
||||
return rows
|
||||
|
||||
def make_style(icon_csv,folder,catalog,route_styles=()):
|
||||
"""Selectable OsmAnd map style that maps BGM indexes to built-in icons.
|
||||
|
||||
The .obf itself only embeds geometry/tags. Android requires installing and
|
||||
selecting this renderer; stock app ignores bgm_poi and bgm_route styling.
|
||||
def disk_preflight(out, source_bytes, extra_gb=0):
|
||||
"""Conservative warning/blocking check before disk-heavy MapCreator stage.
|
||||
In experience 6 GiB OSM has required >25 GiB scratch. 5x + 1 GiB
|
||||
is intentionally approximate, not a guarantee of completion.
|
||||
"""
|
||||
known={r['code'] for r in catalog}
|
||||
with Path(icon_csv).open(encoding='utf-8-sig',newline='') as f:
|
||||
rows=list(csv.DictReader(f))
|
||||
visible=[]
|
||||
for r in rows:
|
||||
code='0x'+format(int(r['code'],0),'x')
|
||||
if code not in known or r['show_in_custom_style'].strip().lower() not in ('yes','true','1'):
|
||||
continue
|
||||
name=r['osmand_builtin_icon'].strip()
|
||||
if not name or not name.replace('_','').replace('-','').isalnum():
|
||||
raise SystemExit('Invalid OsmAnd icon: '+repr(name)+' code='+code)
|
||||
zoom=int(r['minzoom'])
|
||||
if not 1<=zoom<=21: raise SystemExit('Bad icon zoom: '+code)
|
||||
visible.append((code,name,zoom))
|
||||
root=ET.Element('renderingStyle',name='BGMountains outdoor v0.9',
|
||||
depends='default',defaultColor='#fff8ee',version='1')
|
||||
order=ET.SubElement(root,'order');points=ET.SubElement(root,'point')
|
||||
lines=ET.SubElement(root,'line')
|
||||
for code,name,zoom in visible:
|
||||
ET.SubElement(order,'case',tag='bgm_poi',value='type_'+code,
|
||||
minzoom=str(zoom),point='true',objectType='1',order='120')
|
||||
ET.SubElement(points,'case',tag='bgm_poi',value='type_'+code,minzoom=str(zoom),
|
||||
icon=name,iconOrder='11',iconVisibleSize='16',intersectionSizeFactor='0.7')
|
||||
route_count=0
|
||||
for route in route_styles:
|
||||
if route.get('show','yes').lower() not in ('yes','true','1'):continue
|
||||
route_count+=1
|
||||
code=route['code'];zoom=route['display_minzoom']
|
||||
ET.SubElement(order,'case',tag='bgm_route',value='type_'+code,
|
||||
minzoom=zoom,objectType='2',order='125')
|
||||
# Color and width faithfully recover TYP base line color, but NOT its bitmap
|
||||
# segment pattern. A precise pattern renderer requires a custom app build.
|
||||
ET.SubElement(lines,'case',tag='bgm_route',value='type_'+code,
|
||||
minzoom=zoom,color=route['color'],strokeWidth=route['width'],cap='ROUND')
|
||||
path=folder/'BGMountains-outdoor.render.xml'
|
||||
write_xml(ET.ElementTree(root),path)
|
||||
return path,len(visible),route_count
|
||||
|
||||
def obtain_creator(cache, offline=False):
|
||||
"""Opt-in download from official OsmAnd nightly build (large file)."""
|
||||
import zipfile
|
||||
link='https://download.osmand.net/latest-night-build/OsmAndMapCreator-main.zip'
|
||||
target=cache/'OsmAndMapCreator-main.zip'
|
||||
extracted=cache/'unpacked'
|
||||
found=list(extracted.rglob('utilities.sh')) if extracted.is_dir() else []
|
||||
if found:return found[0].parent
|
||||
if offline:raise SystemExit('Cannot download MapCreator with --offline; pass --creator')
|
||||
cache.mkdir(parents=True,exist_ok=True)
|
||||
if not target.is_file():
|
||||
print('Downloading official OsmAndMapCreator (~150 MB) ...',flush=True)
|
||||
try:
|
||||
with urllib.request.urlopen(link,timeout=180) as src,target.open('wb') as dest:
|
||||
shutil.copyfileobj(src,dest,length=2**20)
|
||||
except Exception as ex:
|
||||
target.unlink(missing_ok=True)
|
||||
raise SystemExit('Cannot download OsmAndMapCreator: '+str(ex)+'\nDownload manually: '+link)
|
||||
print('Extracting MapCreator...',flush=True)
|
||||
extracted.mkdir(parents=True,exist_ok=True)
|
||||
with zipfile.ZipFile(target) as z:
|
||||
for m in z.infolist():
|
||||
rel=Path(m.filename)
|
||||
if rel.is_absolute() or '..' in rel.parts:
|
||||
raise SystemExit('Unsafe file in MapCreator archive: '+m.filename)
|
||||
z.extract(m,extracted)
|
||||
found=list(extracted.rglob('utilities.sh'))
|
||||
if not found:raise SystemExit('MapCreator ZIP extracted but utilities.sh not found')
|
||||
for name in ('utilities.sh','inspector.sh'):
|
||||
for f in extracted.rglob(name):f.chmod(f.stat().st_mode|0o111)
|
||||
return found[0].parent
|
||||
|
||||
def resolve_creator(path):
|
||||
name = path or os.environ.get('OSMAND_MAPCREATOR')
|
||||
if not name:return None
|
||||
creator=Path(name).expanduser().resolve()
|
||||
if creator.is_file() and creator.name=='utilities.sh':creator=creator.parent
|
||||
if not (creator/'utilities.sh').is_file():
|
||||
raise SystemExit('Cannot find utilities.sh under '+str(creator))
|
||||
return creator
|
||||
gib=1024**3
|
||||
free=shutil.disk_usage(out).free
|
||||
required=max(int(2*gib),int(source_bytes*5 + gib),int(extra_gb*gib))
|
||||
if free<required:
|
||||
raise RuntimeError(f'Insufficient free space for OBF build: free={free/gib:.2f} GiB; '
|
||||
f'estimated minimum={required/gib:.2f} GiB. '
|
||||
'Use --prepare-only or set --ignore-disk-check if scratch resides on another filesystem.')
|
||||
print(f'Disk preflight OK: free={free/gib:.1f} GiB, estimated minimum={required/gib:.1f} GiB',flush=True)
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
inp=p.add_mutually_exclusive_group(required=True)
|
||||
inp.add_argument('--img',help='Original Garmin IMG; converter is compiled and executed')
|
||||
inp.add_argument('--osm',help='Already converted OSM XML; must contain bgm_poi=type_0x... tags (v0.7+)')
|
||||
p.add_argument('--creator',help='OsmAndMapCreator directory containing utilities.sh (or OSMAND_MAPCREATOR env)')
|
||||
p.add_argument('--download-creator',action='store_true',help='Download official OsmAndMapCreator ZIP automatically (~150 MB) when no --creator is set')
|
||||
p.add_argument('--out',default='build/bgmountains-output',help='Output folder')
|
||||
p.add_argument('--jobs',type=int,default=1,help='C++ IMG reader threads (opt-in)')
|
||||
p.add_argument('--bbox',help='Optional W,S,E,N bounds for small-area test')
|
||||
p.add_argument('--threshold',type=int,default=100,help='Frequent-type occurrence threshold')
|
||||
p.add_argument('--frequent-zoom',type=int,default=15,help='Store frequent raw map icons at zoom N and above')
|
||||
p.add_argument('--rare-zoom',type=int,default=17,help='Store rare raw map icons at zoom N and above')
|
||||
p.add_argument('--icons-config',default=str(ROOT/'config/osmand-icons.csv'),help='OsmAnd icon mappings')
|
||||
p.add_argument('--trail-styles',default=str(ROOT/'config/trail-styles.csv'),help='Garmin TYP coloured route overlay styles')
|
||||
p.add_argument('--poi-overrides',default=str(ROOT/'config/poi-overrides-all-types.csv'))
|
||||
p.add_argument('--poi-name-fixes',default=str(ROOT/'config/poi-name-fixes.csv'))
|
||||
p.add_argument('--poi-labels',default=str(ROOT/'config/poi-labels-bg.csv'),help='Bulgarian prefixes for stable generated names on nameless Garmin POIs')
|
||||
p.add_argument('--poi-additions',default=str(ROOT/'config/poi-additions.csv'))
|
||||
p.add_argument('--poi-base',help='Official upstream poi_types.xml; if omitted use cache/download')
|
||||
p.add_argument('--rendering-base',help='Official upstream rendering_types.xml; if omitted use cache/download')
|
||||
p.add_argument('--offline',action='store_true',help='Never download upstream XML')
|
||||
p.add_argument('--prepare-only',action='store_true',help='Do not run OsmAndMapCreator, but generate OSM and XMLs')
|
||||
p.add_argument('--skip-qa',action='store_true',help='Skip OSM comparison for faster repeat runs')
|
||||
p.add_argument('--java-opts',default='-Xms512M -Xmx8192M',help='JAVA_OPTS for OsmAndMapCreator')
|
||||
p.add_argument('--skip-cpp-build',action='store_true')
|
||||
p.add_argument('--catalog',help='Preexisting POI catalog when supplying --osm (otherwise use packaged nationwide catalog)')
|
||||
p.add_argument('--frequency-catalog',default=str(ROOT/'data/BGMountains-unique-poi-types.csv'),help='Reference nationwide counts for map index zoom decisions even in --bbox builds')
|
||||
args=p.parse_args()
|
||||
if args.jobs<1 or args.jobs>64 or args.threshold<1 or not 1<=args.frequent_zoom<=21 or not 1<=args.rare_zoom<=21: p.error('--jobs 1..64; threshold >=1; zoom 1..21')
|
||||
dest=Path(args.out).expanduser().resolve();dest.mkdir(parents=True,exist_ok=True)
|
||||
creator=resolve_creator(args.creator)
|
||||
if not creator and args.download_creator and not args.prepare_only:
|
||||
creator=obtain_creator(ROOT/'build'/'mapcreator-download',args.offline)
|
||||
if not creator and not args.prepare_only:p.error('Supply --creator /path/to/OsmAndMapCreator (or OSMAND_MAPCREATOR env) or use --prepare-only')
|
||||
if args.img:
|
||||
if not Path(args.img).is_file():p.error('IMG file missing: '+args.img)
|
||||
binary=ROOT/'build'/'garmin2osm'
|
||||
if not args.skip_cpp_build or not binary.is_file():
|
||||
call(['cmake','-S',ROOT,'-B',ROOT/'build','-DCMAKE_BUILD_TYPE=Release'])
|
||||
call(['cmake','--build',ROOT/'build','--config','Release','-j','2'])
|
||||
osm=dest/'BGMountains.osm'
|
||||
cmd=[binary,'--input',Path(args.img).resolve(),'--output',osm,'--poi-overrides',Path(args.poi_overrides).resolve(),
|
||||
'--poi-name-fixes',Path(args.poi_name_fixes).resolve(),'--poi-labels',Path(args.poi_labels).resolve(),'--poi-additions',Path(args.poi_additions).resolve(),
|
||||
'--poi-manifest',dest/'source-pois.csv','--poi-types-out',dest/'poi-types.csv','--jobs',args.jobs]
|
||||
if args.bbox:cmd+=['--bbox',args.bbox]
|
||||
call(cmd,log=dest/'converter.log')
|
||||
catalog=dest/'poi-types.csv'
|
||||
if not args.skip_qa:
|
||||
call([sys.executable,ROOT/'tools/validate_pois.py','compare','--source',dest/'source-pois.csv','--osm',osm,
|
||||
'--rules',Path(args.poi_overrides).resolve(),'--report',dest/'validation.json','--mismatches',dest/'mismatches.csv'],
|
||||
log=dest/'validator.log')
|
||||
else:
|
||||
osm=Path(args.osm).expanduser().resolve()
|
||||
if not osm.is_file():p.error('OSM file missing: '+str(osm))
|
||||
catalog=Path(args.catalog).expanduser().resolve() if args.catalog else ROOT/'data'/'BGMountains-unique-poi-types.csv'
|
||||
call([sys.executable,ROOT/'tools/audit_icon_coverage.py', '--osm',osm, '--catalog',catalog,
|
||||
'--csv',dest/'poi-coverage.csv','--json',dest/'poi-coverage.json'],
|
||||
log=dest/'poi-coverage.log')
|
||||
rows=catalog_rows(catalog)
|
||||
nationwide={r['code']:r['count'] for r in catalog_rows(args.frequency_catalog)} if args.frequency_catalog else {}
|
||||
for row in rows:
|
||||
row['sample_count']=row['count']
|
||||
row['count']=nationwide.get(row['code'],row['count'])
|
||||
# A separately supplied legacy OSM might lack our new classifier.
|
||||
# For --img, the v0.7 converter above just generated it by construction.
|
||||
if args.osm:
|
||||
import gzip
|
||||
opener=gzip.open if osm.suffix=='.gz' else open
|
||||
with opener(osm,'rt',encoding='utf-8',errors='replace') as f:
|
||||
found=any('k="bgm_poi"' in line for line in f)
|
||||
if not found:
|
||||
raise SystemExit('Existing OSM lacks bgm_poi; re-extract with v0.7 C++ converter.')
|
||||
folder=dest/'osmand-config';folder.mkdir(exist_ok=True)
|
||||
poi_base=base_xml(args.poi_base,creator,'poi',dest/'upstream',args.offline)
|
||||
rendering_base=base_xml(args.rendering_base,creator,'rendering',dest/'upstream',args.offline)
|
||||
route_styles=read_route_styles(args.trail_styles)
|
||||
poi_cfg,render_cfg,summary=make_configs(poi_base,rendering_base,rows,folder,args.threshold,args.frequent_zoom,args.rare_zoom,route_styles)
|
||||
style,nicons,nlines=make_style(args.icons_config,folder,rows,route_styles)
|
||||
print('Indexed POI types:',summary['catalog_types'],'frequent:',summary['types_at_or_above_threshold'],'custom icons:',nicons,'colored route types:',nlines,flush=True)
|
||||
if args.prepare_only:
|
||||
print('PREPARED (no OBF generated). Files at:',dest)
|
||||
p.add_argument('--img',help='Original Garmin IMG')
|
||||
p.add_argument('--osm',help='Already generated original Garmin-derived raw OSM')
|
||||
p.add_argument('--out',default='build/bgmountains-v095')
|
||||
p.add_argument('--poi-config',default=str(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'))
|
||||
p.add_argument('--line-config',default=str(ROOT/'config/line-mapping.csv'))
|
||||
p.add_argument('--jobs',type=int,default=1,help='IMG workers: default 1; explicitly set N to parallelize')
|
||||
p.add_argument('--bbox',help='W,S,E,N region for smoke tests')
|
||||
p.add_argument('--template',action='store_true',help='Generate blank new-map mapping templates using this IMG (not BGMountains defaults)')
|
||||
p.add_argument('--attach-routes',action='store_true',help='Opt in to high-memory matching of trail overlays to physical paths (not recommended on full map)')
|
||||
p.add_argument('--no-route-attachment',action='store_true',help='Deprecated: attachment is disabled by default')
|
||||
p.add_argument('--prepare-only',action='store_true',help='Extract/translate/validate, do not compile OBF')
|
||||
p.add_argument('--creator',help='Existing OsmAndMapCreator directory containing utilities.sh')
|
||||
p.add_argument('--download-creator',action='store_true',help='Obtain official OsmAndMapCreator from download.osmand.net')
|
||||
p.add_argument('--java-opts',default='-Xms512M -Xmx8192M')
|
||||
p.add_argument('--keep-raw',action='store_true',help='Keep pre-translation OSM for debugging')
|
||||
p.add_argument('--ignore-disk-check',action='store_true',help='Override conservative OBF scratch space safety check')
|
||||
p.add_argument('--disk-headroom-gb',type=float,default=0,help='Minimum free GiB required before Java OBF build (0=auto estimate)')
|
||||
a=p.parse_args()
|
||||
if bool(a.img)==bool(a.osm):p.error('Provide exactly one of --img or --osm')
|
||||
if a.jobs<1 or a.jobs>64:p.error('--jobs must be 1..64')
|
||||
if not a.template:
|
||||
for typ,pth in (('POI',a.poi_config),('line',a.line_config)):
|
||||
if not pathlib.Path(pth).is_file():p.error(f'{typ} mapping file not found: {pth}')
|
||||
out=pathlib.Path(a.out).resolve();out.mkdir(parents=True,exist_ok=True)
|
||||
src=pathlib.Path(a.osm).resolve() if a.osm else None
|
||||
poi=pathlib.Path(a.poi_config).resolve();line=pathlib.Path(a.line_config).resolve()
|
||||
# Resolve MapCreator before running an expensive full-country extraction.
|
||||
# Missing download/network credentials must fail before consuming gigabytes.
|
||||
creator=None
|
||||
if not a.prepare_only and not a.template:
|
||||
creator=creator_dir(a.creator or os.environ.get('OSMAND_MAPCREATOR'))
|
||||
if not creator and a.download_creator:
|
||||
creator=fetch_creator(ROOT/'build'/'mapcreator')
|
||||
if not creator:
|
||||
p.error('OBF requires --creator DIR or --download-creator; for testing use --prepare-only.')
|
||||
bin_path=ROOT/'build'/'garmin2osm'
|
||||
if a.img:
|
||||
run('cmake','-S',ROOT,'-B',ROOT/'build','-DCMAKE_BUILD_TYPE=Release')
|
||||
run('cmake','--build',ROOT/'build','-j','2','--target','garmin2osm')
|
||||
src=out/'IMG-raw.osm'
|
||||
cmd=[str(bin_path),'--input',str(pathlib.Path(a.img).resolve()),'--output',str(src),
|
||||
'--poi-types-out',str(out/'discovered-pois.csv'),'--poi-manifest',str(out/'source-pois.csv'),
|
||||
'--jobs',str(a.jobs)]
|
||||
if a.bbox:cmd+=['--bbox',a.bbox]
|
||||
if not a.template:
|
||||
run(sys.executable,TRANSLATOR,'prepare','--poi',poi,'--lines',line,'--out',out)
|
||||
cmd+=['--poi-overrides',str(out/'cpp-poi-override.csv'),'--poi-labels',str(out/'cpp-name-labels.csv')]
|
||||
run(*cmd,log=out/'extract.log')
|
||||
if a.template:
|
||||
if not a.img:p.error('--template requires --img')
|
||||
run(sys.executable,TRANSLATOR,'template','--poi',out/'poi-mapping.TEMPLATE.csv',
|
||||
'--lines',out/'line-mapping.TEMPLATE.csv','--catalog',out/'discovered-pois.csv','--osm',src)
|
||||
print('Templates created. Review both files before using them for native OSM conversion.')
|
||||
return 0
|
||||
before={str(i.resolve()) for i in dest.glob('*.obf')}
|
||||
# Save Java logs, and preserve OsmAndMapCreator temp data inside user-selected output dir.
|
||||
env=os.environ.copy();env['JAVA_OPTS']=args.java_opts
|
||||
program=creator/'utilities.sh'
|
||||
cmd=[str(program),'generate-obf-no-address',str(osm),
|
||||
'--poi-types='+str(poi_cfg),'--rendering-types='+str(render_cfg)]
|
||||
call(cmd,cwd=dest,log=dest/'mapcreator.log',env=env)
|
||||
after=[i for i in dest.glob('*.obf') if str(i.resolve()) not in before]
|
||||
if not after:
|
||||
# Creator sometimes writes next to the .osm source, even with cwd changed.
|
||||
alt=list(osm.parent.glob('*.obf'))
|
||||
after=[i for i in alt if i.is_file()]
|
||||
if not after:raise SystemExit('MapCreator returned success but no .obf was found; see '+str(dest/'mapcreator.log'))
|
||||
built=max(after,key=lambda p:p.stat().st_mtime)
|
||||
print('OBF created:',built,'bytes:',built.stat().st_size)
|
||||
if (creator/'inspector.sh').is_file():
|
||||
call([str(creator/'inspector.sh'),'-vpoi',str(built)],cwd=dest,log=dest/'obf-poi-index.txt',env=env)
|
||||
(dest/'build-manifest.json').write_text(json.dumps({
|
||||
'created_at_utc':dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'obf':str(built),'obf_bytes':built.stat().st_size,
|
||||
'input_osm':str(osm),'poi_types':str(poi_cfg),'rendering_types':str(render_cfg),
|
||||
'custom_style':str(style), 'custom_style_icon_rules':nicons, 'custom_route_rules':nlines,
|
||||
'summary':summary
|
||||
},ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
dst=out/'BGMountains-native.osm'
|
||||
run(sys.executable,TRANSLATOR,'transform','--poi',poi,'--lines',line,'--osm',src,
|
||||
'--out',dst,'--report',out/'route-report.json',log=out/'translate.log')
|
||||
if a.attach_routes and not a.no_route_attachment:
|
||||
attached=out/'BGMountains-native-attached.osm'
|
||||
run(sys.executable,ROOT/'scripts/attach_native_routes.py','--osm',dst,'--out',attached,
|
||||
'--lines',line,'--report',out/'route-attachment.json',log=out/'route-attachment.log')
|
||||
dst.unlink();attached.rename(dst)
|
||||
run(sys.executable,TRANSLATOR,'audit','--poi',poi,'--lines',line,'--osm',dst,
|
||||
'--out',out/'semantic-audit.json',log=out/'audit.log')
|
||||
# Fail closed if the translated file is not well-formed XML, including relation refs.
|
||||
import xml.etree.ElementTree as ET
|
||||
from native_translate import top_level_elements
|
||||
n=0
|
||||
for e in top_level_elements(dst):
|
||||
if e.tag=='relation':n+=1
|
||||
if not a.keep_raw and a.img:src.unlink(missing_ok=True)
|
||||
if a.prepare_only:
|
||||
print('PREPARED OSM:',dst,'route relations:',n,'No OBF compiled.')
|
||||
return 0
|
||||
if not a.ignore_disk_check:disk_preflight(out,dst.stat().st_size,a.disk_headroom_gb)
|
||||
env=os.environ.copy();env['JAVA_OPTS']=a.java_opts
|
||||
# utilities.sh versions differ in whether JAVA_OPTS is honored; this flag is
|
||||
# explicitly recognized by the JVM. It also reaches spawned Java processes.
|
||||
env['JAVA_TOOL_OPTIONS']=a.java_opts
|
||||
# Native types are built into OsmAnd: NO --poi-types/--rendering-types override,
|
||||
# NO custom .render.xml and NO extracted Garmin icons.
|
||||
started=time.time()
|
||||
run(creator/'utilities.sh','generate-obf-no-address',dst,log=out/'mapcreator.log',cwd=out,env=env)
|
||||
obfs=[f for f in out.glob('*.obf') if f.stat().st_mtime >= started-2 and f.stat().st_size>0]
|
||||
if not obfs:raise RuntimeError('Creator returned no OBF in '+str(out))
|
||||
obf=max(obfs,key=lambda x:x.stat().st_mtime)
|
||||
if (creator/'inspector.sh').exists():
|
||||
run(creator/'inspector.sh','-vpoi',obf,log=out/'obf-inspection.txt',cwd=out,env=env)
|
||||
(out/'build-manifest.json').write_text(json.dumps({
|
||||
'version':'v0.95','input':a.img or a.osm,'poi_config':str(poi),'line_config':str(line),
|
||||
'poi_config_sha256':hashlib.sha256(poi.read_bytes()).hexdigest(),
|
||||
'line_config_sha256':hashlib.sha256(line.read_bytes()).hexdigest(),
|
||||
'output_obf':str(obf),'obf_bytes':obf.stat().st_size,
|
||||
'compiled_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'notice':'Native stock rendering. POI display zoom controlled by OsmAnd; no claim that all tags get icons.'},ensure_ascii=False,indent=2)+'\n')
|
||||
print('BUILT OBF:',obf)
|
||||
return 0
|
||||
if __name__=='__main__':
|
||||
sys.exit(main())
|
||||
if __name__=='__main__':sys.exit(main())
|
||||
|
||||
+2
-13
@@ -1,15 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compatibility shim. No manually supplied POI XML files are required anymore.
|
||||
# Preferred: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out
|
||||
set -euo pipefail
|
||||
if (( $# < 2 )); then
|
||||
echo "Usage: $0 CREATOR_DIR INPUT_v0.7.osm[.gz] [OUTPUT_DIR]" >&2
|
||||
echo "Recommended instead: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out" >&2
|
||||
exit 2
|
||||
fi
|
||||
creator="$1"; osm="$2"; out="${3:-$(dirname "$osm")/rebuilt-obf}"
|
||||
root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
source_dir="$(dirname "$osm")"
|
||||
args=(--osm "$osm" --creator "$creator" --out "$out")
|
||||
if [[ -f "$source_dir/poi-types.csv" ]]; then args+=(--catalog "$source_dir/poi-types.csv"); fi
|
||||
exec python3 "$root/scripts/build_bgmountains.py" "${args[@]}"
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/build_bgmountains.py "$@"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""v0.93: streaming IMG/OSM native-OsmAnd translation and honest QA.
|
||||
Only user-editable controls: config/poi-mapping.csv and config/line-mapping.csv.
|
||||
"""v0.95: streaming IMG/OSM native-OsmAnd translation and honest QA.
|
||||
Only user-editable controls: config/BGMountains-native-poi-mapping-v094.csv and config/line-mapping.csv.
|
||||
"""
|
||||
import argparse,base64,collections,csv,hashlib,json,pathlib,re,sqlite3,sys,xml.etree.ElementTree as ET
|
||||
from xml.sax.saxutils import escape,quoteattr
|
||||
@@ -34,14 +34,42 @@ def read_csv(p):
|
||||
lines=(line for line in f if not line.startswith('#'))
|
||||
yield from csv.DictReader(lines)
|
||||
def code(value):return hex(int(value,0)).lower()
|
||||
def read_poi_configuration(path):
|
||||
"""Read one canonical POI config: unique base rule per code and optional
|
||||
ordered name_contains rules (a v0.94 feature previously lost in Python).
|
||||
The v0.93/0.94 15-column files remain backward-compatible.
|
||||
"""
|
||||
bases={}; conditional=[]
|
||||
for line_no,row in enumerate(read_csv(path),start=2):
|
||||
try:c=code(row['code'])
|
||||
except Exception as ex:raise ValueError(f'Invalid POI code at {path}:{line_no}: {row}') from ex
|
||||
row=dict(row);row['code']=c
|
||||
act=(row.get('action') or 'replace').strip().lower()
|
||||
if act not in ('replace','append','raw','drop'):
|
||||
raise ValueError(f'Invalid action at {path}:{line_no}: {act}')
|
||||
tags_from_string(row.get('osm_tags',''))
|
||||
if row.get('name_contains',''):
|
||||
conditional.append(row)
|
||||
else:
|
||||
if c in bases:raise ValueError(f'Duplicate POI base mapping {c} at {path}:{line_no}')
|
||||
bases[c]=row
|
||||
for row in conditional:
|
||||
if row['code'] not in bases:raise ValueError(f'Conditional rule has no base: {row["code"]}')
|
||||
return bases,conditional
|
||||
|
||||
def mapping(path):
|
||||
"""Base POI map; conditional rules are applied by the C++ override stage."""
|
||||
return read_poi_configuration(path)[0]
|
||||
|
||||
def line_mapping(path):
|
||||
ret={}
|
||||
for row in read_csv(path):
|
||||
try:c=code(row['code'])
|
||||
except Exception as ex:raise ValueError('Invalid code in '+str(path)+': '+str(row)) from ex
|
||||
if c in ret:raise ValueError(f'Duplicate code {c} in {path}')
|
||||
except Exception as ex:raise ValueError('Invalid line code in '+str(path)+': '+str(row)) from ex
|
||||
if c in ret:raise ValueError(f'Duplicate line code {c} in {path}')
|
||||
ret[c]=row
|
||||
return ret
|
||||
|
||||
def tags_from_string(v):
|
||||
d={}
|
||||
for x in v.split(';'):
|
||||
@@ -86,7 +114,7 @@ def add_native_unknown_fallback(elem,tags,rule):
|
||||
tags['bgm:generated_name']='yes'
|
||||
return True
|
||||
|
||||
def make_cpp_rules(pois,out,labels):
|
||||
def make_cpp_rules(pois,out,labels,conditional=()):
|
||||
"""C++ v0.9 uses these internal stage files; they are NOT user control files."""
|
||||
with open(out,'w',encoding='utf-8',newline='') as f,open(labels,'w',encoding='utf-8',newline='') as g:
|
||||
w=csv.writer(f);w.writerow(['code','name_contains','mode','tags','note'])
|
||||
@@ -99,6 +127,15 @@ def make_cpp_rules(pois,out,labels):
|
||||
if action=='drop':vals={}
|
||||
w.writerow([c,'',action,';'.join(k+'='+v for k,v in vals.items()),'v0.92 compiled from poi-mapping.csv'])
|
||||
if row.get('label_bg'):l.writerow([c,row['label_bg']])
|
||||
# Order matters: base rule first, then the explicit name-specific rules
|
||||
# in CSV order. C++ evaluates these against ORIGINAL Garmin names.
|
||||
for row in conditional:
|
||||
c=row['code'];action=(row.get('action') or 'replace').strip().lower()
|
||||
vals=tags_from_string(row.get('osm_tags',''))
|
||||
if action=='raw':action='keep_raw';vals={}
|
||||
w.writerow([c,row['name_contains'],action,
|
||||
';'.join(k+'='+v for k,v in vals.items()),
|
||||
'v0.95 name-specific rule: '+row.get('note','')])
|
||||
|
||||
def inspect_counts(osm,pois,lines,out,strict=False):
|
||||
seen=collections.Counter();unresolved=collections.Counter();annotations=collections.Counter();fallback=collections.Counter();bad=collections.Counter();routes=collections.Counter();examples={};category=collections.Counter();n=0
|
||||
@@ -143,7 +180,7 @@ def transform(source,dest,pois,lines,report,strict=False):
|
||||
db.execute('CREATE TABLE edges (code TEXT, way INTEGER, a INTEGER, b INTEGER)')
|
||||
poi_count=0; ways=0; candidate_edges=0; emitted=0; synthetic=0; kinds=collections.Counter();seen=collections.Counter();gaps=collections.Counter()
|
||||
with open(source,'rb') as inp,open(dest,'w',encoding='utf-8') as out:
|
||||
out.write('<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="garmin2osm-v0.93" upload="never">\n')
|
||||
out.write('<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="garmin2osm-v0.95" upload="never">\n')
|
||||
for el in top_level_elements(inp):
|
||||
if el.tag=='node':
|
||||
tags=tagdict(el)
|
||||
@@ -268,10 +305,10 @@ def main():
|
||||
a=p.parse_args()
|
||||
if a.mode=='template':res=template(a.catalog,a.poi,a.lines,a.osm)
|
||||
else:
|
||||
ps=mapping(a.poi);ls=mapping(a.lines)
|
||||
ps,conditional=read_poi_configuration(a.poi);ls=line_mapping(a.lines)
|
||||
if a.mode=='prepare':
|
||||
out=pathlib.Path(a.out);out.mkdir(parents=True,exist_ok=True)
|
||||
make_cpp_rules(ps,out/'cpp-poi-override.csv',out/'cpp-name-labels.csv');res={'poi_types':len(ps),'line_types':len(ls)}
|
||||
make_cpp_rules(ps,out/'cpp-poi-override.csv',out/'cpp-name-labels.csv',conditional);res={'poi_types':len(ps),'name_specific_rules':len(conditional),'line_types':len(ls)}
|
||||
elif a.mode=='transform':res=transform(a.osm,a.out,ps,ls,a.report)
|
||||
else:res=inspect_counts(a.osm,ps,ls,a.out,a.strict)
|
||||
print(json.dumps(res,ensure_ascii=False,indent=2));return 0
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import unittest, tempfile, csv, pathlib, sys
|
||||
sys.path.insert(0,str(pathlib.Path(__file__).resolve().parents[1]/"tools"))
|
||||
import audit_osmand_poi_mapping as a
|
||||
class TestPoiRegistry(unittest.TestCase):
|
||||
def test_registry_and_semantics(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
p=pathlib.Path(t)/'types.xml'
|
||||
p.write_text('''<poi_types><poi_type name="water" tag="amenity" value="drinking_water"/><poi_type name="locality" tag="place" value="locality" no_indx="true"/><poi_additional name="dwater" tag="drinking_water" value="yes"/></poi_types>''')
|
||||
reg,ex=a.registry(p)
|
||||
base={'code':'0x6414','source_count':'32922','priority':'critical','osm_tags':'amenity=drinking_water;drinking_water=yes'}
|
||||
self.assertEqual(a.interpret(base,reg,ex)['status'],'native_indexable_type')
|
||||
self.assertEqual(a.interpret({**base,'osm_tags':'place=locality'},reg,ex)['status'],'only_nonindexed_type')
|
||||
self.assertEqual(a.interpret({**base,'osm_tags':'drinking_water=yes'},reg,ex)['status'],'additional_without_primary_type')
|
||||
self.assertEqual(a.interpret({**base,'osm_tags':'man_made=water_intake'},reg,ex)['status'],'no_matching_native_type')
|
||||
self.assertEqual(a.interpret({**base,'code':'0x6600','priority':'terrain','osm_tags':''},reg,ex)['severity'],'terrain_exemption')
|
||||
if __name__=='__main__':unittest.main()
|
||||
+8
-8
@@ -4,7 +4,7 @@ import csv,importlib.util,json,pathlib,subprocess,sys,tempfile,unittest,xml.etre
|
||||
ROOT=pathlib.Path(__file__).resolve().parents[1]
|
||||
spec=importlib.util.spec_from_file_location('translator',ROOT/'scripts/native_translate.py')
|
||||
t=importlib.util.module_from_spec(spec);spec.loader.exec_module(t)
|
||||
POI=ROOT/'config/poi-mapping.csv';LINES=ROOT/'config/line-mapping.csv'
|
||||
POI=ROOT/'config/BGMountains-native-poi-mapping-v094.csv';LINES=ROOT/'config/line-mapping.csv'
|
||||
class Native092(unittest.TestCase):
|
||||
def test_native_source_semantics(self):
|
||||
rows=t.mapping(POI)
|
||||
@@ -15,17 +15,17 @@ class Native092(unittest.TestCase):
|
||||
self.assertIn(rows['0x6600']['action'],('append','raw'))
|
||||
self.assertFalse(any('leisure=marina' in r['osm_tags'] for r in rows.values()))
|
||||
self.assertFalse(any('tourism=information;information=board' in r['osm_tags'] for r in rows.values()))
|
||||
self.assertTrue(all('garmin' not in r['osm_tags'] for r in rows.values()))
|
||||
self.assertTrue(all('garmin:type' not in r['osm_tags'] for r in rows.values()))
|
||||
self.assertTrue(all(int(rows[k]['zoom_hint'])<=12 for k in ('0x6414','0x6511','0x6500','0x660a')))
|
||||
def test_config_only_two_and_types(self):
|
||||
self.assertEqual(set(p.name for p in (ROOT/'config').glob('*.csv')),{'poi-mapping.csv','line-mapping.csv'})
|
||||
self.assertTrue(any(x.get('route_mode')=='relation' for x in t.mapping(LINES).values()))
|
||||
for line in t.mapping(LINES).values():
|
||||
self.assertEqual(set(p.name for p in (ROOT/'config').glob('*.csv')),{'BGMountains-native-poi-mapping-v094.csv','line-mapping.csv'})
|
||||
self.assertTrue(any(x.get('route_mode')=='relation' for x in t.line_mapping(LINES).values()))
|
||||
for line in t.line_mapping(LINES).values():
|
||||
if line['route_mode']=='relation':self.assertTrue(line['osmc_symbol'])
|
||||
def test_native_relation_members_and_poi_provenance(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
d=pathlib.Path(directory);src=d/'sample.osm';dst=d/'native.osm';report=d/'report.json'
|
||||
line=next(r for r in t.mapping(LINES).values() if r['route_mode']=='relation')
|
||||
line=next(r for r in t.line_mapping(LINES).values() if r['route_mode']=='relation')
|
||||
typ=line['code']
|
||||
xml=['<?xml version="1.0" encoding="UTF-8"?><osm version="0.6">']
|
||||
for i in range(1,5):xml.append(f'<node id="{i}" lat="41.{i}" lon="23.{i}"/>')
|
||||
@@ -33,7 +33,7 @@ class Native092(unittest.TestCase):
|
||||
for i,refs in [(10,[1,2]),(11,[2,3])]:
|
||||
xml.append(f'<way id="{i}">'+''.join(f'<nd ref="{n}"/>' for n in refs)+f'<tag k="garmin:feature" v="extended_polyline"/><tag k="garmin:type" v="{typ}"/><tag k="bgm_route" v="type_{typ}"/></way>')
|
||||
xml.append('</osm>');src.write_text('\n'.join(xml))
|
||||
result=t.transform(src,dst,t.mapping(POI),t.mapping(LINES),report)
|
||||
result=t.transform(src,dst,t.mapping(POI),t.line_mapping(LINES),report)
|
||||
self.assertEqual(result['route_candidate_ways'],2);self.assertEqual(result['route_relations'],1)
|
||||
root=ET.parse(dst).getroot();rel=root.find('relation');tags=t.tagdict(rel)
|
||||
self.assertEqual(tags['route'],'hiking');self.assertTrue(tags['osmc:symbol'].endswith('_bar'))
|
||||
@@ -45,7 +45,7 @@ class Native092(unittest.TestCase):
|
||||
def test_route_attachment_to_real_way_only(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
d=pathlib.Path(directory);src=d/'src.osm';dst=d/'out.osm';rep=d/'report.json'
|
||||
typ=next(row['code'] for row in t.mapping(LINES).values() if row['route_mode']=='relation')
|
||||
typ=next(row['code'] for row in t.line_mapping(LINES).values() if row['route_mode']=='relation')
|
||||
parts=['<osm version="0.6">','<node id="1" lat="41" lon="23"/>','<node id="2" lat="41.1" lon="23.1"/>',
|
||||
'<node id="3" lat="41.2" lon="23.2"/>','<way id="22"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="highway" v="path"/></way>',
|
||||
'<way id="33"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="garmin:type" v="'+typ+'"/><tag k="garmin:feature" v="extended_polyline"/></way>',
|
||||
|
||||
+8
-8
@@ -10,25 +10,25 @@ class Reliability093(unittest.TestCase):
|
||||
out=pathlib.Path(d)
|
||||
src=out/'input.osm';dst=out/'output.osm';report=out/'report.json'
|
||||
src.write_text('''<osm version="0.6"><node id="5" lat="42.1345678" lon="24.2345678"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x9999"/><tag k="garmin:mapset" v="1234"/><tag k="bgm:generated_name" v="yes"/><tag k="name" v="Garmin обект 0x9999 - Q3ABCDEFG9"/></node><node id="6" lat="42.2" lon="24.2"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="176"/><tag k="name" v="176"/></node></osm>''')
|
||||
t.transform(src,dst,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
||||
t.transform(src,dst,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
||||
nodes=ET.parse(dst).getroot().findall('node');a=t.tagdict(nodes[0]);b=t.tagdict(nodes[1])
|
||||
self.assertEqual(a['place'],'locality');self.assertEqual(a['name'],'Неуточнено - Q3ABCDEFG9')
|
||||
self.assertEqual(a['garmin:type'],'0x9999');self.assertEqual(a['bgm:classification'],'unverified')
|
||||
self.assertNotIn('place',b);self.assertEqual(b['ele'],'176')
|
||||
data=t.inspect_counts(dst,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),out/'audit.json')
|
||||
data=t.inspect_counts(dst,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),out/'audit.json')
|
||||
self.assertEqual(data['synthetic_locality_fallback_total'],1)
|
||||
|
||||
def test_known_bgm_types_are_not_invented_localities(self):
|
||||
cfg=t.mapping(ROOT/'config/poi-mapping.csv')
|
||||
self.assertEqual(cfg['0x6600']['osm_tags'],'')
|
||||
cfg=t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv')
|
||||
self.assertIn('bgm:spot_height=yes',cfg['0x6600']['osm_tags']);self.assertEqual(cfg['0x6600']['action'],'append')
|
||||
self.assertNotIn('place=locality',cfg['0x6600']['osm_tags'])
|
||||
self.assertEqual(cfg['0x650b']['osm_tags'],'tourism=information;information=route_marker;hiking=yes')
|
||||
self.assertEqual(cfg['0x660b']['osm_tags'],'tourism=information;information=trail_blaze;hiking=yes')
|
||||
self.assertIn('information=trail_blaze',cfg['0x650b']['osm_tags'])
|
||||
self.assertIn('information=trail_blaze',cfg['0x660b']['osm_tags'])
|
||||
def test_fallback_does_not_overwrite_known_native_types(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
src=pathlib.Path(d)/'before.osm';out=pathlib.Path(d)/'after.osm';rep=pathlib.Path(d)/'rep.json'
|
||||
src.write_text('''<osm><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6400"/><tag k="landuse" v="quarry"/><tag k="name" v="Кариера П"/></node><node id="2" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6406"/><tag k="barrier" v="gate"/></node><node id="3" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x640d"/><tag k="aerialway" v="station"/></node></osm>''')
|
||||
t.transform(src,out,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),rep)
|
||||
t.transform(src,out,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),rep)
|
||||
nodes=ET.parse(out).getroot().findall('node')
|
||||
self.assertTrue(all('place' not in t.tagdict(e) for e in nodes))
|
||||
self.assertEqual(t.tagdict(nodes[0])['name'],'Кариера П')
|
||||
@@ -53,7 +53,7 @@ class Reliability093(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path=pathlib.Path(d)/'sample.osm';report=pathlib.Path(d)/'report.json'
|
||||
path.write_text('''<osm version="0.6"><node id="1" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="132"/></node><node id="2" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x650b"/><tag k="tourism" v="information"/><tag k="information" v="route_marker"/></node><node id="3" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6502"/><tag k="ford" v="yes"/></node></osm>''')
|
||||
stats=t.inspect_counts(path,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
||||
stats=t.inspect_counts(path,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
||||
self.assertEqual(stats['unclassified_total'],0)
|
||||
self.assertEqual(stats['annotations_not_missing_pois']['spot_height'],1)
|
||||
if __name__=='__main__':unittest.main()
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Version 0.95 regression checks, including its REAL published CLI contract.
|
||||
|
||||
Only mock OsmAndMapCreator is used for the orchestration test, not for proving
|
||||
OBF binary correctness. The real IMG tests were run separately during release.
|
||||
"""
|
||||
import csv,importlib.util,json,pathlib,subprocess,sys,tempfile,unittest,xml.etree.ElementTree as ET
|
||||
ROOT=pathlib.Path(__file__).resolve().parents[1]
|
||||
SPEC=importlib.util.spec_from_file_location('nt',ROOT/'scripts/native_translate.py')
|
||||
t=importlib.util.module_from_spec(SPEC);SPEC.loader.exec_module(t)
|
||||
POI=ROOT/'config/BGMountains-native-poi-mapping-v094.csv'
|
||||
LINES=ROOT/'config/line-mapping.csv'
|
||||
SCRIPT=ROOT/'scripts/build_bgmountains.py'
|
||||
class V095(unittest.TestCase):
|
||||
def test_mapping_handoff_has_all_named_exceptions(self):
|
||||
bases,conditional=t.read_poi_configuration(POI)
|
||||
self.assertEqual(len(bases),1871)
|
||||
self.assertEqual(len(conditional),79)
|
||||
self.assertEqual(len(t.line_mapping(LINES)),50)
|
||||
self.assertEqual(sum(x['code']=='0x6414' and x['name_contains']=='Сух' for x in conditional),1)
|
||||
with tempfile.TemporaryDirectory() as dd:
|
||||
d=pathlib.Path(dd)
|
||||
t.make_cpp_rules(bases,d/'cpp.csv',d/'labels.csv',conditional)
|
||||
rows=list(csv.DictReader((d/'cpp.csv').open()))
|
||||
self.assertEqual(len(rows),1950)
|
||||
self.assertEqual(next(x for x in rows if x['code']=='0x6414' and x['name_contains']=='')['tags'].split(';')[:2],['amenity=drinking_water','man_made=water_tap'])
|
||||
self.assertEqual(next(x for x in rows if x['code']=='0x6414' and x['name_contains']=='Сух')['tags'].split(';')[:2],['man_made=water_tap','drinking_water=no'])
|
||||
def test_argument_contract_with_users_flags(self):
|
||||
helptext=subprocess.run([sys.executable,SCRIPT,'--help'],capture_output=True,text=True,check=True).stdout
|
||||
for flag in ('--poi-config','--line-config','--download-creator','--jobs','--java-opts','--bbox','--prepare-only','--creator','--template'):
|
||||
self.assertIn(flag,helptext)
|
||||
self.assertNotIn('unrecognized arguments',helptext)
|
||||
def test_smoke_full_orchestration_with_mock_creator(self):
|
||||
with tempfile.TemporaryDirectory() as dd:
|
||||
d=pathlib.Path(dd)
|
||||
creator=d/'creator';creator.mkdir()
|
||||
(creator/'utilities.sh').write_text('#!/bin/sh\nset -eu\n[ "$1" = "generate-obf-no-address" ]\n[ -f "$2" ]\n[ "${JAVA_TOOL_OPTIONS:-}" = "-Xms512M -Xmx25G" ]\nprintf MOCK-OBF > "${2%.osm}.obf"\n')
|
||||
(creator/'utilities.sh').chmod(0o755)
|
||||
(creator/'inspector.sh').write_text('#!/bin/sh\n[ "$1" = "-vpoi" ]\nprintf MOCK-INSPECTION\n')
|
||||
(creator/'inspector.sh').chmod(0o755)
|
||||
raw=d/'minimal.osm';raw.write_text('''<osm version="0.6"><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6414"/><tag k="amenity" v="drinking_water"/><tag k="man_made" v="water_tap"/><tag k="name" v="Чешма"/></node></osm>''',encoding='utf-8')
|
||||
cmd=[sys.executable,str(SCRIPT),'--osm',str(raw),'--poi-config',str(POI),'--line-config',str(LINES),'--out',str(d/'out'),'--jobs','8','--java-opts','-Xms512M -Xmx25G','--creator',str(creator)]
|
||||
run=subprocess.run(cmd,cwd=ROOT,capture_output=True,text=True)
|
||||
self.assertEqual(run.returncode,0,run.stderr+'\n'+run.stdout)
|
||||
self.assertTrue((d/'out'/'BGMountains-native.obf').is_file())
|
||||
self.assertEqual(json.loads((d/'out'/'build-manifest.json').read_text())['version'],'v0.95')
|
||||
self.assertEqual(json.loads((d/'out'/'semantic-audit.json').read_text())['status'],'PASS')
|
||||
tags=t.tagdict(ET.parse(d/'out'/'BGMountains-native.osm').getroot().find('node'))
|
||||
self.assertEqual(tags['amenity'],'drinking_water')
|
||||
self.assertEqual(tags['man_made'],'water_tap')
|
||||
def test_future_template_does_not_require_existing_configs(self):
|
||||
with tempfile.TemporaryDirectory() as dd:
|
||||
d=pathlib.Path(dd)
|
||||
# No fake success from a malformed configuration: expected exit nonzero.
|
||||
p=subprocess.run([sys.executable,SCRIPT,'--osm',str(d/'notthere.osm'),'--poi-config',str(d/'missing.csv'),'--line-config',str(LINES),'--prepare-only'],capture_output=True,text=True)
|
||||
self.assertNotEqual(p.returncode,0)
|
||||
self.assertIn('mapping file not found',p.stderr)
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit Garmin -> OSM POI mappings against an exact OsmAnd poi_types.xml.
|
||||
|
||||
No OBF creation. Exits 0 for ordinary data regressions unless --strict is requested.
|
||||
The existence of a poi_type does NOT guarantee map icon visibility at any zoom.
|
||||
"""
|
||||
import argparse,csv,json,collections,sys
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
def registry(path):
|
||||
root=ET.parse(path).getroot()
|
||||
main=collections.defaultdict(list);extras=collections.defaultdict(list)
|
||||
for node in root.iter():
|
||||
if node.tag not in ('poi_type','poi_additional'):continue
|
||||
t=node.attrib.get('tag');v=node.attrib.get('value')
|
||||
if not t or v is None:continue
|
||||
d=main if node.tag=='poi_type' else extras
|
||||
d[f'{t}={v}'].append(node.attrib)
|
||||
return main,extras
|
||||
|
||||
def interpret(row,main,extra):
|
||||
tags=[v.strip() for v in row['osm_tags'].split(';') if '=' in v]
|
||||
supported=[];noindex=[];additional=[];missing=[]
|
||||
for tv in tags:
|
||||
candidates=main.get(tv,[])
|
||||
if any(x.get('no_indx')!='true' for x in candidates):supported.append(tv)
|
||||
elif candidates:noindex.append(tv)
|
||||
elif tv in extra:additional.append(tv)
|
||||
else:missing.append(tv)
|
||||
if supported:kind='native_indexable_type'
|
||||
elif noindex:kind='only_nonindexed_type'
|
||||
elif additional:kind='additional_without_primary_type'
|
||||
else:kind='no_matching_native_type'
|
||||
is_terrain=(row.get('priority')=='terrain' or row.get('code')=='0x6600')
|
||||
priority=row.get('priority','')
|
||||
cnt=int(row.get('source_count') or 0)
|
||||
if is_terrain:severity='terrain_exemption'
|
||||
elif kind=='native_indexable_type':severity='ok'
|
||||
elif priority=='critical':severity='critical'
|
||||
elif cnt>=100:severity='high'
|
||||
else:severity='low'
|
||||
return {'code':row.get('code',''),'count':cnt,'priority':priority,'legend_bg':row.get('legend_bg',''),
|
||||
'osm_tags':row.get('osm_tags',''), 'status':kind, 'severity':severity,
|
||||
'indexable_poi_tags':';'.join(supported),'nonindexed_poi_tags':';'.join(noindex),
|
||||
'additional_only_tags':';'.join(additional),'unrecognized_tags':';'.join(missing),
|
||||
'sample_lat':row.get('example_lat',''),'sample_lon':row.get('example_lon',''),
|
||||
'explanation':'Registry compatibility only; does not guarantee rendering zoom or OBF preservation'}
|
||||
|
||||
def main(argv=None):
|
||||
ap=argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--mapping',required=True,help='Single canonical POI mapping CSV')
|
||||
ap.add_argument('--poi-types',required=True,help='OsmAnd official poi_types.xml, preferably pinned version')
|
||||
ap.add_argument('--report',default='poi-native-index-audit.csv')
|
||||
ap.add_argument('--json',default='poi-native-index-audit.json')
|
||||
ap.add_argument('--strict',action='store_true',help='Exit nonzero for unindexable critical/high types')
|
||||
args=ap.parse_args(argv)
|
||||
native,extra=registry(args.poi_types)
|
||||
with open(args.mapping,encoding='utf8-sig',newline='') as f:
|
||||
allrows=list(csv.DictReader(f))
|
||||
# Name-conditional rows are not additional Garmin types; counting them as
|
||||
# independent types would multiply source_count and inflate POI statistics.
|
||||
rows=[r for r in allrows if not r.get('name_contains','')]
|
||||
out=[interpret(r,native,extra) for r in rows]
|
||||
with open(args.report,'w',newline='',encoding='utf8') as f:
|
||||
w=csv.DictWriter(f,fieldnames=out[0].keys());w.writeheader();w.writerows(out)
|
||||
stats=collections.Counter(r['status'] for r in out)
|
||||
weighted=collections.Counter()
|
||||
for r in out:weighted[r['status']]+=r['count']
|
||||
risk=[r for r in out if r['severity'] in ('critical','high')]
|
||||
payload={'type_counts':dict(stats),'weighted_source_counts':dict(weighted),'critical_or_high':len(risk),
|
||||
'top_regressions':sorted(risk,key=lambda z:-z['count'])[:30],
|
||||
'warning':'POI type registry check ONLY. Does not guarantee OBF map index presence, symbol visibility, zoom or Nearby settings.'}
|
||||
Path(args.json).write_text(json.dumps(payload,ensure_ascii=False,indent=2),encoding='utf8')
|
||||
print(f"Inspected {len(out)} Garmin types: {dict(stats)}, high/critical issues: {len(risk)}")
|
||||
print(f"Wrote {args.report} and {args.json}")
|
||||
return 2 if args.strict and risk else 0
|
||||
|
||||
if __name__=='__main__':sys.exit(main())
|
||||
Reference in New Issue
Block a user