Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3433135790 |
+7
-7
@@ -1,5 +1,5 @@
|
|||||||
cmake_minimum_required(VERSION 3.16)
|
cmake_minimum_required(VERSION 3.16)
|
||||||
project(garmin2osmand VERSION 0.8.0 LANGUAGES CXX)
|
project(garmin2osmand VERSION 0.95.0 LANGUAGES CXX)
|
||||||
set(CMAKE_CXX_STANDARD 17)
|
set(CMAKE_CXX_STANDARD 17)
|
||||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||||
find_package(PNG REQUIRED)
|
find_package(PNG REQUIRED)
|
||||||
@@ -15,18 +15,18 @@ else()
|
|||||||
target_compile_options(garmin2osmand PRIVATE -Wall -Wextra -Wpedantic -O2)
|
target_compile_options(garmin2osmand PRIVATE -Wall -Wextra -Wpedantic -O2)
|
||||||
endif()
|
endif()
|
||||||
enable_testing()
|
enable_testing()
|
||||||
add_test(NAME help COMMAND garmin2osmand --help)
|
add_test(NAME help COMMAND garmin2osm --help)
|
||||||
|
|
||||||
find_package(Python3 COMPONENTS Interpreter QUIET)
|
find_package(Python3 COMPONENTS Interpreter QUIET)
|
||||||
if(Python3_Interpreter_FOUND)
|
if(Python3_Interpreter_FOUND)
|
||||||
add_test(NAME classic_and_gmp_end_to_end COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_smoke.py $<TARGET_FILE:garmin2osmand>)
|
add_test(NAME legacy_native_vector_parser COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_vector.py $<TARGET_FILE:garmin2osm>)
|
||||||
add_test(NAME native_vector_export COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_vector.py $<TARGET_FILE:garmin2osm>)
|
add_test(NAME v092_native_osm_translation COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v092.py)
|
||||||
add_test(NAME bgmountains_legend_audit COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_legend_audit.py)
|
|
||||||
endif()
|
endif()
|
||||||
if(Python3_Interpreter_FOUND)
|
if(Python3_Interpreter_FOUND)
|
||||||
add_test(NAME osmand_obf_pipeline_xml COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_obf_pipeline.py)
|
add_test(NAME v093_memory_and_mapping_regressions COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v093.py)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(Python3_Interpreter_FOUND)
|
if(Python3_Interpreter_FOUND)
|
||||||
add_test(NAME bgmountains_v08_display_mappings COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v08_config.py)
|
add_test(NAME v095_cli_config_and_obf_orchestration COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_v095.py)
|
||||||
|
add_test(NAME native_registry_audit COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_registry_audit.py)
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
@@ -1,103 +1,169 @@
|
|||||||
# BGMountains Garmin IMG → OSM → OsmAnd OBF v0.8
|
# BGMountains Garmin IMG → native OsmAnd OBF — v0.95
|
||||||
|
|
||||||
**This release fixes the v0.7 icon-visibility blind spot and anonymous POIs.**
|
**This is the integrated source release.** It replaces the v0.92/v0.93 Python
|
||||||
|
pipeline and v0.94 standalone CSVs. Do not combine scripts from older ZIPs:
|
||||||
|
those versions accept different flags and do not apply the 79 name-conditional
|
||||||
|
rules. This archive has **one POI config and one line config**.
|
||||||
|
|
||||||
- Every one of the **263 validated retained Garmin POI types** has at least one conventional OSM semantic/display tag in `config/poi-overrides-all-types.csv`, in addition to raw `bgm_poi`/`garmin:*` provenance tags.
|
## Quick start (Ubuntu/Debian)
|
||||||
- **87 codes** occur at least 100 times in the source catalog (409,154 source POIs). **46** use conventional feature categories recognized by stock OsmAnd in principle; **41** use an explicitly marked, synthetic `tourism=information;information=board` **local-OsmAnd display fallback**. Some additional lower-frequency codes also need fallback. This DOES NOT assert that the physical object is an information board.
|
|
||||||
- Missing original IMG names become stable Bulgarian descriptive names, e.g. `Вода - Каптаж - 1THF1SDNTKJ`, with `bgm:generated_name=yes`. Original nonempty names are preserved.
|
|
||||||
- `--jobs N` enables parallel extraction, **default one thread**.
|
|
||||||
- New `tools/audit_icon_coverage.py` verifies source OSM classification coverage, names and produces per-type coordinate report. It **does not** prove OBF index presence or visual display; use an OBF inspector and OsmAnd for that.
|
|
||||||
- Map-derived icon art and datasets remain governed by BGMountains' applicable copyright conditions. These synthetic POI tags are **not appropriate for uploading unchanged to the real OpenStreetMap database**.
|
|
||||||
|
|
||||||
## Quick build (Ubuntu/Debian)
|
```bash
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo apt update
|
sudo apt update
|
||||||
sudo apt install -y build-essential cmake libpng-dev libsqlite3-dev python3 openjdk-21-jre-headless unzip
|
sudo apt install -y cmake g++ make libpng-dev libsqlite3-dev python3 openjdk-21-jre-headless unzip
|
||||||
|
unzip BGMountains-garmin2osm-v0.95-integrated.zip
|
||||||
|
cd BGMountains-garmin2osm-v0.95
|
||||||
|
```
|
||||||
|
|
||||||
|
Use **exactly** this from the release root (where `scripts/` and `config/` are):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nohup python3 scripts/build_bgmountains.py \
|
||||||
|
--img "$HOME/gmapsupp.img" \
|
||||||
|
--poi-config config/BGMountains-native-poi-mapping-v094.csv \
|
||||||
|
--line-config config/line-mapping.csv \
|
||||||
|
--out bg-mountains-v095-build \
|
||||||
|
--download-creator \
|
||||||
|
--jobs 8 \
|
||||||
|
--java-opts '-Xms512M -Xmx25G' > bg-v095.log 2>&1 &
|
||||||
|
```
|
||||||
|
|
||||||
|
The `--poi-config` and `--line-config` arguments are **implemented and tested**.
|
||||||
|
Other supported flags: `--creator /path/to/OsmAndMapCreator`, `--bbox W,S,E,N`,
|
||||||
|
`--prepare-only`, `--keep-raw`, `--template`, `--attach-routes`,
|
||||||
|
`--disk-headroom-gb N`, `--ignore-disk-check`, `--jobs N` (default 1).
|
||||||
|
|
||||||
|
**Check your actual disk space first.** Full-country MapCreator compilation has
|
||||||
|
previously required over 25 GiB of temporary disk space in addition to the
|
||||||
|
source OSM (6+ GiB) on your machine. The script uses a conservative estimate
|
||||||
|
`max(2 GiB, 5 × native-OSM-size + 1 GiB)`, unless `--disk-headroom-gb`
|
||||||
|
requires more. These figures are not upper bounds. Choose an output filesystem
|
||||||
|
with generous free space. `-Xmx25G` controls heap RAM, **not disk usage**.
|
||||||
|
|
||||||
|
## Safe regional test, no Java or extra download
|
||||||
|
|
||||||
|
```bash
|
||||||
python3 scripts/build_bgmountains.py \
|
python3 scripts/build_bgmountains.py \
|
||||||
--img /absolute/path/to/gmapsupp.img \
|
--img "$HOME/gmapsupp.img" \
|
||||||
--out ./BGMountains-out \
|
--poi-config config/BGMountains-native-poi-mapping-v094.csv \
|
||||||
--download-creator
|
--line-config config/line-mapping.csv \
|
||||||
```
|
--bbox '25.05,42.43,25.16,42.54' \
|
||||||
|
--jobs 8 \
|
||||||
`--download-creator` is for the official OsmAndMapCreator, and the pipeline downloads the original official `poi_types.xml` and `rendering_types.xml` the first time as needed. Network access is required for these unless you already supply the files. **This runtime did not have MapCreator, so full OBF compilation was not executed here.**
|
--out ./regional-test \
|
||||||
|
|
||||||
Already installed OsmAndMapCreator:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
python3 scripts/build_bgmountains.py --img ./gmapsupp.img --out ./BGMountains-out \
|
|
||||||
--creator /path/to/OsmAndMapCreator
|
|
||||||
```
|
|
||||||
|
|
||||||
Opt-in parallelism: add `--jobs 4`. The output directory must be fresh: the converter refuses to overwrite an existing OSM or staging files.
|
|
||||||
|
|
||||||
Prepare and validate the OSM without invoking the Java OBF builder:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
python3 scripts/build_bgmountains.py --img ./gmapsupp.img --out ./BGMountains-preview \
|
|
||||||
--prepare-only
|
--prepare-only
|
||||||
```
|
```
|
||||||
|
|
||||||
**IMPORTANT: You must REBUILD the OSM and OBF.** This update cannot modify an already compiled `.obf` or give v0.7 output its new tags in place.
|
Inspect `regional-test/BGMountains-native.osm`, `semantic-audit.json`,
|
||||||
|
`route-report.json`, `extract.log`, `translate.log`, `audit.log`. The tool
|
||||||
|
must exit nonzero on a failed conversion subprocess. You can instead pass an
|
||||||
|
existing MapCreator with `--creator ...` (omit `--prepare-only`) for a small
|
||||||
|
real OBF build.
|
||||||
|
|
||||||
## What the configuration files mean
|
## Two editable configuration files
|
||||||
|
|
||||||
| File | Purpose |
|
### 1. `config/BGMountains-native-poi-mapping-v094.csv`
|
||||||
|---|---|
|
|
||||||
| `config/poi-overrides-all-types.csv` | **Active** 1,871-row config, including 263 retained types with revised tags. `mode=replace` replaces old hardcoded semantics, retaining provenance/name. All non-present legacy type rows remain for historical comparison. |
|
|
||||||
| `config/poi-overrides-all-types-v07-user-upload.csv` | Exact backup of the CSV you supplied. |
|
|
||||||
| `config/poi-labels-bg.csv` | New 263-row Bulgarian fallback name prefixes, editable. |
|
|
||||||
| `config/poi-icon-audit.csv` | All 263 types, count, original tags vs new tags, classification confidence and sample coordinates. |
|
|
||||||
| `config/osmand-icons.csv` | Optional **custom render style** icon names/visibility; never controls the default `tourism=information` fallback injected through the active overrides. |
|
|
||||||
| `data/BGMountains-frequent-poi-types-100plus-v08.csv` | All 87 frequent types, with reviewed tags, classification mode and coordinates. |
|
|
||||||
| `tools/audit_icon_coverage.py` | Independent POI OSM field audit; one row per present type. |
|
|
||||||
|
|
||||||
**How to edit a type:** find `code=0x6500` in active overrides. Edit the **tags** column, leaving a `key=value;key=value` list. Change `mode=replace` only if you understand how the C++ built-in classifications merge. Set a Bulgarian text prefix in `config/poi-labels-bg.csv`. Run again from scratch to rebuild OSM then OBF.
|
1,871 base hex-code entries plus 79 **name-specific** exceptions (1,950 data
|
||||||
|
rows). Columns include: `code`, `source_count`, `legend_bg`, `legend_en`,
|
||||||
|
`meaning_confidence`, `osm_tags`, `label_bg`, `priority`, `zoom_hint`, `action`,
|
||||||
|
`note`, examples, and optional `name_contains`. For conditional matching,
|
||||||
|
set `name_contains` in an additional row with the same `code`. The matcher uses
|
||||||
|
case-sensitive substring matching on the **original Garmin name**, with rules
|
||||||
|
applied in file order after the base rule. `replace` changes the converter's
|
||||||
|
semantic defaults; `append` retains previous tags; `raw` leaves unspecified
|
||||||
|
features unclassified until private-map fallback; `drop` discards the feature.
|
||||||
|
|
||||||
### Why 0x6500 is not `amenity=drinking_water`
|
The Python pipeline automatically creates `cpp-poi-override.csv` and
|
||||||
|
`cpp-name-labels.csv` **inside the build output directory**. These are generated
|
||||||
|
intermediates, **not third and fourth user configuration files**. There is no
|
||||||
|
need to manually supply them or an icon pack to OsmAnd.
|
||||||
|
|
||||||
The original legend labels this a **каптаж** (water intake/catchment), not necessarily a drinking-water source. v0.8 records `man_made=water_intake` but adds an explicit *presentation-only* `tourism=information;information=board;bgm:synthetic_icon=yes` for common OsmAnd rendering. The "information board" is NOT a physical claim. The default Bulgarian name prefix is `Вода - Каптаж`. If you want `Вода - Резервоар`, just edit the label CSV; do not misrepresent a small intake as an artificial lake (`natural=water;water=reservoir`).
|
Important native examples:
|
||||||
|
|
||||||
`0x6602` already has `tourism=viewpoint` and does not use a synthetic board marker. Its failure to appear in a particular existing OBF could be an old build, incomplete POI indexing, or display suppression—only inspecting that actual OBF can distinguish these.
|
- `0x6414`: `amenity=drinking_water;man_made=water_tap` (legacy v18 dual
|
||||||
|
classification); named dry sources such as `(Сух.)` remove drinking-water
|
||||||
|
classification and get `drinking_water=no`.
|
||||||
|
- `0x6500`: `man_made=water_intake;amenity=water_point;drinking_water=unknown`
|
||||||
|
for **private-map index compatibility**, not evidence that its water is
|
||||||
|
potable, available or accessible. Named shaft (`Шах...`) exceptions remove
|
||||||
|
the compatibility alias.
|
||||||
|
- `0x6511`: `natural=spring` (potability not asserted).
|
||||||
|
- `0x6402`: `building=yes` by default, explicitly named `Заслон` or `Хижа`
|
||||||
|
may become shelter/hut (availability/access not verified).
|
||||||
|
- `0x661d`: agricultural structures remain agricultural unless specifically
|
||||||
|
identified otherwise.
|
||||||
|
- `0x6600`: keep numeric spot height `ele=*`, **no fake settlement**.
|
||||||
|
- Unknown POIs: `place=locality`, `Неуточнено - <stable hash>` if unnamed,
|
||||||
|
and `bgm:synthetic_fallback=place_locality`; this private-map fallback **does
|
||||||
|
not guarantee Nearby POI indexing**.
|
||||||
|
|
||||||
### Stable anonymous names
|
`priority` and `zoom_hint` are documented classification hints only. They do
|
||||||
|
**not force stock OsmAnd to show an icon at zoom 12**. Likewise `garmin:type`
|
||||||
|
remains a provenance tag, not a display rule. Only stock OsmAnd classifications
|
||||||
|
and rendering apply to the resulting OBF.
|
||||||
|
|
||||||
For an unnamed POI, `name = Bulgarian_label + " - " + FNV1a_base36_11(mapset|type|quantized_lat|quantized_lon)`. The output is deterministic for an identical IMG and coordinates across threads and builds. A very small collision probability remains; the coordinates/mapset/type remain available. These generated names are for an offline map, **not actual surveyed names**.
|
### 2. `config/line-mapping.csv`
|
||||||
|
|
||||||
### Test against your three sample coordinates
|
50 Garmin line types (20 currently have `route_mode=relation`, 30 preserve
|
||||||
|
source geometry). Columns include Garmin `code`, legend, color,
|
||||||
|
`osm_way_tags`, `route_mode`, `osmc_symbol`, `network`, confidence and priority.
|
||||||
|
Selected source overlay segments are grouped into native
|
||||||
|
`type=route;route=hiking` relations with `osmc:symbol`. Garmin source color is
|
||||||
|
preserved as relation `colour=*`. The script does not create duplicate routable
|
||||||
|
paths from decoration lines. Opt-in `--attach-routes` matches overlays to real
|
||||||
|
existing path geometries, but can use significant memory; it is off by default.
|
||||||
|
**Existing physical path routing remains in the data.** Exact phone-side color,
|
||||||
|
icon and at-zoom visibility still depend on stock OsmAnd.
|
||||||
|
|
||||||
| Type | Coordinates | Expected tags | v0.8 generated name |
|
## New or unknown IMG maps: generate configuration templates
|
||||||
|---|---|---|---|
|
|
||||||
| `0x6602` | `42.0346141,24.6528697` | `tourism=viewpoint` | `Панорама - Гледка - 25IVP49H823` |
|
|
||||||
| `0x6500` | `42.0347643,24.6522689` | `man_made=water_intake;tourism=information;information=board` | `Вода - Каптаж - 1THF1SDNTKJ` |
|
|
||||||
| `0x6500` | `42.0340991,24.6520114` | Same | `Вода - Каптаж - ED03D455VTZ` |
|
|
||||||
|
|
||||||
These examples were directly extracted from the 422MiB IMG into a Plovdiv test OSM and compared to a source manifest. All 57 POIs in that test passed exact semantic-tag checks, with **zero missing and extra**. Serial vs `--jobs 4` POI identities/tags were equal.
|
```bash
|
||||||
|
python3 scripts/build_bgmountains.py --img /path/to/unknown.img \
|
||||||
## Review before copying `.obf` into OsmAnd
|
--out ./template-newmap --bbox '23.0,42.0,24.0,43.0' --template
|
||||||
|
|
||||||
Pipeline artifacts in `--out`:
|
|
||||||
- `BGMountains.osm` — exact source XML for the OBF builder
|
|
||||||
- `source-pois.csv`, `poi-types.csv` — extraction manifest and type counts
|
|
||||||
- `validation.json`, `mismatches.csv` — original-source to OSM audit
|
|
||||||
- `poi-coverage.csv`, `poi-coverage.json` — export display mapping coverage
|
|
||||||
- `osmand-config/poi_types.bgmountains.xml`, `rendering_types.bgmountains.xml` — registrations added to **complete upstream XML**
|
|
||||||
- `osmand-config/BGMountains-custom.render.xml` — optional style; exact Garmin TYP art is NOT embedded into stock OsmAnd
|
|
||||||
- `*.obf` plus `mapcreator.log`, `obf-poi-index.txt` (only after a successful Java compilation)
|
|
||||||
|
|
||||||
The `tourism=information` fallback should give OsmAnd a native searchable POI category in an appropriate compatible build. **Neither an OSM tag nor an indexed OBF proves a symbol will be drawn at every zoom**: the map style, profile, POI filters, icon availability, density/collision and selected offline map still affect visibility. Test at zoom 17–20 with POI layers enabled. Avoid having an old overlapping BGMountains OBF selected at the same time.
|
|
||||||
|
|
||||||
To inspect your built file with OsmAndMapCreator:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
/path/to/OsmAndMapCreator/inspector.sh -vpoi -vpoiobjects \
|
|
||||||
-bbox=24.64,42.05,24.665,42.02 ./BGMountains-out/*.obf \
|
|
||||||
> ./BGMountains-out/plovdiv-obf-inspector.txt
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Inspect output for the three named POIs. **There is no tested generic parser of OsmAndMapCreator's inspector text in this package**, and we have not verified a full-country OBF built from v0.8 in this runtime. If OSM has the POI but the inspector doesn't, focus on indexing or generated XML; if inspector has it but the screen doesn't, focus on style/POI display/filter/zoom.
|
Produces `poi-mapping.TEMPLATE.csv` and `line-mapping.TEMPLATE.csv` with
|
||||||
|
observed types and sample coordinates. Fill them manually and run with
|
||||||
|
`--poi-config ./template-newmap/poi-mapping.TEMPLATE.csv` and
|
||||||
|
`--line-config ./template-newmap/line-mapping.TEMPLATE.csv`. **Do not assume
|
||||||
|
Garmin codes have the same meaning in unrelated maps.** If you need a complete
|
||||||
|
catalog for a new national map, generate its templates without `--bbox`, but
|
||||||
|
this can require significant time and intermediate disk space.
|
||||||
|
|
||||||
### Local-only licensing warning
|
## Validation and provenance
|
||||||
|
|
||||||
Do not upload the generated synthetic display tags to OpenStreetMap or assert they accurately describe physical information boards. The original IMG, derived icons and extracts can have third-party copyright restrictions. The source-only code path should be distributed separately from map-derived sample assets until rights are confirmed.
|
```bash
|
||||||
|
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
|
||||||
|
cmake --build build -j 4
|
||||||
|
ctest --test-dir build --output-on-failure
|
||||||
|
```
|
||||||
|
|
||||||
|
There is also `tools/audit_osmand_poi_mapping.py`, which takes the official
|
||||||
|
version-matched OsmAnd `poi_types.xml` and produces a registry compatibility
|
||||||
|
report, **not a claim of final icon visibility**:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 tools/audit_osmand_poi_mapping.py \
|
||||||
|
--mapping config/BGMountains-native-poi-mapping-v094.csv \
|
||||||
|
--poi-types /path/to/poi_types.xml \
|
||||||
|
--report native-poi-audit.csv --json native-poi-audit.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Source: https://github.com/osmandapp/OsmAnd-resources/blob/master/poi/poi_types.xml
|
||||||
|
|
||||||
|
## Tested, and NOT tested
|
||||||
|
|
||||||
|
- C++ Release build, 6 CTest test groups, conditional POI CSV generation,
|
||||||
|
CLI options, and simulated MapCreator orchestration passed.
|
||||||
|
- Original 422 MiB IMG: regional `--jobs 8` and serial preparation succeeded
|
||||||
|
with identical 427 POI coordinate/name/tag records and 6 route relations.
|
||||||
|
- Separate region: `(Сух.)` water-tap conditional correctly produced
|
||||||
|
`man_made=water_tap;drinking_water=no`, and no `amenity=drinking_water`.
|
||||||
|
- **A real OsmAndMapCreator binary was not available here**; its host was
|
||||||
|
unreachable. The OBF orchestration unit test uses a **mock**, so native
|
||||||
|
nationwide OBF compilation and phone rendering are not verified. Do not
|
||||||
|
mistake that mock for real OBF validation.
|
||||||
|
|
||||||
|
License: the source bundle contains historically BGMountains-derived
|
||||||
|
classification information. Treat that data as private-map material under
|
||||||
|
applicable source licenses; the package is **not** authorization to republish
|
||||||
|
BGMountains geography or to import its features into the public OSM database.
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"release": "v0.95",
|
||||||
|
"code_groups": 6,
|
||||||
|
"test_groups_passed": 6,
|
||||||
|
"real_img_source": "gmapsupp(2).img (original supplied, 441942016 bytes)",
|
||||||
|
"real_regional_bbox": "25.05,42.43,25.16,42.54",
|
||||||
|
"real_regional_pois": 427,
|
||||||
|
"real_regional_routes": 6,
|
||||||
|
"parallel_workers_tested": [
|
||||||
|
1,
|
||||||
|
8
|
||||||
|
],
|
||||||
|
"serial_parallel_poi_tags_names_equal": true,
|
||||||
|
"conditional_dry_water_result": "man_made=water_tap;drinking_water=no; amenity=drinking_water absent",
|
||||||
|
"java_real_obf_build": "NOT TESTED (official host unavailable)",
|
||||||
|
"obf_mock_orchestration": "PASSED; not a real OBF binary"
|
||||||
|
}
|
||||||
Binary file not shown.
+144
-326
@@ -1,341 +1,159 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""One-command BGMountains IMG -> reviewed OSM -> custom indexed OsmAnd OBF.
|
"""BGMountains v0.95 native OsmAnd pipeline: only TWO editable mapping files.
|
||||||
|
|
||||||
Only Python 3 stdlib needed for orchestration; CMake/C++/OsmAndMapCreator are external.
|
Never needs extracted Garmin icons or custom styles on Android.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import argparse
|
import argparse,datetime,hashlib,json,os,pathlib,shutil,subprocess,sys,time,urllib.request,zipfile
|
||||||
import csv
|
ROOT=pathlib.Path(__file__).resolve().parents[1]
|
||||||
import datetime as dt
|
TRANSLATOR=ROOT/'scripts/native_translate.py'
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import urllib.request
|
|
||||||
import xml.etree.ElementTree as ET
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
def run(*args,log=None,cwd=None,env=None):
|
||||||
UPSTREAM = {
|
cmd=[str(x) for x in args];print('+',' '.join(cmd),flush=True)
|
||||||
'poi': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/poi/poi_types.xml',
|
|
||||||
'rendering': 'https://raw.githubusercontent.com/osmandapp/OsmAnd-resources/master/obf_creation/rendering_types.xml'
|
|
||||||
}
|
|
||||||
|
|
||||||
def call(argv, cwd=None, log=None, env=None):
|
|
||||||
argv = [str(s) for s in argv]
|
|
||||||
print('+', ' '.join(shlex_quote(s) for s in argv), flush=True)
|
|
||||||
if log:
|
if log:
|
||||||
with Path(log).open('w', encoding='utf-8') as output:
|
with open(log,'w',encoding='utf-8') as f:
|
||||||
r = subprocess.run(argv, cwd=cwd, env=env, stdout=output, stderr=subprocess.STDOUT)
|
p=subprocess.run(cmd,stdout=f,stderr=subprocess.STDOUT,cwd=cwd,env=env)
|
||||||
if r.returncode:
|
if p.returncode:
|
||||||
print('Command failed; last log lines:', file=sys.stderr)
|
print(''.join(pathlib.Path(log).read_text(errors='replace').splitlines(True)[-40:]),file=sys.stderr)
|
||||||
print('\n'.join(Path(log).read_text(errors='replace').splitlines()[-35:]), file=sys.stderr)
|
raise RuntimeError('Command failed, see '+str(log))
|
||||||
raise SystemExit(r.returncode)
|
else:subprocess.run(cmd,check=True,cwd=cwd,env=env)
|
||||||
else:
|
|
||||||
subprocess.run(argv, cwd=cwd, env=env, check=True)
|
|
||||||
|
|
||||||
def shlex_quote(s):
|
def creator_dir(x):
|
||||||
import shlex
|
if not x:return None
|
||||||
return shlex.quote(s)
|
p=pathlib.Path(x).expanduser().resolve();p=p.parent if p.name=='utilities.sh' else p
|
||||||
|
if not (p/'utilities.sh').exists():raise FileNotFoundError('MapCreator utilities.sh missing: '+str(p))
|
||||||
|
return p
|
||||||
|
|
||||||
def base_xml(provided, creator, key, cache, offline):
|
def fetch_creator(target):
|
||||||
if provided:
|
extracted=target/'unpacked';c=list(extracted.rglob('utilities.sh')) if extracted.exists() else []
|
||||||
p = Path(provided).expanduser().resolve()
|
if c:return c[0].parent
|
||||||
if not p.is_file(): raise SystemExit('Upstream XML not found: '+str(p))
|
target.mkdir(parents=True,exist_ok=True)
|
||||||
return p
|
url='https://download.osmand.net/latest-night-build/OsmAndMapCreator-main.zip'
|
||||||
names = ['poi_types.xml'] if key == 'poi' else ['rendering_types.xml']
|
zip_path=target/'OsmAndMapCreator-main.zip'
|
||||||
if creator:
|
if not zip_path.exists():
|
||||||
for prefix in [creator, creator/'resources', creator/'config', creator/'poi', creator/'obf_creation']:
|
print('Fetching official MapCreator:',url)
|
||||||
for name in names:
|
with urllib.request.urlopen(url,timeout=120) as src,open(zip_path,'wb') as dst:shutil.copyfileobj(src,dst)
|
||||||
candidate = prefix/name
|
with zipfile.ZipFile(zip_path) as z:
|
||||||
if candidate.is_file(): return candidate.resolve()
|
for entry in z.infolist():
|
||||||
target = cache/names[0]
|
parts=pathlib.PurePosixPath(entry.filename)
|
||||||
if target.is_file(): return target
|
if parts.is_absolute() or '..' in parts.parts:raise ValueError('Unsafe creator ZIP entry')
|
||||||
if offline:
|
z.extractall(extracted)
|
||||||
raise SystemExit('Missing upstream '+names[0]+': pass --'+key+'-base FILE or allow download')
|
c=list(extracted.rglob('utilities.sh'))
|
||||||
target.parent.mkdir(parents=True,exist_ok=True)
|
if not c:raise FileNotFoundError('Creator ZIP lacks utilities.sh')
|
||||||
print('Fetching official OsmAnd '+names[0]+' (first run only) ...', flush=True)
|
for f in extracted.rglob('*.sh'):f.chmod(f.stat().st_mode|0o111)
|
||||||
try:
|
return c[0].parent
|
||||||
with urllib.request.urlopen(UPSTREAM[key], timeout=50) as u, target.open('wb') as out:
|
|
||||||
shutil.copyfileobj(u,out)
|
|
||||||
except Exception as ex:
|
|
||||||
target.unlink(missing_ok=True)
|
|
||||||
raise SystemExit('Cannot fetch official OsmAnd '+names[0]+': '+str(ex)+'\nProvide --'+key+'-base FILE manually.')
|
|
||||||
return target
|
|
||||||
|
|
||||||
def catalog_rows(p):
|
def disk_preflight(out, source_bytes, extra_gb=0):
|
||||||
with Path(p).open('r',encoding='utf-8-sig',newline='') as f:
|
"""Conservative warning/blocking check before disk-heavy MapCreator stage.
|
||||||
rows=list(csv.DictReader(f))
|
In experience 6 GiB OSM has required >25 GiB scratch. 5x + 1 GiB
|
||||||
if not rows or not {'code','count','has_typ_icon'}.issubset(rows[0]):
|
is intentionally approximate, not a guarantee of completion.
|
||||||
raise SystemExit('Invalid POI catalog (expected code,count,has_typ_icon): '+str(p))
|
|
||||||
out=[]
|
|
||||||
seen=set()
|
|
||||||
for r in rows:
|
|
||||||
code='0x'+format(int(r['code'],0),'x')
|
|
||||||
if code in seen:raise SystemExit('Duplicate code in catalog: '+code)
|
|
||||||
seen.add(code)
|
|
||||||
out.append(dict(r,code=code,count=int(r['count'])))
|
|
||||||
return sorted(out,key=lambda r:(-r['count'],int(r['code'],0)))
|
|
||||||
|
|
||||||
def write_xml(tree,path):
|
|
||||||
ET.indent(tree,space=' ')
|
|
||||||
tree.write(path,encoding='utf-8',xml_declaration=True)
|
|
||||||
ET.parse(path)
|
|
||||||
|
|
||||||
def make_configs(poi_base,render_base,catalog,folder,threshold,frequent_zoom=15,rare_zoom=17,route_styles=()):
|
|
||||||
# Keep entire upstream definitions! Replacing with a standalone tiny XML is
|
|
||||||
# precisely what used to make ordinary semantic POIs disappear.
|
|
||||||
poi=ET.parse(poi_base); proot=poi.getroot()
|
|
||||||
if proot.tag!='poi_types':raise SystemExit('Not an OsmAnd poi_types.xml: '+str(poi_base))
|
|
||||||
if any(c.get('name')=='bgmountains_raw' for c in proot.findall('poi_category')):
|
|
||||||
raise SystemExit('Input poi_types.xml already has a bgmountains_raw category')
|
|
||||||
cat=ET.SubElement(proot,'poi_category',name='bgmountains_raw', default_tag='bgm_poi', no_edit='true')
|
|
||||||
for r in catalog:
|
|
||||||
code=r['code']
|
|
||||||
ET.SubElement(cat,'poi_type',name='bgm_'+code.replace('0x',''),tag='bgm_poi',value='type_'+code,no_edit='true')
|
|
||||||
pa=folder/'poi_types.bgmountains.xml'
|
|
||||||
write_xml(poi,pa)
|
|
||||||
|
|
||||||
rendering=ET.parse(render_base); rroot=rendering.getroot()
|
|
||||||
if rroot.tag != 'osmand_types':
|
|
||||||
# OsmAnd resources use the tag 'rendering_types'; fail instead of dropping all built-in types.
|
|
||||||
raise SystemExit('Unexpected OsmAnd rendering_types.xml root: '+rroot.tag)
|
|
||||||
cats=[c for c in rroot.iter('category') if c.get('name')=='user_defined']
|
|
||||||
target=cats[0] if cats else ET.SubElement(rroot,'category',name='user_defined')
|
|
||||||
for r in catalog:
|
|
||||||
code=r['code']
|
|
||||||
ET.SubElement(target,'type',tag='bgm_poi',value='type_'+code,
|
|
||||||
minzoom=str(frequent_zoom) if r['count']>=threshold else str(rare_zoom))
|
|
||||||
# Route overlays have their OWN map-index main types, not only an ignored metadata tag.
|
|
||||||
# Do not map decorative overlays to 'highway=path': that would create bogus routing.
|
|
||||||
for route in route_styles:
|
|
||||||
if route.get('show','yes').lower() not in ('yes','1','true'): continue
|
|
||||||
ET.SubElement(target,'type',tag='bgm_route',value='type_'+route['code'],minzoom=route['display_minzoom'])
|
|
||||||
rp=folder/'rendering_types.bgmountains.xml'
|
|
||||||
write_xml(rendering,rp)
|
|
||||||
summary={
|
|
||||||
'catalog_types':len(catalog), 'route_overlay_types':len(route_styles),
|
|
||||||
'poi_count_in_catalog':sum(r.get('sample_count',r['count']) for r in catalog),
|
|
||||||
'types_at_or_above_threshold':sum(r['count']>=threshold for r in catalog),
|
|
||||||
'frequent_index_minzoom':frequent_zoom, 'rare_index_minzoom':rare_zoom,
|
|
||||||
'items_at_or_above_threshold':sum(r.get('sample_count',r['count']) for r in catalog if r['count']>=threshold),
|
|
||||||
'types_with_extracted_icon':sum(r['has_typ_icon']=='yes' for r in catalog),
|
|
||||||
'poi_base_sha256':hashlib.sha256(Path(poi_base).read_bytes()).hexdigest(),
|
|
||||||
'rendering_base_sha256':hashlib.sha256(Path(render_base).read_bytes()).hexdigest(),
|
|
||||||
'note':'Local-only compat data; synthetic information icons are not truth about an informational board. All Garmin hex POI types indexed; high-frequency raw types inserted into OBF map index from selected frequent zoom; low-frequency from selected rare zoom. Rendering exact Garmin icon PNGs requires installing compatible icons/style in OsmAnd, not merely compiling an OBF.'
|
|
||||||
}
|
|
||||||
(folder/'type-index-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
|
||||||
return pa,rp,summary
|
|
||||||
|
|
||||||
def read_route_styles(path):
|
|
||||||
with Path(path).open(encoding='utf-8-sig',newline='') as f: rows=list(csv.DictReader(f))
|
|
||||||
if not rows or not {'code','color','show','display_minzoom','width'}.issubset(rows[0]):
|
|
||||||
raise SystemExit('Invalid trail-styles.csv: '+str(path))
|
|
||||||
seen=set()
|
|
||||||
for r in rows:
|
|
||||||
code='0x'+format(int(r['code'],0),'x')
|
|
||||||
if code in seen: raise SystemExit('Duplicate route style type: '+code)
|
|
||||||
seen.add(code);r['code']=code
|
|
||||||
if not (len(r['color'])==7 and r['color'][0]=='#' and all(c in '0123456789abcdefABCDEF' for c in r['color'][1:])):
|
|
||||||
raise SystemExit('Invalid CSS color in route type '+code)
|
|
||||||
if not (0.2<=float(r['width'])<=15 and 1<=int(r['display_minzoom'])<=21):
|
|
||||||
raise SystemExit('Invalid route width/zoom '+code)
|
|
||||||
return rows
|
|
||||||
|
|
||||||
def make_style(icon_csv,folder,catalog,route_styles=()):
|
|
||||||
"""Selectable OsmAnd map style that maps BGM indexes to built-in icons.
|
|
||||||
|
|
||||||
The .obf itself only embeds geometry/tags. Android requires installing and
|
|
||||||
selecting this renderer; stock app ignores bgm_poi and bgm_route styling.
|
|
||||||
"""
|
"""
|
||||||
known={r['code'] for r in catalog}
|
gib=1024**3
|
||||||
with Path(icon_csv).open(encoding='utf-8-sig',newline='') as f:
|
free=shutil.disk_usage(out).free
|
||||||
rows=list(csv.DictReader(f))
|
required=max(int(2*gib),int(source_bytes*5 + gib),int(extra_gb*gib))
|
||||||
visible=[]
|
if free<required:
|
||||||
for r in rows:
|
raise RuntimeError(f'Insufficient free space for OBF build: free={free/gib:.2f} GiB; '
|
||||||
code='0x'+format(int(r['code'],0),'x')
|
f'estimated minimum={required/gib:.2f} GiB. '
|
||||||
if code not in known or r['show_in_custom_style'].strip().lower() not in ('yes','true','1'):
|
'Use --prepare-only or set --ignore-disk-check if scratch resides on another filesystem.')
|
||||||
continue
|
print(f'Disk preflight OK: free={free/gib:.1f} GiB, estimated minimum={required/gib:.1f} GiB',flush=True)
|
||||||
name=r['osmand_builtin_icon'].strip()
|
|
||||||
if not name or not name.replace('_','').replace('-','').isalnum():
|
|
||||||
raise SystemExit('Invalid OsmAnd icon: '+repr(name)+' code='+code)
|
|
||||||
zoom=int(r['minzoom'])
|
|
||||||
if not 1<=zoom<=21: raise SystemExit('Bad icon zoom: '+code)
|
|
||||||
visible.append((code,name,zoom))
|
|
||||||
root=ET.Element('renderingStyle',name='BGMountains outdoor v0.9',
|
|
||||||
depends='default',defaultColor='#fff8ee',version='1')
|
|
||||||
order=ET.SubElement(root,'order');points=ET.SubElement(root,'point')
|
|
||||||
lines=ET.SubElement(root,'line')
|
|
||||||
for code,name,zoom in visible:
|
|
||||||
ET.SubElement(order,'case',tag='bgm_poi',value='type_'+code,
|
|
||||||
minzoom=str(zoom),point='true',objectType='1',order='120')
|
|
||||||
ET.SubElement(points,'case',tag='bgm_poi',value='type_'+code,minzoom=str(zoom),
|
|
||||||
icon=name,iconOrder='11',iconVisibleSize='16',intersectionSizeFactor='0.7')
|
|
||||||
route_count=0
|
|
||||||
for route in route_styles:
|
|
||||||
if route.get('show','yes').lower() not in ('yes','true','1'):continue
|
|
||||||
route_count+=1
|
|
||||||
code=route['code'];zoom=route['display_minzoom']
|
|
||||||
ET.SubElement(order,'case',tag='bgm_route',value='type_'+code,
|
|
||||||
minzoom=zoom,objectType='2',order='125')
|
|
||||||
# Color and width faithfully recover TYP base line color, but NOT its bitmap
|
|
||||||
# segment pattern. A precise pattern renderer requires a custom app build.
|
|
||||||
ET.SubElement(lines,'case',tag='bgm_route',value='type_'+code,
|
|
||||||
minzoom=zoom,color=route['color'],strokeWidth=route['width'],cap='ROUND')
|
|
||||||
path=folder/'BGMountains-outdoor.render.xml'
|
|
||||||
write_xml(ET.ElementTree(root),path)
|
|
||||||
return path,len(visible),route_count
|
|
||||||
|
|
||||||
def obtain_creator(cache, offline=False):
|
|
||||||
"""Opt-in download from official OsmAnd nightly build (large file)."""
|
|
||||||
import zipfile
|
|
||||||
link='https://download.osmand.net/latest-night-build/OsmAndMapCreator-main.zip'
|
|
||||||
target=cache/'OsmAndMapCreator-main.zip'
|
|
||||||
extracted=cache/'unpacked'
|
|
||||||
found=list(extracted.rglob('utilities.sh')) if extracted.is_dir() else []
|
|
||||||
if found:return found[0].parent
|
|
||||||
if offline:raise SystemExit('Cannot download MapCreator with --offline; pass --creator')
|
|
||||||
cache.mkdir(parents=True,exist_ok=True)
|
|
||||||
if not target.is_file():
|
|
||||||
print('Downloading official OsmAndMapCreator (~150 MB) ...',flush=True)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(link,timeout=180) as src,target.open('wb') as dest:
|
|
||||||
shutil.copyfileobj(src,dest,length=2**20)
|
|
||||||
except Exception as ex:
|
|
||||||
target.unlink(missing_ok=True)
|
|
||||||
raise SystemExit('Cannot download OsmAndMapCreator: '+str(ex)+'\nDownload manually: '+link)
|
|
||||||
print('Extracting MapCreator...',flush=True)
|
|
||||||
extracted.mkdir(parents=True,exist_ok=True)
|
|
||||||
with zipfile.ZipFile(target) as z:
|
|
||||||
for m in z.infolist():
|
|
||||||
rel=Path(m.filename)
|
|
||||||
if rel.is_absolute() or '..' in rel.parts:
|
|
||||||
raise SystemExit('Unsafe file in MapCreator archive: '+m.filename)
|
|
||||||
z.extract(m,extracted)
|
|
||||||
found=list(extracted.rglob('utilities.sh'))
|
|
||||||
if not found:raise SystemExit('MapCreator ZIP extracted but utilities.sh not found')
|
|
||||||
for name in ('utilities.sh','inspector.sh'):
|
|
||||||
for f in extracted.rglob(name):f.chmod(f.stat().st_mode|0o111)
|
|
||||||
return found[0].parent
|
|
||||||
|
|
||||||
def resolve_creator(path):
|
|
||||||
name = path or os.environ.get('OSMAND_MAPCREATOR')
|
|
||||||
if not name:return None
|
|
||||||
creator=Path(name).expanduser().resolve()
|
|
||||||
if creator.is_file() and creator.name=='utilities.sh':creator=creator.parent
|
|
||||||
if not (creator/'utilities.sh').is_file():
|
|
||||||
raise SystemExit('Cannot find utilities.sh under '+str(creator))
|
|
||||||
return creator
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
p=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
p=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||||
inp=p.add_mutually_exclusive_group(required=True)
|
p.add_argument('--img',help='Original Garmin IMG')
|
||||||
inp.add_argument('--img',help='Original Garmin IMG; converter is compiled and executed')
|
p.add_argument('--osm',help='Already generated original Garmin-derived raw OSM')
|
||||||
inp.add_argument('--osm',help='Already converted OSM XML; must contain bgm_poi=type_0x... tags (v0.7+)')
|
p.add_argument('--out',default='build/bgmountains-v095')
|
||||||
p.add_argument('--creator',help='OsmAndMapCreator directory containing utilities.sh (or OSMAND_MAPCREATOR env)')
|
p.add_argument('--poi-config',default=str(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'))
|
||||||
p.add_argument('--download-creator',action='store_true',help='Download official OsmAndMapCreator ZIP automatically (~150 MB) when no --creator is set')
|
p.add_argument('--line-config',default=str(ROOT/'config/line-mapping.csv'))
|
||||||
p.add_argument('--out',default='build/bgmountains-output',help='Output folder')
|
p.add_argument('--jobs',type=int,default=1,help='IMG workers: default 1; explicitly set N to parallelize')
|
||||||
p.add_argument('--jobs',type=int,default=1,help='C++ IMG reader threads (opt-in)')
|
p.add_argument('--bbox',help='W,S,E,N region for smoke tests')
|
||||||
p.add_argument('--bbox',help='Optional W,S,E,N bounds for small-area test')
|
p.add_argument('--template',action='store_true',help='Generate blank new-map mapping templates using this IMG (not BGMountains defaults)')
|
||||||
p.add_argument('--threshold',type=int,default=100,help='Frequent-type occurrence threshold')
|
p.add_argument('--attach-routes',action='store_true',help='Opt in to high-memory matching of trail overlays to physical paths (not recommended on full map)')
|
||||||
p.add_argument('--frequent-zoom',type=int,default=15,help='Store frequent raw map icons at zoom N and above')
|
p.add_argument('--no-route-attachment',action='store_true',help='Deprecated: attachment is disabled by default')
|
||||||
p.add_argument('--rare-zoom',type=int,default=17,help='Store rare raw map icons at zoom N and above')
|
p.add_argument('--prepare-only',action='store_true',help='Extract/translate/validate, do not compile OBF')
|
||||||
p.add_argument('--icons-config',default=str(ROOT/'config/osmand-icons.csv'),help='OsmAnd icon mappings')
|
p.add_argument('--creator',help='Existing OsmAndMapCreator directory containing utilities.sh')
|
||||||
p.add_argument('--trail-styles',default=str(ROOT/'config/trail-styles.csv'),help='Garmin TYP coloured route overlay styles')
|
p.add_argument('--download-creator',action='store_true',help='Obtain official OsmAndMapCreator from download.osmand.net')
|
||||||
p.add_argument('--poi-overrides',default=str(ROOT/'config/poi-overrides-all-types.csv'))
|
p.add_argument('--java-opts',default='-Xms512M -Xmx8192M')
|
||||||
p.add_argument('--poi-name-fixes',default=str(ROOT/'config/poi-name-fixes.csv'))
|
p.add_argument('--keep-raw',action='store_true',help='Keep pre-translation OSM for debugging')
|
||||||
p.add_argument('--poi-labels',default=str(ROOT/'config/poi-labels-bg.csv'),help='Bulgarian prefixes for stable generated names on nameless Garmin POIs')
|
p.add_argument('--ignore-disk-check',action='store_true',help='Override conservative OBF scratch space safety check')
|
||||||
p.add_argument('--poi-additions',default=str(ROOT/'config/poi-additions.csv'))
|
p.add_argument('--disk-headroom-gb',type=float,default=0,help='Minimum free GiB required before Java OBF build (0=auto estimate)')
|
||||||
p.add_argument('--poi-base',help='Official upstream poi_types.xml; if omitted use cache/download')
|
a=p.parse_args()
|
||||||
p.add_argument('--rendering-base',help='Official upstream rendering_types.xml; if omitted use cache/download')
|
if bool(a.img)==bool(a.osm):p.error('Provide exactly one of --img or --osm')
|
||||||
p.add_argument('--offline',action='store_true',help='Never download upstream XML')
|
if a.jobs<1 or a.jobs>64:p.error('--jobs must be 1..64')
|
||||||
p.add_argument('--prepare-only',action='store_true',help='Do not run OsmAndMapCreator, but generate OSM and XMLs')
|
if not a.template:
|
||||||
p.add_argument('--skip-qa',action='store_true',help='Skip OSM comparison for faster repeat runs')
|
for typ,pth in (('POI',a.poi_config),('line',a.line_config)):
|
||||||
p.add_argument('--java-opts',default='-Xms512M -Xmx8192M',help='JAVA_OPTS for OsmAndMapCreator')
|
if not pathlib.Path(pth).is_file():p.error(f'{typ} mapping file not found: {pth}')
|
||||||
p.add_argument('--skip-cpp-build',action='store_true')
|
out=pathlib.Path(a.out).resolve();out.mkdir(parents=True,exist_ok=True)
|
||||||
p.add_argument('--catalog',help='Preexisting POI catalog when supplying --osm (otherwise use packaged nationwide catalog)')
|
src=pathlib.Path(a.osm).resolve() if a.osm else None
|
||||||
p.add_argument('--frequency-catalog',default=str(ROOT/'data/BGMountains-unique-poi-types.csv'),help='Reference nationwide counts for map index zoom decisions even in --bbox builds')
|
poi=pathlib.Path(a.poi_config).resolve();line=pathlib.Path(a.line_config).resolve()
|
||||||
args=p.parse_args()
|
# Resolve MapCreator before running an expensive full-country extraction.
|
||||||
if args.jobs<1 or args.jobs>64 or args.threshold<1 or not 1<=args.frequent_zoom<=21 or not 1<=args.rare_zoom<=21: p.error('--jobs 1..64; threshold >=1; zoom 1..21')
|
# Missing download/network credentials must fail before consuming gigabytes.
|
||||||
dest=Path(args.out).expanduser().resolve();dest.mkdir(parents=True,exist_ok=True)
|
creator=None
|
||||||
creator=resolve_creator(args.creator)
|
if not a.prepare_only and not a.template:
|
||||||
if not creator and args.download_creator and not args.prepare_only:
|
creator=creator_dir(a.creator or os.environ.get('OSMAND_MAPCREATOR'))
|
||||||
creator=obtain_creator(ROOT/'build'/'mapcreator-download',args.offline)
|
if not creator and a.download_creator:
|
||||||
if not creator and not args.prepare_only:p.error('Supply --creator /path/to/OsmAndMapCreator (or OSMAND_MAPCREATOR env) or use --prepare-only')
|
creator=fetch_creator(ROOT/'build'/'mapcreator')
|
||||||
if args.img:
|
if not creator:
|
||||||
if not Path(args.img).is_file():p.error('IMG file missing: '+args.img)
|
p.error('OBF requires --creator DIR or --download-creator; for testing use --prepare-only.')
|
||||||
binary=ROOT/'build'/'garmin2osm'
|
bin_path=ROOT/'build'/'garmin2osm'
|
||||||
if not args.skip_cpp_build or not binary.is_file():
|
if a.img:
|
||||||
call(['cmake','-S',ROOT,'-B',ROOT/'build','-DCMAKE_BUILD_TYPE=Release'])
|
run('cmake','-S',ROOT,'-B',ROOT/'build','-DCMAKE_BUILD_TYPE=Release')
|
||||||
call(['cmake','--build',ROOT/'build','--config','Release','-j','2'])
|
run('cmake','--build',ROOT/'build','-j','2','--target','garmin2osm')
|
||||||
osm=dest/'BGMountains.osm'
|
src=out/'IMG-raw.osm'
|
||||||
cmd=[binary,'--input',Path(args.img).resolve(),'--output',osm,'--poi-overrides',Path(args.poi_overrides).resolve(),
|
cmd=[str(bin_path),'--input',str(pathlib.Path(a.img).resolve()),'--output',str(src),
|
||||||
'--poi-name-fixes',Path(args.poi_name_fixes).resolve(),'--poi-labels',Path(args.poi_labels).resolve(),'--poi-additions',Path(args.poi_additions).resolve(),
|
'--poi-types-out',str(out/'discovered-pois.csv'),'--poi-manifest',str(out/'source-pois.csv'),
|
||||||
'--poi-manifest',dest/'source-pois.csv','--poi-types-out',dest/'poi-types.csv','--jobs',args.jobs]
|
'--jobs',str(a.jobs)]
|
||||||
if args.bbox:cmd+=['--bbox',args.bbox]
|
if a.bbox:cmd+=['--bbox',a.bbox]
|
||||||
call(cmd,log=dest/'converter.log')
|
if not a.template:
|
||||||
catalog=dest/'poi-types.csv'
|
run(sys.executable,TRANSLATOR,'prepare','--poi',poi,'--lines',line,'--out',out)
|
||||||
if not args.skip_qa:
|
cmd+=['--poi-overrides',str(out/'cpp-poi-override.csv'),'--poi-labels',str(out/'cpp-name-labels.csv')]
|
||||||
call([sys.executable,ROOT/'tools/validate_pois.py','compare','--source',dest/'source-pois.csv','--osm',osm,
|
run(*cmd,log=out/'extract.log')
|
||||||
'--rules',Path(args.poi_overrides).resolve(),'--report',dest/'validation.json','--mismatches',dest/'mismatches.csv'],
|
if a.template:
|
||||||
log=dest/'validator.log')
|
if not a.img:p.error('--template requires --img')
|
||||||
else:
|
run(sys.executable,TRANSLATOR,'template','--poi',out/'poi-mapping.TEMPLATE.csv',
|
||||||
osm=Path(args.osm).expanduser().resolve()
|
'--lines',out/'line-mapping.TEMPLATE.csv','--catalog',out/'discovered-pois.csv','--osm',src)
|
||||||
if not osm.is_file():p.error('OSM file missing: '+str(osm))
|
print('Templates created. Review both files before using them for native OSM conversion.')
|
||||||
catalog=Path(args.catalog).expanduser().resolve() if args.catalog else ROOT/'data'/'BGMountains-unique-poi-types.csv'
|
|
||||||
call([sys.executable,ROOT/'tools/audit_icon_coverage.py', '--osm',osm, '--catalog',catalog,
|
|
||||||
'--csv',dest/'poi-coverage.csv','--json',dest/'poi-coverage.json'],
|
|
||||||
log=dest/'poi-coverage.log')
|
|
||||||
rows=catalog_rows(catalog)
|
|
||||||
nationwide={r['code']:r['count'] for r in catalog_rows(args.frequency_catalog)} if args.frequency_catalog else {}
|
|
||||||
for row in rows:
|
|
||||||
row['sample_count']=row['count']
|
|
||||||
row['count']=nationwide.get(row['code'],row['count'])
|
|
||||||
# A separately supplied legacy OSM might lack our new classifier.
|
|
||||||
# For --img, the v0.7 converter above just generated it by construction.
|
|
||||||
if args.osm:
|
|
||||||
import gzip
|
|
||||||
opener=gzip.open if osm.suffix=='.gz' else open
|
|
||||||
with opener(osm,'rt',encoding='utf-8',errors='replace') as f:
|
|
||||||
found=any('k="bgm_poi"' in line for line in f)
|
|
||||||
if not found:
|
|
||||||
raise SystemExit('Existing OSM lacks bgm_poi; re-extract with v0.7 C++ converter.')
|
|
||||||
folder=dest/'osmand-config';folder.mkdir(exist_ok=True)
|
|
||||||
poi_base=base_xml(args.poi_base,creator,'poi',dest/'upstream',args.offline)
|
|
||||||
rendering_base=base_xml(args.rendering_base,creator,'rendering',dest/'upstream',args.offline)
|
|
||||||
route_styles=read_route_styles(args.trail_styles)
|
|
||||||
poi_cfg,render_cfg,summary=make_configs(poi_base,rendering_base,rows,folder,args.threshold,args.frequent_zoom,args.rare_zoom,route_styles)
|
|
||||||
style,nicons,nlines=make_style(args.icons_config,folder,rows,route_styles)
|
|
||||||
print('Indexed POI types:',summary['catalog_types'],'frequent:',summary['types_at_or_above_threshold'],'custom icons:',nicons,'colored route types:',nlines,flush=True)
|
|
||||||
if args.prepare_only:
|
|
||||||
print('PREPARED (no OBF generated). Files at:',dest)
|
|
||||||
return 0
|
return 0
|
||||||
before={str(i.resolve()) for i in dest.glob('*.obf')}
|
dst=out/'BGMountains-native.osm'
|
||||||
# Save Java logs, and preserve OsmAndMapCreator temp data inside user-selected output dir.
|
run(sys.executable,TRANSLATOR,'transform','--poi',poi,'--lines',line,'--osm',src,
|
||||||
env=os.environ.copy();env['JAVA_OPTS']=args.java_opts
|
'--out',dst,'--report',out/'route-report.json',log=out/'translate.log')
|
||||||
program=creator/'utilities.sh'
|
if a.attach_routes and not a.no_route_attachment:
|
||||||
cmd=[str(program),'generate-obf-no-address',str(osm),
|
attached=out/'BGMountains-native-attached.osm'
|
||||||
'--poi-types='+str(poi_cfg),'--rendering-types='+str(render_cfg)]
|
run(sys.executable,ROOT/'scripts/attach_native_routes.py','--osm',dst,'--out',attached,
|
||||||
call(cmd,cwd=dest,log=dest/'mapcreator.log',env=env)
|
'--lines',line,'--report',out/'route-attachment.json',log=out/'route-attachment.log')
|
||||||
after=[i for i in dest.glob('*.obf') if str(i.resolve()) not in before]
|
dst.unlink();attached.rename(dst)
|
||||||
if not after:
|
run(sys.executable,TRANSLATOR,'audit','--poi',poi,'--lines',line,'--osm',dst,
|
||||||
# Creator sometimes writes next to the .osm source, even with cwd changed.
|
'--out',out/'semantic-audit.json',log=out/'audit.log')
|
||||||
alt=list(osm.parent.glob('*.obf'))
|
# Fail closed if the translated file is not well-formed XML, including relation refs.
|
||||||
after=[i for i in alt if i.is_file()]
|
import xml.etree.ElementTree as ET
|
||||||
if not after:raise SystemExit('MapCreator returned success but no .obf was found; see '+str(dest/'mapcreator.log'))
|
from native_translate import top_level_elements
|
||||||
built=max(after,key=lambda p:p.stat().st_mtime)
|
n=0
|
||||||
print('OBF created:',built,'bytes:',built.stat().st_size)
|
for e in top_level_elements(dst):
|
||||||
if (creator/'inspector.sh').is_file():
|
if e.tag=='relation':n+=1
|
||||||
call([str(creator/'inspector.sh'),'-vpoi',str(built)],cwd=dest,log=dest/'obf-poi-index.txt',env=env)
|
if not a.keep_raw and a.img:src.unlink(missing_ok=True)
|
||||||
(dest/'build-manifest.json').write_text(json.dumps({
|
if a.prepare_only:
|
||||||
'created_at_utc':dt.datetime.now(dt.timezone.utc).isoformat(),
|
print('PREPARED OSM:',dst,'route relations:',n,'No OBF compiled.')
|
||||||
'obf':str(built),'obf_bytes':built.stat().st_size,
|
return 0
|
||||||
'input_osm':str(osm),'poi_types':str(poi_cfg),'rendering_types':str(render_cfg),
|
if not a.ignore_disk_check:disk_preflight(out,dst.stat().st_size,a.disk_headroom_gb)
|
||||||
'custom_style':str(style), 'custom_style_icon_rules':nicons, 'custom_route_rules':nlines,
|
env=os.environ.copy();env['JAVA_OPTS']=a.java_opts
|
||||||
'summary':summary
|
# utilities.sh versions differ in whether JAVA_OPTS is honored; this flag is
|
||||||
},ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
# explicitly recognized by the JVM. It also reaches spawned Java processes.
|
||||||
|
env['JAVA_TOOL_OPTIONS']=a.java_opts
|
||||||
|
# Native types are built into OsmAnd: NO --poi-types/--rendering-types override,
|
||||||
|
# NO custom .render.xml and NO extracted Garmin icons.
|
||||||
|
started=time.time()
|
||||||
|
run(creator/'utilities.sh','generate-obf-no-address',dst,log=out/'mapcreator.log',cwd=out,env=env)
|
||||||
|
obfs=[f for f in out.glob('*.obf') if f.stat().st_mtime >= started-2 and f.stat().st_size>0]
|
||||||
|
if not obfs:raise RuntimeError('Creator returned no OBF in '+str(out))
|
||||||
|
obf=max(obfs,key=lambda x:x.stat().st_mtime)
|
||||||
|
if (creator/'inspector.sh').exists():
|
||||||
|
run(creator/'inspector.sh','-vpoi',obf,log=out/'obf-inspection.txt',cwd=out,env=env)
|
||||||
|
(out/'build-manifest.json').write_text(json.dumps({
|
||||||
|
'version':'v0.95','input':a.img or a.osm,'poi_config':str(poi),'line_config':str(line),
|
||||||
|
'poi_config_sha256':hashlib.sha256(poi.read_bytes()).hexdigest(),
|
||||||
|
'line_config_sha256':hashlib.sha256(line.read_bytes()).hexdigest(),
|
||||||
|
'output_obf':str(obf),'obf_bytes':obf.stat().st_size,
|
||||||
|
'compiled_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||||
|
'notice':'Native stock rendering. POI display zoom controlled by OsmAnd; no claim that all tags get icons.'},ensure_ascii=False,indent=2)+'\n')
|
||||||
|
print('BUILT OBF:',obf)
|
||||||
return 0
|
return 0
|
||||||
if __name__=='__main__':
|
if __name__=='__main__':sys.exit(main())
|
||||||
sys.exit(main())
|
|
||||||
|
|||||||
+2
-13
@@ -1,15 +1,4 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Compatibility shim. No manually supplied POI XML files are required anymore.
|
|
||||||
# Preferred: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
if (( $# < 2 )); then
|
cd "$(dirname "$0")/.."
|
||||||
echo "Usage: $0 CREATOR_DIR INPUT_v0.7.osm[.gz] [OUTPUT_DIR]" >&2
|
exec python3 scripts/build_bgmountains.py "$@"
|
||||||
echo "Recommended instead: python3 scripts/build_bgmountains.py --img gmapsupp.img --download-creator --out ./build/out" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
creator="$1"; osm="$2"; out="${3:-$(dirname "$osm")/rebuilt-obf}"
|
|
||||||
root="$(cd "$(dirname "$0")/.." && pwd)"
|
|
||||||
source_dir="$(dirname "$osm")"
|
|
||||||
args=(--osm "$osm" --creator "$creator" --out "$out")
|
|
||||||
if [[ -f "$source_dir/poi-types.csv" ]]; then args+=(--catalog "$source_dir/poi-types.csv"); fi
|
|
||||||
exec python3 "$root/scripts/build_bgmountains.py" "${args[@]}"
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""v0.93: streaming IMG/OSM native-OsmAnd translation and honest QA.
|
"""v0.95: streaming IMG/OSM native-OsmAnd translation and honest QA.
|
||||||
Only user-editable controls: config/poi-mapping.csv and config/line-mapping.csv.
|
Only user-editable controls: config/BGMountains-native-poi-mapping-v094.csv and config/line-mapping.csv.
|
||||||
"""
|
"""
|
||||||
import argparse,base64,collections,csv,hashlib,json,pathlib,re,sqlite3,sys,xml.etree.ElementTree as ET
|
import argparse,base64,collections,csv,hashlib,json,pathlib,re,sqlite3,sys,xml.etree.ElementTree as ET
|
||||||
from xml.sax.saxutils import escape,quoteattr
|
from xml.sax.saxutils import escape,quoteattr
|
||||||
@@ -34,14 +34,42 @@ def read_csv(p):
|
|||||||
lines=(line for line in f if not line.startswith('#'))
|
lines=(line for line in f if not line.startswith('#'))
|
||||||
yield from csv.DictReader(lines)
|
yield from csv.DictReader(lines)
|
||||||
def code(value):return hex(int(value,0)).lower()
|
def code(value):return hex(int(value,0)).lower()
|
||||||
|
def read_poi_configuration(path):
|
||||||
|
"""Read one canonical POI config: unique base rule per code and optional
|
||||||
|
ordered name_contains rules (a v0.94 feature previously lost in Python).
|
||||||
|
The v0.93/0.94 15-column files remain backward-compatible.
|
||||||
|
"""
|
||||||
|
bases={}; conditional=[]
|
||||||
|
for line_no,row in enumerate(read_csv(path),start=2):
|
||||||
|
try:c=code(row['code'])
|
||||||
|
except Exception as ex:raise ValueError(f'Invalid POI code at {path}:{line_no}: {row}') from ex
|
||||||
|
row=dict(row);row['code']=c
|
||||||
|
act=(row.get('action') or 'replace').strip().lower()
|
||||||
|
if act not in ('replace','append','raw','drop'):
|
||||||
|
raise ValueError(f'Invalid action at {path}:{line_no}: {act}')
|
||||||
|
tags_from_string(row.get('osm_tags',''))
|
||||||
|
if row.get('name_contains',''):
|
||||||
|
conditional.append(row)
|
||||||
|
else:
|
||||||
|
if c in bases:raise ValueError(f'Duplicate POI base mapping {c} at {path}:{line_no}')
|
||||||
|
bases[c]=row
|
||||||
|
for row in conditional:
|
||||||
|
if row['code'] not in bases:raise ValueError(f'Conditional rule has no base: {row["code"]}')
|
||||||
|
return bases,conditional
|
||||||
|
|
||||||
def mapping(path):
|
def mapping(path):
|
||||||
|
"""Base POI map; conditional rules are applied by the C++ override stage."""
|
||||||
|
return read_poi_configuration(path)[0]
|
||||||
|
|
||||||
|
def line_mapping(path):
|
||||||
ret={}
|
ret={}
|
||||||
for row in read_csv(path):
|
for row in read_csv(path):
|
||||||
try:c=code(row['code'])
|
try:c=code(row['code'])
|
||||||
except Exception as ex:raise ValueError('Invalid code in '+str(path)+': '+str(row)) from ex
|
except Exception as ex:raise ValueError('Invalid line code in '+str(path)+': '+str(row)) from ex
|
||||||
if c in ret:raise ValueError(f'Duplicate code {c} in {path}')
|
if c in ret:raise ValueError(f'Duplicate line code {c} in {path}')
|
||||||
ret[c]=row
|
ret[c]=row
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
def tags_from_string(v):
|
def tags_from_string(v):
|
||||||
d={}
|
d={}
|
||||||
for x in v.split(';'):
|
for x in v.split(';'):
|
||||||
@@ -86,7 +114,7 @@ def add_native_unknown_fallback(elem,tags,rule):
|
|||||||
tags['bgm:generated_name']='yes'
|
tags['bgm:generated_name']='yes'
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def make_cpp_rules(pois,out,labels):
|
def make_cpp_rules(pois,out,labels,conditional=()):
|
||||||
"""C++ v0.9 uses these internal stage files; they are NOT user control files."""
|
"""C++ v0.9 uses these internal stage files; they are NOT user control files."""
|
||||||
with open(out,'w',encoding='utf-8',newline='') as f,open(labels,'w',encoding='utf-8',newline='') as g:
|
with open(out,'w',encoding='utf-8',newline='') as f,open(labels,'w',encoding='utf-8',newline='') as g:
|
||||||
w=csv.writer(f);w.writerow(['code','name_contains','mode','tags','note'])
|
w=csv.writer(f);w.writerow(['code','name_contains','mode','tags','note'])
|
||||||
@@ -99,6 +127,15 @@ def make_cpp_rules(pois,out,labels):
|
|||||||
if action=='drop':vals={}
|
if action=='drop':vals={}
|
||||||
w.writerow([c,'',action,';'.join(k+'='+v for k,v in vals.items()),'v0.92 compiled from poi-mapping.csv'])
|
w.writerow([c,'',action,';'.join(k+'='+v for k,v in vals.items()),'v0.92 compiled from poi-mapping.csv'])
|
||||||
if row.get('label_bg'):l.writerow([c,row['label_bg']])
|
if row.get('label_bg'):l.writerow([c,row['label_bg']])
|
||||||
|
# Order matters: base rule first, then the explicit name-specific rules
|
||||||
|
# in CSV order. C++ evaluates these against ORIGINAL Garmin names.
|
||||||
|
for row in conditional:
|
||||||
|
c=row['code'];action=(row.get('action') or 'replace').strip().lower()
|
||||||
|
vals=tags_from_string(row.get('osm_tags',''))
|
||||||
|
if action=='raw':action='keep_raw';vals={}
|
||||||
|
w.writerow([c,row['name_contains'],action,
|
||||||
|
';'.join(k+'='+v for k,v in vals.items()),
|
||||||
|
'v0.95 name-specific rule: '+row.get('note','')])
|
||||||
|
|
||||||
def inspect_counts(osm,pois,lines,out,strict=False):
|
def inspect_counts(osm,pois,lines,out,strict=False):
|
||||||
seen=collections.Counter();unresolved=collections.Counter();annotations=collections.Counter();fallback=collections.Counter();bad=collections.Counter();routes=collections.Counter();examples={};category=collections.Counter();n=0
|
seen=collections.Counter();unresolved=collections.Counter();annotations=collections.Counter();fallback=collections.Counter();bad=collections.Counter();routes=collections.Counter();examples={};category=collections.Counter();n=0
|
||||||
@@ -143,7 +180,7 @@ def transform(source,dest,pois,lines,report,strict=False):
|
|||||||
db.execute('CREATE TABLE edges (code TEXT, way INTEGER, a INTEGER, b INTEGER)')
|
db.execute('CREATE TABLE edges (code TEXT, way INTEGER, a INTEGER, b INTEGER)')
|
||||||
poi_count=0; ways=0; candidate_edges=0; emitted=0; synthetic=0; kinds=collections.Counter();seen=collections.Counter();gaps=collections.Counter()
|
poi_count=0; ways=0; candidate_edges=0; emitted=0; synthetic=0; kinds=collections.Counter();seen=collections.Counter();gaps=collections.Counter()
|
||||||
with open(source,'rb') as inp,open(dest,'w',encoding='utf-8') as out:
|
with open(source,'rb') as inp,open(dest,'w',encoding='utf-8') as out:
|
||||||
out.write('<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="garmin2osm-v0.93" upload="never">\n')
|
out.write('<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="garmin2osm-v0.95" upload="never">\n')
|
||||||
for el in top_level_elements(inp):
|
for el in top_level_elements(inp):
|
||||||
if el.tag=='node':
|
if el.tag=='node':
|
||||||
tags=tagdict(el)
|
tags=tagdict(el)
|
||||||
@@ -268,10 +305,10 @@ def main():
|
|||||||
a=p.parse_args()
|
a=p.parse_args()
|
||||||
if a.mode=='template':res=template(a.catalog,a.poi,a.lines,a.osm)
|
if a.mode=='template':res=template(a.catalog,a.poi,a.lines,a.osm)
|
||||||
else:
|
else:
|
||||||
ps=mapping(a.poi);ls=mapping(a.lines)
|
ps,conditional=read_poi_configuration(a.poi);ls=line_mapping(a.lines)
|
||||||
if a.mode=='prepare':
|
if a.mode=='prepare':
|
||||||
out=pathlib.Path(a.out);out.mkdir(parents=True,exist_ok=True)
|
out=pathlib.Path(a.out);out.mkdir(parents=True,exist_ok=True)
|
||||||
make_cpp_rules(ps,out/'cpp-poi-override.csv',out/'cpp-name-labels.csv');res={'poi_types':len(ps),'line_types':len(ls)}
|
make_cpp_rules(ps,out/'cpp-poi-override.csv',out/'cpp-name-labels.csv',conditional);res={'poi_types':len(ps),'name_specific_rules':len(conditional),'line_types':len(ls)}
|
||||||
elif a.mode=='transform':res=transform(a.osm,a.out,ps,ls,a.report)
|
elif a.mode=='transform':res=transform(a.osm,a.out,ps,ls,a.report)
|
||||||
else:res=inspect_counts(a.osm,ps,ls,a.out,a.strict)
|
else:res=inspect_counts(a.osm,ps,ls,a.out,a.strict)
|
||||||
print(json.dumps(res,ensure_ascii=False,indent=2));return 0
|
print(json.dumps(res,ensure_ascii=False,indent=2));return 0
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import unittest, tempfile, csv, pathlib, sys
|
||||||
|
sys.path.insert(0,str(pathlib.Path(__file__).resolve().parents[1]/"tools"))
|
||||||
|
import audit_osmand_poi_mapping as a
|
||||||
|
class TestPoiRegistry(unittest.TestCase):
|
||||||
|
def test_registry_and_semantics(self):
|
||||||
|
with tempfile.TemporaryDirectory() as t:
|
||||||
|
p=pathlib.Path(t)/'types.xml'
|
||||||
|
p.write_text('''<poi_types><poi_type name="water" tag="amenity" value="drinking_water"/><poi_type name="locality" tag="place" value="locality" no_indx="true"/><poi_additional name="dwater" tag="drinking_water" value="yes"/></poi_types>''')
|
||||||
|
reg,ex=a.registry(p)
|
||||||
|
base={'code':'0x6414','source_count':'32922','priority':'critical','osm_tags':'amenity=drinking_water;drinking_water=yes'}
|
||||||
|
self.assertEqual(a.interpret(base,reg,ex)['status'],'native_indexable_type')
|
||||||
|
self.assertEqual(a.interpret({**base,'osm_tags':'place=locality'},reg,ex)['status'],'only_nonindexed_type')
|
||||||
|
self.assertEqual(a.interpret({**base,'osm_tags':'drinking_water=yes'},reg,ex)['status'],'additional_without_primary_type')
|
||||||
|
self.assertEqual(a.interpret({**base,'osm_tags':'man_made=water_intake'},reg,ex)['status'],'no_matching_native_type')
|
||||||
|
self.assertEqual(a.interpret({**base,'code':'0x6600','priority':'terrain','osm_tags':''},reg,ex)['severity'],'terrain_exemption')
|
||||||
|
if __name__=='__main__':unittest.main()
|
||||||
+8
-8
@@ -4,7 +4,7 @@ import csv,importlib.util,json,pathlib,subprocess,sys,tempfile,unittest,xml.etre
|
|||||||
ROOT=pathlib.Path(__file__).resolve().parents[1]
|
ROOT=pathlib.Path(__file__).resolve().parents[1]
|
||||||
spec=importlib.util.spec_from_file_location('translator',ROOT/'scripts/native_translate.py')
|
spec=importlib.util.spec_from_file_location('translator',ROOT/'scripts/native_translate.py')
|
||||||
t=importlib.util.module_from_spec(spec);spec.loader.exec_module(t)
|
t=importlib.util.module_from_spec(spec);spec.loader.exec_module(t)
|
||||||
POI=ROOT/'config/poi-mapping.csv';LINES=ROOT/'config/line-mapping.csv'
|
POI=ROOT/'config/BGMountains-native-poi-mapping-v094.csv';LINES=ROOT/'config/line-mapping.csv'
|
||||||
class Native092(unittest.TestCase):
|
class Native092(unittest.TestCase):
|
||||||
def test_native_source_semantics(self):
|
def test_native_source_semantics(self):
|
||||||
rows=t.mapping(POI)
|
rows=t.mapping(POI)
|
||||||
@@ -15,17 +15,17 @@ class Native092(unittest.TestCase):
|
|||||||
self.assertIn(rows['0x6600']['action'],('append','raw'))
|
self.assertIn(rows['0x6600']['action'],('append','raw'))
|
||||||
self.assertFalse(any('leisure=marina' in r['osm_tags'] for r in rows.values()))
|
self.assertFalse(any('leisure=marina' in r['osm_tags'] for r in rows.values()))
|
||||||
self.assertFalse(any('tourism=information;information=board' in r['osm_tags'] for r in rows.values()))
|
self.assertFalse(any('tourism=information;information=board' in r['osm_tags'] for r in rows.values()))
|
||||||
self.assertTrue(all('garmin' not in r['osm_tags'] for r in rows.values()))
|
self.assertTrue(all('garmin:type' not in r['osm_tags'] for r in rows.values()))
|
||||||
self.assertTrue(all(int(rows[k]['zoom_hint'])<=12 for k in ('0x6414','0x6511','0x6500','0x660a')))
|
self.assertTrue(all(int(rows[k]['zoom_hint'])<=12 for k in ('0x6414','0x6511','0x6500','0x660a')))
|
||||||
def test_config_only_two_and_types(self):
|
def test_config_only_two_and_types(self):
|
||||||
self.assertEqual(set(p.name for p in (ROOT/'config').glob('*.csv')),{'poi-mapping.csv','line-mapping.csv'})
|
self.assertEqual(set(p.name for p in (ROOT/'config').glob('*.csv')),{'BGMountains-native-poi-mapping-v094.csv','line-mapping.csv'})
|
||||||
self.assertTrue(any(x.get('route_mode')=='relation' for x in t.mapping(LINES).values()))
|
self.assertTrue(any(x.get('route_mode')=='relation' for x in t.line_mapping(LINES).values()))
|
||||||
for line in t.mapping(LINES).values():
|
for line in t.line_mapping(LINES).values():
|
||||||
if line['route_mode']=='relation':self.assertTrue(line['osmc_symbol'])
|
if line['route_mode']=='relation':self.assertTrue(line['osmc_symbol'])
|
||||||
def test_native_relation_members_and_poi_provenance(self):
|
def test_native_relation_members_and_poi_provenance(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
d=pathlib.Path(directory);src=d/'sample.osm';dst=d/'native.osm';report=d/'report.json'
|
d=pathlib.Path(directory);src=d/'sample.osm';dst=d/'native.osm';report=d/'report.json'
|
||||||
line=next(r for r in t.mapping(LINES).values() if r['route_mode']=='relation')
|
line=next(r for r in t.line_mapping(LINES).values() if r['route_mode']=='relation')
|
||||||
typ=line['code']
|
typ=line['code']
|
||||||
xml=['<?xml version="1.0" encoding="UTF-8"?><osm version="0.6">']
|
xml=['<?xml version="1.0" encoding="UTF-8"?><osm version="0.6">']
|
||||||
for i in range(1,5):xml.append(f'<node id="{i}" lat="41.{i}" lon="23.{i}"/>')
|
for i in range(1,5):xml.append(f'<node id="{i}" lat="41.{i}" lon="23.{i}"/>')
|
||||||
@@ -33,7 +33,7 @@ class Native092(unittest.TestCase):
|
|||||||
for i,refs in [(10,[1,2]),(11,[2,3])]:
|
for i,refs in [(10,[1,2]),(11,[2,3])]:
|
||||||
xml.append(f'<way id="{i}">'+''.join(f'<nd ref="{n}"/>' for n in refs)+f'<tag k="garmin:feature" v="extended_polyline"/><tag k="garmin:type" v="{typ}"/><tag k="bgm_route" v="type_{typ}"/></way>')
|
xml.append(f'<way id="{i}">'+''.join(f'<nd ref="{n}"/>' for n in refs)+f'<tag k="garmin:feature" v="extended_polyline"/><tag k="garmin:type" v="{typ}"/><tag k="bgm_route" v="type_{typ}"/></way>')
|
||||||
xml.append('</osm>');src.write_text('\n'.join(xml))
|
xml.append('</osm>');src.write_text('\n'.join(xml))
|
||||||
result=t.transform(src,dst,t.mapping(POI),t.mapping(LINES),report)
|
result=t.transform(src,dst,t.mapping(POI),t.line_mapping(LINES),report)
|
||||||
self.assertEqual(result['route_candidate_ways'],2);self.assertEqual(result['route_relations'],1)
|
self.assertEqual(result['route_candidate_ways'],2);self.assertEqual(result['route_relations'],1)
|
||||||
root=ET.parse(dst).getroot();rel=root.find('relation');tags=t.tagdict(rel)
|
root=ET.parse(dst).getroot();rel=root.find('relation');tags=t.tagdict(rel)
|
||||||
self.assertEqual(tags['route'],'hiking');self.assertTrue(tags['osmc:symbol'].endswith('_bar'))
|
self.assertEqual(tags['route'],'hiking');self.assertTrue(tags['osmc:symbol'].endswith('_bar'))
|
||||||
@@ -45,7 +45,7 @@ class Native092(unittest.TestCase):
|
|||||||
def test_route_attachment_to_real_way_only(self):
|
def test_route_attachment_to_real_way_only(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
d=pathlib.Path(directory);src=d/'src.osm';dst=d/'out.osm';rep=d/'report.json'
|
d=pathlib.Path(directory);src=d/'src.osm';dst=d/'out.osm';rep=d/'report.json'
|
||||||
typ=next(row['code'] for row in t.mapping(LINES).values() if row['route_mode']=='relation')
|
typ=next(row['code'] for row in t.line_mapping(LINES).values() if row['route_mode']=='relation')
|
||||||
parts=['<osm version="0.6">','<node id="1" lat="41" lon="23"/>','<node id="2" lat="41.1" lon="23.1"/>',
|
parts=['<osm version="0.6">','<node id="1" lat="41" lon="23"/>','<node id="2" lat="41.1" lon="23.1"/>',
|
||||||
'<node id="3" lat="41.2" lon="23.2"/>','<way id="22"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="highway" v="path"/></way>',
|
'<node id="3" lat="41.2" lon="23.2"/>','<way id="22"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="highway" v="path"/></way>',
|
||||||
'<way id="33"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="garmin:type" v="'+typ+'"/><tag k="garmin:feature" v="extended_polyline"/></way>',
|
'<way id="33"><nd ref="1"/><nd ref="2"/><nd ref="3"/><tag k="garmin:type" v="'+typ+'"/><tag k="garmin:feature" v="extended_polyline"/></way>',
|
||||||
|
|||||||
+8
-8
@@ -10,25 +10,25 @@ class Reliability093(unittest.TestCase):
|
|||||||
out=pathlib.Path(d)
|
out=pathlib.Path(d)
|
||||||
src=out/'input.osm';dst=out/'output.osm';report=out/'report.json'
|
src=out/'input.osm';dst=out/'output.osm';report=out/'report.json'
|
||||||
src.write_text('''<osm version="0.6"><node id="5" lat="42.1345678" lon="24.2345678"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x9999"/><tag k="garmin:mapset" v="1234"/><tag k="bgm:generated_name" v="yes"/><tag k="name" v="Garmin обект 0x9999 - Q3ABCDEFG9"/></node><node id="6" lat="42.2" lon="24.2"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="176"/><tag k="name" v="176"/></node></osm>''')
|
src.write_text('''<osm version="0.6"><node id="5" lat="42.1345678" lon="24.2345678"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x9999"/><tag k="garmin:mapset" v="1234"/><tag k="bgm:generated_name" v="yes"/><tag k="name" v="Garmin обект 0x9999 - Q3ABCDEFG9"/></node><node id="6" lat="42.2" lon="24.2"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="176"/><tag k="name" v="176"/></node></osm>''')
|
||||||
t.transform(src,dst,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
t.transform(src,dst,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
||||||
nodes=ET.parse(dst).getroot().findall('node');a=t.tagdict(nodes[0]);b=t.tagdict(nodes[1])
|
nodes=ET.parse(dst).getroot().findall('node');a=t.tagdict(nodes[0]);b=t.tagdict(nodes[1])
|
||||||
self.assertEqual(a['place'],'locality');self.assertEqual(a['name'],'Неуточнено - Q3ABCDEFG9')
|
self.assertEqual(a['place'],'locality');self.assertEqual(a['name'],'Неуточнено - Q3ABCDEFG9')
|
||||||
self.assertEqual(a['garmin:type'],'0x9999');self.assertEqual(a['bgm:classification'],'unverified')
|
self.assertEqual(a['garmin:type'],'0x9999');self.assertEqual(a['bgm:classification'],'unverified')
|
||||||
self.assertNotIn('place',b);self.assertEqual(b['ele'],'176')
|
self.assertNotIn('place',b);self.assertEqual(b['ele'],'176')
|
||||||
data=t.inspect_counts(dst,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),out/'audit.json')
|
data=t.inspect_counts(dst,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),out/'audit.json')
|
||||||
self.assertEqual(data['synthetic_locality_fallback_total'],1)
|
self.assertEqual(data['synthetic_locality_fallback_total'],1)
|
||||||
|
|
||||||
def test_known_bgm_types_are_not_invented_localities(self):
|
def test_known_bgm_types_are_not_invented_localities(self):
|
||||||
cfg=t.mapping(ROOT/'config/poi-mapping.csv')
|
cfg=t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv')
|
||||||
self.assertEqual(cfg['0x6600']['osm_tags'],'')
|
self.assertIn('bgm:spot_height=yes',cfg['0x6600']['osm_tags']);self.assertEqual(cfg['0x6600']['action'],'append')
|
||||||
self.assertNotIn('place=locality',cfg['0x6600']['osm_tags'])
|
self.assertNotIn('place=locality',cfg['0x6600']['osm_tags'])
|
||||||
self.assertEqual(cfg['0x650b']['osm_tags'],'tourism=information;information=route_marker;hiking=yes')
|
self.assertIn('information=trail_blaze',cfg['0x650b']['osm_tags'])
|
||||||
self.assertEqual(cfg['0x660b']['osm_tags'],'tourism=information;information=trail_blaze;hiking=yes')
|
self.assertIn('information=trail_blaze',cfg['0x660b']['osm_tags'])
|
||||||
def test_fallback_does_not_overwrite_known_native_types(self):
|
def test_fallback_does_not_overwrite_known_native_types(self):
|
||||||
with tempfile.TemporaryDirectory() as d:
|
with tempfile.TemporaryDirectory() as d:
|
||||||
src=pathlib.Path(d)/'before.osm';out=pathlib.Path(d)/'after.osm';rep=pathlib.Path(d)/'rep.json'
|
src=pathlib.Path(d)/'before.osm';out=pathlib.Path(d)/'after.osm';rep=pathlib.Path(d)/'rep.json'
|
||||||
src.write_text('''<osm><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6400"/><tag k="landuse" v="quarry"/><tag k="name" v="Кариера П"/></node><node id="2" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6406"/><tag k="barrier" v="gate"/></node><node id="3" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x640d"/><tag k="aerialway" v="station"/></node></osm>''')
|
src.write_text('''<osm><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6400"/><tag k="landuse" v="quarry"/><tag k="name" v="Кариера П"/></node><node id="2" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6406"/><tag k="barrier" v="gate"/></node><node id="3" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x640d"/><tag k="aerialway" v="station"/></node></osm>''')
|
||||||
t.transform(src,out,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),rep)
|
t.transform(src,out,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),rep)
|
||||||
nodes=ET.parse(out).getroot().findall('node')
|
nodes=ET.parse(out).getroot().findall('node')
|
||||||
self.assertTrue(all('place' not in t.tagdict(e) for e in nodes))
|
self.assertTrue(all('place' not in t.tagdict(e) for e in nodes))
|
||||||
self.assertEqual(t.tagdict(nodes[0])['name'],'Кариера П')
|
self.assertEqual(t.tagdict(nodes[0])['name'],'Кариера П')
|
||||||
@@ -53,7 +53,7 @@ class Reliability093(unittest.TestCase):
|
|||||||
with tempfile.TemporaryDirectory() as d:
|
with tempfile.TemporaryDirectory() as d:
|
||||||
path=pathlib.Path(d)/'sample.osm';report=pathlib.Path(d)/'report.json'
|
path=pathlib.Path(d)/'sample.osm';report=pathlib.Path(d)/'report.json'
|
||||||
path.write_text('''<osm version="0.6"><node id="1" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="132"/></node><node id="2" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x650b"/><tag k="tourism" v="information"/><tag k="information" v="route_marker"/></node><node id="3" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6502"/><tag k="ford" v="yes"/></node></osm>''')
|
path.write_text('''<osm version="0.6"><node id="1" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6600"/><tag k="ele" v="132"/></node><node id="2" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x650b"/><tag k="tourism" v="information"/><tag k="information" v="route_marker"/></node><node id="3" lat="43" lon="22"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6502"/><tag k="ford" v="yes"/></node></osm>''')
|
||||||
stats=t.inspect_counts(path,t.mapping(ROOT/'config/poi-mapping.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
stats=t.inspect_counts(path,t.mapping(ROOT/'config/BGMountains-native-poi-mapping-v094.csv'),t.mapping(ROOT/'config/line-mapping.csv'),report)
|
||||||
self.assertEqual(stats['unclassified_total'],0)
|
self.assertEqual(stats['unclassified_total'],0)
|
||||||
self.assertEqual(stats['annotations_not_missing_pois']['spot_height'],1)
|
self.assertEqual(stats['annotations_not_missing_pois']['spot_height'],1)
|
||||||
if __name__=='__main__':unittest.main()
|
if __name__=='__main__':unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Version 0.95 regression checks, including its REAL published CLI contract.
|
||||||
|
|
||||||
|
Only mock OsmAndMapCreator is used for the orchestration test, not for proving
|
||||||
|
OBF binary correctness. The real IMG tests were run separately during release.
|
||||||
|
"""
|
||||||
|
import csv,importlib.util,json,pathlib,subprocess,sys,tempfile,unittest,xml.etree.ElementTree as ET
|
||||||
|
ROOT=pathlib.Path(__file__).resolve().parents[1]
|
||||||
|
SPEC=importlib.util.spec_from_file_location('nt',ROOT/'scripts/native_translate.py')
|
||||||
|
t=importlib.util.module_from_spec(SPEC);SPEC.loader.exec_module(t)
|
||||||
|
POI=ROOT/'config/BGMountains-native-poi-mapping-v094.csv'
|
||||||
|
LINES=ROOT/'config/line-mapping.csv'
|
||||||
|
SCRIPT=ROOT/'scripts/build_bgmountains.py'
|
||||||
|
class V095(unittest.TestCase):
|
||||||
|
def test_mapping_handoff_has_all_named_exceptions(self):
|
||||||
|
bases,conditional=t.read_poi_configuration(POI)
|
||||||
|
self.assertEqual(len(bases),1871)
|
||||||
|
self.assertEqual(len(conditional),79)
|
||||||
|
self.assertEqual(len(t.line_mapping(LINES)),50)
|
||||||
|
self.assertEqual(sum(x['code']=='0x6414' and x['name_contains']=='Сух' for x in conditional),1)
|
||||||
|
with tempfile.TemporaryDirectory() as dd:
|
||||||
|
d=pathlib.Path(dd)
|
||||||
|
t.make_cpp_rules(bases,d/'cpp.csv',d/'labels.csv',conditional)
|
||||||
|
rows=list(csv.DictReader((d/'cpp.csv').open()))
|
||||||
|
self.assertEqual(len(rows),1950)
|
||||||
|
self.assertEqual(next(x for x in rows if x['code']=='0x6414' and x['name_contains']=='')['tags'].split(';')[:2],['amenity=drinking_water','man_made=water_tap'])
|
||||||
|
self.assertEqual(next(x for x in rows if x['code']=='0x6414' and x['name_contains']=='Сух')['tags'].split(';')[:2],['man_made=water_tap','drinking_water=no'])
|
||||||
|
def test_argument_contract_with_users_flags(self):
|
||||||
|
helptext=subprocess.run([sys.executable,SCRIPT,'--help'],capture_output=True,text=True,check=True).stdout
|
||||||
|
for flag in ('--poi-config','--line-config','--download-creator','--jobs','--java-opts','--bbox','--prepare-only','--creator','--template'):
|
||||||
|
self.assertIn(flag,helptext)
|
||||||
|
self.assertNotIn('unrecognized arguments',helptext)
|
||||||
|
def test_smoke_full_orchestration_with_mock_creator(self):
|
||||||
|
with tempfile.TemporaryDirectory() as dd:
|
||||||
|
d=pathlib.Path(dd)
|
||||||
|
creator=d/'creator';creator.mkdir()
|
||||||
|
(creator/'utilities.sh').write_text('#!/bin/sh\nset -eu\n[ "$1" = "generate-obf-no-address" ]\n[ -f "$2" ]\n[ "${JAVA_TOOL_OPTIONS:-}" = "-Xms512M -Xmx25G" ]\nprintf MOCK-OBF > "${2%.osm}.obf"\n')
|
||||||
|
(creator/'utilities.sh').chmod(0o755)
|
||||||
|
(creator/'inspector.sh').write_text('#!/bin/sh\n[ "$1" = "-vpoi" ]\nprintf MOCK-INSPECTION\n')
|
||||||
|
(creator/'inspector.sh').chmod(0o755)
|
||||||
|
raw=d/'minimal.osm';raw.write_text('''<osm version="0.6"><node id="1" lat="42" lon="24"><tag k="garmin:feature" v="poi"/><tag k="garmin:type" v="0x6414"/><tag k="amenity" v="drinking_water"/><tag k="man_made" v="water_tap"/><tag k="name" v="Чешма"/></node></osm>''',encoding='utf-8')
|
||||||
|
cmd=[sys.executable,str(SCRIPT),'--osm',str(raw),'--poi-config',str(POI),'--line-config',str(LINES),'--out',str(d/'out'),'--jobs','8','--java-opts','-Xms512M -Xmx25G','--creator',str(creator)]
|
||||||
|
run=subprocess.run(cmd,cwd=ROOT,capture_output=True,text=True)
|
||||||
|
self.assertEqual(run.returncode,0,run.stderr+'\n'+run.stdout)
|
||||||
|
self.assertTrue((d/'out'/'BGMountains-native.obf').is_file())
|
||||||
|
self.assertEqual(json.loads((d/'out'/'build-manifest.json').read_text())['version'],'v0.95')
|
||||||
|
self.assertEqual(json.loads((d/'out'/'semantic-audit.json').read_text())['status'],'PASS')
|
||||||
|
tags=t.tagdict(ET.parse(d/'out'/'BGMountains-native.osm').getroot().find('node'))
|
||||||
|
self.assertEqual(tags['amenity'],'drinking_water')
|
||||||
|
self.assertEqual(tags['man_made'],'water_tap')
|
||||||
|
def test_future_template_does_not_require_existing_configs(self):
|
||||||
|
with tempfile.TemporaryDirectory() as dd:
|
||||||
|
d=pathlib.Path(dd)
|
||||||
|
# No fake success from a malformed configuration: expected exit nonzero.
|
||||||
|
p=subprocess.run([sys.executable,SCRIPT,'--osm',str(d/'notthere.osm'),'--poi-config',str(d/'missing.csv'),'--line-config',str(LINES),'--prepare-only'],capture_output=True,text=True)
|
||||||
|
self.assertNotEqual(p.returncode,0)
|
||||||
|
self.assertIn('mapping file not found',p.stderr)
|
||||||
|
if __name__=='__main__':unittest.main()
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Audit Garmin -> OSM POI mappings against an exact OsmAnd poi_types.xml.
|
||||||
|
|
||||||
|
No OBF creation. Exits 0 for ordinary data regressions unless --strict is requested.
|
||||||
|
The existence of a poi_type does NOT guarantee map icon visibility at any zoom.
|
||||||
|
"""
|
||||||
|
import argparse,csv,json,collections,sys
|
||||||
|
from pathlib import Path
|
||||||
|
from xml.etree import ElementTree as ET
|
||||||
|
|
||||||
|
def registry(path):
|
||||||
|
root=ET.parse(path).getroot()
|
||||||
|
main=collections.defaultdict(list);extras=collections.defaultdict(list)
|
||||||
|
for node in root.iter():
|
||||||
|
if node.tag not in ('poi_type','poi_additional'):continue
|
||||||
|
t=node.attrib.get('tag');v=node.attrib.get('value')
|
||||||
|
if not t or v is None:continue
|
||||||
|
d=main if node.tag=='poi_type' else extras
|
||||||
|
d[f'{t}={v}'].append(node.attrib)
|
||||||
|
return main,extras
|
||||||
|
|
||||||
|
def interpret(row,main,extra):
|
||||||
|
tags=[v.strip() for v in row['osm_tags'].split(';') if '=' in v]
|
||||||
|
supported=[];noindex=[];additional=[];missing=[]
|
||||||
|
for tv in tags:
|
||||||
|
candidates=main.get(tv,[])
|
||||||
|
if any(x.get('no_indx')!='true' for x in candidates):supported.append(tv)
|
||||||
|
elif candidates:noindex.append(tv)
|
||||||
|
elif tv in extra:additional.append(tv)
|
||||||
|
else:missing.append(tv)
|
||||||
|
if supported:kind='native_indexable_type'
|
||||||
|
elif noindex:kind='only_nonindexed_type'
|
||||||
|
elif additional:kind='additional_without_primary_type'
|
||||||
|
else:kind='no_matching_native_type'
|
||||||
|
is_terrain=(row.get('priority')=='terrain' or row.get('code')=='0x6600')
|
||||||
|
priority=row.get('priority','')
|
||||||
|
cnt=int(row.get('source_count') or 0)
|
||||||
|
if is_terrain:severity='terrain_exemption'
|
||||||
|
elif kind=='native_indexable_type':severity='ok'
|
||||||
|
elif priority=='critical':severity='critical'
|
||||||
|
elif cnt>=100:severity='high'
|
||||||
|
else:severity='low'
|
||||||
|
return {'code':row.get('code',''),'count':cnt,'priority':priority,'legend_bg':row.get('legend_bg',''),
|
||||||
|
'osm_tags':row.get('osm_tags',''), 'status':kind, 'severity':severity,
|
||||||
|
'indexable_poi_tags':';'.join(supported),'nonindexed_poi_tags':';'.join(noindex),
|
||||||
|
'additional_only_tags':';'.join(additional),'unrecognized_tags':';'.join(missing),
|
||||||
|
'sample_lat':row.get('example_lat',''),'sample_lon':row.get('example_lon',''),
|
||||||
|
'explanation':'Registry compatibility only; does not guarantee rendering zoom or OBF preservation'}
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
ap=argparse.ArgumentParser(description=__doc__)
|
||||||
|
ap.add_argument('--mapping',required=True,help='Single canonical POI mapping CSV')
|
||||||
|
ap.add_argument('--poi-types',required=True,help='OsmAnd official poi_types.xml, preferably pinned version')
|
||||||
|
ap.add_argument('--report',default='poi-native-index-audit.csv')
|
||||||
|
ap.add_argument('--json',default='poi-native-index-audit.json')
|
||||||
|
ap.add_argument('--strict',action='store_true',help='Exit nonzero for unindexable critical/high types')
|
||||||
|
args=ap.parse_args(argv)
|
||||||
|
native,extra=registry(args.poi_types)
|
||||||
|
with open(args.mapping,encoding='utf8-sig',newline='') as f:
|
||||||
|
allrows=list(csv.DictReader(f))
|
||||||
|
# Name-conditional rows are not additional Garmin types; counting them as
|
||||||
|
# independent types would multiply source_count and inflate POI statistics.
|
||||||
|
rows=[r for r in allrows if not r.get('name_contains','')]
|
||||||
|
out=[interpret(r,native,extra) for r in rows]
|
||||||
|
with open(args.report,'w',newline='',encoding='utf8') as f:
|
||||||
|
w=csv.DictWriter(f,fieldnames=out[0].keys());w.writeheader();w.writerows(out)
|
||||||
|
stats=collections.Counter(r['status'] for r in out)
|
||||||
|
weighted=collections.Counter()
|
||||||
|
for r in out:weighted[r['status']]+=r['count']
|
||||||
|
risk=[r for r in out if r['severity'] in ('critical','high')]
|
||||||
|
payload={'type_counts':dict(stats),'weighted_source_counts':dict(weighted),'critical_or_high':len(risk),
|
||||||
|
'top_regressions':sorted(risk,key=lambda z:-z['count'])[:30],
|
||||||
|
'warning':'POI type registry check ONLY. Does not guarantee OBF map index presence, symbol visibility, zoom or Nearby settings.'}
|
||||||
|
Path(args.json).write_text(json.dumps(payload,ensure_ascii=False,indent=2),encoding='utf8')
|
||||||
|
print(f"Inspected {len(out)} Garmin types: {dict(stats)}, high/critical issues: {len(risk)}")
|
||||||
|
print(f"Wrote {args.report} and {args.json}")
|
||||||
|
return 2 if args.strict and risk else 0
|
||||||
|
|
||||||
|
if __name__=='__main__':sys.exit(main())
|
||||||
Reference in New Issue
Block a user