diff --git a/Docs/ServerConfig/cache/Caddyfile b/Docs/ServerConfig/cache/Caddyfile new file mode 100644 index 000000000..b818fae82 --- /dev/null +++ b/Docs/ServerConfig/cache/Caddyfile @@ -0,0 +1,14 @@ +cache.mapcomplete.org { + reverse_proxy /summary/* { + to http://127.0.0.1:2345 + } + + reverse_proxy /extractgraph { + to http://127.0.0.1:2346 + } + + reverse_proxy /* { + to http://127.0.0.1:7800 + } + +} diff --git a/Docs/ServerConfig/cache/cache.txt b/Docs/ServerConfig/cache/cache.txt new file mode 100644 index 000000000..ddda8f29a --- /dev/null +++ b/Docs/ServerConfig/cache/cache.txt @@ -0,0 +1,15 @@ +# Cache.mapComplete.org server config + +The "cache"-server is hosted at nerdlab. + +It has a full OSM-copy on disk, and has a Postgis/Postgres database with a table for every layer for quick access. It should run the tileserver and the summaryserver + +## Dyndns + +https://dynamicdns.park-your-domain.com/update?host=cache&domain=mapcomplete.org&password=[ddns_password] + +## Setup + +See SettingUpPSQL.md + + diff --git a/scripts/hetzner/config/Caddyfile b/Docs/ServerConfig/hetzner/Caddyfile similarity index 100% rename from scripts/hetzner/config/Caddyfile rename to Docs/ServerConfig/hetzner/Caddyfile diff --git a/scripts/hetzner/config/csp-logger-config.json b/Docs/ServerConfig/hetzner/csp-logger-config.json similarity index 100% rename from scripts/hetzner/config/csp-logger-config.json rename to Docs/ServerConfig/hetzner/csp-logger-config.json diff --git a/Docs/ServerConfig/hetzner/hetzner.txt b/Docs/ServerConfig/hetzner/hetzner.txt new file mode 100644 index 000000000..0dae8e283 --- /dev/null +++ b/Docs/ServerConfig/hetzner/hetzner.txt @@ -0,0 +1,5 @@ +# Hetzner + +This server hosts the studio files and is used for expermintal builds. + +For used hosts, see the Caddyfile diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md new file mode 100644 index 000000000..6a7f79c9a --- /dev/null +++ b/Docs/SettingUpPSQL.md @@ -0,0 +1,75 @@ +# Setting up a synced OSM-server for quick layer access + +## Setting up the SQL-server: + +`sudo docker run --name some-postgis -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -d -p 5444:5432 -v /home/pietervdvn/data/pgsql/:/var/lib/postgresql/data postgis/postgis` + +Then, connect to this databank with PGAdmin, create a database within it. +Then activate following extensions for this database (right click > Create > Extension): + +- Postgis activeren (rechtsklikken > Create > extension) +- HStore activeren + +Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot. + +- Open PGAdmin, open the PGSQL-tool (CLI-button at the top) +- Run `max_connections = 2000;` and `show config_file;` to get the config file location (in docker). This is probably `/var/lib/postgresql/data/postgresql.conf` +- In a terminal, run `sudo docker exec -i bash` (run `sudo docker ps` to get the container id) +- `sed -i "s/max_connections = 100/max_connections = 5000/" /var/lib/postgresql/data/postgresql.conf` +- Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"` +- `sudo docker restart ` + +## Create export scripts for every layer + +Use `vite-node ./scripts/osm2pgsql/generateBuildDbScript.ts` + +## Importing data + +Install osm2pgsql (hint: compile from source is painless) +To seed the database: + +```` +osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi .osm.pbf +```` +Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm + +Belgium (~555mb) takes 15m +World (80GB) should take 15m*160 = 2400m = 40hr + +73G Jan 23 00:22 planet-240115.osm.pbf: 2024-02-10 16:45:11 osm2pgsql took 871615s (242h 6m 55s; 10 days) overall on lain.local with RAID5 on 4 HDD disks, database is over 1Terrabyte (!) + +Server specs + +Lenovo thinkserver RD350, Intel Xeon E5-2600, 2Rx4 PC3 + 11 watt powered off, 73 watt idle, ~100 watt when importing + +HP ProLiant DL360 G7 (1U): 2Rx4 DDR3-memory (PC3) + Intel Xeon X56** + + +## Updating data + +`osm2pgsql-replication update -d postgresql://user:password@localhost:5444/osm-poi -- -O flex -S build_db.lua -s --flat-nodes=import-help-file` + + +## Deploying a tile server + +pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv + +```` +export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi +nohup ./pg_tileserv & +```` + +Tiles are available at: +```` +map.addSource("drinking_water", { +"type": "vector", +"tiles": ["http://127.0.0.2:7800/public.drinking_water/{z}/{x}/{y}.pbf"] // http://127.0.0.2:7800/public.drinking_water.json", +}) +```` + +# Rebooting: + +-> Restart the docker container +-> diff --git a/assets/layers/climbing_opportunity/climbing_opportunity.json b/assets/layers/climbing_opportunity/climbing_opportunity.json index 11d772791..974d1fe65 100644 --- a/assets/layers/climbing_opportunity/climbing_opportunity.json +++ b/assets/layers/climbing_opportunity/climbing_opportunity.json @@ -35,6 +35,7 @@ } }, "minzoom": 19, + "doCount": false, "title": { "render": { "en": "Climbing opportunity?", diff --git a/assets/layers/etymology/etymology.json b/assets/layers/etymology/etymology.json index 45d9bdf68..443c1f705 100644 --- a/assets/layers/etymology/etymology.json +++ b/assets/layers/etymology/etymology.json @@ -30,7 +30,7 @@ } }, "calculatedTags": [ - "_same_name_ids=closestn(feat)('*', 250, undefined, 5000)?.filter(f => f.feat.properties.name === feat.properties.name)?.map(f => f.feat.properties.id)??[]" + "_same_name_ids=closestn(feat)('*', 250, undefined, 3000)?.filter(f => f.feat.properties.name === feat.properties.name)?.map(f => f.feat.properties.id)??[]" ], "minzoom": 12, "title": { diff --git a/assets/layers/ice_cream/ice_cream.json b/assets/layers/ice_cream/ice_cream.json index b6d6576a9..8b5ea0568 100644 --- a/assets/layers/ice_cream/ice_cream.json +++ b/assets/layers/ice_cream/ice_cream.json @@ -4,6 +4,7 @@ "en": "Ice cream parlors", "de": "Eisdielen" }, + "minzoom": 14, "description": { "en": "A place where ice cream is sold over the counter", "de": "Ein Ort, an dem Eiscreme an der Theke verkauft wird" diff --git a/assets/layers/item_with_image/item_with_image.json b/assets/layers/item_with_image/item_with_image.json index 27c6ac1b3..8209d2cd1 100644 --- a/assets/layers/item_with_image/item_with_image.json +++ b/assets/layers/item_with_image/item_with_image.json @@ -22,10 +22,14 @@ "render": { "en": "POI with image" }, - "mappings": [{ - "if": "name~*", - "then": {"*": "name"} - }] + "mappings": [ + { + "if": "name~*", + "then": { + "*": "name" + } + } + ] }, "name": { "en": "Items with at least one image" diff --git a/assets/layers/last_click/last_click.json b/assets/layers/last_click/last_click.json index c99a4ce5b..335731bda 100644 --- a/assets/layers/last_click/last_click.json +++ b/assets/layers/last_click/last_click.json @@ -1,7 +1,7 @@ { "id": "last_click", "name": null, - "description": "This layer defines how to render the 'last click'-location. By default, it will show a marker with the possibility to add a new point (if there are some presets) and/or to add a new note (if the 'note' layer attribute is set). If none are possible, this layer won't show up", + "description": "This 'layer' is not really a layer, but contains part of the code how the popup to 'add a new marker' is displayed", "source": "special", "isShown": { "or": [ diff --git a/assets/layers/shower/shower.json b/assets/layers/shower/shower.json index 759004606..50fb25ed9 100644 --- a/assets/layers/shower/shower.json +++ b/assets/layers/shower/shower.json @@ -17,7 +17,7 @@ "source": { "osmTags": "amenity=shower" }, - "minzoom": 12, + "minzoom": 8, "title": { "render": { "en": "Shower", diff --git a/assets/layers/summary/summary.json b/assets/layers/summary/summary.json new file mode 100644 index 000000000..deccd5e6b --- /dev/null +++ b/assets/layers/summary/summary.json @@ -0,0 +1,27 @@ +{ + "id": "summary", + "description": "Special layer which shows `count`", + "source": "special", + "title": { + "render": { + "en": "Summary" + } + }, + "tagRenderings": [ + "all_tags" + ], + "pointRendering": [ + { + "location": [ + "point", + "centroid" + ], + "iconSize": "40,40", + "label": { + "render": "{total_metric}" + }, + "labelCss": "background: #ffffffbb", + "labelCssClasses": "w-12 text-lg rounded-xl p-1 px-2" + } + ] +} diff --git a/assets/layers/walls_and_buildings/walls_and_buildings.json b/assets/layers/walls_and_buildings/walls_and_buildings.json index f8368a464..53a482adc 100644 --- a/assets/layers/walls_and_buildings/walls_and_buildings.json +++ b/assets/layers/walls_and_buildings/walls_and_buildings.json @@ -31,6 +31,7 @@ ], "minzoom": 18, "shownByDefault": false, + "isCounted": false, "title": { "render": { "en": "Wall or building", diff --git a/assets/themes/advertising/advertising.json b/assets/themes/advertising/advertising.json index 46718f5cd..0b00bf07a 100644 --- a/assets/themes/advertising/advertising.json +++ b/assets/themes/advertising/advertising.json @@ -47,4 +47,4 @@ ], "widenFactor": 0.01, "maintainer": "Offsel" -} \ No newline at end of file +} diff --git a/assets/themes/atm/atm.json b/assets/themes/atm/atm.json index fc6f4e707..2a625cd04 100644 --- a/assets/themes/atm/atm.json +++ b/assets/themes/atm/atm.json @@ -58,7 +58,8 @@ "name": null, "filter": { "sameAs": "bank_with_atm" - } + }, + "doCount": false } }, { diff --git a/assets/themes/bag/bag.json b/assets/themes/bag/bag.json index 1aa324bc3..d9a70c590 100644 --- a/assets/themes/bag/bag.json +++ b/assets/themes/bag/bag.json @@ -45,7 +45,7 @@ "hideFromOverview": true, "layers": [ { - "id": "osm:buildings", + "id": "osm_buildings", "name": "OSM Buildings", "title": "OSM Building", "description": "Layer showing buildings that are in OpenStreetMap", @@ -54,6 +54,7 @@ "maxCacheAge": 0 }, "minzoom": 18, + "doCount": false, "calculatedTags": [ "_surface:strict:=feat(get)('_surface')" ], @@ -147,7 +148,7 @@ ] }, { - "id": "osm:adresses", + "id": "osm_adresses", "name": "OSM Adresses", "title": "OSM Adress", "description": "Layer showing adresses that are in OpenStreetMap", @@ -164,6 +165,7 @@ "maxCacheAge": 0 }, "minzoom": 18, + "doCount": false, "pointRendering": [ { "label": { @@ -185,7 +187,7 @@ ] }, { - "id": "bag:pand", + "id": "bag_pand", "name": "BAG Buildings", "title": "BAG Building", "description": { @@ -207,7 +209,7 @@ }, "minzoom": 18, "calculatedTags": [ - "_overlaps_with_buildings=overlapWith(feat)('osm:buildings').filter(f => f.feat.properties.id.indexOf('-') < 0)", + "_overlaps_with_buildings=overlapWith(feat)('osm_buildings').filter(f => f.feat.properties.id.indexOf('-') < 0)", "_overlaps_with=feat(get)('_overlaps_with_buildings').find(f => f.overlap > 1 /* square meter */ )", "_overlaps_with_properties=feat(get)('_overlaps_with')?.feat?.properties", "_overlap_percentage=Math.round(100 * (feat(get)('_overlaps_with')?.overlap / feat(get)('_overlaps_with_properties')['_surface:strict']))", @@ -228,7 +230,7 @@ "render": { "special": { "type": "import_way_button", - "targetLayer": "osm:buildings", + "targetLayer": "osm_buildings", "tags": "building=$_bag_obj:building; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date", "text": { "*": "Upload this building to OpenStreetMap" @@ -258,7 +260,7 @@ }, { "if": "_overlaps_with!=", - "then": "{conflate_button(osm:buildings, building=$_bag_obj:building; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Replace the geometry in OpenStreetMap, , _osm_obj:id)}" + "then": "{conflate_button(osm_buildings, building=$_bag_obj:building; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Replace the geometry in OpenStreetMap, , _osm_obj:id)}" }, { "if": { @@ -268,7 +270,7 @@ "_bag_obj:in_construction=true" ] }, - "then": "{import_way_button(osm:buildings, building=$_bag_obj:building; construction=$_bag_obj:construction; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Upload this building to OpenStreetMap)}" + "then": "{import_way_button(osm_buildings, building=$_bag_obj:building; construction=$_bag_obj:construction; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Upload this building to OpenStreetMap)}" } ] }, @@ -348,7 +350,7 @@ }, { "id": "Overlapping building", - "render": "
The overlapping osm:buildings is a {_osm_obj:building} and covers {_overlap_percentage}% of the BAG building.
The BAG-building covers {_reverse_overlap_percentage}% of the OSM building

BAG geometry:

{minimap(21, id):height:10rem;border-radius:1rem;overflow:hidden}

OSM geometry:

{minimap(21,_osm_obj:id):height:10rem;border-radius:1rem;overflow:hidden}
", + "render": "
The overlapping osm_buildings is a {_osm_obj:building} and covers {_overlap_percentage}% of the BAG building.
The BAG-building covers {_reverse_overlap_percentage}% of the OSM building

BAG geometry:

{minimap(21, id):height:10rem;border-radius:1rem;overflow:hidden}

OSM geometry:

{minimap(21,_osm_obj:id):height:10rem;border-radius:1rem;overflow:hidden}
", "condition": "_overlaps_with!=" }, { @@ -386,7 +388,7 @@ ] }, { - "id": "bag:verblijfsobject", + "id": "bag_verblijfsobject", "name": "BAG Addresses", "title": "BAG Address", "description": "Address information from the BAG register", @@ -398,7 +400,7 @@ }, "minzoom": 18, "calculatedTags": [ - "_closed_osm_addr:=closest(feat)('osm:adresses').properties", + "_closed_osm_addr:=closest(feat)('osm_adresses').properties", "_bag_obj:addr:housenumber=`${feat.properties.huisnummer}${feat.properties.huisletter}${(feat.properties.toevoeging != '') ? '-' : ''}${feat.properties.toevoeging}`", "_bag_obj:ref:bag=Number(feat.properties.identificatie)", "_bag_obj:source:date=new Date().toISOString().split('T')[0]", @@ -411,7 +413,7 @@ "tagRenderings": [ { "id": "Import button", - "render": "{import_button(osm:adresses, addr:city=$woonplaats; addr:housenumber=$_bag_obj:addr:housenumber; addr:postcode=$postcode; addr:street=$openbare_ruimte; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date, Upload this adress to OpenStreetMap)}", + "render": "{import_button(osm_adresses, addr:city=$woonplaats; addr:housenumber=$_bag_obj:addr:housenumber; addr:postcode=$postcode; addr:street=$openbare_ruimte; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date, Upload this adress to OpenStreetMap)}", "condition": "_imported_osm_object_found=false" }, { diff --git a/assets/themes/benches/benches.json b/assets/themes/benches/benches.json index 5161ecaa7..d1ea28881 100644 --- a/assets/themes/benches/benches.json +++ b/assets/themes/benches/benches.json @@ -73,4 +73,4 @@ "bench_at_pt" ], "widenFactor": 1.5 -} \ No newline at end of file +} diff --git a/assets/themes/climbing/climbing.json b/assets/themes/climbing/climbing.json index f080d16fa..31b513d86 100644 --- a/assets/themes/climbing/climbing.json +++ b/assets/themes/climbing/climbing.json @@ -394,6 +394,7 @@ }, "minzoom": 16, "name": null, + "doCount": false, "+tagRenderings": [ { "id": "repairs_climbing_shoes", @@ -460,6 +461,7 @@ ], "override": { "minzoom": 15, + "doCount": false, "pointRendering": [ { "iconSize": "30,30" diff --git a/assets/themes/cycle_highways/cycle_highways.json b/assets/themes/cycle_highways/cycle_highways.json index 2365eaade..5c4408bbd 100644 --- a/assets/themes/cycle_highways/cycle_highways.json +++ b/assets/themes/cycle_highways/cycle_highways.json @@ -266,12 +266,6 @@ ] } ], - "enableDownload": true, - "enablePdfDownload": true, "overpassTimeout": 60, - "widenFactor": 1.1, - "#overpassUrl": "https://overpass.kumi.systems/api/interpreter", - "clustering": { - "maxZoom": 1 - } + "widenFactor": 1.1 } diff --git a/assets/themes/cyclenodes/cyclenodes.json b/assets/themes/cyclenodes/cyclenodes.json index f77308652..94c17b99d 100644 --- a/assets/themes/cyclenodes/cyclenodes.json +++ b/assets/themes/cyclenodes/cyclenodes.json @@ -32,7 +32,7 @@ "hideFromOverview": true, "layers": [ { - "id": "node2node", + "id": "node2node_bicycle", "name": { "en": "Node to node links", "de": "Knotenpunktverbindungen", @@ -126,7 +126,7 @@ "pointRendering": null }, { - "id": "node", + "id": "node_bicycle", "name": { "en": "Nodes", "de": "Knotenpunkte", @@ -327,6 +327,7 @@ ], "override": { "minzoom": 16, + "id": "bicycle_guidepost", "source": { "osmTags": { "and": [ diff --git a/assets/themes/cyclestreets/cyclestreets.json b/assets/themes/cyclestreets/cyclestreets.json index 7b8ce2fc8..da41a8deb 100644 --- a/assets/themes/cyclestreets/cyclestreets.json +++ b/assets/themes/cyclestreets/cyclestreets.json @@ -290,7 +290,7 @@ ] }, { - "id": "all_streets", + "id": "not_cyclestreets", "name": { "nl": "Alle straten", "en": "All streets", @@ -403,7 +403,8 @@ }, "width": "5" } - ] + ], + "isCounted": false } ], "overrideAll": { @@ -768,9 +769,5 @@ } ] }, - "widenFactor": 2, - "clustering": { - "maxZoom": 12, - "minNeededElements": 200 - } + "widenFactor": 2 } diff --git a/assets/themes/etymology/etymology.json b/assets/themes/etymology/etymology.json index 877cdcd8b..3928fd728 100644 --- a/assets/themes/etymology/etymology.json +++ b/assets/themes/etymology/etymology.json @@ -68,6 +68,7 @@ "pl": "Ulice bez informacji o etymologii" }, "minzoom": 15, + "isCounted": false, "source": { "=osmTags": { "and": [ @@ -99,6 +100,7 @@ "pl": "Parki i lasy bez informacji o etymologii" }, "minzoom": 18, + "isCounted": false, "source": { "osmTags": { "and": [ @@ -131,6 +133,7 @@ "pl": "Instytucje edukacyjne bez informacji o etymologii" }, "minzoom": 18, + "isCounted": false, "source": { "osmTags": { "and": [ @@ -166,6 +169,7 @@ "pl": "Miejsca kulturowe bez informacji o etymologii" }, "minzoom": 18, + "isCounted": false, "source": { "osmTags": { "and": [ @@ -201,6 +205,7 @@ "pl": "Miejsca turystyczne bez informacji o etymologii" }, "minzoom": 18, + "isCounted": false, "source": { "osmTags": { "and": [ @@ -235,6 +240,7 @@ "pl": "Miejsca związane ze zdrowiem i społeczeństwem bez informacji o etymologii" }, "minzoom": 18, + "isCounted": false, "source": { "osmTags": { "and": [ @@ -268,6 +274,7 @@ "pl": "Miejsca sportowe bez informacji o etymologii" }, "minzoom": 18, + "isCounted": false, "source": { "osmTags": { "and": [ @@ -284,10 +291,5 @@ } } } - ], - "widenFactor": 2, - "clustering": { - "maxZoom": 14, - "minNeededElements": 250 - } -} \ No newline at end of file + ] +} diff --git a/assets/themes/fritures/fritures.json b/assets/themes/fritures/fritures.json index 1543f2b9a..05b5b8949 100644 --- a/assets/themes/fritures/fritures.json +++ b/assets/themes/fritures/fritures.json @@ -96,7 +96,8 @@ "override": { "minzoom": 18, "filter": null, - "name": null + "name": null, + "isCounted": false } } ], diff --git a/assets/themes/ghostbikes/ghostbikes.json b/assets/themes/ghostbikes/ghostbikes.json index 045e57d72..841f11480 100644 --- a/assets/themes/ghostbikes/ghostbikes.json +++ b/assets/themes/ghostbikes/ghostbikes.json @@ -43,8 +43,5 @@ "layers": [ "ghost_bike" ], - "widenFactor": 5, - "clustering": { - "maxZoom": 0 - } -} \ No newline at end of file + "widenFactor": 5 +} diff --git a/assets/themes/ghostsigns/ghostsigns.json b/assets/themes/ghostsigns/ghostsigns.json index 21577bb77..e95b54369 100644 --- a/assets/themes/ghostsigns/ghostsigns.json +++ b/assets/themes/ghostsigns/ghostsigns.json @@ -140,7 +140,12 @@ "source": { "osmTags": "advertising=wall_painting" }, + "id": "advertising_wall_paintings", "minzoom": 18, + "name": { + "en": "All advertentie wall paintings", + "nl": "Alle adverterende muurschilderingen" + }, "+tagRenderings": [ { "id": "historic", @@ -172,8 +177,9 @@ { "iconSize": "20,20" } - ] + ], + "isCounted": false } } ] -} \ No newline at end of file +} diff --git a/assets/themes/grb/grb.json b/assets/themes/grb/grb.json index c4c955660..89461b7ac 100644 --- a/assets/themes/grb/grb.json +++ b/assets/themes/grb/grb.json @@ -29,7 +29,7 @@ "hideFromOverview": true, "layers": [ { - "id": "osm-buildings", + "id": "osm_buildings_no_points", "name": "All OSM-buildings", "source": { "osmTags": { @@ -296,7 +296,7 @@ "name": "GRB geometries", "title": "GRB outline", "calculatedTags": [ - "_overlaps_with_buildings=overlapWith(feat)('osm-buildings').filter(f => f.feat.properties.id.indexOf('-') < 0) ?? []", + "_overlaps_with_buildings=overlapWith(feat)('osm_buildings_no_points').filter(f => f.feat.properties.id.indexOf('-') < 0) ?? []", "_overlaps_with=get(feat)('_overlaps_with_buildings').find(f => f.overlap > 1 /* square meter */ )", "_osm_obj:source:ref=get(feat)('_overlaps_with')?.feat?.properties['source:geometry:ref']", "_osm_obj:id=get(feat)('_overlaps_with')?.feat?.properties?.id", @@ -319,7 +319,7 @@ "tagRenderings": [ { "id": "Import-button", - "render": "{import_way_button(osm-buildings,building=$building;man_made=$man_made; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber; building:min_level=$_building:min_level, Upload this building to OpenStreetMap,,_is_part_of_building=true,1,_moveable=true)}", + "render": "{import_way_button(osm_buildings_no_points,building=$building;man_made=$man_made; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber; building:min_level=$_building:min_level, Upload this building to OpenStreetMap,,_is_part_of_building=true,1,_moveable=true)}", "mappings": [ { "#": "Failsafe", @@ -371,7 +371,7 @@ "addr:housenumber!:={_osm_obj:addr:housenumber}" ] }, - "then": "{conflate_button(osm-buildings,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber, Replace the geometry in OpenStreetMap and add the address,,_osm_obj:id)}" + "then": "{conflate_button(osm_buildings_no_points,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber, Replace the geometry in OpenStreetMap and add the address,,_osm_obj:id)}" }, { "if": { @@ -380,7 +380,7 @@ "_reverse_overlap_percentage>50" ] }, - "then": "{conflate_button(osm-buildings,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref, Replace the geometry in OpenStreetMap,,_osm_obj:id)}" + "then": "{conflate_button(osm_buildings_no_points,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref, Replace the geometry in OpenStreetMap,,_osm_obj:id)}" } ] }, @@ -612,7 +612,7 @@ "builtin": "crab_address", "override": { "calculatedTags+": [ - "_embedded_in=overlapWith(feat)('osm-buildings').filter(b => /* Do not match newly created objects */ b.feat.properties.id.indexOf('-') < 0)[0]?.feat?.properties ?? {}", + "_embedded_in=overlapWith(feat)('osm_buildings_no_points').filter(b => /* Do not match newly created objects */ b.feat.properties.id.indexOf('-') < 0)[0]?.feat?.properties ?? {}", "_embedding_nr=get(feat)('_embedded_in')['addr:housenumber']+(get(feat)('_embedded_in')['addr:unit'] ?? '')", "_embedding_street=get(feat)('_embedded_in')['addr:street']", "_embedding_id=get(feat)('_embedded_in').id", @@ -709,7 +709,7 @@ "text": { "nl": "Voeg dit adres als een nieuw adrespunt toe" }, - "snap_onto_layers": "osm-buildings" + "snap_onto_layers": "osm_buildings_no_points" } }, "mappings": [ @@ -785,12 +785,10 @@ } ], "overrideAll": { - "minzoom": 17 + "minzoom": 17, + "doCount": false }, "widenFactor": 2, "overpassMaxZoom": 15, - "osmApiTileSize": 17, - "clustering": { - "maxZoom": 15 - } + "osmApiTileSize": 17 } diff --git a/assets/themes/healthcare/healthcare.json b/assets/themes/healthcare/healthcare.json index dba448616..cd2c8d874 100644 --- a/assets/themes/healthcare/healthcare.json +++ b/assets/themes/healthcare/healthcare.json @@ -39,7 +39,7 @@ { "builtin": "shops", "override": { - "id": "medical-shops", + "id": "medical_shops", "minzoom": 13, "=filter": [ "open_now", @@ -113,6 +113,7 @@ "override": { "=presets": [], "name": null, + "doCount": false, "minzoom": 18 } } diff --git a/assets/themes/icecream/icecream.json b/assets/themes/icecream/icecream.json index 483b5a991..509fd9e86 100644 --- a/assets/themes/icecream/icecream.json +++ b/assets/themes/icecream/icecream.json @@ -17,6 +17,5 @@ "icon": "./assets/layers/ice_cream/ice_cream.svg", "layers": [ "ice_cream" - ], - "minzoom": "14" + ] } diff --git a/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json b/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json index 108cfd410..ce049f837 100644 --- a/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json +++ b/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json @@ -46,6 +46,7 @@ { "builtin": "crossings", "override": { + "id": "crossings_no_traffic_lights", "=presets": [ { "title": { @@ -87,4 +88,4 @@ }, "kerbs" ] -} \ No newline at end of file +} diff --git a/assets/themes/mapcomplete-changes/mapcomplete-changes.json b/assets/themes/mapcomplete-changes/mapcomplete-changes.json index 3d4f53082..591381bdf 100644 --- a/assets/themes/mapcomplete-changes/mapcomplete-changes.json +++ b/assets/themes/mapcomplete-changes/mapcomplete-changes.json @@ -1,19 +1,13 @@ { "id": "mapcomplete-changes", "title": { - "en": "Changes made with MapComplete", - "de": "Änderungen mit MapComplete", - "es": "Cambios hechos con MapComplete" + "en": "Changes made with MapComplete" }, "shortDescription": { - "en": "Shows changes made by MapComplete", - "de": "Änderungen von MapComplete anzeigen", - "es": "Muestra los cambios hechos por MapComplete" + "en": "Shows changes made by MapComplete" }, "description": { - "en": "This maps shows all the changes made with MapComplete", - "de": "Diese Karte zeigt alle mit MapComplete vorgenommenen Änderungen", - "es": "Este mapa muestra todos los cambios hechos con MapComplete" + "en": "This maps shows all the changes made with MapComplete" }, "icon": "./assets/svg/logo.svg", "hideFromOverview": true, @@ -26,9 +20,7 @@ { "id": "mapcomplete-changes", "name": { - "en": "Changeset centers", - "de": "Zentrum der Änderungssätze", - "es": "Centro del conjunto de cambios" + "en": "Changeset centers" }, "minzoom": 0, "source": { @@ -39,55 +31,41 @@ }, "title": { "render": { - "en": "Changeset for {theme}", - "de": "Änderungssatz für {theme}", - "es": "Conjunto de cambios para {theme}" + "en": "Changeset for {theme}" } }, "description": { - "en": "Shows all MapComplete changes", - "de": "Alle MapComplete-Änderungen anzeigen", - "es": "Muestra todos los cambios de MapComplete" + "en": "Shows all MapComplete changes" }, "tagRenderings": [ { "id": "show_changeset_id", "render": { - "en": "Changeset {id}", - "de": "Änderungssatz {id}", - "es": "Conjunto de cambios {id}" + "en": "Changeset {id}" } }, { "id": "contributor", "question": { - "en": "What contributor did make this change?", - "de": "Wer hat diese Änderung vorgenommen?", - "es": "¿Quién realizó este cambio?" + "en": "What contributor did make this change?" }, "freeform": { "key": "user" }, "render": { - "en": "Change made by {user}", - "de": "Änderung von {user}", - "es": "Cambio hecho por {user}" + "en": "Change made by {user}" } }, { "id": "theme-id", "question": { - "en": "What theme was used to make this change?", - "de": "Welches Thema wurde für die Änderung verwendet?", - "es": "¿Qué tema se utilizó para realizar este cambio?" + "en": "What theme was used to make this change?" }, "freeform": { "key": "theme" }, "render": { - "en": "Change with theme {theme}", - "de": "Geändert mit Thema {theme}", - "es": "Cambio con el tema {theme}" + "en": "Change with theme {theme}" } }, { @@ -96,27 +74,19 @@ "key": "locale" }, "question": { - "en": "What locale (language) was this change made in?", - "de": "In welcher Benutzersprache wurde die Änderung vorgenommen?", - "es": "¿En qué configuración regional (idioma) se realizó este cambio?" + "en": "What locale (language) was this change made in?" }, "render": { - "en": "User locale is {locale}", - "de": "Benutzersprache {locale}", - "es": "La configuración regional del usuario es {locale}" + "en": "User locale is {locale}" } }, { "id": "host", "render": { - "en": "Change with with {host}", - "de": "Änderung über {host}", - "es": "Cambio con {host}" + "en": "Change with with {host}" }, "question": { - "en": "What host (website) was this change made with?", - "de": "Über welchen Host (Webseite) wurde diese Änderung vorgenommen?", - "es": "¿Con qué host (página web) se realizó este cambio?" + "en": "What host (website) was this change made with?" }, "freeform": { "key": "host" @@ -137,14 +107,10 @@ { "id": "version", "question": { - "en": "What version of MapComplete was used to make this change?", - "de": "Mit welcher MapComplete Version wurde die Änderung vorgenommen?", - "es": "¿Qué versión de MapComplete se usó para realizar este cambio?" + "en": "What version of MapComplete was used to make this change?" }, "render": { - "en": "Made with {editor}", - "de": "Erstellt mit {editor}", - "es": "Hecho con {editor}" + "en": "Made with {editor}" }, "freeform": { "key": "editor" @@ -518,9 +484,7 @@ } ], "question": { - "en": "Themename contains {search}", - "de": "Themename enthält {search}", - "es": "El nombre del tema contiene {search}" + "en": "Themename contains {search}" } } ] @@ -536,9 +500,7 @@ } ], "question": { - "en": "Themename does not contain {search}", - "de": "Der Name enthält nicht {search}", - "es": "El nombre del tema no contiene {search}" + "en": "Themename does not contain {search}" } } ] @@ -554,9 +516,7 @@ } ], "question": { - "en": "Made by contributor {search}", - "de": "Erstellt vom Mitwirkenden {search}", - "es": "Hecho por el colaborador {search}" + "en": "Made by contributor {search}" } } ] @@ -572,9 +532,7 @@ } ], "question": { - "en": "Not made by contributor {search}", - "de": "Nicht erstellt von Mitwirkendem {search}", - "es": "No hecho por el colaborador {search}" + "en": "Not made by contributor {search}" } } ] @@ -591,9 +549,7 @@ } ], "question": { - "en": "Made before {search}", - "de": "Erstellt vor {search}", - "es": "Hecho antes de {search}" + "en": "Made before {search}" } } ] @@ -610,9 +566,7 @@ } ], "question": { - "en": "Made after {search}", - "de": "Erstellt nach {search}", - "es": "Hecho después de {search}" + "en": "Made after {search}" } } ] @@ -628,9 +582,7 @@ } ], "question": { - "en": "User language (iso-code) {search}", - "de": "Benutzersprache (ISO-Code) {search}", - "es": "Idioma del usuario (código ISO) {search}" + "en": "User language (iso-code) {search}" } } ] @@ -646,9 +598,7 @@ } ], "question": { - "en": "Made with host {search}", - "de": "Erstellt mit host {search}", - "es": "Hecho con el host {search}" + "en": "Made with host {search}" } } ] @@ -659,9 +609,7 @@ { "osmTags": "add-image>0", "question": { - "en": "Changeset added at least one image", - "de": "Im Änderungssatz wurde mindestens ein Bild hinzugefügt", - "es": "El conjunto de cambios ha añadido al menos una imagen" + "en": "Changeset added at least one image" } } ] @@ -672,9 +620,7 @@ { "osmTags": "theme!=grb", "question": { - "en": "Exclude GRB theme", - "de": "GRB-Thema ausschließen", - "es": "Excluir el tema del GRB" + "en": "Exclude GRB theme" } } ] @@ -685,9 +631,7 @@ { "osmTags": "theme!=etymology", "question": { - "en": "Exclude etymology theme", - "de": "Etymologie-Thema ausschließen", - "es": "Excluir el tema de la etimología" + "en": "Exclude etymology theme" } } ] @@ -702,9 +646,7 @@ { "id": "link_to_more", "render": { - "en": "More statistics can be found here", - "de": "Weitere Statistiken gibt es hier", - "es": "Puede encontrar más estadísticas aquí" + "en": "More statistics can be found here" } }, { diff --git a/assets/themes/openlovemap/openlovemap.json b/assets/themes/openlovemap/openlovemap.json index 135a77be4..b065d55f8 100644 --- a/assets/themes/openlovemap/openlovemap.json +++ b/assets/themes/openlovemap/openlovemap.json @@ -18,7 +18,7 @@ { "builtin": "shops", "override": { - "id": "erotic-shop", + "id": "erotic_shop", "source": { "osmTags": "shop=erotic" }, @@ -51,6 +51,7 @@ "minzoom": 18, "=presets": [], "=name": null, + "doCount": false, "=filter": { "sameAs": "erotic-shop" } @@ -125,6 +126,7 @@ "override": { "minzoom": 18, "=presets": [], + "doCount": false, "=name": null } }, @@ -203,6 +205,7 @@ "override": { "minzoom": 18, "=presets": [], + "doCount": false, "=name": null } }, @@ -237,6 +240,7 @@ ] } ], + "doCount": false, "=presets": [] } } diff --git a/assets/themes/shops/shops.json b/assets/themes/shops/shops.json index b19b67260..99503d0f2 100644 --- a/assets/themes/shops/shops.json +++ b/assets/themes/shops/shops.json @@ -55,5 +55,7 @@ "ice_cream", "trolley_bay" ], - "widenFactor": 3 -} \ No newline at end of file + "overrideAll": { + "minzoom": 16 + } +} diff --git a/assets/themes/toilets/toilets.json b/assets/themes/toilets/toilets.json index 3879b0385..87afd0068 100644 --- a/assets/themes/toilets/toilets.json +++ b/assets/themes/toilets/toilets.json @@ -44,4 +44,4 @@ "shower" ], "widenFactor": 3 -} \ No newline at end of file +} diff --git a/assets/themes/transit/transit.json b/assets/themes/transit/transit.json index 84c4a6abb..5676d6802 100644 --- a/assets/themes/transit/transit.json +++ b/assets/themes/transit/transit.json @@ -53,6 +53,7 @@ "builtin": "shelter", "override": { "minzoom": 18, + "id": "pt_shelter", "source": { "osmTags": { "and": [ diff --git a/assets/themes/uk_addresses/uk_addresses.json b/assets/themes/uk_addresses/uk_addresses.json index 0a05b9665..97bf12a81 100644 --- a/assets/themes/uk_addresses/uk_addresses.json +++ b/assets/themes/uk_addresses/uk_addresses.json @@ -185,7 +185,7 @@ ] }, { - "id": "address", + "id": "uk_address", "name": { "en": "Known addresses in OSM" }, diff --git a/assets/themes/velopark/velopark.json b/assets/themes/velopark/velopark.json index 2d43fdcbb..2beb58305 100644 --- a/assets/themes/velopark/velopark.json +++ b/assets/themes/velopark/velopark.json @@ -284,6 +284,7 @@ "bike_parking" ], "override": { + "doCount": false, "minzoom": 14 } }, @@ -294,6 +295,7 @@ "bicycle_rental" ], "override": { + "doCount": false, "minzoom": 18 } } diff --git a/assets/themes/vending_machine/vending_machine.json b/assets/themes/vending_machine/vending_machine.json index 55e92092b..6e562630d 100644 --- a/assets/themes/vending_machine/vending_machine.json +++ b/assets/themes/vending_machine/vending_machine.json @@ -31,6 +31,7 @@ "override": { "id": "all_vending_machine", "name": null, + "doCount": false, "filter": { "sameAs": "vending_machine" }, @@ -49,6 +50,7 @@ { "builtin": "parking_ticket_machine", "override": { + "doCount": false, "name": null, "minzoom": 18 } @@ -56,6 +58,7 @@ { "builtin": "elongated_coin", "override": { + "doCount": false, "name": null, "minzoom": 18 } @@ -63,9 +66,10 @@ { "builtin": "ticket_machine", "override": { + "doCount": false, "name": null, "minzoom": 18 } } ] -} \ No newline at end of file +} diff --git a/assets/themes/walkingnodes/walkingnodes.json b/assets/themes/walkingnodes/walkingnodes.json index cd1f73bf8..bd59bf50b 100644 --- a/assets/themes/walkingnodes/walkingnodes.json +++ b/assets/themes/walkingnodes/walkingnodes.json @@ -23,7 +23,7 @@ "hideFromOverview": true, "layers": [ { - "id": "node2node", + "id": "node2node_hiking", "name": { "en": "Node to node links", "de": "Knotenpunktverbindungen", @@ -122,7 +122,7 @@ ] }, { - "id": "node", + "id": "node_hiking", "name": { "en": "Nodes", "de": "Knotenpunkte", @@ -293,6 +293,7 @@ ], "override": { "minzoom": 16, + "id": "guidepost_hiking", "source": { "osmTags": { "and": [ diff --git a/langs/en.json b/langs/en.json index 02f158baa..6979f1fe8 100644 --- a/langs/en.json +++ b/langs/en.json @@ -214,6 +214,7 @@ "current_view_generic": "Export a PDF off the current view for {paper_size} in {orientation} orientation" }, "title": "Download", + "toMuch": "There are to much features to download them all", "uploadGpx": "Upload your track to OpenStreetMap" }, "enableGeolocationForSafari": "Did you not get the popup to ask for geopermission?", @@ -784,7 +785,9 @@ "feedback": "This is not a valid web address" }, "wikidata": { - "description": "A Wikidata identifier" + "description": "A Wikidata identifier", + "empty": "Please, enter some wikidata-entries", + "startsWithQ": "A wikidata identifier starts with Q and is followed by a number" } } } diff --git a/langs/layers/ca.json b/langs/layers/ca.json index 73cdc5531..bd8aa1b30 100644 --- a/langs/layers/ca.json +++ b/langs/layers/ca.json @@ -7100,6 +7100,15 @@ } } }, + "souvenir_note": { + "tagRenderings": { + "designs": { + "freeform": { + "placeholder": "Nombre de dissenys (p. e. 5)" + } + } + } + }, "speed_camera": { "description": "Capa que mostra càmeres de velocitat", "name": "Càmera de velocitat", diff --git a/langs/layers/cs.json b/langs/layers/cs.json index f2eddfbec..874ef3089 100644 --- a/langs/layers/cs.json +++ b/langs/layers/cs.json @@ -7396,6 +7396,15 @@ } } }, + "souvenir_note": { + "tagRenderings": { + "designs": { + "freeform": { + "placeholder": "Počet vzorů (např. 5)" + } + } + } + }, "speed_camera": { "description": "Vrstva zobrazující rychlostní radary", "name": "Rychlostní radar", diff --git a/langs/layers/de.json b/langs/layers/de.json index d6be3ee6d..5c2f8a7f4 100644 --- a/langs/layers/de.json +++ b/langs/layers/de.json @@ -8563,6 +8563,15 @@ "render": "Geschwindigkeitsreduzierte Straße" } }, + "souvenir_note": { + "tagRenderings": { + "designs": { + "freeform": { + "placeholder": "Motivanzahl (z.B. 5)" + } + } + } + }, "speed_camera": { "description": "Ebene mit Blitzern", "name": "Blitzer", diff --git a/langs/layers/en.json b/langs/layers/en.json index 09f6d317b..4d05671c4 100644 --- a/langs/layers/en.json +++ b/langs/layers/en.json @@ -5658,6 +5658,12 @@ "render": "Information board" } }, + "item_with_image": { + "name": "Items with at least one image", + "title": { + "render": "POI with image" + } + }, "kerbs": { "description": "A layer showing kerbs.", "filter": { @@ -6993,6 +6999,101 @@ "render": "Playground" } }, + "playground_equipment": { + "description": "Layer showing playground equipment", + "name": "Playground equipment", + "presets": { + "0": { + "description": "An exact type is asked later", + "title": "a playground device" + } + }, + "tagRenderings": { + "type": { + "freeform": { + "placeholder": "Type of device" + }, + "mappings": { + "0": { + "then": "This is a swing" + }, + "1": { + "then": "This is a structure consisting of several connected playground devices" + }, + "2": { + "then": "This is a slide" + }, + "3": { + "then": "This is a sand pit" + }, + "4": { + "then": "This is a spring rider" + }, + "5": { + "then": "This is a climbing frame" + }, + "6": { + "then": "This is a seesaw" + }, + "7": { + "then": "This is a playhouse" + }, + "8": { + "then": "This is a roundabout" + }, + "9": { + "then": "This is a basket swing" + }, + "10": { + "then": "This is a zip wire" + }, + "11": { + "then": "This is a horizontal bar" + }, + "12": { + "then": "This is a hopscotch" + }, + "13": { + "then": "This is a splash pad" + }, + "14": { + "then": "This is a climbing wall" + }, + "15": { + "then": "This is a map" + }, + "16": { + "then": "This is a bridge (either as a standalone device or as part of a larger structure)" + }, + "17": { + "then": "This is a bouncy cushion" + }, + "18": { + "then": "This is an activity panel" + }, + "19": { + "then": "This is a teen shelter" + }, + "20": { + "then": "This is a funnel used to play with funnel ball" + }, + "21": { + "then": "This is a spinning circle" + } + }, + "question": "What kind of device is this?", + "render": "This is a {playground}" + }, + "wheelchair-access": { + "override": { + "question": "Is this device accessible by wheelchair?" + } + } + }, + "title": { + "render": "Playground device" + } + }, "postboxes": { "description": "The layer showing postboxes.", "name": "Postboxes", @@ -7007,6 +7108,43 @@ }, "postoffices": { "description": "A layer showing post offices.", + "filter": { + "1": { + "options": { + "0": { + "question": "Offers letter posting" + } + } + }, + "2": { + "options": { + "0": { + "question": "Offers parcel posting" + } + } + }, + "3": { + "options": { + "0": { + "question": "Offers pickup of missed parcels" + } + } + }, + "4": { + "options": { + "0": { + "question": "Accepts pickup of parcels sent here" + } + } + }, + "5": { + "options": { + "0": { + "question": "Sells stamps" + } + } + } + }, "name": "Post offices", "presets": { "0": { @@ -8563,6 +8701,125 @@ "render": "Slow road" } }, + "souvenir_coin": { + "description": "Layer showing machines selling souvenir coins", + "name": "Souvenir Coin Machines", + "presets": { + "0": { + "description": "Add a machine selling souvenir coins", + "title": "a souvenir coin machine" + } + }, + "tagRenderings": { + "charge": { + "freeform": { + "placeholder": "Cost (e.g. 2 EUR)" + }, + "mappings": { + "0": { + "then": "A souvenir coin costs 2 euro" + } + }, + "question": "How much does a souvenir coin cost?", + "render": "A souvenir coins costs {charge}" + }, + "designs": { + "override": { + "mappings": { + "0": { + "then": "This machine has one design available" + }, + "1": { + "then": "This machine has two designs available" + }, + "2": { + "then": "This machine has three designs available" + }, + "3": { + "then": "This machine has four designs available" + } + }, + "render": "This machine has {coin:design_count} designs available" + } + }, + "indoor": { + "mappings": { + "0": { + "then": "This machine is located indoors." + }, + "1": { + "then": "This machine is located outdoors." + } + }, + "question": "Is this machine located indoors?" + } + }, + "title": { + "render": "Souvenir Coin Machine" + } + }, + "souvenir_note": { + "description": "Layer showing machines selling souvenir banknotes", + "name": "Souvenir Banknote Machines", + "presets": { + "0": { + "description": "Add a machine selling souvenir banknotes", + "title": "a souvenir banknote machine" + } + }, + "tagRenderings": { + "charge": { + "freeform": { + "placeholder": "Cost (e.g. 2 EUR)" + }, + "mappings": { + "0": { + "then": "A souvenir note costs 2 euro" + }, + "1": { + "then": "A souvenir note costs 3 euro" + } + }, + "question": "How much does a souvenir note cost?", + "render": "A souvenir note costs {charge}" + }, + "designs": { + "freeform": { + "placeholder": "Number of designs (e.g. 5)" + }, + "mappings": { + "0": { + "then": "This machine has one design available." + }, + "1": { + "then": "This machine has two designs available." + }, + "2": { + "then": "This machine has three designs available." + }, + "3": { + "then": "This machine has four designs available." + } + }, + "question": "How many designs are available?", + "render": "This machine has {note:design_count} designs available." + }, + "indoor": { + "mappings": { + "0": { + "then": "This machine is located indoors." + }, + "1": { + "then": "This machine is located outdoors." + } + }, + "question": "Is this machine located indoors?" + } + }, + "title": { + "render": "Souvenir Banknote Machine" + } + }, "speed_camera": { "description": "Layer showing speed cameras", "name": "Speed Camera", @@ -9063,6 +9320,11 @@ "render": "Stripclub" } }, + "summary": { + "title": { + "render": "Summary" + } + }, "surveillance_camera": { "description": "This layer shows surveillance cameras and allows a contributor to update information and add new cameras", "name": "Surveillance camera's", diff --git a/langs/layers/es.json b/langs/layers/es.json index 1297516d2..cb4b1ebab 100644 --- a/langs/layers/es.json +++ b/langs/layers/es.json @@ -4023,6 +4023,15 @@ } } }, + "souvenir_note": { + "tagRenderings": { + "designs": { + "freeform": { + "placeholder": "Número de diseños (por ejemplo, 5)" + } + } + } + }, "speed_camera": { "description": "Capa con cámaras de velocidad", "name": "Cámara de velocidad", diff --git a/langs/layers/nl.json b/langs/layers/nl.json index 4e838823d..ffd5547aa 100644 --- a/langs/layers/nl.json +++ b/langs/layers/nl.json @@ -6065,6 +6065,33 @@ "render": "Speeltuin" } }, + "playground_equipment": { + "tagRenderings": { + "type": { + "mappings": { + "0": { + "then": "Dit is een schommel" + }, + "3": { + "then": "Dit is een zandbak" + }, + "4": { + "then": "Dit is een veertoestel" + }, + "5": { + "then": "Dit is een klimrek" + }, + "6": { + "then": "Dit is een wipwap" + }, + "11": { + "then": "Dit is een rekstok" + } + }, + "question": "Wat voor speeltoestel is dit?" + } + } + }, "postboxes": { "description": "Deze laag toont brievenbussen.", "name": "Brievenbussen", diff --git a/langs/themes/en.json b/langs/themes/en.json index be7c82c18..060fbe7c3 100644 --- a/langs/themes/en.json +++ b/langs/themes/en.json @@ -810,6 +810,71 @@ "description": "A ghost bike is a memorial for a cyclist who died in a traffic accident, in the form of a white bicycle placed permanently near the accident location.

On this map, one can see all the ghost bikes which are known by OpenStreetMap. Is a ghost bike missing? Everyone can add or update information here - you only need to have a (free) OpenStreetMap account.

There exists an automated account on Mastodon which posts a monthly overview of ghost bikes worldwide

", "title": "Ghost bikes" }, + "ghostsigns": { + "description": "A map showing disused signs on buildings", + "layers": { + "0": { + "description": "Layer showing disused signs on buildings", + "name": "Ghost Signs", + "presets": { + "0": { + "title": "a ghost sign" + } + }, + "tagRenderings": { + "brand": { + "freeform": { + "placeholder": "Business name" + }, + "question": "For what business was this sign made?", + "render": "This sign was made for: {brand}" + }, + "historic": { + "mappings": { + "0": { + "then": "This is a ghost sign" + }, + "1": { + "then": "This is not a ghost sign, answering this will hide the sign from the map" + } + }, + "question": "Is this a ghost sign?", + "questionHint": "Is this sign for a business that no longer exists or no longer being maintained?" + }, + "inscription": { + "freeform": { + "placeholder": "Text on the sign" + }, + "question": "What is the text on the sign?", + "render": "The text on the sign is: {inscription}" + } + }, + "title": { + "render": "Ghost Sign" + } + }, + "1": { + "override": { + "+tagRenderings": { + "0": { + "mappings": { + "0": { + "then": "This is a ghost sign" + }, + "1": { + "then": "This is not a ghost sign" + } + }, + "question": "Is this a ghost sign?", + "questionHint": "Is this sign for a business that no longer exists or no longer being maintained?" + } + }, + "name": "All advertentie wall paintings" + } + } + }, + "title": "Ghost Signs" + }, "grb": { "description": "This theme is an attempt to help automating the GRB import.", "layers": { @@ -886,6 +951,10 @@ "description": "On this map, publicly accessible indoor places are shown", "title": "Indoors" }, + "items_with_image": { + "description": "A map showing all items on OSM which have an image. This theme is a very bad fit for MapComplete as someone is not able to directly add a picture. However, this theme is mostly here to include this all into the database, which'll allow this to quickly fetch images nearby for other features", + "title": "All items with images" + }, "kerbs_and_crossings": { "description": "A map showing kerbs and crossings.", "layers": { @@ -1262,6 +1331,32 @@ }, "postboxes": { "description": "On this map you can find and add data of post offices and post boxes. You can use this map to find where you can mail your next postcard! :)
Spotted an error or is a post box missing? You can edit this map with a free OpenStreetMap account.", + "layers": { + "3": { + "override": { + "+tagRenderings": { + "0": { + "mappings": { + "0": { + "then": "This shop is a post partner" + }, + "1": { + "then": "This shop is not a post partner" + } + }, + "question": "Is this shop a post partner?" + } + }, + "=presets": { + "0": { + "description": "If a shop is not yet on the map and is a post partner, you can add it here.", + "title": "a missing shop that is a post partner" + } + }, + "description": "Add a new post partner to the map in an existing shop" + } + } + }, "shortDescription": "A map showing postboxes and post offices", "title": "Postbox and Post Office Map" }, diff --git a/langs/themes/nl.json b/langs/themes/nl.json index 43e12c89e..5af27d779 100644 --- a/langs/themes/nl.json +++ b/langs/themes/nl.json @@ -721,6 +721,15 @@ "description": "Een Witte Fiets of Spookfiets is een aandenken aan een fietser die bij een verkeersongeval om het leven kwam. Het gaat om een fiets die volledig wit is geschilderd en in de buurt van het ongeval werd geinstalleerd.

Op deze kaart zie je alle witte fietsen die door OpenStreetMap gekend zijn. Ontbreekt er een Witte Fiets of wens je informatie aan te passen? Meld je dan aan met een (gratis) OpenStreetMap account.", "title": "Witte Fietsen" }, + "ghostsigns": { + "layers": { + "1": { + "override": { + "name": "Alle adverterende muurschilderingen" + } + } + } + }, "grb": { "description": "Dit thema helpt het GRB importeren.", "layers": { diff --git a/package-lock.json b/package-lock.json index edabe369d..123278a2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@turf/length": "^6.5.0", "@turf/turf": "^6.5.0", "@types/dompurify": "^3.0.2", + "@types/pg": "^8.10.9", "@types/qrcode-generator": "^1.0.6", "@types/showdown": "^2.0.0", "chart.js": "^3.8.0", @@ -51,6 +52,8 @@ "osmtogeojson": "^3.0.0-beta.5", "panzoom": "^9.4.3", "papaparse": "^5.3.1", + "pbf": "^3.2.1", + "pg": "^8.11.3", "pic4carto": "^2.1.15", "prompt-sync": "^4.2.0", "qrcode-generator": "^1.4.4", @@ -4237,6 +4240,68 @@ "resolved": "https://registry.npmjs.org/@types/pbf/-/pbf-3.0.3.tgz", "integrity": "sha512-hw6bDMjvm+QTvEC+pRLpnTknQXoPu8Fnf+A+zX9HB7j/7RfYajFSbdukabo3adPwvvEHhIMafQl0R0Tpej7clQ==" }, + "node_modules/@types/pg": { + "version": "8.10.9", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.10.9.tgz", + "integrity": "sha512-UksbANNE/f8w0wOMxVKKIrLCbEMV+oM1uKejmwXr39olg4xqcfBDbXxObJAt6XxHbDa4XTKOlUEcEltXDX+XLQ==", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^4.0.1" + } + }, + "node_modules/@types/pg/node_modules/pg-types": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.0.1.tgz", + "integrity": "sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==", + "dependencies": { + "pg-int8": "1.0.1", + "pg-numeric": "1.0.2", + "postgres-array": "~3.0.1", + "postgres-bytea": "~3.0.0", + "postgres-date": "~2.0.1", + "postgres-interval": "^3.0.0", + "postgres-range": "^1.1.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@types/pg/node_modules/postgres-array": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", + "integrity": "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==", + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-bytea": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", + "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", + "dependencies": { + "obuf": "~1.1.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/pg/node_modules/postgres-date": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.0.1.tgz", + "integrity": "sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-interval": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", + "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==", + "engines": { + "node": ">=12" + } + }, "node_modules/@types/prompt-sync": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@types/prompt-sync/-/prompt-sync-4.2.0.tgz", @@ -5291,6 +5356,14 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/bytewise": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/bytewise/-/bytewise-1.1.0.tgz", @@ -9625,6 +9698,11 @@ "node": ">= 0.4" } }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==" + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -9864,6 +9942,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/panzoom": { "version": "9.4.3", "resolved": "https://registry.npmjs.org/panzoom/-/panzoom-9.4.3.tgz", @@ -9973,6 +10056,97 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, + "node_modules/pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-numeric": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", + "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/pic4carto": { "version": "2.1.15", "resolved": "https://registry.npmjs.org/pic4carto/-/pic4carto-2.1.15.tgz", @@ -10157,6 +10331,46 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-range": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.3.tgz", + "integrity": "sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==" + }, "node_modules/potpack": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.0.0.tgz", @@ -11480,6 +11694,14 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -16869,6 +17091,55 @@ "resolved": "https://registry.npmjs.org/@types/pbf/-/pbf-3.0.3.tgz", "integrity": "sha512-hw6bDMjvm+QTvEC+pRLpnTknQXoPu8Fnf+A+zX9HB7j/7RfYajFSbdukabo3adPwvvEHhIMafQl0R0Tpej7clQ==" }, + "@types/pg": { + "version": "8.10.9", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.10.9.tgz", + "integrity": "sha512-UksbANNE/f8w0wOMxVKKIrLCbEMV+oM1uKejmwXr39olg4xqcfBDbXxObJAt6XxHbDa4XTKOlUEcEltXDX+XLQ==", + "requires": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^4.0.1" + }, + "dependencies": { + "pg-types": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.0.1.tgz", + "integrity": "sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==", + "requires": { + "pg-int8": "1.0.1", + "pg-numeric": "1.0.2", + "postgres-array": "~3.0.1", + "postgres-bytea": "~3.0.0", + "postgres-date": "~2.0.1", + "postgres-interval": "^3.0.0", + "postgres-range": "^1.1.1" + } + }, + "postgres-array": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", + "integrity": "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==" + }, + "postgres-bytea": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", + "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", + "requires": { + "obuf": "~1.1.2" + } + }, + "postgres-date": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.0.1.tgz", + "integrity": "sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==" + }, + "postgres-interval": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", + "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==" + } + } + }, "@types/prompt-sync": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@types/prompt-sync/-/prompt-sync-4.2.0.tgz", @@ -17642,6 +17913,11 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "bytewise": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/bytewise/-/bytewise-1.1.0.tgz", @@ -20913,6 +21189,11 @@ "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, + "obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -21088,6 +21369,11 @@ "p-limit": "^3.0.2" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "panzoom": { "version": "9.4.3", "resolved": "https://registry.npmjs.org/panzoom/-/panzoom-9.4.3.tgz", @@ -21173,6 +21459,73 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, + "pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-cloudflare": "^1.1.1", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + }, + "pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-numeric": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", + "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==" + }, + "pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "requires": {} + }, + "pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + } + }, "pic4carto": { "version": "2.1.15", "resolved": "https://registry.npmjs.org/pic4carto/-/pic4carto-2.1.15.tgz", @@ -21277,6 +21630,34 @@ "util-deprecate": "^1.0.2" } }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "postgres-range": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.3.tgz", + "integrity": "sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==" + }, "potpack": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.0.0.tgz", @@ -22234,6 +22615,11 @@ } } }, + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + }, "sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", diff --git a/package.json b/package.json index 8eeef3fea..8eec40b88 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mapcomplete", - "version": "0.38.0", + "version": "0.40.0", "repository": "https://github.com/pietervdvn/MapComplete", "description": "A small website to edit OSM easily", "bugs": "https://github.com/pietervdvn/MapComplete/issues", @@ -21,6 +21,7 @@ "oauth_secret": "NBWGhWDrD3QDB35xtVuxv4aExnmIt4FA_WgeLtwxasg", "url": "https://www.openstreetmap.org" }, + "mvt_layer_server": "https://cache.mapcomplete.org/public.{type}_{layer}/{z}/{x}/{y}.pbf", "disabled:oauth_credentials": { "##": "DEV", "#": "This client-id is registered by 'MapComplete' on https://master.apis.dev.openstreetmap.org/", @@ -59,8 +60,6 @@ "reset:translations": "vite-node scripts/generateTranslations.ts -- --ignore-weblate", "generate:layouts": "vite-node scripts/generateLayouts.ts", "generate:docs": "rm -rf Docs/Themes/* && rm -rf Docs/Layers/* && rm -rf Docs/TagInfo && mkdir Docs/TagInfo && vite-node scripts/generateDocs.ts && vite-node scripts/generateTaginfoProjectFiles.ts", - "generate:cache:speelplekken": "npm run generate:layeroverview && vite-node scripts/generateCache.ts -- speelplekken 14 ../MapComplete-data/speelplekken_cache/ 51.20 4.35 51.09 4.56", - "generate:cache:natuurpunt": "npm run generate:layeroverview && vite-node scripts/generateCache.ts -- natuurpunt 12 ../MapComplete-data/natuurpunt_cache/ 50.40 2.1 51.54 6.4 --generate-point-overview nature_reserve,visitor_information_centre", "generate:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts", "velopark": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --force --themes=velopark --layers=bike_parking,maproulette_challenge", "generate:mapcomplete-changes-theme": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --generate-change-map", @@ -93,6 +92,8 @@ "weblate-fix-heavy": "git fetch weblate-hosted-layers; git fetch weblate-hosted-core; git merge weblate-hosted-layers/master weblate-hosted-core/master ", "housekeeping": "git pull && npx update-browserslist-db@latest && npm run weblate-fix-heavy && npm run generate && npm run generate:docs && npm run generate:contributor-list && vite-node scripts/fetchLanguages.ts && npm run format && git add assets/ langs/ Docs/ **/*.ts Docs/* src/* && git commit -m 'chore: automated housekeeping...'", "reuse-compliance": "reuse lint", + "summary-server": "vite-node scripts/osm2pgsql/tilecountServer.ts", + "ldjson-server": "vite-node scripts/serverLdScrape.ts", "backup:images": "vite-node scripts/generateImageAnalysis.ts -- ~/data/imgur-image-backup/" }, "keywords": [ @@ -119,6 +120,7 @@ "@turf/length": "^6.5.0", "@turf/turf": "^6.5.0", "@types/dompurify": "^3.0.2", + "@types/pg": "^8.10.9", "@types/qrcode-generator": "^1.0.6", "@types/showdown": "^2.0.0", "chart.js": "^3.8.0", @@ -150,6 +152,8 @@ "osmtogeojson": "^3.0.0-beta.5", "panzoom": "^9.4.3", "papaparse": "^5.3.1", + "pbf": "^3.2.1", + "pg": "^8.11.3", "pic4carto": "^2.1.15", "prompt-sync": "^4.2.0", "qrcode-generator": "^1.4.4", diff --git a/public/css/index-tailwind-output.css b/public/css/index-tailwind-output.css index 828557555..81bf9ff94 100644 --- a/public/css/index-tailwind-output.css +++ b/public/css/index-tailwind-output.css @@ -777,6 +777,10 @@ video { float: left; } +.m-8 { + margin: 2rem; +} + .m-4 { margin: 1rem; } @@ -789,10 +793,6 @@ video { margin: 0px; } -.m-8 { - margin: 2rem; -} - .m-2 { margin: 0.5rem; } @@ -896,6 +896,10 @@ video { margin-right: 4rem; } +.mb-4 { + margin-bottom: 1rem; +} + .mt-4 { margin-top: 1rem; } @@ -928,10 +932,6 @@ video { margin-right: 0.25rem; } -.mb-4 { - margin-bottom: 1rem; -} - .ml-1 { margin-left: 0.25rem; } @@ -1127,14 +1127,14 @@ video { height: 50%; } -.h-7 { - height: 1.75rem; -} - .h-3 { height: 0.75rem; } +.h-7 { + height: 1.75rem; +} + .h-11 { height: 2.75rem; } @@ -1163,6 +1163,10 @@ video { height: 20rem; } +.h-5\/6 { + height: 83.333333%; +} + .h-56 { height: 14rem; } @@ -1233,14 +1237,14 @@ video { width: 1rem; } -.w-7 { - width: 1.75rem; -} - .w-3 { width: 0.75rem; } +.w-7 { + width: 1.75rem; +} + .w-11 { width: 2.75rem; } @@ -1266,6 +1270,14 @@ video { width: 4rem; } +.w-5\/6 { + width: 83.333333%; +} + +.w-1\/6 { + width: 16.666667%; +} + .w-min { width: -webkit-min-content; width: min-content; diff --git a/scripts/Script.ts b/scripts/Script.ts index 48a727b22..25b0b14e7 100644 --- a/scripts/Script.ts +++ b/scripts/Script.ts @@ -13,8 +13,15 @@ export default abstract class Script { ScriptUtils.fixUtils() const args = [...process.argv] args.splice(0, 2) + const start = new Date() this.main(args) - .then((_) => console.log("All done")) + .then((_) =>{ + const end = new Date() + const millisNeeded = end.getTime() - start.getTime() + + const green = (s) => "\x1b[92m" + s + "\x1b[0m" + console.log(green("All done! (" + millisNeeded + " ms)")) + }) .catch((e) => console.log("ERROR:", e)) } diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts deleted file mode 100644 index 44ac75577..000000000 --- a/scripts/generateCache.ts +++ /dev/null @@ -1,571 +0,0 @@ -/** - * Generates a collection of geojson files based on an overpass query for a given theme - */ -import { Utils } from "../src/Utils" -import { Overpass } from "../src/Logic/Osm/Overpass" -import { existsSync, readFileSync, writeFileSync } from "fs" -import { TagsFilter } from "../src/Logic/Tags/TagsFilter" -import { Or } from "../src/Logic/Tags/Or" -import { AllKnownLayouts } from "../src/Customizations/AllKnownLayouts" -import * as OsmToGeoJson from "osmtogeojson" -import MetaTagging from "../src/Logic/MetaTagging" -import { UIEventSource } from "../src/Logic/UIEventSource" -import { TileRange, Tiles } from "../src/Models/TileRange" -import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig" -import ScriptUtils from "./ScriptUtils" -import PerLayerFeatureSourceSplitter from "../src/Logic/FeatureSource/PerLayerFeatureSourceSplitter" -import FilteredLayer from "../src/Models/FilteredLayer" -import StaticFeatureSource from "../src/Logic/FeatureSource/Sources/StaticFeatureSource" -import Constants from "../src/Models/Constants" -import { GeoOperations } from "../src/Logic/GeoOperations" -import SimpleMetaTaggers, { ReferencingWaysMetaTagger } from "../src/Logic/SimpleMetaTagger" -import FilteringFeatureSource from "../src/Logic/FeatureSource/Sources/FilteringFeatureSource" -import { Feature } from "geojson" -import { BBox } from "../src/Logic/BBox" -import { FeatureSource } from "../src/Logic/FeatureSource/FeatureSource" -import OsmObjectDownloader from "../src/Logic/Osm/OsmObjectDownloader" -import FeaturePropertiesStore from "../src/Logic/FeatureSource/Actors/FeaturePropertiesStore" - -ScriptUtils.fixUtils() - -function createOverpassObject(theme: LayoutConfig, backend: string) { - let filters: TagsFilter[] = [] - let extraScripts: string[] = [] - for (const layer of theme.layers) { - if (typeof layer === "string") { - throw "A layer was not expanded!" - } - if (layer.doNotDownload) { - continue - } - if (!layer.source) { - continue - } - if (layer.source.geojsonSource) { - // This layer defines a geoJson-source - // SHould it be cached? - if (layer.source.isOsmCacheLayer !== true) { - continue - } - } - - filters.push(layer.source.osmTags) - } - filters = Utils.NoNull(filters) - extraScripts = Utils.NoNull(extraScripts) - if (filters.length + extraScripts.length === 0) { - throw "Nothing to download! The theme doesn't declare anything to download" - } - return new Overpass(new Or(filters), extraScripts, backend, new UIEventSource(60)) -} - -function rawJsonName(targetDir: string, x: number, y: number, z: number): string { - return targetDir + "_" + z + "_" + x + "_" + y + ".json" -} - -function geoJsonName(targetDir: string, x: number, y: number, z: number): string { - return targetDir + "_" + z + "_" + x + "_" + y + ".geojson" -} - -/// Downloads the given tilerange from overpass and saves them to disk -async function downloadRaw( - targetdir: string, - r: TileRange, - theme: LayoutConfig -): Promise<{ failed: number; skipped: number }> { - let downloaded = 0 - let failed = 0 - let skipped = 0 - const startTime = new Date().getTime() - for (let x = r.xstart; x <= r.xend; x++) { - for (let y = r.ystart; y <= r.yend; y++) { - downloaded++ - const filename = rawJsonName(targetdir, x, y, r.zoomlevel) - if (existsSync(filename)) { - console.log("Already exists (not downloading again): ", filename) - skipped++ - continue - } - const runningSeconds = (new Date().getTime() - startTime) / 1000 - const resting = failed + (r.total - downloaded) - const perTile = runningSeconds / (downloaded - skipped) - const estimated = Math.floor(resting * perTile) - console.log( - "total: ", - downloaded, - "/", - r.total, - "failed: ", - failed, - "skipped: ", - skipped, - "running time: ", - Utils.toHumanTime(runningSeconds) + "s", - "estimated left: ", - Utils.toHumanTime(estimated), - "(" + Math.floor(perTile) + "s/tile)" - ) - - const boundsArr = Tiles.tile_bounds(r.zoomlevel, x, y) - const bounds = { - north: Math.max(boundsArr[0][0], boundsArr[1][0]), - south: Math.min(boundsArr[0][0], boundsArr[1][0]), - east: Math.max(boundsArr[0][1], boundsArr[1][1]), - west: Math.min(boundsArr[0][1], boundsArr[1][1]), - } - const overpass = createOverpassObject( - theme, - Constants.defaultOverpassUrls[failed % Constants.defaultOverpassUrls.length] - ) - const url = overpass.buildQuery( - "[bbox:" + - bounds.south + - "," + - bounds.west + - "," + - bounds.north + - "," + - bounds.east + - "]" - ) - - try { - const json = await Utils.downloadJson(url) - if ((json.remark ?? "").startsWith("runtime error")) { - console.error("Got a runtime error: ", json.remark) - failed++ - } else if (json.elements.length === 0) { - console.log("Got an empty response! Writing anyway") - } - - console.log( - "Got the response - writing ", - json.elements.length, - " elements to ", - filename - ) - writeFileSync(filename, JSON.stringify(json, null, " ")) - } catch (err) { - console.log(url) - console.log( - "Could not download - probably hit the rate limit; waiting a bit. (" + err + ")" - ) - failed++ - await ScriptUtils.sleep(1) - } - } - } - - return { failed: failed, skipped: skipped } -} - -/* - * Downloads extra geojson sources and returns the features. - * Extra geojson layers should not be tiled - */ -async function downloadExtraData(theme: LayoutConfig) /* : any[] */ { - const allFeatures: any[] = [] - for (const layer of theme.layers) { - if (!layer.source?.geojsonSource) { - continue - } - const source = layer.source.geojsonSource - if (layer.source.isOsmCacheLayer !== undefined && layer.source.isOsmCacheLayer !== false) { - // Cached layers are not considered here - continue - } - if (source.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")) { - // We ignore map notes - continue - } - console.log("Downloading extra data: ", source) - await Utils.downloadJson(source).then((json) => allFeatures.push(...json.features)) - } - return allFeatures -} - -function loadAllTiles( - targetdir: string, - r: TileRange, - theme: LayoutConfig, - extraFeatures: any[] -): FeatureSource { - let allFeatures = [...extraFeatures] - let processed = 0 - for (let x = r.xstart; x <= r.xend; x++) { - for (let y = r.ystart; y <= r.yend; y++) { - processed++ - const filename = rawJsonName(targetdir, x, y, r.zoomlevel) - console.log(" Loading and processing", processed, "/", r.total, filename) - if (!existsSync(filename)) { - console.error("Not found - and not downloaded. Run this script again!: " + filename) - continue - } - - // We read the raw OSM-file and convert it to a geojson - const rawOsm = JSON.parse(readFileSync(filename, { encoding: "utf8" })) - - // Create and save the geojson file - which is the main chunk of the data - const geojson = OsmToGeoJson.default(rawOsm) - console.log(" which as", geojson.features.length, "features") - - allFeatures.push(...geojson.features) - } - } - return StaticFeatureSource.fromGeojson(allFeatures) -} - -/** - * Load all the tiles into memory from disk - */ -async function sliceToTiles( - allFeatures: FeatureSource, - theme: LayoutConfig, - targetdir: string, - pointsOnlyLayers: string[], - clip: boolean, - targetzoomLevel: number = 9 -) { - const skippedLayers = new Set() - - const indexedFeatures: Map = new Map() - let indexisBuilt = false - const osmObjectDownloader = new OsmObjectDownloader() - - function buildIndex() { - for (const f of allFeatures.features.data) { - indexedFeatures.set(f.properties.id, f) - } - indexisBuilt = true - } - - function getFeatureById(id) { - if (!indexisBuilt) { - buildIndex() - } - return indexedFeatures.get(id) - } - - const flayers: FilteredLayer[] = theme.layers.map((l) => new FilteredLayer(l)) - const perLayer = new PerLayerFeatureSourceSplitter(flayers, allFeatures) - for (const [layerId, source] of perLayer.perLayer) { - const layer = flayers.find((flayer) => flayer.layerDef.id === layerId).layerDef - const targetZoomLevel = layer.source.geojsonZoomLevel ?? targetzoomLevel - - if (layer.source.geojsonSource && layer.source.isOsmCacheLayer !== true) { - console.log("Skipping layer ", layerId, ": not a caching layer") - skippedLayers.add(layer.id) - continue - } - const flayer: FilteredLayer = new FilteredLayer(layer) - console.log( - "Handling layer ", - layerId, - "which has", - source.features.data.length, - "features" - ) - if (source.features.data.length === 0) { - continue - } - const featureProperties: FeaturePropertiesStore = new FeaturePropertiesStore(source) - - MetaTagging.addMetatags( - source.features.data, - { - getFeaturesWithin: (_) => { - return [allFeatures.features.data] - }, - getFeatureById: getFeatureById, - }, - layer, - theme, - osmObjectDownloader, - featureProperties, - { - includeDates: false, - includeNonDates: true, - evaluateStrict: true, - } - ) - - while (SimpleMetaTaggers.country.runningTasks.size > 0) { - console.log( - "Still waiting for ", - SimpleMetaTaggers.country.runningTasks.size, - " features which don't have a country yet" - ) - await ScriptUtils.sleep(250) - } - - const createdTiles = [] - // At this point, we have all the features of the entire area. - // However, we want to export them per tile of a fixed size, so we use a dynamicTileSOurce to split it up - const features = source.features.data - const perBbox = GeoOperations.spreadIntoBboxes(features, targetZoomLevel) - - for (let [tileIndex, features] of perBbox) { - const bbox = BBox.fromTileIndex(tileIndex).asGeoJson({}) - console.log("Got tile:", tileIndex, layer.id) - if (features.length === 0) { - continue - } - const filteredTile = new FilteringFeatureSource( - flayer, - new StaticFeatureSource(features) - ) - console.log( - "Tile " + - layer.id + - "." + - tileIndex + - " contains " + - filteredTile.features.data.length + - " features after filtering (" + - features.length + - ") features before" - ) - if (filteredTile.features.data.length === 0) { - continue - } - - let strictlyCalculated = 0 - let featureCount = 0 - - for (const feature of features) { - // Some cleanup - - if (layer.calculatedTags !== undefined) { - // Evaluate all the calculated tags strictly - const calculatedTagKeys = layer.calculatedTags.map((ct) => ct[0]) - featureCount++ - const props = feature.properties - for (const calculatedTagKey of calculatedTagKeys) { - const strict = props[calculatedTagKey] - - if (props.hasOwnProperty(calculatedTagKey)) { - delete props[calculatedTagKey] - } - - props[calculatedTagKey] = strict - strictlyCalculated++ - if (strictlyCalculated % 100 === 0) { - console.log( - "Strictly calculated ", - strictlyCalculated, - "values for tile", - tileIndex, - ": now at ", - featureCount, - "/", - filteredTile.features.data.length, - "examle value: ", - strict - ) - } - } - } - delete feature["bbox"] - } - - if (clip) { - console.log("Clipping features") - features = [].concat( - ...features.map((f: Feature) => GeoOperations.clipWith(f, bbox)) - ) - } - // Lets save this tile! - const [z, x, y] = Tiles.tile_from_index(tileIndex) - // console.log("Writing tile ", z, x, y, layerId) - const targetPath = geoJsonName(targetdir + "_" + layerId, x, y, z) - createdTiles.push(tileIndex) - // This is the geojson file containing all features for this tile - writeFileSync( - targetPath, - JSON.stringify( - { - type: "FeatureCollection", - features, - }, - null, - " " - ) - ) - console.log("Written tile", targetPath, "with", filteredTile.features.data.length) - } - - // All the tiles are written at this point - // Only thing left to do is to create the index - const path = targetdir + "_" + layerId + "_" + targetZoomLevel + "_overview.json" - const perX = {} - createdTiles - .map((i) => Tiles.tile_from_index(i)) - .forEach(([z, x, y]) => { - const key = "" + x - if (perX[key] === undefined) { - perX[key] = [] - } - perX[key].push(y) - }) - console.log("Written overview: ", path, "with ", createdTiles.length, "tiles") - writeFileSync(path, JSON.stringify(perX)) - - // And, if needed, to create a points-only layer - if (pointsOnlyLayers.indexOf(layer.id) >= 0) { - const filtered = new FilteringFeatureSource(flayer, source) - const features = filtered.features.data - - const points = features.map((feature) => GeoOperations.centerpoint(feature)) - console.log("Writing points overview for ", layerId) - const targetPath = targetdir + "_" + layerId + "_points.geojson" - // This is the geojson file containing all features for this tile - writeFileSync( - targetPath, - JSON.stringify( - { - type: "FeatureCollection", - features: points, - }, - null, - " " - ) - ) - } - } - - const skipped = Array.from(skippedLayers) - if (skipped.length > 0) { - console.warn( - "Did not save any cache files for layers " + - skipped.join(", ") + - " as these didn't set the flag `isOsmCache` to true" - ) - } -} - -export async function main(args: string[]) { - console.log("Cache builder started with args ", args.join(" ")) - ReferencingWaysMetaTagger.enabled = false - if (args.length < 6) { - console.error( - "Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] [--clip]" + - "--force-zoom-level causes non-cached-layers to be donwnloaded\n" + - "--clip will erase parts of the feature falling outside of the bounding box" - ) - return - } - const themeName = args[0] - const zoomlevel = Number(args[1]) - console.log( - "Target zoomlevel for the tiles is", - zoomlevel, - "; this can be overridden by the individual layers" - ) - - const targetdir = args[2] + "/" + themeName - if (!existsSync(args[2])) { - console.log("Directory not found") - throw `The directory ${args[2]} does not exist` - } - - const lat0 = Number(args[3]) - const lon0 = Number(args[4]) - const lat1 = Number(args[5]) - const lon1 = Number(args[6]) - const clip = args.indexOf("--clip") >= 0 - - if (isNaN(lat0)) { - throw "The first number (a latitude) is not a valid number" - } - - if (isNaN(lon0)) { - throw "The second number (a longitude) is not a valid number" - } - if (isNaN(lat1)) { - throw "The third number (a latitude) is not a valid number" - } - - if (isNaN(lon1)) { - throw "The fourth number (a longitude) is not a valid number" - } - - const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1) - - if (isNaN(tileRange.total)) { - throw "Something has gone wrong: tilerange is NAN" - } - - if (tileRange.total === 0) { - console.log("Tilerange has zero tiles - this is probably an error") - return - } - - const theme = AllKnownLayouts.allKnownLayouts.get(themeName) - if (theme === undefined) { - const keys = Array.from(AllKnownLayouts.allKnownLayouts.keys()) - console.error("The theme " + themeName + " was not found; try one of ", keys) - return - } - - theme.layers = theme.layers.filter( - (l) => - Constants.priviliged_layers.indexOf(l.id) < 0 && !l.id.startsWith("note_import_") - ) - console.log("Layers to download:", theme.layers.map((l) => l.id).join(", ")) - - let generatePointLayersFor = [] - if (args[7] == "--generate-point-overview") { - if (args[8] === undefined) { - throw "--generate-point-overview needs a list of layers to generate the overview for (or * for all)" - } else if (args[8] === "*") { - generatePointLayersFor = theme.layers.map((l) => l.id) - } else { - generatePointLayersFor = args[8].split(",") - } - console.log( - "Also generating a point overview for layers ", - generatePointLayersFor.join(",") - ) - } - { - const index = args.indexOf("--force-zoom-level") - if (index >= 0) { - const forcedZoomLevel = Number(args[index + 1]) - for (const layer of theme.layers) { - layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson" - layer.source.isOsmCacheLayer = true - layer.source.geojsonZoomLevel = forcedZoomLevel - } - } - } - - let failed = 0 - do { - try { - const cachingResult = await downloadRaw(targetdir, tileRange, theme) - failed = cachingResult.failed - if (failed > 0) { - await ScriptUtils.sleep(30000) - } - } catch (e) { - console.error(e) - return - } - } while (failed > 0) - - const extraFeatures = await downloadExtraData(theme) - const allFeaturesSource = loadAllTiles(targetdir, tileRange, theme, extraFeatures) - await sliceToTiles(allFeaturesSource, theme, targetdir, generatePointLayersFor, clip, zoomlevel) -} - -let args = [...process.argv] -if (!args[1]?.endsWith("test/TestAll.ts")) { - args.splice(0, 2) - try { - main(args) - .then(() => console.log("All done!")) - .catch((e) => console.error("Error building cache:", e)) - } catch (e) { - console.error("Error building cache:", e) - } -} diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index 4eac23395..263b9b148 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -10,6 +10,7 @@ import { PrevalidateTheme, ValidateLayer, ValidateThemeAndLayers, + ValidateThemeEnsemble, } from "../src/Models/ThemeConfig/Conversion/Validation" import { Translation } from "../src/UI/i18n/Translation" import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer" @@ -29,6 +30,8 @@ import LayerConfig from "../src/Models/ThemeConfig/LayerConfig" import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig" import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext" import { GenerateFavouritesLayer } from "./generateFavouritesLayer" +import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig" +import { TagsFilter } from "../src/Logic/Tags/TagsFilter" // This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files. // It spits out an overview of those to be used to load them @@ -367,7 +370,6 @@ class LayerOverviewUtils extends Script { ?.split(",") ?? [] ) - const start = new Date() const forceReload = args.some((a) => a == "--force") const licensePaths = new Set() @@ -397,6 +399,10 @@ class LayerOverviewUtils extends Script { themeWhitelist ) + new ValidateThemeEnsemble().convertStrict( + Array.from(sharedThemes.values()).map((th) => new LayoutConfig(th, true)) + ) + if (recompiledThemes.length > 0) { writeFileSync( "./src/assets/generated/known_layers.json", @@ -458,17 +464,8 @@ class LayerOverviewUtils extends Script { ) } - const end = new Date() - const millisNeeded = end.getTime() - start.getTime() if (AllSharedLayers.getSharedLayersConfigs().size == 0) { - console.error( - "This was a bootstrapping-run. Run generate layeroverview again!(" + - millisNeeded + - " ms)" - ) - } else { - const green = (s) => "\x1b[92m" + s + "\x1b[0m" - console.log(green("All done! (" + millisNeeded + " ms)")) + console.error("This was a bootstrapping-run. Run generate layeroverview again!") } } diff --git a/scripts/generateLayouts.ts b/scripts/generateLayouts.ts index 7242d3888..54cf8e66e 100644 --- a/scripts/generateLayouts.ts +++ b/scripts/generateLayouts.ts @@ -279,6 +279,7 @@ async function generateCsp( "https://www.openstreetmap.org", "https://api.openstreetmap.org", "https://pietervdvn.goatcounter.com", + "https://cache.mapcomplete.org", ].concat(...(await eliUrls())) SpecialVisualizations.specialVisualizations.forEach((sv) => { diff --git a/scripts/hetzner/deployHetzner.sh b/scripts/hetzner/deployHetzner.sh index 61c8d34b7..f3df27b9c 100755 --- a/scripts/hetzner/deployHetzner.sh +++ b/scripts/hetzner/deployHetzner.sh @@ -16,9 +16,9 @@ npm run test && npm run prepare-deploy && zip dist.zip -r dist/* && mv config.json.bu config.json && -scp ./scripts/hetzner/config/* hetzner:/root/ && +scp ./Docs/ServerConfig/hetzner/* hetzner:/root/ && rsync -rzh --progress dist.zip hetzner:/root/ && echo "Upload completed, deploying config and booting" && ssh hetzner -t "unzip dist.zip && rm dist.zip && rm -rf public/ && mv dist public && caddy stop && caddy start" && -rm dist.zip +# rm dist.zip npm run clean diff --git a/scripts/osm2pgsql/generateBuildDbScript.ts b/scripts/osm2pgsql/generateBuildDbScript.ts new file mode 100644 index 000000000..7ddd217f0 --- /dev/null +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -0,0 +1,306 @@ +import { TagsFilter } from "../../src/Logic/Tags/TagsFilter" +import { Tag } from "../../src/Logic/Tags/Tag" +import { And } from "../../src/Logic/Tags/And" +import Script from "../Script" +import fs from "fs" +import { Or } from "../../src/Logic/Tags/Or" +import { RegexTag } from "../../src/Logic/Tags/RegexTag" +import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation" +import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts" +import { OsmObject } from "../../src/Logic/Osm/OsmObject" + +class LuaSnippets { + public static helpers = [ + "function countTbl(tbl)\n" + + " local c = 0\n" + + " for n in pairs(tbl) do \n" + + " c = c + 1 \n" + + " end\n" + + " return c\n" + + "end", + ].join("\n") + + public static isPolygonFeature(): { blacklist: TagsFilter; whitelisted: TagsFilter } { + const dict = OsmObject.polygonFeatures + const or: TagsFilter[] = [] + const blacklisted: TagsFilter[] = [] + dict.forEach(({ values, blacklist }, k) => { + if (blacklist) { + if (values === undefined) { + blacklisted.push(new RegexTag(k, /.+/is)) + return + } + values.forEach((v) => { + blacklisted.push(new RegexTag(k, v)) + }) + return + } + if (values === undefined || values === null) { + or.push(new RegexTag(k, /.+/is)) + return + } + values.forEach((v) => { + or.push(new RegexTag(k, v)) + }) + }) + console.log( + "Polygon features are:", + or.map((t) => t.asHumanString(false, false, {})) + ) + return { blacklist: new Or(blacklisted), whitelisted: new Or(or) } + } + + public static toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { + if (tag instanceof Tag) { + return `object.tags["${tag.key}"] == "${tag.value}"` + } + if (tag instanceof And) { + const expr = tag.and.map((t) => this.toLuaFilter(t, true)).join(" and ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof Or) { + const expr = tag.or.map((t) => this.toLuaFilter(t, true)).join(" or ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof RegexTag) { + let expr = LuaSnippets.regexTagToLua(tag) + if (useParens) { + expr = "(" + expr + ")" + } + return expr + } + let msg = "Could not handle" + tag.asHumanString(false, false, {}) + console.error(msg) + throw msg + } + + private static regexTagToLua(tag: RegexTag) { + if (typeof tag.value === "string" && tag.invert) { + return `object.tags["${tag.key}"] ~= "${tag.value}"` + } + + if (typeof tag.value === "string" && !tag.invert) { + return `object.tags["${tag.key}"] == "${tag.value}"` + } + + let v: string = (tag.value).source.replace(/\\\//g, "/") + + if ("" + tag.value === "/.+/is" && !tag.invert) { + return `object.tags["${tag.key}"] ~= nil` + } + + if ("" + tag.value === "/.+/is" && tag.invert) { + return `object.tags["${tag.key}"] == nil` + } + + if (tag.matchesEmpty && !tag.invert) { + return `object.tags["${tag.key}"] == nil or object.tags["${tag.key}"] == ""` + } + + if (tag.matchesEmpty && tag.invert) { + return `object.tags["${tag.key}"] ~= nil or object.tags["${tag.key}"] ~= ""` + } + + let head = "^((.*;)?" + let tail = "(;.*)?)$" + if (v.startsWith(head)) { + v = "(" + v.substring(head.length) + } + if (v.endsWith(tail)) { + v = v.substring(0, v.length - tail.length) + ")" + // We basically remove the optional parts at the start and the end, as object.find has this freedom anyway. + // This might result in _some_ incorrect values that end up in the database (e.g. when matching 'friture', it might als match "abc;foo_friture_bar;xyz", but the frontend will filter this out + } + + if (v.indexOf(")?") > 0) { + throw ( + "LUA regexes have a bad support for (optional) capture groups, as such, " + + v + + " is not supported" + ) + } + + if (tag.invert) { + return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${v}")` + } + + return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))` + } +} + +class GenerateLayerLua { + private readonly _id: string + private readonly _tags: TagsFilter + private readonly _foundInThemes: string[] + + constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) { + this._tags = tags + this._id = id + this._foundInThemes = foundInThemes + } + + public generateTables(): string { + if (!this._tags) { + return undefined + } + return [ + `db_tables.pois_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'pois_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'point', projection = 4326, not_null = true },", + " }", + "})", + "", + `db_tables.lines_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'lines_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'linestring', projection = 4326, not_null = true },", + " }", + "})", + + `db_tables.polygons_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'polygons_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'polygon', projection = 4326, not_null = true },", + " }", + "})", + "", + ].join("\n") + } +} + +class GenerateBuildDbScript extends Script { + constructor() { + super("Generates a .lua-file to use with osm2pgsql") + } + + async main(args: string[]) { + const allNeededLayers = new ValidateThemeEnsemble().convertStrict( + AllKnownLayouts.allKnownLayouts.values() + ) + + const generators: GenerateLayerLua[] = [] + + allNeededLayers.forEach(({ tags, foundInTheme }, layerId) => { + generators.push(new GenerateLayerLua(layerId, tags, foundInTheme)) + }) + + const script = [ + "local db_tables = {}", + LuaSnippets.helpers, + ...generators.map((g) => g.generateTables()), + this.generateProcessPoi(allNeededLayers), + this.generateProcessWay(allNeededLayers), + ].join("\n\n\n") + const path = "build_db.lua" + fs.writeFileSync(path, script, "utf-8") + console.log("Written", path) + console.log( + allNeededLayers.size + + " layers will be created with 3 tables each. Make sure to set 'max_connections' to at least " + + (10 + 3 * allNeededLayers.size) + ) + } + + private earlyAbort() { + return [" if countTbl(object.tags) == 0 then", " return", " end", ""].join("\n") + } + + private generateProcessPoi( + allNeededLayers: Map + ) { + const body: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + body.push(this.insertInto(tags, layerId, "pois_").join("\n")) + }) + + return [ + "function osm2pgsql.process_node(object)", + this.earlyAbort(), + " local geom = object:as_point()", + " local matches_filter = false", + body.join("\n"), + "end", + ].join("\n") + } + + /** + * If matches_filter + * @param tags + * @param layerId + * @param tableprefix + * @private + */ + private insertInto( + tags: TagsFilter, + layerId: string, + tableprefix: "pois_" | "lines_" | "polygons_" + ) { + const filter = LuaSnippets.toLuaFilter(tags) + return [ + " matches_filter = " + filter, + " if matches_filter then", + " db_tables." + tableprefix + layerId + ":insert({", + " geom = geom,", + " tags = object.tags", + " })", + " end", + ] + } + + private generateProcessWay(allNeededLayers: Map) { + const bodyLines: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + bodyLines.push(this.insertInto(tags, layerId, "lines_").join("\n")) + }) + + const bodyPolygons: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + bodyPolygons.push(this.insertInto(tags, layerId, "polygons_").join("\n")) + }) + + const isPolygon = LuaSnippets.isPolygonFeature() + return [ + "function process_polygon(object, geom)", + " local matches_filter", + ...bodyPolygons, + "end", + "function process_linestring(object, geom)", + " local matches_filter", + ...bodyLines, + "end", + "", + "function osm2pgsql.process_way(object)", + this.earlyAbort(), + " local object_is_line = not object.is_closed or " + + LuaSnippets.toLuaFilter(isPolygon.blacklist), + ` local object_is_area = object.is_closed and (object.tags["area"] == "yes" or (not object_is_line and ${LuaSnippets.toLuaFilter( + isPolygon.whitelisted, + true + )}))`, + " if object_is_area then", + " process_polygon(object, object:as_polygon())", + " else", + " process_linestring(object, object:as_linestring())", + " end", + "end", + ].join("\n") + } +} + +new GenerateBuildDbScript().run() diff --git a/scripts/osm2pgsql/tilecountServer.ts b/scripts/osm2pgsql/tilecountServer.ts new file mode 100644 index 000000000..86aa8c4d7 --- /dev/null +++ b/scripts/osm2pgsql/tilecountServer.ts @@ -0,0 +1,217 @@ +import { Client } from "pg" +import { Tiles } from "../../src/Models/TileRange" +import { Server } from "../server" + +/** + * Just the OSM2PGSL default database + */ +interface PoiDatabaseMeta { + attributes + current_timestamp + db_format + flat_node_file + import_timestamp + output + prefix + replication_base_url + replication_sequence_number + replication_timestamp + style + updatable + version +} + +/** + * Connects with a Postgis database, gives back how much items there are within the given BBOX + */ +class OsmPoiDatabase { + private static readonly prefixes: ReadonlyArray = ["pois", "lines", "polygons"] + private readonly _client: Client + private isConnected = false + private supportedLayers: Set = undefined + private supportedLayersDate: Date = undefined + private metaCache: PoiDatabaseMeta = undefined + private metaCacheDate: Date = undefined + + constructor(connectionString: string) { + this._client = new Client(connectionString) + } + + async getCount( + layer: string, + bbox: [[number, number], [number, number]] = undefined + ): Promise<{ count: number; lat: number; lon: number }> { + if (!this.isConnected) { + await this._client.connect() + this.isConnected = true + } + + let total: number = 0 + let latSum = 0 + let lonSum = 0 + for (const prefix of OsmPoiDatabase.prefixes) { + let query = + "SELECT COUNT(*), ST_AsText(ST_Centroid(ST_Collect(geom))) FROM " + + prefix + + "_" + + layer + + if (bbox) { + query += ` WHERE ST_MakeEnvelope (${bbox[0][0]}, ${bbox[0][1]}, ${bbox[1][0]}, ${bbox[1][1]}, 4326) ~ geom` + } + const result = await this._client.query(query) + const count = Number(result.rows[0].count) + let point = result.rows[0].st_astext + if (count === 0) { + continue + } + total += count + if (!point) { + continue + } + point = point.substring(6, point.length - 1) + const [lon, lat] = point.split(" ") + latSum += lat * count + lonSum += lon * count + } + + return { count: total, lat: latSum / total, lon: lonSum / total } + } + + disconnect() { + this._client.end() + } + + async getLayers(): Promise> { + if ( + this.supportedLayers !== undefined && + new Date().getTime() - this.supportedLayersDate.getTime() < 1000 * 60 * 60 * 24 + ) { + return this.supportedLayers + } + const q = + "SELECT table_name \n" + + "FROM information_schema.tables \n" + + "WHERE table_schema = 'public' AND table_name LIKE 'lines_%';" + const result = await this._client.query(q) + const layers = result.rows.map((r) => r.table_name.substring("lines_".length)) + this.supportedLayers = new Set(layers) + this.supportedLayersDate = new Date() + return this.supportedLayers + } + + async getMeta(): Promise { + const now = new Date() + if (this.metaCache !== undefined) { + const diffSec = (this.metaCacheDate.getTime() - now.getTime()) / 1000 + if (diffSec < 120) { + return this.metaCache + } + } + const result = await this._client.query("SELECT * FROM public.osm2pgsql_properties") + const meta = {} + for (const { property, value } of result.rows) { + meta[property] = value + } + this.metaCacheDate = now + this.metaCache = meta + return this.metaCache + } +} + +class CachedSqlCount { + private readonly _cache: Record< + string, + Record< + number, + { + date: Date + entry: { count: number; lat: number; lon: number } + } + > + > = {} + + private readonly _poiDatabase: OsmPoiDatabase + private readonly _maxAge: number + + constructor(poiDatabase: OsmPoiDatabase, maxAge: number) { + this._poiDatabase = poiDatabase + this._maxAge = maxAge + } + + public async getCount( + layer: string, + tileId: number + ): Promise<{ count: number; lat: number; lon: number }> { + const cachedEntry = this._cache[layer]?.[tileId] + if (cachedEntry) { + const age = (new Date().getTime() - cachedEntry.date.getTime()) / 1000 + if (age < this._maxAge) { + return cachedEntry.entry + } + } + const bbox = Tiles.tile_bounds_lon_lat(...Tiles.tile_from_index(tileId)) + const count = await this._poiDatabase.getCount(layer, bbox) + if (!this._cache[layer]) { + this._cache[layer] = {} + } + this._cache[layer][tileId] = { entry: count, date: new Date() } + return count + } +} + +const connectionString = "postgresql://user:password@localhost:5444/osm-poi" +const tcs = new OsmPoiDatabase(connectionString) +const withCache = new CachedSqlCount(tcs, 14 * 60 * 60 * 24) +new Server(2345, { ignorePathPrefix: ["summary"] }, [ + { + mustMatch: "status.json", + mimetype: "application/json", + handle: async (path: string) => { + const layers = await tcs.getLayers() + const meta = await tcs.getMeta() + return JSON.stringify({ meta, layers: Array.from(layers) }) + }, + }, + { + mustMatch: /[a-zA-Z0-9+_-]+\/[0-9]+\/[0-9]+\/[0-9]+\.json/, + mimetype: "application/json", // "application/vnd.geo+json", + async handle(path) { + const [layers, z, x, y] = path.split(".")[0].split("/") + + let sum = 0 + let properties: Record = {} + const availableLayers = await tcs.getLayers() + let latSum = 0 + let lonSum = 0 + for (const layer of layers.split("+")) { + if (!availableLayers.has(layer)) { + continue + } + const count = await withCache.getCount( + layer, + Tiles.tile_index(Number(z), Number(x), Number(y)) + ) + + properties[layer] = count.count + if (count.count !== 0) { + latSum += count.lat * count.count + lonSum += count.lon * count.count + sum += count.count + } + } + + properties["lon"] = lonSum / sum + properties["lat"] = latSum / sum + + return JSON.stringify({ ...properties, total: sum }) + }, + }, +]) +console.log( + ">>>", + await tcs.getCount("drinking_water", [ + [3.194358020772171, 51.228073636083394], + [3.2839964396059145, 51.172701162680994], + ]) +) diff --git a/scripts/postal_code_tools/genPostal.sh b/scripts/postal_code_tools/genPostal.sh deleted file mode 100755 index 1784daba4..000000000 --- a/scripts/postal_code_tools/genPostal.sh +++ /dev/null @@ -1,5 +0,0 @@ -#! /bin/bash - -# npm run generate:layeroverview -cd ../.. -ts-node scripts/generateCache.ts postal_codes 8 /home/pietervdvn/Downloads/postal_codes 49.69606181911566 2.373046875 51.754240074033525 6.459960937499999 --generate-point-overview '*' --force-zoom-level 1 \ No newline at end of file diff --git a/scripts/server.ts b/scripts/server.ts new file mode 100644 index 000000000..8bed5bc18 --- /dev/null +++ b/scripts/server.ts @@ -0,0 +1,112 @@ +import http from "node:http" + +export class Server { + constructor( + port: number, + options: { + ignorePathPrefix?: string[] + }, + handle: { + mustMatch: string | RegExp + mimetype: string + handle: (path: string, queryParams: URLSearchParams) => Promise + }[] + ) { + handle.push({ + mustMatch: "", + mimetype: "text/html", + handle: async () => { + return `Supported endpoints are
    ${handle + .filter((h) => h.mustMatch !== "") + .map((h) => { + let l = h.mustMatch + if (typeof h.mustMatch === "string") { + l = `${l}` + } + return "
  • " + l + "
  • " + }) + .join("")}
` + }, + }) + http.createServer(async (req: http.IncomingMessage, res) => { + try { + console.log( + req.method + " " + req.url, + "from:", + req.headers.origin, + new Date().toISOString() + ) + + const url = new URL(`http://127.0.0.1/` + req.url) + let path = url.pathname + while (path.startsWith("/")) { + path = path.substring(1) + } + if (options?.ignorePathPrefix) { + for (const toIgnore of options.ignorePathPrefix) { + if (path.startsWith(toIgnore)) { + path = path.substring(toIgnore.length + 1) + break + } + } + } + const handler = handle.find((h) => { + if (typeof h.mustMatch === "string") { + return h.mustMatch === path + } + if (path.match(h.mustMatch)) { + return true + } + }) + + if (handler === undefined || handler === null) { + res.writeHead(404, { "Content-Type": "text/html" }) + res.write("

Not found...

") + res.end() + return + } + + res.setHeader( + "Access-Control-Allow-Headers", + "Origin, X-Requested-With, Content-Type, Accept" + ) + res.setHeader("Access-Control-Allow-Origin", req.headers.origin ?? "*") + if (req.method === "OPTIONS") { + res.setHeader( + "Access-Control-Allow-Methods", + "POST, GET, OPTIONS, DELETE, UPDATE" + ) + res.writeHead(204, { "Content-Type": handler.mimetype }) + res.end() + return + } + if (req.method === "POST" || req.method === "UPDATE") { + return + } + + if (req.method === "DELETE") { + return + } + + try { + const result = await handler.handle(path, url.searchParams) + res.writeHead(200, { "Content-Type": handler.mimetype }) + res.write(result) + res.end() + } catch (e) { + console.error("Could not handle request:", e) + res.writeHead(500) + res.write(e) + res.end() + } + } catch (e) { + console.error("FATAL:", e) + res.end() + } + }).listen(port) + console.log( + "Server is running on port " + port, + ". Supported endpoints are: " + handle.map((h) => h.mustMatch).join(", ") + ) + } +} diff --git a/scripts/serverLdScrape.ts b/scripts/serverLdScrape.ts new file mode 100644 index 000000000..1157ef1c2 --- /dev/null +++ b/scripts/serverLdScrape.ts @@ -0,0 +1,54 @@ +import Script from "../scripts/Script" +import { Server } from "../scripts/server" +import { Utils } from "../src/Utils" +import parse from "node-html-parser" +class ServerLdScrape extends Script { + constructor() { + super("Starts a server which fetches a webpage and returns embedded LD+JSON") + } + async main(args: string[]): Promise { + const port = Number(args[0] ?? 2346) + + const cache: Record = {} + + new Server(port, {}, [ + { + mustMatch: "extractgraph", + mimetype: "application/ld+json", + async handle(content, searchParams: URLSearchParams) { + const url = searchParams.get("url") + if (cache[url] !== undefined) { + const { date, contents } = cache[url] + // In seconds + const tdiff = (new Date().getTime() - date.getTime()) / 1000 + if (tdiff < 24 * 60 * 60) { + return contents + } + } + const dloaded = await Utils.download(url, { + "User-Agent": + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete", + }) + const parsed = parse(dloaded) + const scripts = Array.from(parsed.getElementsByTagName("script")) + const snippets = [] + for (const script of scripts) { + const tp = script.attributes["type"] + if (tp !== "application/ld+json") { + continue + } + try { + snippets.push(JSON.parse(script.textContent)) + } catch (e) { + console.error(e) + } + } + cache[url] = { contents: snippets, date: new Date() } + return JSON.stringify(snippets) + }, + }, + ]) + } +} + +new ServerLdScrape().run() diff --git a/src/Logic/FeatureSource/FeatureSource.ts b/src/Logic/FeatureSource/FeatureSource.ts index 3b9798dbb..a0a32d29c 100644 --- a/src/Logic/FeatureSource/FeatureSource.ts +++ b/src/Logic/FeatureSource/FeatureSource.ts @@ -5,6 +5,13 @@ import { Feature } from "geojson" export interface FeatureSource { features: Store } + +export interface UpdatableFeatureSource extends FeatureSource { + /** + * Forces an update and downloads the data, even if the feature source is supposed to be active + */ + updateAsync() +} export interface WritableFeatureSource extends FeatureSource { features: UIEventSource } @@ -16,6 +23,11 @@ export interface FeatureSourceForLayer extends Feat readonly layer: FilteredLayer } +export interface FeatureSourceForTile extends FeatureSource { + readonly x: number + readonly y: number + readonly z: number +} /** * A feature source which is aware of the indexes it contains */ diff --git a/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts b/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts index 951b688c5..3ba2c785c 100644 --- a/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts +++ b/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts @@ -1,44 +1,55 @@ import { Store, UIEventSource } from "../../UIEventSource" -import { FeatureSource, IndexedFeatureSource } from "../FeatureSource" +import { FeatureSource, IndexedFeatureSource, UpdatableFeatureSource } from "../FeatureSource" import { Feature } from "geojson" import { Utils } from "../../../Utils" /** - * + * The featureSourceMerger receives complete geometries from various sources. + * If multiple sources contain the same object (as determined by 'id'), only one copy of them is retained */ -export default class FeatureSourceMerger implements IndexedFeatureSource { +export default class FeatureSourceMerger + implements IndexedFeatureSource +{ public features: UIEventSource = new UIEventSource([]) public readonly featuresById: Store> - private readonly _featuresById: UIEventSource> - private readonly _sources: FeatureSource[] = [] + protected readonly _featuresById: UIEventSource> + protected readonly _sources: Src[] + /** * Merges features from different featureSources. * In case that multiple features have the same id, the latest `_version_number` will be used. Otherwise, we will take the last one */ - constructor(...sources: FeatureSource[]) { + constructor(...sources: Src[]) { this._featuresById = new UIEventSource>(new Map()) this.featuresById = this._featuresById const self = this sources = Utils.NoNull(sources) for (let source of sources) { source.features.addCallback(() => { - self.addData(sources.map((s) => s.features.data)) + self.addDataFromSources(sources) }) } - this.addData(sources.map((s) => s.features.data)) this._sources = sources + this.addDataFromSources(sources) } - public addSource(source: FeatureSource) { + public addSource(source: Src) { if (!source) { return } + if (!source.features) { + console.error("No source found in", source) + } this._sources.push(source) source.features.addCallbackAndRun(() => { - this.addData(this._sources.map((s) => s.features.data)) + this.addDataFromSources(this._sources) }) } + protected addDataFromSources(sources: Src[]) { + this.addData(sources.map((s) => s.features.data)) + } + protected addData(sources: Feature[][]) { sources = Utils.NoNull(sources) let somethingChanged = false @@ -56,7 +67,7 @@ export default class FeatureSourceMerger implements IndexedFeatureSource { const id = f.properties.id unseen.delete(id) if (!all.has(id)) { - // This is a new feature + // This is a new, previously unseen feature somethingChanged = true all.set(id, f) continue @@ -81,11 +92,23 @@ export default class FeatureSourceMerger implements IndexedFeatureSource { return } - const newList = [] - all.forEach((value) => { - newList.push(value) - }) + const newList = Array.from(all.values()) + this.features.setData(newList) this._featuresById.setData(all) } } + +export class UpdatableFeatureSourceMerger< + Src extends UpdatableFeatureSource = UpdatableFeatureSource + > + extends FeatureSourceMerger + implements IndexedFeatureSource, UpdatableFeatureSource +{ + constructor(...sources: Src[]) { + super(...sources) + } + async updateAsync() { + await Promise.all(this._sources.map((src) => src.updateAsync())) + } +} diff --git a/src/Logic/FeatureSource/Sources/GeoJsonSource.ts b/src/Logic/FeatureSource/Sources/GeoJsonSource.ts index 998f88310..dd5abbbab 100644 --- a/src/Logic/FeatureSource/Sources/GeoJsonSource.ts +++ b/src/Logic/FeatureSource/Sources/GeoJsonSource.ts @@ -11,9 +11,14 @@ import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" import { Tiles } from "../../../Models/TileRange" export default class GeoJsonSource implements FeatureSource { - public readonly features: Store + private readonly _features: UIEventSource = new UIEventSource(undefined) + public readonly features: Store = this._features private readonly seenids: Set private readonly idKey?: string + private readonly url: string + private readonly layer: LayerConfig + private _isDownloaded = false + private currentlyRunning: Promise public constructor( layer: LayerConfig, @@ -30,6 +35,7 @@ export default class GeoJsonSource implements FeatureSource { this.idKey = layer.source.idKey this.seenids = options?.featureIdBlacklist ?? new Set() let url = layer.source.geojsonSource.replace("{layer}", layer.id) + this.layer = layer let zxy = options?.zxy if (zxy !== undefined) { let tile_bbox: BBox @@ -57,94 +63,88 @@ export default class GeoJsonSource implements FeatureSource { .replace("{x_min}", "" + bounds.minLon) .replace("{x_max}", "" + bounds.maxLon) } + this.url = url - const eventsource = new UIEventSource([]) if (options?.isActive !== undefined) { options.isActive.addCallbackAndRunD(async (active) => { if (!active) { return } - this.LoadJSONFrom(url, eventsource, layer) - .then((fs) => console.debug("Loaded", fs.length, "features from", url)) - .catch((err) => console.warn("Could not load ", url, "due to", err)) + this.updateAsync() return true // data is loaded, we can safely unregister }) } else { - this.LoadJSONFrom(url, eventsource, layer) - .then((fs) => console.debug("Loaded", fs.length, "features from", url)) - .catch((err) => console.warn("Could not load ", url, "due to", err)) + this.updateAsync() } - this.features = eventsource + } + + public async updateAsync(): Promise { + if (!this.currentlyRunning) { + this.currentlyRunning = this.LoadJSONFrom() + } + await this.currentlyRunning } /** * Init the download, write into the specified event source for the given layer. * Note this method caches the requested geojson for five minutes */ - private async LoadJSONFrom( - url: string, - eventSource: UIEventSource, - layer: LayerConfig, - options?: { - maxCacheAgeSec?: number | 300 + private async LoadJSONFrom(options?: { maxCacheAgeSec?: number | 300 }): Promise { + if (this._isDownloaded) { + return } - ): Promise { - const self = this - let json = await Utils.downloadJsonCached(url, (options?.maxCacheAgeSec ?? 300) * 1000) + const url = this.url + try { + let json = await Utils.downloadJsonCached(url, (options?.maxCacheAgeSec ?? 300) * 1000) - if (json.features === undefined || json.features === null) { - json.features = [] - } - - if (layer.source.mercatorCrs) { - json = GeoOperations.GeoJsonToWGS84(json) - } - - const time = new Date() - const newFeatures: Feature[] = [] - let i = 0 - let skipped = 0 - for (const feature of json.features) { - if (feature.geometry.type === "Point") { - // See https://github.com/maproulette/maproulette-backend/issues/242 - feature.geometry.coordinates = feature.geometry.coordinates.map(Number) + if (json.features === undefined || json.features === null) { + json.features = [] } - const props = feature.properties - for (const key in props) { - if (props[key] === null) { - delete props[key] + + if (this.layer.source.mercatorCrs) { + json = GeoOperations.GeoJsonToWGS84(json) + } + + const newFeatures: Feature[] = [] + let i = 0 + for (const feature of json.features) { + if (feature.geometry.type === "Point") { + // See https://github.com/maproulette/maproulette-backend/issues/242 + feature.geometry.coordinates = feature.geometry.coordinates.map(Number) + } + const props = feature.properties + for (const key in props) { + if (props[key] === null) { + delete props[key] + } + + if (typeof props[key] !== "string") { + // Make sure all the values are string, it crashes stuff otherwise + props[key] = JSON.stringify(props[key]) + } } - if (typeof props[key] !== "string") { - // Make sure all the values are string, it crashes stuff otherwise - props[key] = JSON.stringify(props[key]) + if (this.idKey !== undefined) { + props.id = props[this.idKey] } + + if (props.id === undefined) { + props.id = url + "/" + i + feature.id = url + "/" + i + i++ + } + if (this.seenids.has(props.id)) { + continue + } + this.seenids.add(props.id) + newFeatures.push(feature) } - if (self.idKey !== undefined) { - props.id = props[self.idKey] - } - - if (props.id === undefined) { - props.id = url + "/" + i - feature.id = url + "/" + i - i++ - } - if (self.seenids.has(props.id)) { - skipped++ - continue - } - self.seenids.add(props.id) - - let freshness: Date = time - if (feature.properties["_last_edit:timestamp"] !== undefined) { - freshness = new Date(props["_last_edit:timestamp"]) - } - - newFeatures.push(feature) + this._features.setData(newFeatures) + this._isDownloaded = true + return newFeatures + } catch (e) { + console.warn("Could not load ", url, "due to", e) } - - eventSource.setData(newFeatures) - return newFeatures } } diff --git a/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts b/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts index c95857d70..3fe9b79e9 100644 --- a/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts +++ b/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts @@ -11,16 +11,15 @@ import { OsmTags } from "../../../Models/OsmFeature" * Highly specialized feature source. * Based on a lon/lat UIEVentSource, will generate the corresponding feature with the correct properties */ -export class LastClickFeatureSource implements WritableFeatureSource { - public readonly features: UIEventSource = new UIEventSource([]) - public readonly hasNoteLayer: boolean +export class LastClickFeatureSource { public readonly renderings: string[] - public readonly hasPresets: boolean private i: number = 0 + private readonly hasPresets: boolean + private readonly hasNoteLayer: boolean - constructor(location: Store<{ lon: number; lat: number }>, layout: LayoutConfig) { - this.hasNoteLayer = layout.layers.some((l) => l.id === "note") - this.hasPresets = layout.layers.some((l) => l.presets?.length > 0) + constructor(layout: LayoutConfig) { + this.hasNoteLayer = layout.hasNoteLayer() + this.hasPresets = layout.hasPresets() const allPresets: BaseUIElement[] = [] for (const layer of layout.layers) for (let i = 0; i < (layer.presets ?? []).length; i++) { @@ -43,16 +42,11 @@ export class LastClickFeatureSource implements WritableFeatureSource { Utils.runningFromConsole ? "" : uiElem.ConstructElement().innerHTML ) ) - - location.addCallbackAndRunD(({ lon, lat }) => { - this.features.setData([this.createFeature(lon, lat)]) - }) } public createFeature(lon: number, lat: number): Feature { const properties: OsmTags = { - lastclick: "yes", - id: "last_click_" + this.i, + id: "new_point_dialog", has_note_layer: this.hasNoteLayer ? "yes" : "no", has_presets: this.hasPresets ? "yes" : "no", renderings: this.renderings.join(""), diff --git a/src/Logic/FeatureSource/Sources/LayoutSource.ts b/src/Logic/FeatureSource/Sources/LayoutSource.ts index c9377cfca..dcb19e089 100644 --- a/src/Logic/FeatureSource/Sources/LayoutSource.ts +++ b/src/Logic/FeatureSource/Sources/LayoutSource.ts @@ -1,16 +1,17 @@ import GeoJsonSource from "./GeoJsonSource" import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" -import { FeatureSource } from "../FeatureSource" +import { UpdatableFeatureSource } from "../FeatureSource" import { Or } from "../../Tags/Or" import FeatureSwitchState from "../../State/FeatureSwitchState" import OverpassFeatureSource from "./OverpassFeatureSource" import { Store, UIEventSource } from "../../UIEventSource" import OsmFeatureSource from "./OsmFeatureSource" -import FeatureSourceMerger from "./FeatureSourceMerger" import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSource" import { BBox } from "../../BBox" import LocalStorageFeatureSource from "../TiledFeatureSource/LocalStorageFeatureSource" import FullNodeDatabaseSource from "../TiledFeatureSource/FullNodeDatabaseSource" +import DynamicMvtileSource from "../TiledFeatureSource/DynamicMvtTileSource" +import FeatureSourceMerger from "./FeatureSourceMerger" /** * This source will fetch the needed data from various sources for the given layout. @@ -18,19 +19,24 @@ import FullNodeDatabaseSource from "../TiledFeatureSource/FullNodeDatabaseSource * Note that special layers (with `source=null` will be ignored) */ export default class LayoutSource extends FeatureSourceMerger { - private readonly _isLoading: UIEventSource = new UIEventSource(false) /** * Indicates if a data source is loading something */ - public readonly isLoading: Store = this._isLoading + public readonly isLoading: Store + + private readonly supportsForceDownload: UpdatableFeatureSource[] + constructor( layers: LayerConfig[], featureSwitches: FeatureSwitchState, mapProperties: { bounds: Store; zoom: Store }, backend: string, isDisplayed: (id: string) => Store, + mvtAvailableLayers: Set, fullNodeDatabaseSource?: FullNodeDatabaseSource ) { + const supportsForceDownload: UpdatableFeatureSource[] = [] + const { bounds, zoom } = mapProperties // remove all 'special' layers layers = layers.filter((layer) => layer.source !== null && layer.source !== undefined) @@ -44,8 +50,13 @@ export default class LayoutSource extends FeatureSourceMerger { maxAge: l.maxAgeOfCache, }) ) + const mvtSources: UpdatableFeatureSource[] = osmLayers + .filter((f) => mvtAvailableLayers.has(f.id)) + .map((l) => LayoutSource.setupMvtSource(l, mapProperties, isDisplayed(l.id))) + const nonMvtSources = [] + const nonMvtLayers = osmLayers.filter((l) => !mvtAvailableLayers.has(l.id)) - const overpassSource = LayoutSource.setupOverpass(osmLayers, bounds, zoom, featureSwitches) + const isLoading = new UIEventSource(false) const osmApiSource = LayoutSource.setupOsmApiSource( osmLayers, @@ -55,27 +66,53 @@ export default class LayoutSource extends FeatureSourceMerger { featureSwitches, fullNodeDatabaseSource ) - const geojsonSources: FeatureSource[] = geojsonlayers.map((l) => - LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)) - ) + nonMvtSources.push(osmApiSource) - super(overpassSource, osmApiSource, ...geojsonSources, ...fromCache) + let overpassSource: OverpassFeatureSource = undefined + if (nonMvtLayers.length > 0) { + console.log( + "Layers ", + nonMvtLayers.map((l) => l.id), + " cannot be fetched from the cache server, defaulting to overpass/OSM-api" + ) + overpassSource = LayoutSource.setupOverpass(osmLayers, bounds, zoom, featureSwitches) + nonMvtSources.push(overpassSource) + supportsForceDownload.push(overpassSource) + } - const self = this function setIsLoading() { const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data - self._isLoading.setData(loading) + isLoading.setData(loading) } overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading()) osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading()) + + const geojsonSources: UpdatableFeatureSource[] = geojsonlayers.map((l) => + LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)) + ) + + super(...geojsonSources, ...fromCache, ...mvtSources, ...nonMvtSources) + + this.isLoading = isLoading + supportsForceDownload.push(...geojsonSources) + supportsForceDownload.push(...mvtSources) // Non-mvt sources are handled by overpass + this.supportsForceDownload = supportsForceDownload + } + + private static setupMvtSource( + layer: LayerConfig, + mapProperties: { zoom: Store; bounds: Store }, + isActive?: Store + ): UpdatableFeatureSource { + return new DynamicMvtileSource(layer, mapProperties, { isActive }) } private static setupGeojsonSource( layer: LayerConfig, mapProperties: { zoom: Store; bounds: Store }, isActive?: Store - ): FeatureSource { + ): UpdatableFeatureSource { const source = layer.source isActive = mapProperties.zoom.map( (z) => (isActive?.data ?? true) && z >= layer.minzoom, @@ -161,4 +198,10 @@ export default class LayoutSource extends FeatureSourceMerger { } ) } + + public async downloadAll() { + console.log("Downloading all data") + await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync())) + console.log("Done") + } } diff --git a/src/Logic/FeatureSource/Sources/MvtSource.ts b/src/Logic/FeatureSource/Sources/MvtSource.ts new file mode 100644 index 000000000..ed66dcd51 --- /dev/null +++ b/src/Logic/FeatureSource/Sources/MvtSource.ts @@ -0,0 +1,464 @@ +import { Feature as GeojsonFeature, Geometry } from "geojson" + +import { Store, UIEventSource } from "../../UIEventSource" +import { FeatureSourceForTile, UpdatableFeatureSource } from "../FeatureSource" +import Pbf from "pbf" + +type Coords = [number, number][] + +class MvtFeatureBuilder { + private static readonly geom_types = ["Unknown", "Point", "LineString", "Polygon"] as const + private readonly _size: number + private readonly _x0: number + private readonly _y0: number + + constructor(extent: number, x: number, y: number, z: number) { + this._size = extent * Math.pow(2, z) + this._x0 = extent * x + this._y0 = extent * y + } + + private static signedArea(ring: Coords): number { + let sum = 0 + const len = ring.length + // J is basically (i - 1) % len + let j = len - 1 + let p1 + let p2 + for (let i = 0; i < len; i++) { + p1 = ring[i] + p2 = ring[j] + sum += (p2.x - p1.x) * (p1.y + p2.y) + j = i + } + return sum + } + + /** + * + * const rings = [ [ [ 3.208361864089966, 51.186908820014736 ], [ 3.2084155082702637, 51.18689537073311 ], [ 3.208436965942383, 51.186888646090836 ], [ 3.2084155082702637, 51.18686174751187 ], [ 3.2084155082702637, 51.18685502286465 ], [ 3.2083725929260254, 51.18686847215807 ], [ 3.2083404064178467, 51.18687519680333 ], [ 3.208361864089966, 51.186908820014736 ] ] ] + * MvtFeatureBuilder.classifyRings(rings) // => [rings] + */ + private static classifyRings(rings: Coords[]): Coords[][] { + if (rings.length <= 0) { + throw "Now rings in polygon found" + } + if (rings.length == 1) { + return [rings] + } + + const polygons: Coords[][] = [] + let currentPolygon: Coords[] + + for (let i = 0; i < rings.length; i++) { + let ring = rings[i] + const area = this.signedArea(ring) + if (area === 0) { + // Weird, degenerate ring + continue + } + const ccw = area < 0 + + if (ccw === area < 0) { + if (currentPolygon) { + polygons.push(currentPolygon) + } + currentPolygon = [ring] + } else { + currentPolygon.push(ring) + } + } + if (currentPolygon) { + polygons.push(currentPolygon) + } + + return polygons + } + + public toGeoJson(geometry: number[], typeIndex: 1 | 2 | 3, properties: any): GeojsonFeature { + let coords: Coords[] = this.encodeGeometry(geometry) + let classified = undefined + switch (typeIndex) { + case 1: + const points = [] + for (let i = 0; i < coords.length; i++) { + points[i] = coords[i][0] + } + coords = points + this.project(coords) + break + + case 2: + for (let i = 0; i < coords.length; i++) { + this.project(coords[i]) + } + break + + case 3: + classified = MvtFeatureBuilder.classifyRings(coords) + for (let i = 0; i < classified.length; i++) { + for (let j = 0; j < classified[i].length; j++) { + this.project(classified[i][j]) + } + } + break + } + + let type: string = MvtFeatureBuilder.geom_types[typeIndex] + let polygonCoords: Coords | Coords[] | Coords[][] + if (coords.length === 1) { + polygonCoords = (classified ?? coords)[0] + } else { + polygonCoords = classified ?? coords + type = "Multi" + type + } + + return { + type: "Feature", + geometry: { + type: type, + coordinates: polygonCoords, + }, + properties, + } + } + + /** + * + * const geometry = [9,233,8704,130,438,1455,270,653,248,423,368,493,362,381,330,267,408,301,406,221,402,157,1078,429,1002,449,1036,577,800,545,1586,1165,164,79,40] + * const builder = new MvtFeatureBuilder(4096, 66705, 43755, 17) + * const expected = [[3.2106759399175644,51.213658395282124],[3.2108227908611298,51.21396418776169],[3.2109133154153824,51.21410154168976],[3.210996463894844,51.214190590500664],[3.211119845509529,51.214294340548975],[3.211241215467453,51.2143745681588],[3.2113518565893173,51.21443085341426],[3.211488649249077,51.21449427925393],[3.2116247713565826,51.214540903490956],[3.211759552359581,51.21457408647774],[3.2121209800243378,51.214664394485254],[3.212456926703453,51.21475890267553],[3.2128042727708817,51.214880292910834],[3.213072493672371,51.214994962285544],[3.2136042416095734,51.21523984134939],[3.2136592268943787,51.21525664260963],[3.213672637939453,51.21525664260963]] + * builder.project(builder.encodeGeometry(geometry)[0]) // => expected + * @param geometry + * @private + */ + private encodeGeometry(geometry: number[]): Coords[] { + let cX = 0 + let cY = 0 + let coordss: Coords[] = [] + let currentRing: Coords = [] + for (let i = 0; i < geometry.length; i++) { + let commandInteger = geometry[i] + let commandId = commandInteger & 0x7 + let commandCount = commandInteger >> 3 + /* + Command Id Parameters Parameter Count + MoveTo 1 dX, dY 2 + LineTo 2 dX, dY 2 + ClosePath 7 No parameters 0 + */ + if (commandId === 1) { + // MoveTo means: we start a new ring + if (currentRing.length !== 0) { + coordss.push(currentRing) + currentRing = [] + } + } + if (commandId === 1 || commandId === 2) { + for (let j = 0; j < commandCount; j++) { + const dx = geometry[i + j * 2 + 1] + cX += (dx >> 1) ^ -(dx & 1) + const dy = geometry[i + j * 2 + 2] + cY += (dy >> 1) ^ -(dy & 1) + currentRing.push([cX, cY]) + } + i += commandCount * 2 + } + if (commandId === 7) { + currentRing.push([...currentRing[0]]) + i++ + } + } + if (currentRing.length > 0) { + coordss.push(currentRing) + } + return coordss + } + + /** + * Inline replacement of the location by projecting + * @param line the line which will be rewritten inline + * @return line + */ + private project(line: Coords) { + const y0 = this._y0 + const x0 = this._x0 + const size = this._size + for (let i = 0; i < line.length; i++) { + let p = line[i] + let y2 = 180 - ((p[1] + y0) * 360) / size + line[i] = [ + ((p[0] + x0) * 360) / size - 180, + (360 / Math.PI) * Math.atan(Math.exp((y2 * Math.PI) / 180)) - 90, + ] + } + return line + } +} + +class Layer { + public static read(pbf, end) { + return pbf.readFields( + Layer._readField, + { version: 0, name: "", features: [], keys: [], values: [], extent: 0 }, + end + ) + } + + static _readField(tag, obj, pbf) { + if (tag === 15) obj.version = pbf.readVarint() + else if (tag === 1) obj.name = pbf.readString() + else if (tag === 2) obj.features.push(Feature.read(pbf, pbf.readVarint() + pbf.pos)) + else if (tag === 3) obj.keys.push(pbf.readString()) + else if (tag === 4) obj.values.push(Value.read(pbf, pbf.readVarint() + pbf.pos)) + else if (tag === 5) obj.extent = pbf.readVarint() + } + + public static write(obj, pbf) { + if (obj.version) pbf.writeVarintField(15, obj.version) + if (obj.name) pbf.writeStringField(1, obj.name) + if (obj.features) + for (var i = 0; i < obj.features.length; i++) + pbf.writeMessage(2, Feature.write, obj.features[i]) + if (obj.keys) for (i = 0; i < obj.keys.length; i++) pbf.writeStringField(3, obj.keys[i]) + if (obj.values) + for (i = 0; i < obj.values.length; i++) pbf.writeMessage(4, Value.write, obj.values[i]) + if (obj.extent) pbf.writeVarintField(5, obj.extent) + } +} + +class Feature { + static read(pbf, end) { + return pbf.readFields(Feature._readField, { id: 0, tags: [], type: 0, geometry: [] }, end) + } + + static _readField(tag, obj, pbf) { + if (tag === 1) obj.id = pbf.readVarint() + else if (tag === 2) pbf.readPackedVarint(obj.tags) + else if (tag === 3) obj.type = pbf.readVarint() + else if (tag === 4) pbf.readPackedVarint(obj.geometry) + } + + public static write(obj, pbf) { + if (obj.id) pbf.writeVarintField(1, obj.id) + if (obj.tags) pbf.writePackedVarint(2, obj.tags) + if (obj.type) pbf.writeVarintField(3, obj.type) + if (obj.geometry) pbf.writePackedVarint(4, obj.geometry) + } +} + +class Value { + public static read(pbf, end) { + return pbf.readFields( + Value._readField, + { + string_value: "", + float_value: 0, + double_value: 0, + int_value: 0, + uint_value: 0, + sint_value: 0, + bool_value: false, + }, + end + ) + } + + static _readField = function (tag, obj, pbf) { + if (tag === 1) obj.string_value = pbf.readString() + else if (tag === 2) obj.float_value = pbf.readFloat() + else if (tag === 3) obj.double_value = pbf.readDouble() + else if (tag === 4) obj.int_value = pbf.readVarint(true) + else if (tag === 5) obj.uint_value = pbf.readVarint() + else if (tag === 6) obj.sint_value = pbf.readSVarint() + else if (tag === 7) obj.bool_value = pbf.readBoolean() + } + + public static write(obj, pbf) { + if (obj.string_value) pbf.writeStringField(1, obj.string_value) + if (obj.float_value) pbf.writeFloatField(2, obj.float_value) + if (obj.double_value) pbf.writeDoubleField(3, obj.double_value) + if (obj.int_value) pbf.writeVarintField(4, obj.int_value) + if (obj.uint_value) pbf.writeVarintField(5, obj.uint_value) + if (obj.sint_value) pbf.writeSVarintField(6, obj.sint_value) + if (obj.bool_value) pbf.writeBooleanField(7, obj.bool_value) + } +} + +class Tile { + // code generated by pbf v3.2.1 + + static GeomType = { + UNKNOWN: { + value: 0, + options: {}, + }, + POINT: { + value: 1, + options: {}, + }, + LINESTRING: { + value: 2, + options: {}, + }, + POLYGON: { + value: 3, + options: {}, + }, + } + + public static read(pbf, end) { + return pbf.readFields(Tile._readField, { layers: [] }, end) + } + + static _readField(tag, obj, pbf) { + if (tag === 3) obj.layers.push(Layer.read(pbf, pbf.readVarint() + pbf.pos)) + } + + static write(obj, pbf) { + if (obj.layers) + for (var i = 0; i < obj.layers.length; i++) + pbf.writeMessage(3, Layer.write, obj.layers[i]) + } +} + +export default class MvtSource implements FeatureSourceForTile, UpdatableFeatureSource { + public readonly features: Store[]> + public readonly x: number + public readonly y: number + public readonly z: number + private readonly _url: string + private readonly _layerName: string + private readonly _features: UIEventSource< + GeojsonFeature< + Geometry, + { + [name: string]: any + } + >[] + > = new UIEventSource[]>([]) + private currentlyRunning: Promise + + constructor( + url: string, + x: number, + y: number, + z: number, + layerName?: string, + isActive?: Store + ) { + this._url = url + this._layerName = layerName + this.x = x + this.y = y + this.z = z + this.updateAsync() + this.features = this._features.map( + (fs) => { + if (fs === undefined || isActive?.data === false) { + return [] + } + return fs + }, + [isActive] + ) + } + + async updateAsync() { + if (!this.currentlyRunning) { + this.currentlyRunning = this.download() + } + await this.currentlyRunning + } + + private getValue(v: { + // Exactly one of these values must be present in a valid message + string_value?: string + float_value?: number + double_value?: number + int_value?: number + uint_value?: number + sint_value?: number + bool_value?: boolean + }): string | number | undefined | boolean { + if (v.string_value !== "") { + return v.string_value + } + if (v.double_value !== 0) { + return v.double_value + } + if (v.float_value !== 0) { + return v.float_value + } + if (v.int_value !== 0) { + return v.int_value + } + if (v.uint_value !== 0) { + return v.uint_value + } + if (v.sint_value !== 0) { + return v.sint_value + } + if (v.bool_value !== false) { + return v.bool_value + } + return undefined + } + + private async download(): Promise { + try { + const result = await fetch(this._url) + if (result.status !== 200) { + console.error("Could not download tile " + this._url) + return + } + const buffer = await result.arrayBuffer() + const data = Tile.read(new Pbf(buffer), undefined) + const layers = data.layers + let layer = data.layers[0] + if (layers.length > 1) { + if (!this._layerName) { + throw "Multiple layers in the downloaded tile, but no layername is given to choose from" + } + layer = layers.find((l) => l.name === this._layerName) + } + if (!layer) { + return + } + const builder = new MvtFeatureBuilder(layer.extent, this.x, this.y, this.z) + const features: GeojsonFeature[] = [] + + for (const feature of layer.features) { + const properties = this.inflateProperties(feature.tags, layer.keys, layer.values) + features.push(builder.toGeoJson(feature.geometry, feature.type, properties)) + } + this._features.setData(features) + } catch (e) { + console.error("Could not download MVT tile due to", e) + } + } + + private inflateProperties(tags: number[], keys: string[], values: { string_value: string }[]) { + const properties = {} + for (let i = 0; i < tags.length; i += 2) { + properties[keys[tags[i]]] = this.getValue(values[tags[i + 1]]) + } + let type: string + switch (properties["osm_type"]) { + case "N": + type = "node" + break + case "W": + type = "way" + break + case "R": + type = "relation" + break + } + properties["id"] = type + "/" + properties["osm_id"] + delete properties["osm_id"] + delete properties["osm_type"] + + return properties + } +} diff --git a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts index 02ef73c43..af8fdbbf9 100644 --- a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts +++ b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts @@ -1,5 +1,5 @@ import { Feature } from "geojson" -import { FeatureSource } from "../FeatureSource" +import { UpdatableFeatureSource } from "../FeatureSource" import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource" import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" import { Or } from "../../Tags/Or" @@ -12,7 +12,7 @@ import { BBox } from "../../BBox" * A wrapper around the 'Overpass'-object. * It has more logic and will automatically fetch the data for the right bbox and the active layers */ -export default class OverpassFeatureSource implements FeatureSource { +export default class OverpassFeatureSource implements UpdatableFeatureSource { /** * The last loaded features, as geojson */ @@ -59,61 +59,12 @@ export default class OverpassFeatureSource implements FeatureSource { }) } - /** - * Creates the 'Overpass'-object for the given layers - * @param interpreterUrl - * @param layersToDownload - * @constructor - * @private - */ - private GetFilter(interpreterUrl: string, layersToDownload: LayerConfig[]): Overpass { - let filters: TagsFilter[] = layersToDownload.map((layer) => layer.source.osmTags) - filters = Utils.NoNull(filters) - if (filters.length === 0) { - return undefined - } - return new Overpass(new Or(filters), [], interpreterUrl, this.state.overpassTimeout) - } - - /** - * - * @private - */ - private async updateAsyncIfNeeded(): Promise { - if (!this._isActive?.data) { - return - } - if (this.runningQuery.data) { - console.log("Still running a query, not updating") - return undefined - } - - if (this.timeout.data > 0) { - console.log("Still in timeout - not updating") - return undefined - } - const requestedBounds = this.state.bounds.data - if ( - this._lastQueryBBox !== undefined && - requestedBounds.isContainedIn(this._lastQueryBBox) - ) { - return undefined - } - const result = await this.updateAsync() - if (!result) { - return - } - const [bounds, _, __] = result - this._lastQueryBBox = bounds - } - /** * Download the relevant data from overpass. Attempt to use a different server; only downloads the relevant layers * @private */ - private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> { + public async updateAsync(): Promise { let data: any = undefined - let date: Date = undefined let lastUsed = 0 const layersToDownload = [] @@ -172,7 +123,7 @@ export default class OverpassFeatureSource implements FeatureSource { return undefined } this.runningQuery.setData(true) - ;[data, date] = await overpass.queryGeoJson(bounds) + data = (await overpass.queryGeoJson(bounds))[0] } catch (e) { self.retries.data++ self.retries.ping() @@ -205,13 +156,55 @@ export default class OverpassFeatureSource implements FeatureSource { console.log("Overpass returned", data.features.length, "features") self.features.setData(data.features) - return [bounds, date, layersToDownload] + this._lastQueryBBox = bounds } catch (e) { console.error("Got the overpass response, but could not process it: ", e, e.stack) - return undefined } finally { self.retries.setData(0) self.runningQuery.setData(false) } } + + /** + * Creates the 'Overpass'-object for the given layers + * @param interpreterUrl + * @param layersToDownload + * @constructor + * @private + */ + private GetFilter(interpreterUrl: string, layersToDownload: LayerConfig[]): Overpass { + let filters: TagsFilter[] = layersToDownload.map((layer) => layer.source.osmTags) + filters = Utils.NoNull(filters) + if (filters.length === 0) { + return undefined + } + return new Overpass(new Or(filters), [], interpreterUrl, this.state.overpassTimeout) + } + + /** + * + * @private + */ + private async updateAsyncIfNeeded(): Promise { + if (!this._isActive?.data) { + return + } + if (this.runningQuery.data) { + console.log("Still running a query, not updating") + return undefined + } + + if (this.timeout.data > 0) { + console.log("Still in timeout - not updating") + return undefined + } + const requestedBounds = this.state.bounds.data + if ( + this._lastQueryBBox !== undefined && + requestedBounds.isContainedIn(this._lastQueryBBox) + ) { + return undefined + } + await this.updateAsync() + } } diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts index 88455e9bb..7cb342856 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts @@ -1,11 +1,11 @@ -import { Store } from "../../UIEventSource" -import DynamicTileSource from "./DynamicTileSource" +import { ImmutableStore, Store } from "../../UIEventSource" +import { UpdatableDynamicTileSource } from "./DynamicTileSource" import { Utils } from "../../../Utils" import GeoJsonSource from "../Sources/GeoJsonSource" import { BBox } from "../../BBox" import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" -export default class DynamicGeoJsonTileSource extends DynamicTileSource { +export default class DynamicGeoJsonTileSource extends UpdatableDynamicTileSource { private static whitelistCache = new Map() constructor( @@ -65,7 +65,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { const blackList = new Set() super( - source.geojsonZoomLevel, + new ImmutableStore(source.geojsonZoomLevel), layer.minzoom, (zxy) => { if (whitelist !== undefined) { diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts new file mode 100644 index 000000000..22b18b258 --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts @@ -0,0 +1,132 @@ +import { Store } from "../../UIEventSource" +import { UpdatableDynamicTileSource } from "./DynamicTileSource" +import { Utils } from "../../../Utils" +import { BBox } from "../../BBox" +import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" +import MvtSource from "../Sources/MvtSource" +import { Tiles } from "../../../Models/TileRange" +import Constants from "../../../Models/Constants" +import { UpdatableFeatureSourceMerger } from "../Sources/FeatureSourceMerger" +import { LineSourceMerger } from "./LineSourceMerger" +import { PolygonSourceMerger } from "./PolygonSourceMerger" + +class PolygonMvtSource extends PolygonSourceMerger { + constructor( + layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + } + ) { + const roundedZoom = mapProperties.zoom.mapD((z) => Math.min(Math.floor(z / 2) * 2, 14)) + super( + roundedZoom, + layer.minzoom, + (zxy) => { + const [z, x, y] = Tiles.tile_from_index(zxy) + const url = Utils.SubstituteKeys(Constants.VectorTileServer, { + z, + x, + y, + layer: layer.id, + type: "polygons", + }) + return new MvtSource(url, x, y, z) + }, + mapProperties, + { + isActive: options?.isActive, + } + ) + } +} + +class LineMvtSource extends LineSourceMerger { + constructor( + layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + } + ) { + const roundedZoom = mapProperties.zoom.mapD((z) => Math.min(Math.floor(z / 2) * 2, 14)) + super( + roundedZoom, + layer.minzoom, + (zxy) => { + const [z, x, y] = Tiles.tile_from_index(zxy) + const url = Utils.SubstituteKeys(Constants.VectorTileServer, { + z, + x, + y, + layer: layer.id, + type: "lines", + }) + return new MvtSource(url, x, y, z) + }, + mapProperties, + { + isActive: options?.isActive, + } + ) + } +} + +class PointMvtSource extends UpdatableDynamicTileSource { + constructor( + layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + } + ) { + const roundedZoom = mapProperties.zoom.mapD((z) => Math.min(Math.floor(z / 2) * 2, 14)) + super( + roundedZoom, + layer.minzoom, + (zxy) => { + const [z, x, y] = Tiles.tile_from_index(zxy) + const url = Utils.SubstituteKeys(Constants.VectorTileServer, { + z, + x, + y, + layer: layer.id, + type: "pois", + }) + return new MvtSource(url, x, y, z) + }, + mapProperties, + { + isActive: options?.isActive, + } + ) + } +} + +export default class DynamicMvtileSource extends UpdatableFeatureSourceMerger { + constructor( + layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + } + ) { + super( + new PointMvtSource(layer, mapProperties, options), + new LineMvtSource(layer, mapProperties, options), + new PolygonMvtSource(layer, mapProperties, options) + ) + } +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts index e8e69ee2a..ea8040832 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts @@ -1,71 +1,124 @@ import { Store, Stores } from "../../UIEventSource" import { Tiles } from "../../../Models/TileRange" import { BBox } from "../../BBox" -import { FeatureSource } from "../FeatureSource" +import { FeatureSource, UpdatableFeatureSource } from "../FeatureSource" import FeatureSourceMerger from "../Sources/FeatureSourceMerger" /*** * A tiled source which dynamically loads the required tiles at a fixed zoom level. * A single featureSource will be initialized for every tile in view; which will later be merged into this featureSource */ -export default class DynamicTileSource extends FeatureSourceMerger { +export default class DynamicTileSource< + Src extends FeatureSource = FeatureSource +> extends FeatureSourceMerger { + private readonly loadedTiles = new Set() + private readonly zDiff: number + private readonly zoomlevel: Store + private readonly constructSource: (tileIndex: number) => Src + private readonly bounds: Store + + /** + * + * @param zoomlevel If {z} is specified in the source, the 'zoomlevel' will be used as zoomlevel to download from + * @param minzoom Only activate this feature source if zoomed in further then this + * @param constructSource + * @param mapProperties + * @param options + */ constructor( - zoomlevel: number, + zoomlevel: Store, minzoom: number, - constructSource: (tileIndex) => FeatureSource, + constructSource: (tileIndex: number) => Src, mapProperties: { bounds: Store zoom: Store }, options?: { isActive?: Store + zDiff?: number } ) { super() - const loadedTiles = new Set() + this.constructSource = constructSource + this.zoomlevel = zoomlevel + this.zDiff = options?.zDiff ?? 0 + this.bounds = mapProperties.bounds + const neededTiles: Store = Stores.ListStabilized( mapProperties.bounds - .mapD( - (bounds) => { - if (options?.isActive && !options?.isActive.data) { - return undefined - } + .mapD(() => { + if (options?.isActive && !options?.isActive.data) { + return undefined + } - if (mapProperties.zoom.data < minzoom) { - return undefined - } - const tileRange = Tiles.TileRangeBetween( - zoomlevel, - bounds.getNorth(), - bounds.getEast(), - bounds.getSouth(), - bounds.getWest() - ) - if (tileRange.total > 500) { - console.warn( - "Got a really big tilerange, bounds and location might be out of sync" - ) - return undefined - } - - const needed = Tiles.MapRange(tileRange, (x, y) => - Tiles.tile_index(zoomlevel, x, y) - ).filter((i) => !loadedTiles.has(i)) - if (needed.length === 0) { - return undefined - } - return needed - }, - [options?.isActive, mapProperties.zoom] - ) + if (mapProperties.zoom.data < minzoom) { + return undefined + } + return this.getNeededTileIndices() + }, [options?.isActive, mapProperties.zoom]) .stabilized(250) ) - neededTiles.addCallbackAndRunD((neededIndexes) => { - for (const neededIndex of neededIndexes) { - loadedTiles.add(neededIndex) - super.addSource(constructSource(neededIndex)) - } - }) + neededTiles.addCallbackAndRunD((neededIndexes) => this.downloadTiles(neededIndexes)) + } + + protected downloadTiles(neededIndexes: number[]): Src[] { + const sources: Src[] = [] + for (const neededIndex of neededIndexes) { + this.loadedTiles.add(neededIndex) + const src = this.constructSource(neededIndex) + super.addSource(src) + sources.push(src) + } + return sources + } + + protected getNeededTileIndices() { + const bounds = this.bounds.data + const z = Math.floor(this.zoomlevel.data) + this.zDiff + const tileRange = Tiles.TileRangeBetween( + z, + bounds.getNorth(), + bounds.getEast(), + bounds.getSouth(), + bounds.getWest() + ) + if (tileRange.total > 500) { + console.warn("Got a really big tilerange, bounds and location might be out of sync") + return [] + } + const needed = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(z, x, y)).filter( + (i) => !this.loadedTiles.has(i) + ) + if (needed.length === 0) { + return [] + } + return needed + } +} + +export class UpdatableDynamicTileSource + extends DynamicTileSource + implements UpdatableFeatureSource +{ + constructor( + zoomlevel: Store, + minzoom: number, + constructSource: (tileIndex: number) => Src, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + zDiff?: number + } + ) { + super(zoomlevel, minzoom, constructSource, mapProperties, options) + } + + async updateAsync() { + const sources = super.downloadTiles(super.getNeededTileIndices()) + await Promise.all(sources.map((src) => src.updateAsync())) } } diff --git a/src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts b/src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts new file mode 100644 index 000000000..03e26ac3f --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts @@ -0,0 +1,80 @@ +import { FeatureSourceForTile, UpdatableFeatureSource } from "../FeatureSource" +import { Store } from "../../UIEventSource" +import { BBox } from "../../BBox" +import { Utils } from "../../../Utils" +import { Feature, MultiLineString, Position } from "geojson" +import { GeoOperations } from "../../GeoOperations" +import { UpdatableDynamicTileSource } from "./DynamicTileSource" + +/** + * The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together. + * This is used to reconstruct polygons of vector tiles + */ +export class LineSourceMerger extends UpdatableDynamicTileSource< + FeatureSourceForTile & UpdatableFeatureSource +> { + private readonly _zoomlevel: Store + + constructor( + zoomlevel: Store, + minzoom: number, + constructSource: (tileIndex: number) => FeatureSourceForTile & UpdatableFeatureSource, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + } + ) { + super(zoomlevel, minzoom, constructSource, mapProperties, options) + this._zoomlevel = zoomlevel + } + + protected addDataFromSources(sources: FeatureSourceForTile[]) { + sources = Utils.NoNull(sources) + const all: Map> = new Map() + const currentZoom = this._zoomlevel?.data ?? 0 + for (const source of sources) { + if (source.z != currentZoom) { + continue + } + for (const f of source.features.data) { + const id = f.properties.id + const coordinates: Position[][] = [] + if (f.geometry.type === "LineString") { + coordinates.push(f.geometry.coordinates) + } else if (f.geometry.type === "MultiLineString") { + coordinates.push(...f.geometry.coordinates) + } else { + console.error("Invalid geometry type:", f.geometry.type) + continue + } + const oldV = all.get(id) + if (!oldV) { + all.set(id, { + type: "Feature", + properties: f.properties, + geometry: { + type: "MultiLineString", + coordinates, + }, + }) + continue + } + oldV.geometry.coordinates.push(...coordinates) + } + } + + const keys = Array.from(all.keys()) + for (const key of keys) { + all.set( + key, + GeoOperations.attemptLinearize(>all.get(key)) + ) + } + const newList = Array.from(all.values()) + this.features.setData(newList) + this._featuresById.setData(all) + } +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts index eac9d0859..11ce41080 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts @@ -1,5 +1,5 @@ import DynamicTileSource from "./DynamicTileSource" -import { Store } from "../../UIEventSource" +import { ImmutableStore, Store } from "../../UIEventSource" import { BBox } from "../../BBox" import TileLocalStorage from "../Actors/TileLocalStorage" import { Feature } from "geojson" @@ -27,7 +27,7 @@ export default class LocalStorageFeatureSource extends DynamicTileSource { options?.maxAge ?? 24 * 60 * 60 ) super( - zoomlevel, + new ImmutableStore(zoomlevel), layer.minzoom, (tileIndex) => new StaticFeatureSource( diff --git a/src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts b/src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts new file mode 100644 index 000000000..716db8e27 --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts @@ -0,0 +1,74 @@ +import { FeatureSourceForTile, UpdatableFeatureSource } from "../FeatureSource" +import { Store } from "../../UIEventSource" +import { BBox } from "../../BBox" +import { Utils } from "../../../Utils" +import { Feature } from "geojson" +import { GeoOperations } from "../../GeoOperations" +import DynamicTileSource, { UpdatableDynamicTileSource } from "./DynamicTileSource" + +/** + * The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together. + * This is used to reconstruct polygons of vector tiles + */ +export class PolygonSourceMerger extends UpdatableDynamicTileSource< + FeatureSourceForTile & UpdatableFeatureSource +> { + constructor( + zoomlevel: Store, + minzoom: number, + constructSource: (tileIndex: number) => FeatureSourceForTile & UpdatableFeatureSource, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + } + ) { + super(zoomlevel, minzoom, constructSource, mapProperties, options) + } + + protected addDataFromSources(sources: FeatureSourceForTile[]) { + sources = Utils.NoNull(sources) + const all: Map = new Map() + const zooms: Map = new Map() + + for (const source of sources) { + let z = source.z + for (const f of source.features.data) { + const id = f.properties.id + if (id.endsWith("146616907")) { + console.log("Horeca totaal") + } + if (!all.has(id)) { + // No other parts of this polygon have been seen before, simply add it + all.set(id, f) + zooms.set(id, z) + continue + } + + // A part of this object has been seen before, eventually from a different zoom level + const oldV = all.get(id) + const oldZ = zooms.get(id) + if (oldZ > z) { + // The store contains more detailed information, so we ignore this part which has a lower accuraccy + continue + } + if (oldZ < z) { + // The old value has worse accuracy then what we receive now, we throw it away + all.set(id, f) + zooms.set(id, z) + continue + } + const merged = GeoOperations.union(f, oldV) + merged.properties = oldV.properties + all.set(id, merged) + zooms.set(id, z) + } + } + + const newList = Array.from(all.values()) + this.features.setData(newList) + this._featuresById.setData(all) + } +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts new file mode 100644 index 000000000..81faaa289 --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts @@ -0,0 +1,157 @@ +import DynamicTileSource from "./DynamicTileSource" +import { Store, UIEventSource } from "../../UIEventSource" +import { BBox } from "../../BBox" +import StaticFeatureSource from "../Sources/StaticFeatureSource" +import { Feature, Point } from "geojson" +import { Utils } from "../../../Utils" +import { Tiles } from "../../../Models/TileRange" +import { FeatureSource } from "../FeatureSource" +import FilteredLayer from "../../../Models/FilteredLayer" +import Constants from "../../../Models/Constants" + +export class SummaryTileSourceRewriter implements FeatureSource { + private readonly _features: UIEventSource = new UIEventSource([]) + private filteredLayers: FilteredLayer[] + public readonly features: Store = this._features + private readonly _summarySource: SummaryTileSource + private readonly _totalNumberOfFeatures: UIEventSource = new UIEventSource( + undefined + ) + public readonly totalNumberOfFeatures: Store = this._totalNumberOfFeatures + constructor( + summarySource: SummaryTileSource, + filteredLayers: ReadonlyMap + ) { + this.filteredLayers = Array.from(filteredLayers.values()).filter( + (l) => + Constants.priviliged_layers.indexOf(l.layerDef.id) < 0 && + !l.layerDef.id.startsWith("note_import") + ) + this._summarySource = summarySource + filteredLayers.forEach((v, k) => { + v.isDisplayed.addCallback((_) => this.update()) + }) + this._summarySource.features.addCallbackAndRunD((_) => this.update()) + } + + private update() { + let fullTotal = 0 + const newFeatures: Feature[] = [] + const layersToCount = this.filteredLayers.filter((fl) => fl.isDisplayed.data) + const bitmap = layersToCount.map((l) => (l.isDisplayed.data ? "1" : "0")).join("") + const ids = layersToCount.map((l) => l.layerDef.id) + for (const f of this._summarySource.features.data ?? []) { + let newTotal = 0 + for (const id of ids) { + newTotal += Number(f.properties[id] ?? 0) + } + newFeatures.push({ + ...f, + properties: { + ...f.properties, + id: f.properties.id + bitmap, + total: newTotal, + total_metric: Utils.numberWithMetrixPrefix(newTotal), + }, + }) + fullTotal += newTotal + } + this._features.setData(newFeatures) + this._totalNumberOfFeatures.setData(fullTotal) + } +} + +/** + * Provides features summarizing the total amount of features at a given location + */ +export class SummaryTileSource extends DynamicTileSource { + private static readonly empty = [] + constructor( + cacheserver: string, + layers: string[], + zoomRounded: Store, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + } + ) { + const layersSummed = layers.join("+") + const zDiff = 2 + super( + zoomRounded, + 0, // minzoom + (tileIndex) => { + const [z, x, y] = Tiles.tile_from_index(tileIndex) + let coordinates = Tiles.centerPointOf(z, x, y) + const url = `${cacheserver}/${layersSummed}/${z}/${x}/${y}.json` + const count = UIEventSource.FromPromiseWithErr(Utils.downloadJson(url)) + const features: Store[]> = count.mapD((count) => { + if (count["error"] !== undefined) { + console.error( + "Could not download count for tile", + z, + x, + y, + "due to", + count["error"] + ) + return SummaryTileSource.empty + } + const counts = count["success"] + if (counts === undefined || counts["total"] === 0) { + return SummaryTileSource.empty + } + const lat = counts["lat"] + const lon = counts["lon"] + const total = Number(counts["total"]) + const tileBbox = new BBox(Tiles.tile_bounds_lon_lat(z, x, y)) + if (!tileBbox.contains([lon, lat])) { + console.error( + "Average coordinate is outside of bbox!?", + lon, + lat, + tileBbox, + counts, + url + ) + } else { + coordinates = [lon, lat] + } + return [ + { + type: "Feature", + properties: { + id: "summary_" + tileIndex, + summary: "yes", + ...counts, + total, + total_metric: Utils.numberWithMetrixPrefix(total), + layers: layersSummed, + }, + geometry: { + type: "Point", + coordinates, + }, + }, + ] + }) + return new StaticFeatureSource( + features.map( + (f) => { + if (z - zDiff !== zoomRounded.data) { + return SummaryTileSource.empty + } + return f + }, + [zoomRounded] + ) + ) + }, + mapProperties, + { ...options, zDiff } + ) + } +} diff --git a/src/Logic/GeoOperations.ts b/src/Logic/GeoOperations.ts index 2b0294641..229989f5b 100644 --- a/src/Logic/GeoOperations.ts +++ b/src/Logic/GeoOperations.ts @@ -1,6 +1,6 @@ import { BBox } from "./BBox" import * as turf from "@turf/turf" -import { AllGeoJSON, booleanWithin, Coord } from "@turf/turf" +import { AllGeoJSON, booleanWithin, Coord, Lines } from "@turf/turf" import { Feature, FeatureCollection, @@ -724,6 +724,25 @@ export class GeoOperations { } return kept } + + if (toSplit.geometry.type === "MultiLineString") { + const lines: Feature[][] = toSplit.geometry.coordinates.map( + (coordinates) => + turf.lineSplit({ type: "LineString", coordinates }, boundary) + .features + ) + const splitted: Feature[] = [].concat(...lines) + const kept: Feature[] = [] + for (const f of splitted) { + console.log("Checking", f) + if (!GeoOperations.inside(GeoOperations.centerpointCoordinates(f), boundary)) { + continue + } + f.properties = { ...toSplit.properties } + kept.push(f) + } + return kept + } if (toSplit.geometry.type === "Polygon" || toSplit.geometry.type == "MultiPolygon") { const splitup = turf.intersect(>toSplit, boundary) splitup.properties = { ...toSplit.properties } @@ -1004,6 +1023,68 @@ export class GeoOperations { return GeoOperations.directionsRelative[segment] } + /** + * const coors = [[[3.217198532946432,51.218067],[3.216807134449482,51.21849812105347],[3.2164304037883706,51.2189272]],[[3.2176208,51.21760169669458],[3.217198560167068,51.218067]]] + * const f = {geometry: {coordinates: coors}} + * const merged = GeoOperations.attemptLinearize(f) + * merged.geometry.coordinates // => [[3.2176208,51.21760169669458],[3.217198532946432,51.218067], [3.216807134449482,51.21849812105347],[3.2164304037883706,51.2189272]] + */ + static attemptLinearize( + multiLineStringFeature: Feature + ): Feature { + const coors = multiLineStringFeature.geometry.coordinates + if (coors.length === 0) { + console.error(multiLineStringFeature.geometry) + throw "Error: got degenerate multilinestring" + } + outer: for (let i = coors.length - 1; i >= 0; i--) { + // We try to match the first element of 'i' with another, earlier list `j` + // If a match is found with `j`, j is extended and `i` is scrapped + const iFirst = coors[i][0] + for (let j = 0; j < coors.length; j++) { + if (i == j) { + continue + } + + const jLast = coors[j].at(-1) + if ( + !( + Math.abs(iFirst[0] - jLast[0]) < 0.000001 && + Math.abs(iFirst[1] - jLast[1]) < 0.0000001 + ) + ) { + continue + } + coors[j].splice(coors.length - 1, 1) + coors[j].push(...coors[i]) + coors.splice(i, 1) + continue outer + } + } + if (coors.length === 0) { + throw "No more coordinates found" + } + + if (coors.length === 1) { + return { + type: "Feature", + properties: multiLineStringFeature.properties, + geometry: { + type: "LineString", + coordinates: coors[0], + }, + } + } + return { + type: "Feature", + properties: multiLineStringFeature.properties, + geometry: { + type: "MultiLineString", + coordinates: coors, + }, + } + } + /** * Helper function which does the heavy lifting for 'inside' */ diff --git a/src/Logic/Osm/OsmObject.ts b/src/Logic/Osm/OsmObject.ts index f0e0215cd..d743b58c9 100644 --- a/src/Logic/Osm/OsmObject.ts +++ b/src/Logic/Osm/OsmObject.ts @@ -7,7 +7,7 @@ import { Feature, LineString, Polygon } from "geojson" export abstract class OsmObject { private static defaultBackend = "https://api.openstreetmap.org/" protected static backendURL = OsmObject.defaultBackend - private static polygonFeatures = OsmObject.constructPolygonFeatures() + public static polygonFeatures = OsmObject.constructPolygonFeatures() type: "node" | "way" | "relation" id: number /** diff --git a/src/Logic/UIEventSource.ts b/src/Logic/UIEventSource.ts index 70aa5da87..ea8771f69 100644 --- a/src/Logic/UIEventSource.ts +++ b/src/Logic/UIEventSource.ts @@ -639,8 +639,9 @@ export class UIEventSource extends Store implements Writable { promise: Promise ): UIEventSource<{ success: T } | { error: any } | undefined> { const src = new UIEventSource<{ success: T } | { error: any }>(undefined) - promise?.then((d) => src.setData({ success: d })) - promise?.catch((err) => src.setData({ error: err })) + promise + ?.then((d) => src.setData({ success: d })) + ?.catch((err) => src.setData({ error: err })) return src } diff --git a/src/Models/Constants.ts b/src/Models/Constants.ts index f815756d0..41387bd8b 100644 --- a/src/Models/Constants.ts +++ b/src/Models/Constants.ts @@ -24,6 +24,7 @@ export default class Constants { "range", "last_click", "favourite", + "summary", ] as const /** * Special layers which are not included in a theme by default @@ -156,6 +157,11 @@ export default class Constants { "addSmall", ] as const public static readonly defaultPinIcons: string[] = Constants._defaultPinIcons + /** + * The location that the MVT-layer is hosted. + * This is a MapLibre/MapBox vector tile server which hosts vector tiles for every (official) layer + */ + public static VectorTileServer: string | undefined = Constants.config.mvt_layer_server public static readonly maptilerApiKey = "GvoVAJgu46I5rZapJuAy" private static isRetina(): boolean { diff --git a/src/Models/ThemeConfig/Conversion/LegacyJsonConvert.ts b/src/Models/ThemeConfig/Conversion/LegacyJsonConvert.ts index 3c7cf7efc..4d066b958 100644 --- a/src/Models/ThemeConfig/Conversion/LegacyJsonConvert.ts +++ b/src/Models/ThemeConfig/Conversion/LegacyJsonConvert.ts @@ -269,6 +269,7 @@ class UpdateLegacyTheme extends DesugaringStep { oldThemeConfig.layers = Utils.NoNull(oldThemeConfig.layers) delete oldThemeConfig["language"] delete oldThemeConfig["version"] + delete oldThemeConfig["clustering"] if (oldThemeConfig.startLat === 0) { delete oldThemeConfig.startLat diff --git a/src/Models/ThemeConfig/Conversion/PrepareTheme.ts b/src/Models/ThemeConfig/Conversion/PrepareTheme.ts index 92585607d..cc2c43a41 100644 --- a/src/Models/ThemeConfig/Conversion/PrepareTheme.ts +++ b/src/Models/ThemeConfig/Conversion/PrepareTheme.ts @@ -194,7 +194,6 @@ class AddDefaultLayers extends DesugaringStep { if (v === undefined) { const msg = `Default layer ${layerName} not found. ${state.sharedLayers.size} layers are available` if (layerName === "favourite") { - // context.warn(msg) continue } context.err(msg) diff --git a/src/Models/ThemeConfig/Conversion/Validation.ts b/src/Models/ThemeConfig/Conversion/Validation.ts index 589f11a17..14f9c519c 100644 --- a/src/Models/ThemeConfig/Conversion/Validation.ts +++ b/src/Models/ThemeConfig/Conversion/Validation.ts @@ -283,6 +283,15 @@ export class ValidateTheme extends DesugaringStep { } } + for (let i = 0; i < theme.layers.length; i++) { + const layer = theme.layers[i] + if (!layer.id.match("[a-z][a-z0-9_]*")) { + context + .enters("layers", i, "id") + .err("Invalid ID:" + layer.id + "should match [a-z][a-z0-9_]*") + } + } + return json } } @@ -364,6 +373,34 @@ class MiscThemeChecks extends DesugaringStep { if (json.socialImage === "") { context.warn("Social image for theme " + json.id + " is the emtpy string") } + if (json["clustering"]) { + context.warn("Obsolete field `clustering` is still around") + } + { + for (let i = 0; i < json.layers.length; i++) { + const l = json.layers[i] + if (l["override"]?.["source"] === undefined) { + continue + } + if (l["override"]?.["source"]?.["geoJson"]) { + continue // We don't care about external data as we won't cache it anyway + } + if (l["override"]["id"] !== undefined) { + continue + } + context + .enters("layers", i) + .err("A layer which changes the source-tags must also change the ID") + } + } + + if (json["overideAll"]) { + context + .enter("overideAll") + .err( + "'overrideAll' is spelled with _two_ `r`s. You only wrote a single one of them." + ) + } return json } } @@ -1035,7 +1072,8 @@ export class ValidateTagRenderings extends Fuse { new On("render", new ValidatePossibleLinks()), new On("question", new ValidatePossibleLinks()), new On("questionHint", new ValidatePossibleLinks()), - new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))) + new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))), + new MiscTagRenderingChecks() ) } } @@ -1070,8 +1108,9 @@ export class PrevalidateLayer extends DesugaringStep { if (json.id?.toLowerCase() !== json.id) { context.enter("id").err(`The id of a layer should be lowercase: ${json.id}`) } - if (json.id?.match(/[a-z0-9-_]/) == null) { - context.enter("id").err(`The id of a layer should match [a-z0-9-_]*: ${json.id}`) + const layerRegex = /[a-zA-Z][a-zA-Z_0-9]+/ + if (json.id.match(layerRegex) === null) { + context.enter("id").err("Invalid ID. A layer ID should match " + layerRegex.source) } } @@ -1572,6 +1611,10 @@ export class ValidateLayer extends Conversion< } } + if (json["doCount"]) { + context.enters("doCount").err("Use `isCounted` instead of `doCount`") + } + return { raw: json, parsed: layerConfig } } } @@ -1774,3 +1817,79 @@ export class DetectDuplicatePresets extends DesugaringStep { return json } } + +export class ValidateThemeEnsemble extends Conversion< + LayoutConfig[], + Map< + string, + { + tags: TagsFilter + foundInTheme: string[] + } + > +> { + constructor() { + super( + "Validates that all themes together are logical, i.e. no duplicate ids exists within (overriden) themes", + [], + "ValidateThemeEnsemble" + ) + } + + convert( + json: LayoutConfig[], + context: ConversionContext + ): Map< + string, + { + tags: TagsFilter + foundInTheme: string[] + } + > { + const idToSource = new Map() + + for (const theme of json) { + for (const layer of theme.layers) { + if (typeof layer.source === "string") { + continue + } + if (Constants.priviliged_layers.indexOf(layer.id) >= 0) { + continue + } + if (!layer.source) { + console.log(theme, layer, layer.source) + context.enters(theme.id, "layers", "source", layer.id).err("No source defined") + continue + } + if (layer.source.geojsonSource) { + continue + } + const id = layer.id + const tags = layer.source.osmTags + if (!idToSource.has(id)) { + idToSource.set(id, { tags, foundInTheme: [theme.id] }) + continue + } + + const oldTags = idToSource.get(id).tags + const oldTheme = idToSource.get(id).foundInTheme + if (oldTags.shadows(tags) && tags.shadows(oldTags)) { + // All is good, all is well + oldTheme.push(theme.id) + continue + } + context.err( + [ + "The layer with id '" + + id + + "' is found in multiple themes with different tag definitions:", + "\t In theme " + oldTheme + ":\t" + oldTags.asHumanString(false, false, {}), + "\tIn theme " + theme.id + ":\t" + tags.asHumanString(false, false, {}), + ].join("\n") + ) + } + } + + return idToSource + } +} diff --git a/src/Models/ThemeConfig/Json/LayerConfigJson.ts b/src/Models/ThemeConfig/Json/LayerConfigJson.ts index ef3b43196..0da64a1d9 100644 --- a/src/Models/ThemeConfig/Json/LayerConfigJson.ts +++ b/src/Models/ThemeConfig/Json/LayerConfigJson.ts @@ -176,6 +176,18 @@ export interface LayerConfigJson { */ isShown?: TagConfigJson + /** + * question: should this layer be included in the summary counts? + * + * The layer server can give summary counts for a tile. + * This should however be disabled for some layers, e.g. because there are too many features (walls_and_buildings) or because the count is irrelevant. + * + * ifunset: Do count + * iffalse: Do not include the counts + * iftrue: Do include the count + */ + isCounted?: true | boolean + /** * The minimum needed zoomlevel required to start loading and displaying the data. * This can be used to only show common features (e.g. a bicycle parking) only when the map is zoomed in very much (17). diff --git a/src/Models/ThemeConfig/LayerConfig.ts b/src/Models/ThemeConfig/LayerConfig.ts index b42bc7855..e3a0ac247 100644 --- a/src/Models/ThemeConfig/LayerConfig.ts +++ b/src/Models/ThemeConfig/LayerConfig.ts @@ -28,8 +28,6 @@ import { ImmutableStore } from "../../Logic/UIEventSource" import { OsmTags } from "../OsmFeature" import Constants from "../Constants" import { QuestionableTagRenderingConfigJson } from "./Json/QuestionableTagRenderingConfigJson" -import SvelteUIElement from "../../UI/Base/SvelteUIElement" -import Statistics from "../../assets/svg/Statistics.svelte" export default class LayerConfig extends WithContextLoader { public static readonly syncSelectionAllowed = ["no", "local", "theme-only", "global"] as const @@ -46,7 +44,6 @@ export default class LayerConfig extends WithContextLoader { public readonly isShown: TagsFilter public minzoom: number public minzoomVisible: number - public readonly maxzoom: number public readonly title?: TagRenderingConfig public readonly titleIcons: TagRenderingConfig[] public readonly mapRendering: PointRenderingConfig[] @@ -56,6 +53,7 @@ export default class LayerConfig extends WithContextLoader { public readonly allowMove: MoveConfig | null public readonly allowSplit: boolean public readonly shownByDefault: boolean + public readonly doCount: boolean /** * In seconds */ @@ -161,6 +159,7 @@ export default class LayerConfig extends WithContextLoader { } this.minzoomVisible = json.minzoomVisible ?? this.minzoom this.shownByDefault = json.shownByDefault ?? true + this.doCount = json.isCounted ?? true this.forceLoad = json.forceLoad ?? false if (json.presets === null) json.presets = undefined if (json.presets !== undefined && json.presets?.map === undefined) { @@ -465,9 +464,7 @@ export default class LayerConfig extends WithContextLoader { return [ new Combine([ new Link( - Utils.runningFromConsole - ? "" - : new SvelteUIElement(Statistics, { class: "w-4 h-4 mr-2" }), + "", "https://taginfo.openstreetmap.org/keys/" + values.key + "#values", true ), diff --git a/src/Models/ThemeConfig/LayoutConfig.ts b/src/Models/ThemeConfig/LayoutConfig.ts index 01beaab12..2b50cebfb 100644 --- a/src/Models/ThemeConfig/LayoutConfig.ts +++ b/src/Models/ThemeConfig/LayoutConfig.ts @@ -247,6 +247,14 @@ export default class LayoutConfig implements LayoutInformation { return this.layers.some((l) => l.isLeftRightSensitive()) } + public hasNoteLayer() { + return this.layers.some((l) => l.id === "note") + } + + public hasPresets() { + return this.layers.some((l) => l.presets?.length > 0) + } + public missingTranslations(extraInspection: any): { untranslated: Map total: number diff --git a/src/Models/ThemeConfig/PointRenderingConfig.ts b/src/Models/ThemeConfig/PointRenderingConfig.ts index ca3cd22f5..25526643f 100644 --- a/src/Models/ThemeConfig/PointRenderingConfig.ts +++ b/src/Models/ThemeConfig/PointRenderingConfig.ts @@ -314,7 +314,7 @@ export default class PointRenderingConfig extends WithContextLoader { const label = self.label ?.GetRenderValue(tags) ?.Subs(tags) - ?.SetClass("block center absolute text-center marker-label") + ?.SetClass("flex items-center justify-center absolute marker-label") ?.SetClass(cssClassesLabel) if (cssLabel) { label.SetStyle(cssLabel) diff --git a/src/Models/ThemeViewState.ts b/src/Models/ThemeViewState.ts index 911c4700d..50fe06d71 100644 --- a/src/Models/ThemeViewState.ts +++ b/src/Models/ThemeViewState.ts @@ -62,6 +62,12 @@ import FavouritesFeatureSource from "../Logic/FeatureSource/Sources/FavouritesFe import { ProvidedImage } from "../Logic/ImageProviders/ImageProvider" import { GeolocationControlState } from "../UI/BigComponents/GeolocationControl" import Zoomcontrol from "../UI/Zoomcontrol" +import { + SummaryTileSource, + SummaryTileSourceRewriter, +} from "../Logic/FeatureSource/TiledFeatureSource/SummaryTileSource" +import summaryLayer from "../assets/generated/layers/summary.json" +import { LayerConfigJson } from "./ThemeConfig/Json/LayerConfigJson" import Locale from "../UI/i18n/Locale" /** @@ -107,6 +113,7 @@ export default class ThemeViewState implements SpecialVisualizationState { readonly closestFeatures: NearbyFeatureSource readonly newFeatures: WritableFeatureSource readonly layerState: LayerState + readonly featureSummary: SummaryTileSourceRewriter readonly perLayer: ReadonlyMap readonly perLayerFiltered: ReadonlyMap @@ -139,9 +146,8 @@ export default class ThemeViewState implements SpecialVisualizationState { * Triggered by navigating the map with arrows or by pressing 'space' or 'enter' */ public readonly visualFeedback: UIEventSource = new UIEventSource(false) - private readonly newPointDialog: FilteredLayer - constructor(layout: LayoutConfig) { + constructor(layout: LayoutConfig, mvtAvailableLayers: Set) { Utils.initDomPurify() this.layout = layout this.featureSwitches = new FeatureSwitchState(layout) @@ -224,6 +230,7 @@ export default class ThemeViewState implements SpecialVisualizationState { this.mapProperties, this.osmConnection.Backend(), (id) => self.layerState.filteredLayers.get(id).isDisplayed, + mvtAvailableLayers, this.fullNodeDatabase ) @@ -298,7 +305,6 @@ export default class ThemeViewState implements SpecialVisualizationState { fs.layer.layerDef.maxAgeOfCache ) }) - this.newPointDialog = this.layerState.filteredLayers.get("last_click") this.floors = this.featuresInView.features.stabilized(500).map((features) => { if (!features) { @@ -332,10 +338,7 @@ export default class ThemeViewState implements SpecialVisualizationState { return sorted }) - this.lastClickObject = new LastClickFeatureSource( - this.mapProperties.lastClickLocation, - this.layout - ) + this.lastClickObject = new LastClickFeatureSource(this.layout) this.osmObjectDownloader = new OsmObjectDownloader( this.osmConnection.Backend(), @@ -362,6 +365,7 @@ export default class ThemeViewState implements SpecialVisualizationState { ) this.favourites = new FavouritesFeatureSource(this) + this.featureSummary = this.setupSummaryLayer() this.initActors() this.drawSpecialLayers() this.initHotkeys() @@ -458,16 +462,6 @@ export default class ThemeViewState implements SpecialVisualizationState { this.userRelatedState.markLayoutAsVisited(this.layout) - this.selectedElement.addCallbackAndRunD((feature) => { - // As soon as we have a selected element, we clear the selected element - // This is to work around maplibre, which'll _first_ register the click on the map and only _then_ on the feature - // The only exception is if the last element is the 'add_new'-button, as we don't want it to disappear - if (feature.properties.id === "last_click") { - return - } - this.lastClickObject.features.setData([]) - }) - this.selectedElement.addCallback((selected) => { if (selected === undefined) { Zoomcontrol.resetzoom() @@ -495,13 +489,11 @@ export default class ThemeViewState implements SpecialVisualizationState { if (!toSelect) { return } - const layer = this.layout.getMatchingLayer(toSelect.properties) this.selectedElement.setData(undefined) this.selectedElement.setData(toSelect) }) return } - const layer = this.layout.getMatchingLayer(toSelect.properties) this.selectedElement.setData(undefined) this.selectedElement.setData(toSelect) } @@ -650,6 +642,35 @@ export default class ThemeViewState implements SpecialVisualizationState { ) } + private setupSummaryLayer(): SummaryTileSourceRewriter { + /** + * MaxZoom for the summary layer + */ + const normalLayers = this.layout.layers.filter( + (l) => + Constants.priviliged_layers.indexOf(l.id) < 0 && + !l.id.startsWith("note_import") + ) + const maxzoom = Math.min(...normalLayers.map((l) => l.minzoom)) + + const layers = this.layout.layers.filter( + (l) => + Constants.priviliged_layers.indexOf(l.id) < 0 && + l.source.geojsonSource === undefined && + l.doCount + ) + const url = new URL(Constants.VectorTileServer) + const summaryTileSource = new SummaryTileSource( + url.protocol + "//" + url.host + "/summary", + layers.map((l) => l.id), + this.mapProperties.zoom.map((z) => Math.max(Math.ceil(z), 0)), + this.mapProperties, + { + isActive: this.mapProperties.zoom.map((z) => z <= maxzoom), + } + ) + return new SummaryTileSourceRewriter(summaryTileSource, this.layerState.filteredLayers) + } /** * Add the special layers to the map */ @@ -677,6 +698,7 @@ export default class ThemeViewState implements SpecialVisualizationState { ), current_view: this.currentView, favourite: this.favourites, + summary: this.featureSummary, } this.closestFeatures.registerSource(specialLayers.favourite, "favourite") @@ -714,15 +736,15 @@ export default class ThemeViewState implements SpecialVisualizationState { rangeIsDisplayed?.syncWith(this.featureSwitches.featureSwitchIsTesting, true) } - // enumarate all 'normal' layers and match them with the appropriate 'special' layer - if applicable + // enumerate all 'normal' layers and match them with the appropriate 'special' layer - if applicable this.layerState.filteredLayers.forEach((flayer) => { const id = flayer.layerDef.id const features: FeatureSource = specialLayers[id] if (features === undefined) { return } - if (id === "favourite") { - console.log("Matching special layer", id, flayer) + if (id === "summary") { + return } this.featureProperties.trackFeatureSource(features) @@ -734,6 +756,13 @@ export default class ThemeViewState implements SpecialVisualizationState { selectedElement: this.selectedElement, }) }) + + new ShowDataLayer(this.map, { + features: specialLayers.summary, + layer: new LayerConfig(summaryLayer, "summaryLayer"), + // doShowLayer: this.mapProperties.zoom.map((z) => z < maxzoom), + selectedElement: this.selectedElement, + }) } /** @@ -742,9 +771,6 @@ export default class ThemeViewState implements SpecialVisualizationState { private initActors() { this.selectedElement.addCallback((selected) => { if (selected === undefined) { - console.trace("Unselected") - // We did _unselect_ an item - we always remove the lastclick-object - this.lastClickObject.features.setData([]) this.focusOnMap() } }) diff --git a/src/UI/BigComponents/NewPointLocationInput.svelte b/src/UI/BigComponents/NewPointLocationInput.svelte index 6f0a69f9a..95d1b72cb 100644 --- a/src/UI/BigComponents/NewPointLocationInput.svelte +++ b/src/UI/BigComponents/NewPointLocationInput.svelte @@ -87,6 +87,7 @@ if (snapToLayers?.length > 0) { const snapSources: FeatureSource[] = [] for (const layerId of snapToLayers ?? []) { + // We assume that the layer contains the data, as the OSM-API-Feature-source should have loaded them, even though the layer might not be displayed const layer: FeatureSourceForLayer = state.perLayer.get(layerId) snapSources.push(layer) if (layer.features === undefined) { diff --git a/src/UI/DownloadFlow/DownloadButton.svelte b/src/UI/DownloadFlow/DownloadButton.svelte index 229b74b9c..9ad837de9 100644 --- a/src/UI/DownloadFlow/DownloadButton.svelte +++ b/src/UI/DownloadFlow/DownloadButton.svelte @@ -3,10 +3,8 @@ import { ArrowDownTrayIcon } from "@babeard/svelte-heroicons/mini" import Tr from "../Base/Tr.svelte" import Translations from "../i18n/Translations" - import type { FeatureCollection } from "geojson" import Loading from "../Base/Loading.svelte" import { Translation } from "../i18n/Translation" - import DownloadHelper from "./DownloadHelper" import { Utils } from "../../Utils" import type { PriviligedLayerType } from "../../Models/Constants" import { UIEventSource } from "../../Logic/UIEventSource" @@ -16,14 +14,11 @@ export let extension: string export let mimetype: string export let construct: ( - geojsonCleaned: FeatureCollection, title: string, status?: UIEventSource - ) => (Blob | string) | Promise + ) => Promise export let mainText: Translation export let helperText: Translation - export let metaIsIncluded: boolean - let downloadHelper: DownloadHelper = new DownloadHelper(state) const t = Translations.t.general.download @@ -31,31 +26,21 @@ let isError = false let status: UIEventSource = new UIEventSource(undefined) - async function clicked() { isExporting = true + const gpsLayer = state.layerState.filteredLayers.get("gps_location") - state.lastClickObject.features.setData([]) state.userRelatedState.preferencesAsTags.data["__showTimeSensitiveIcons"] = "no" state.userRelatedState.preferencesAsTags.ping() const gpsIsDisplayed = gpsLayer.isDisplayed.data try { gpsLayer.isDisplayed.setData(false) - const geojson: FeatureCollection = downloadHelper.getCleanGeoJson(metaIsIncluded) const name = state.layout.id const title = `MapComplete_${name}_export_${new Date() .toISOString() .substr(0, 19)}.${extension}` - const promise = construct(geojson, title, status) - let data: Blob | string - if (typeof promise === "string") { - data = promise - } else if (typeof promise["then"] === "function") { - data = await (>promise) - } else { - data = promise - } + const data: Blob | string = await construct(title, status) if (!data) { return } diff --git a/src/UI/DownloadFlow/DownloadHelper.ts b/src/UI/DownloadFlow/DownloadHelper.ts index 80906a1b1..4b61a4cce 100644 --- a/src/UI/DownloadFlow/DownloadHelper.ts +++ b/src/UI/DownloadFlow/DownloadHelper.ts @@ -153,7 +153,7 @@ export default class DownloadHelper { return header + "\n" + elements.join("\n") + "\n" } - public getCleanGeoJsonPerLayer(includeMetaData: boolean): Map { + private getCleanGeoJsonPerLayer(includeMetaData: boolean): Map { const state = this._state const featuresPerLayer = new Map() const neededLayers = state.layout.layers.filter((l) => l.source !== null).map((l) => l.id) @@ -161,6 +161,7 @@ export default class DownloadHelper { for (const neededLayer of neededLayers) { const indexedFeatureSource = state.perLayer.get(neededLayer) + let features = indexedFeatureSource.GetFeaturesWithin(bbox) // The 'indexedFeatureSources' contains _all_ features, they are not filtered yet const filter = state.layerState.filteredLayers.get(neededLayer) diff --git a/src/UI/DownloadFlow/DownloadPanel.svelte b/src/UI/DownloadFlow/DownloadPanel.svelte index 773f4f3e7..2385f6e04 100644 --- a/src/UI/DownloadFlow/DownloadPanel.svelte +++ b/src/UI/DownloadFlow/DownloadPanel.svelte @@ -17,9 +17,16 @@ const downloadHelper = new DownloadHelper(state) let metaIsIncluded = false - const name = state.layout.id - function offerSvg(noSelfIntersectingLines: boolean): string { + let numberOfFeatures = state.featureSummary.totalNumberOfFeatures + + async function getGeojson() { + await state.indexedFeatures.downloadAll() + return downloadHelper.getCleanGeoJson(metaIsIncluded) + } + + async function offerSvg(noSelfIntersectingLines: boolean): Promise { + await state.indexedFeatures.downloadAll() const maindiv = document.getElementById("maindiv") const layers = state.layout.layers.filter((l) => l.source !== null) return downloadHelper.asSvg({ @@ -34,6 +41,8 @@ {#if $isLoading} +{:else if $numberOfFeatures > 100000} + {:else}

@@ -44,20 +53,18 @@ {state} extension="geojson" mimetype="application/vnd.geo+json" - construct={(geojson) => JSON.stringify(geojson)} + construct={async () => JSON.stringify(await getGeojson())} mainText={t.downloadGeojson} helperText={t.downloadGeoJsonHelper} - {metaIsIncluded} /> GeoOperations.toCSV(geojson)} + construct={async () => GeoOperations.toCSV(await getGeojson())} mainText={t.downloadCSV} helperText={t.downloadCSVHelper} - {metaIsIncluded} />

Configuration file
-
+
Below, you'll find the raw configuration file in `.json`-format. This is mostly for debugging purposes, but you can also edit the file directly if you want.
-
- -
- -
- The testobject (which is used to render the questions in the 'information panel' item - has the following tags: -
+
+
+ +
+
+
+ The testobject (which is used to render the questions in the 'information panel' + item has the following tags: +
- + +
+
diff --git a/src/UI/Studio/EditTheme.svelte b/src/UI/Studio/EditTheme.svelte index 16f9e0a36..bd71c0134 100644 --- a/src/UI/Studio/EditTheme.svelte +++ b/src/UI/Studio/EditTheme.svelte @@ -51,7 +51,7 @@
- {Object.keys(perRegion).join(";")} +
Basic properties
@@ -74,16 +74,15 @@
Configuration file
-
+
Below, you'll find the raw configuration file in `.json`-format. This is mostly for debugging purposes, but you can also edit the file directly if you want.
+
- -
diff --git a/src/UI/Test.svelte b/src/UI/Test.svelte index ead85dcd7..a20df370b 100644 --- a/src/UI/Test.svelte +++ b/src/UI/Test.svelte @@ -1,19 +1,4 @@ - +No tests diff --git a/src/UI/ThemeViewGUI.svelte b/src/UI/ThemeViewGUI.svelte index 739f4a9db..2dac52e78 100644 --- a/src/UI/ThemeViewGUI.svelte +++ b/src/UI/ThemeViewGUI.svelte @@ -100,14 +100,18 @@ }) let selectedLayer: Store = state.selectedElement.mapD((element) => { - if (element.properties.id.startsWith("current_view")) { - return currentViewLayer - } - if (element.properties.id === "location_track") { - return layout.layers.find((l) => l.id === "gps_track") - } - return state.layout.getMatchingLayer(element.properties) - }) + if (element.properties.id.startsWith("current_view")) { + return currentViewLayer + } + if(element.properties.id === "new_point_dialog"){ + return layout.layers.find(l => l.id === "last_click") + } + if(element.properties.id === "location_track"){ + return layout.layers.find(l => l.id === "gps_track") + } + return state.layout.getMatchingLayer(element.properties) + }, + ) let currentZoom = state.mapProperties.zoom let showCrosshair = state.userRelatedState.showCrosshair let visualFeedback = state.visualFeedback @@ -266,7 +270,7 @@
- {#if state.lastClickObject.hasPresets || state.lastClickObject.hasNoteLayer} + {#if state.layout.hasPresets() || state.layout.hasNoteLayer()}