LayerServer: fix some bugs in build_db-script, decode MVT-tilesource for more features

This commit is contained in:
Pieter Vander Vennet 2024-02-02 20:04:39 +01:00
parent ee3e000cd1
commit 5b318236bf
8 changed files with 327 additions and 103 deletions

View file

@ -7,6 +7,7 @@ import { Or } from "../../src/Logic/Tags/Or"
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation"
import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts"
import { OsmObject } from "../../src/Logic/Osm/OsmObject"
class LuaSnippets {
@ -20,6 +21,33 @@ class LuaSnippets {
"end",
].join("\n")
public static isPolygonFeature(): { blacklist: TagsFilter, whitelisted: TagsFilter } {
const dict = OsmObject.polygonFeatures
const or: TagsFilter[] = []
const blacklisted : TagsFilter[] = []
dict.forEach(({ values, blacklist }, k) => {
if(blacklist){
if(values === undefined){
blacklisted.push(new RegexTag(k, /.+/is))
return
}
values.forEach(v => {
blacklisted.push(new RegexTag(k, v))
})
return
}
if (values === undefined || values === null) {
or.push(new RegexTag(k, /.+/is))
return
}
values.forEach(v => {
or.push(new RegexTag(k, v))
})
})
console.log("Polygon features are:", or.map(t => t.asHumanString(false, false, {})))
return { blacklist: new Or(blacklisted), whitelisted: new Or(or) }
}
public static toLuaFilter(tag: TagsFilter, useParens: boolean = false): string {
if (tag instanceof Tag) {
return `object.tags["${tag.key}"] == "${tag.value}"`
@ -55,6 +83,10 @@ class LuaSnippets {
return `object.tags["${tag.key}"] ~= "${tag.value}"`
}
if (typeof tag.value === "string" && !tag.invert) {
return `object.tags["${tag.key}"] == "${tag.value}"`
}
const v = (<RegExp>tag.value).source.replace(/\\\//g, "/")
if ("" + tag.value === "/.+/is" && !tag.invert) {
@ -220,6 +252,7 @@ class GenerateBuildDbScript extends Script {
bodyPolygons.push(this.insertInto(tags, layerId, "polygons_").join("\n"))
})
const isPolygon = LuaSnippets.isPolygonFeature()
return [
"function process_polygon(object, geom)",
" local matches_filter",
@ -232,7 +265,9 @@ class GenerateBuildDbScript extends Script {
"",
"function osm2pgsql.process_way(object)",
this.earlyAbort(),
" if object.is_closed then",
" local object_is_line = not object.is_closed or "+LuaSnippets.toLuaFilter(isPolygon.blacklist),
` local object_is_area = object.is_closed and (object.tags["area"] == "yes" or (not object_is_line and ${LuaSnippets.toLuaFilter(isPolygon.whitelisted, true)}))`,
" if object_is_area then",
" process_polygon(object, object:as_polygon())",
" else",
" process_linestring(object, object:as_linestring())",

View file

@ -59,7 +59,7 @@ export default class LayoutSource extends FeatureSourceMerger {
zoom,
featureSwitches
)//*/
/*
const osmApiSource = LayoutSource.setupOsmApiSource(
osmLayers,
bounds,
@ -67,14 +67,14 @@ export default class LayoutSource extends FeatureSourceMerger {
backend,
featureSwitches,
fullNodeDatabaseSource
)
)*/
const geojsonSources: FeatureSource[] = geojsonlayers.map((l) =>
LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id))
)
super(osmApiSource, ...geojsonSources, ...fromCache, ...mvtSources)
super(...geojsonSources, ...fromCache, ...mvtSources)
const self = this
function setIsLoading() {
@ -83,7 +83,7 @@ export default class LayoutSource extends FeatureSourceMerger {
}
// overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading())
osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading())
// osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading())
}
private static setupMvtSource(layer: LayerConfig, mapProperties: { zoom: Store<number>; bounds: Store<BBox> }, isActive?: Store<boolean>): FeatureSource{

View file

@ -1,5 +1,5 @@
import { Store } from "../../UIEventSource"
import DynamicTileSource, { PolygonSourceMerger } from "./DynamicTileSource"
import DynamicTileSource from "./DynamicTileSource"
import { Utils } from "../../../Utils"
import { BBox } from "../../BBox"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
@ -7,6 +7,8 @@ import MvtSource from "../Sources/MvtSource"
import { Tiles } from "../../../Models/TileRange"
import Constants from "../../../Models/Constants"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
import { LineSourceMerger } from "./LineSourceMerger"
import { PolygonSourceMerger } from "./PolygonSourceMerger"
class PolygonMvtSource extends PolygonSourceMerger{
@ -39,6 +41,36 @@ class PolygonMvtSource extends PolygonSourceMerger{
}
class LineMvtSource extends LineSourceMerger{
constructor( layer: LayerConfig,
mapProperties: {
zoom: Store<number>
bounds: Store<BBox>
},
options?: {
isActive?: Store<boolean>
}) {
const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14))
super(
roundedZoom,
layer.minzoom,
(zxy) => {
const [z, x, y] = Tiles.tile_from_index(zxy)
const url = Utils.SubstituteKeys(Constants.VectorTileServer,
{
z, x, y, layer: layer.id,
type: "lines",
})
return new MvtSource(url, x, y, z)
},
mapProperties,
{
isActive: options?.isActive,
})
}
}
class PointMvtSource extends DynamicTileSource {
constructor(
@ -84,9 +116,9 @@ export default class DynamicMvtileSource extends FeatureSourceMerger {
isActive?: Store<boolean>
},
) {
const roundedZoom = mapProperties.zoom.mapD(z => Math.floor(z))
super(
new PointMvtSource(layer, mapProperties, options),
new LineMvtSource(layer, mapProperties, options),
new PolygonMvtSource(layer, mapProperties, options)
)

View file

@ -1,11 +1,8 @@
import { Store, Stores } from "../../UIEventSource"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
import { FeatureSource, FeatureSourceForTile } from "../FeatureSource"
import { FeatureSource } from "../FeatureSource"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
import { Feature } from "geojson"
import { Utils } from "../../../Utils"
import { GeoOperations } from "../../GeoOperations"
/***
@ -84,68 +81,3 @@ export default class DynamicTileSource<Src extends FeatureSource = FeatureSource
}
/**
* The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together.
* This is used to reconstruct polygons of vector tiles
*/
export class PolygonSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
constructor(
zoomlevel: Store<number>,
minzoom: number,
constructSource: (tileIndex: number) => FeatureSourceForTile,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
},
) {
super(zoomlevel, minzoom, constructSource, mapProperties, options)
}
protected addDataFromSources(sources: FeatureSourceForTile[]) {
sources = Utils.NoNull(sources)
const all: Map<string, Feature> = new Map()
const zooms: Map<string, number> = new Map()
for (const source of sources) {
let z = source.z
for (const f of source.features.data) {
const id = f.properties.id
if(id.endsWith("146616907")){
console.log("Horeca totaal")
}
if (!all.has(id)) {
// No other parts of this polygon have been seen before, simply add it
all.set(id, f)
zooms.set(id, z)
continue
}
// A part of this object has been seen before, eventually from a different zoom level
const oldV = all.get(id)
const oldZ = zooms.get(id)
if (oldZ > z) {
// The store contains more detailed information, so we ignore this part which has a lower accuraccy
continue
}
if (oldZ < z) {
// The old value has worse accuracy then what we receive now, we throw it away
all.set(id, f)
zooms.set(id, z)
continue
}
const merged = GeoOperations.union(f, oldV)
merged.properties = oldV.properties
all.set(id, merged)
zooms.set(id, z)
}
}
const newList = Array.from(all.values())
this.features.setData(newList)
this._featuresById.setData(all)
}
}

View file

@ -0,0 +1,80 @@
import { FeatureSourceForTile } from "../FeatureSource"
import { Store } from "../../UIEventSource"
import { BBox } from "../../BBox"
import { Utils } from "../../../Utils"
import { Feature, LineString, MultiLineString, Position } from "geojson"
import { Tiles } from "../../../Models/TileRange"
import { GeoOperations } from "../../GeoOperations"
import DynamicTileSource from "./DynamicTileSource"
/**
* The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together.
* This is used to reconstruct polygons of vector tiles
*/
export class LineSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
private readonly _zoomlevel: Store<number>
constructor(
zoomlevel: Store<number>,
minzoom: number,
constructSource: (tileIndex: number) => FeatureSourceForTile,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
},
) {
super(zoomlevel, minzoom, constructSource, mapProperties, options)
this._zoomlevel = zoomlevel
}
protected addDataFromSources(sources: FeatureSourceForTile[]) {
sources = Utils.NoNull(sources)
const all: Map<string, Feature<MultiLineString>> = new Map()
const currentZoom = this._zoomlevel?.data ?? 0
for (const source of sources) {
if(source.z != currentZoom){
continue
}
const bboxCoors = Tiles.tile_bounds_lon_lat(source.z, source.x, source.y)
const bboxGeo = new BBox(bboxCoors).asGeoJson({})
for (const f of source.features.data) {
const id = f.properties.id
const coordinates : Position[][] = []
if(f.geometry.type === "LineString"){
coordinates.push(f.geometry.coordinates)
}else if(f.geometry.type === "MultiLineString"){
coordinates.push(...f.geometry.coordinates)
}else {
console.error("Invalid geometry type:", f.geometry.type)
continue
}
const oldV = all.get(id)
if(!oldV){
all.set(id, {
type: "Feature",
properties: f.properties,
geometry:{
type:"MultiLineString",
coordinates
}
})
continue
}
oldV.geometry.coordinates.push(...coordinates)
}
}
const keys = Array.from(all.keys())
for (const key of keys) {
all.set(key, <any> GeoOperations.attemptLinearize(<Feature<MultiLineString>>all.get(key)))
}
const newList = Array.from(all.values())
this.features.setData(newList)
this._featuresById.setData(all)
}
}

View file

@ -0,0 +1,73 @@
import { FeatureSourceForTile } from "../FeatureSource"
import { Store } from "../../UIEventSource"
import { BBox } from "../../BBox"
import { Utils } from "../../../Utils"
import { Feature } from "geojson"
import { GeoOperations } from "../../GeoOperations"
import DynamicTileSource from "./DynamicTileSource"
/**
* The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together.
* This is used to reconstruct polygons of vector tiles
*/
export class PolygonSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
constructor(
zoomlevel: Store<number>,
minzoom: number,
constructSource: (tileIndex: number) => FeatureSourceForTile,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
},
) {
super(zoomlevel, minzoom, constructSource, mapProperties, options)
}
protected addDataFromSources(sources: FeatureSourceForTile[]) {
sources = Utils.NoNull(sources)
const all: Map<string, Feature> = new Map()
const zooms: Map<string, number> = new Map()
for (const source of sources) {
let z = source.z
for (const f of source.features.data) {
const id = f.properties.id
if (id.endsWith("146616907")) {
console.log("Horeca totaal")
}
if (!all.has(id)) {
// No other parts of this polygon have been seen before, simply add it
all.set(id, f)
zooms.set(id, z)
continue
}
// A part of this object has been seen before, eventually from a different zoom level
const oldV = all.get(id)
const oldZ = zooms.get(id)
if (oldZ > z) {
// The store contains more detailed information, so we ignore this part which has a lower accuraccy
continue
}
if (oldZ < z) {
// The old value has worse accuracy then what we receive now, we throw it away
all.set(id, f)
zooms.set(id, z)
continue
}
const merged = GeoOperations.union(f, oldV)
merged.properties = oldV.properties
all.set(id, merged)
zooms.set(id, z)
}
}
const newList = Array.from(all.values())
this.features.setData(newList)
this._featuresById.setData(all)
}
}

View file

@ -1,6 +1,6 @@
import { BBox } from "./BBox"
import * as turf from "@turf/turf"
import { AllGeoJSON, booleanWithin, Coord } from "@turf/turf"
import { AllGeoJSON, booleanWithin, Coord, Lines } from "@turf/turf"
import {
Feature,
FeatureCollection,
@ -156,7 +156,7 @@ export class GeoOperations {
const intersection = GeoOperations.calculateIntersection(
feature,
otherFeature,
featureBBox
featureBBox,
)
if (intersection === null) {
continue
@ -195,7 +195,7 @@ export class GeoOperations {
console.error(
"Could not correctly calculate the overlap of ",
feature,
": unsupported type"
": unsupported type",
)
return result
}
@ -224,7 +224,7 @@ export class GeoOperations {
*/
public static inside(
pointCoordinate: [number, number] | Feature<Point>,
feature: Feature
feature: Feature,
): boolean {
// ray-casting algorithm based on
// http://www.ecse.rpi.edu/Homepages/wrf/Research/Short_Notes/pnpoly.html
@ -302,7 +302,7 @@ export class GeoOperations {
*/
public static nearestPoint(
way: Feature<LineString>,
point: [number, number]
point: [number, number],
): Feature<
Point,
{
@ -324,11 +324,11 @@ export class GeoOperations {
public static forceLineString(way: Feature<LineString | Polygon>): Feature<LineString>
public static forceLineString(
way: Feature<MultiLineString | MultiPolygon>
way: Feature<MultiLineString | MultiPolygon>,
): Feature<MultiLineString>
public static forceLineString(
way: Feature<LineString | MultiLineString | Polygon | MultiPolygon>
way: Feature<LineString | MultiLineString | Polygon | MultiPolygon>,
): Feature<LineString | MultiLineString> {
if (way.geometry.type === "Polygon") {
way = { ...way }
@ -448,7 +448,7 @@ export class GeoOperations {
*/
public static LineIntersections(
feature: Feature<LineString | MultiLineString | Polygon | MultiPolygon>,
otherFeature: Feature<LineString | MultiLineString | Polygon | MultiPolygon>
otherFeature: Feature<LineString | MultiLineString | Polygon | MultiPolygon>,
): [number, number][] {
return turf
.lineIntersect(feature, otherFeature)
@ -485,7 +485,7 @@ export class GeoOperations {
locations:
| Feature<LineString>
| Feature<Point, { date?: string; altitude?: number | string }>[],
title?: string
title?: string,
) {
title = title?.trim()
if (title === undefined || title === "") {
@ -506,7 +506,7 @@ export class GeoOperations {
type: "Point",
coordinates: p,
},
}
},
)
}
for (const l of locationsWithMeta) {
@ -521,7 +521,7 @@ export class GeoOperations {
trackPoints.push(trkpt)
}
const header =
'<gpx version="1.1" creator="mapcomplete.org" xmlns="http://www.topografix.com/GPX/1/1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd">'
"<gpx version=\"1.1\" creator=\"mapcomplete.org\" xmlns=\"http://www.topografix.com/GPX/1/1\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd\">"
return (
header +
"\n<name>" +
@ -539,7 +539,7 @@ export class GeoOperations {
*/
public static toGpxPoints(
locations: Feature<Point, { date?: string; altitude?: number | string }>[],
title?: string
title?: string,
) {
title = title?.trim()
if (title === undefined || title === "") {
@ -560,7 +560,7 @@ export class GeoOperations {
trackPoints.push(trkpt)
}
const header =
'<gpx version="1.1" creator="mapcomplete.org" xmlns="http://www.topografix.com/GPX/1/1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd">'
"<gpx version=\"1.1\" creator=\"mapcomplete.org\" xmlns=\"http://www.topografix.com/GPX/1/1\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd\">"
return (
header +
"\n<name>" +
@ -648,7 +648,7 @@ export class GeoOperations {
},
},
distanceMeter,
{ units: "meters" }
{ units: "meters" },
).geometry.coordinates
}
@ -683,7 +683,7 @@ export class GeoOperations {
*/
static completelyWithin(
feature: Feature<Geometry, any>,
possiblyEnclosingFeature: Feature<Polygon | MultiPolygon, any>
possiblyEnclosingFeature: Feature<Polygon | MultiPolygon, any>,
): boolean {
return booleanWithin(feature, possiblyEnclosingFeature)
}
@ -714,6 +714,23 @@ export class GeoOperations {
}
return kept
}
if (toSplit.geometry.type === "MultiLineString") {
const lines: Feature<LineString>[][] = toSplit.geometry.coordinates.map(coordinates =>
turf.lineSplit(<LineString> {type: "LineString", coordinates}, boundary).features )
const splitted: Feature<LineString>[] = [].concat(...lines)
const kept: Feature<LineString>[] = []
for (const f of splitted) {
console.log("Checking", f)
if (!GeoOperations.inside(GeoOperations.centerpointCoordinates(f), boundary)) {
continue
}
f.properties = { ...toSplit.properties }
kept.push(f)
}
console.log(">>>", {lines, splitted, kept})
return kept
}
if (toSplit.geometry.type === "Polygon" || toSplit.geometry.type == "MultiPolygon") {
const splitup = turf.intersect(<Feature<Polygon>>toSplit, boundary)
splitup.properties = { ...toSplit.properties }
@ -739,7 +756,7 @@ export class GeoOperations {
*/
public static featureToCoordinateWithRenderingType(
feature: Feature,
location: "point" | "centroid" | "start" | "end" | "projected_centerpoint" | string
location: "point" | "centroid" | "start" | "end" | "projected_centerpoint" | string,
): [number, number] | undefined {
switch (location) {
case "point":
@ -760,7 +777,7 @@ export class GeoOperations {
const centerpoint = GeoOperations.centerpointCoordinates(feature)
const projected = GeoOperations.nearestPoint(
<Feature<LineString>>feature,
centerpoint
centerpoint,
)
return <[number, number]>projected.geometry.coordinates
}
@ -937,7 +954,7 @@ export class GeoOperations {
* GeoOperations.bearingToHuman(46) // => "NE"
*/
public static bearingToHuman(
bearing: number
bearing: number,
): "N" | "NE" | "E" | "SE" | "S" | "SW" | "W" | "NW" {
while (bearing < 0) {
bearing += 360
@ -956,7 +973,7 @@ export class GeoOperations {
* GeoOperations.bearingToHuman(46) // => "NE"
*/
public static bearingToHumanRelative(
bearing: number
bearing: number,
):
| "straight"
| "slight_right"
@ -975,18 +992,73 @@ export class GeoOperations {
return GeoOperations.directionsRelative[segment]
}
/**
* const coors = [[[3.217198532946432,51.218067],[3.216807134449482,51.21849812105347],[3.2164304037883706,51.2189272]],[[3.2176208,51.21760169669458],[3.217198560167068,51.218067]]]
* const f = <any> {geometry: {coordinates: coors}}
* const merged = GeoOperations.attemptLinearize(f)
* merged.geometry.coordinates // => [[3.2176208,51.21760169669458],[3.217198532946432,51.218067], [3.216807134449482,51.21849812105347],[3.2164304037883706,51.2189272]]
*/
static attemptLinearize(multiLineStringFeature: Feature<MultiLineString>): Feature<LineString | MultiLineString> {
const coors = multiLineStringFeature.geometry.coordinates
if(coors.length === 0) {
console.error(multiLineStringFeature.geometry)
throw "Error: got degenerate multilinestring"
}
outer: for (let i = coors.length - 1; i >= 0; i--) {
// We try to match the first element of 'i' with another, earlier list `j`
// If a match is found with `j`, j is extended and `i` is scrapped
const iFirst = coors[i][0]
for (let j = 0; j < coors.length; j++) {
if (i == j) {
continue
}
const jLast = coors[j].at(-1)
if (!(Math.abs(iFirst[0] - jLast[0]) < 0.000001 && Math.abs(iFirst[1] - jLast[1]) < 0.0000001)) {
continue
}
coors[j].splice(coors.length - 1, 1)
coors[j].push(...coors[i])
coors.splice(i, 1)
continue outer
}
}
if(coors.length === 0) {
throw "No more coordinates found"
}
if (coors.length === 1) {
return {
type: "Feature",
properties: multiLineStringFeature.properties,
geometry: {
type: "LineString",
coordinates: coors[0],
},
}
}
return {
type: "Feature",
properties: multiLineStringFeature.properties,
geometry: {
type: "MultiLineString",
coordinates: coors,
},
}
}
/**
* Helper function which does the heavy lifting for 'inside'
*/
private static pointInPolygonCoordinates(
x: number,
y: number,
coordinates: [number, number][][]
coordinates: [number, number][][],
): boolean {
const inside = GeoOperations.pointWithinRing(
x,
y,
/*This is the outer ring of the polygon */ coordinates[0]
/*This is the outer ring of the polygon */ coordinates[0],
)
if (!inside) {
return false
@ -995,7 +1067,7 @@ export class GeoOperations {
const inHole = GeoOperations.pointWithinRing(
x,
y,
coordinates[i] /* These are inner rings, aka holes*/
coordinates[i], /* These are inner rings, aka holes*/
)
if (inHole) {
return false
@ -1033,7 +1105,7 @@ export class GeoOperations {
feature,
otherFeature,
featureBBox: BBox,
otherFeatureBBox?: BBox
otherFeatureBBox?: BBox,
): number {
if (feature.geometry.type === "LineString") {
otherFeatureBBox = otherFeatureBBox ?? BBox.get(otherFeature)
@ -1082,7 +1154,7 @@ export class GeoOperations {
let intersection = turf.lineSlice(
turf.point(intersectionPointsArray[0]),
turf.point(intersectionPointsArray[1]),
feature
feature,
)
if (intersection == null) {
@ -1103,7 +1175,7 @@ export class GeoOperations {
otherFeature,
feature,
otherFeatureBBox,
featureBBox
featureBBox,
)
}
@ -1123,7 +1195,7 @@ export class GeoOperations {
console.log("Applying fallback intersection...")
const intersection = turf.intersect(
turf.truncate(feature),
turf.truncate(otherFeature)
turf.truncate(otherFeature),
)
if (intersection == null) {
return null

View file

@ -7,7 +7,7 @@ import { Feature, LineString, Polygon } from "geojson"
export abstract class OsmObject {
private static defaultBackend = "https://api.openstreetmap.org/"
protected static backendURL = OsmObject.defaultBackend
private static polygonFeatures = OsmObject.constructPolygonFeatures()
public static polygonFeatures = OsmObject.constructPolygonFeatures()
type: "node" | "way" | "relation"
id: number
/**