From abc4a08b3ae093cdda972ed5438f0b2c06116c8c Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Thu, 24 Mar 2022 19:59:46 +0100 Subject: [PATCH] Fix regression and add tests, add overpass link in layer documentation --- Logic/Osm/Overpass.ts | 42 +++- Logic/Tags/RegexTag.ts | 3 + Logic/Tags/TagUtils.ts | 212 +++++++++--------- Models/ThemeConfig/Conversion/PrepareTheme.ts | 35 +++ Models/ThemeConfig/LayerConfig.ts | 39 ++-- 5 files changed, 201 insertions(+), 130 deletions(-) diff --git a/Logic/Osm/Overpass.ts b/Logic/Osm/Overpass.ts index cb34bb7b73..5463770acd 100644 --- a/Logic/Osm/Overpass.ts +++ b/Logic/Osm/Overpass.ts @@ -4,6 +4,8 @@ import {Utils} from "../../Utils"; import {UIEventSource} from "../UIEventSource"; import {BBox} from "../BBox"; import * as osmtogeojson from "osmtogeojson"; +// @ts-ignore +import {Tag} from "../Tags/Tag"; // used in doctest /** * Interfaces overpass to get all the latest data @@ -16,21 +18,19 @@ export class Overpass { private _includeMeta: boolean; private _relationTracker: RelationsTracker; - constructor(filter: TagsFilter, extraScripts: string[], interpreterUrl: string, - timeout: UIEventSource, - relationTracker: RelationsTracker, + timeout?: UIEventSource, + relationTracker?: RelationsTracker, includeMeta = true) { - this._timeout = timeout; + this._timeout = timeout ?? new UIEventSource(90); this._interpreterUrl = interpreterUrl; const optimized = filter.optimize() if(optimized === true || optimized === false){ throw "Invalid filter: optimizes to true of false" } this._filter = optimized - console.log("Overpass filter is",this._filter) this._extraScripts = extraScripts; this._includeMeta = includeMeta; this._relationTracker = relationTracker @@ -51,23 +51,45 @@ export class Overpass { console.warn("No features for", json) } - self._relationTracker.RegisterRelations(json) + self._relationTracker?.RegisterRelations(json) const geojson = osmtogeojson.default(json); const osmTime = new Date(json.osm3s.timestamp_osm_base); return [geojson, osmTime]; } - buildQuery(bbox: string): string { + /** + * new Overpass(new Tag("key","value"), [], "").buildScript("{{bbox}}") // => `[out:json][timeout:90]{{bbox}};(nwr["key"="value"];);out body;out meta;>;out skel qt;` + */ + public buildScript(bbox: string, postCall: string = "", pretty = false): string { const filters = this._filter.asOverpass() let filter = "" for (const filterOr of filters) { - filter += 'nwr' + filterOr + ';' + if(pretty){ + filter += " " + } + filter += 'nwr' + filterOr + postCall + ';' + if(pretty){ + filter+="\n" + } } for (const extraScript of this._extraScripts) { filter += '(' + extraScript + ');'; } - const query = - `[out:json][timeout:${this._timeout.data}]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;` + return`[out:json][timeout:${this._timeout.data}]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;` + } + + public buildQuery(bbox: string): string { + const query = this.buildScript(bbox) return `${this._interpreterUrl}?data=${encodeURIComponent(query)}` } + + /** + * Little helper method to quickly open overpass-turbo in the browser + */ + public static AsOverpassTurboLink(tags: TagsFilter){ + const overpass = new Overpass(tags, [], "", undefined, undefined, false) + const script = overpass.buildScript("","({{bbox}})", true) + const url = "http://overpass-turbo.eu/?Q=" + return url + encodeURIComponent(script) + } } diff --git a/Logic/Tags/RegexTag.ts b/Logic/Tags/RegexTag.ts index 0ca4d13755..b3a48b6a16 100644 --- a/Logic/Tags/RegexTag.ts +++ b/Logic/Tags/RegexTag.ts @@ -47,6 +47,9 @@ export class RegexTag extends TagsFilter { * * // A wildcard regextag should only give the key * new RegexTag("a", /^..*$/).asOverpass() // => [ `["a"]` ] + * + * // A regextag with a regex key should give correct output + * new RegexTag(/a.*x/, /^..*$/).asOverpass() // => [ `[~"a.*x"~\"^..*$\"]` ] */ asOverpass(): string[] { const inv =this.invert ? "!" : "" diff --git a/Logic/Tags/TagUtils.ts b/Logic/Tags/TagUtils.ts index 9710ee389e..cd4a90f857 100644 --- a/Logic/Tags/TagUtils.ts +++ b/Logic/Tags/TagUtils.ts @@ -181,6 +181,7 @@ export class TagUtils { * TagUtils.Tag("survey:date:={_date:now}") // => new SubstitutingTag("survey:date", "{_date:now}") * TagUtils.Tag("xyz!~\\[\\]") // => new RegexTag("xyz", /^\[\]$/, true) * TagUtils.Tag("tags~(^|.*;)amenity=public_bookcase($|;.*)") // => new RegexTag("tags", /(^|.*;)amenity=public_bookcase($|;.*)/) + * TagUtils.Tag("service:bicycle:.*~~*") // => new RegexTag(/^service:bicycle:.*$/, /^..*$/) */ public static Tag(json: AndOrTagConfigJson | string, context: string = ""): TagsFilter { try { @@ -219,126 +220,125 @@ export class TagUtils { if (json === undefined) { throw `Error while parsing a tag: 'json' is undefined in ${context}. Make sure all the tags are defined and at least one tag is present in a complex expression` } - if (typeof (json) == "string") { - const tag = json as string; + if (typeof (json) != "string") { + if (json.and !== undefined && json.or !== undefined) { + throw `Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined` + } + if (json.and !== undefined) { + return new And(json.and.map(t => TagUtils.Tag(t, context))); + } + if (json.or !== undefined) { + return new Or(json.or.map(t => TagUtils.Tag(t, context))); + } + throw "At " + context + ": unrecognized tag" + } + + + const tag = json as string; + for (const [operator, comparator] of TagUtils.comparators) { + if (tag.indexOf(operator) >= 0) { + const split = Utils.SplitFirst(tag, operator); - for (const [operator, comparator] of TagUtils.comparators) { - if (tag.indexOf(operator) >= 0) { - const split = Utils.SplitFirst(tag, operator); + let val = Number(split[1].trim()) + if (isNaN(val)) { + val = new Date(split[1].trim()).getTime() + } - let val = Number(split[1].trim()) - if (isNaN(val)) { - val = new Date(split[1].trim()).getTime() + const f = (value: string | undefined) => { + if (value === undefined) { + return false; } - - const f = (value: string | undefined) => { - if (value === undefined) { - return false; - } - let b = Number(value?.trim()) + let b = Number(value?.trim()) + if (isNaN(b)) { + b = Utils.ParseDate(value).getTime() if (isNaN(b)) { - b = Utils.ParseDate(value).getTime() - if (isNaN(b)) { - return false - } + return false } - return comparator(b, val) } - return new ComparingTag(split[0], f, operator + val) + return comparator(b, val) } + return new ComparingTag(split[0], f, operator + val) } - - if (tag.indexOf("!~") >= 0) { - const split = Utils.SplitFirst(tag, "!~"); - if (split[1] === "*") { - throw `Don't use 'key!~*' - use 'key=' instead (empty string as value (in the tag ${tag} while parsing ${context})` - } - return new RegexTag( - split[0], - split[1], - true - ); - } - if (tag.indexOf("~~") >= 0) { - const split = Utils.SplitFirst(tag, "~~"); - if (split[1] === "*") { - split[1] = "..*" - } - return new RegexTag( - split[0], - split[1] - ); - } - if (tag.indexOf("!:=") >= 0) { - const split = Utils.SplitFirst(tag, "!:="); - return new SubstitutingTag(split[0], split[1], true); - } - if (tag.indexOf(":=") >= 0) { - const split = Utils.SplitFirst(tag, ":="); - return new SubstitutingTag(split[0], split[1]); - } - - if (tag.indexOf("!=") >= 0) { - const split = Utils.SplitFirst(tag, "!="); - if (split[1] === "*") { - throw "At "+context+": invalid tag "+tag+". To indicate a missing tag, use '"+split[0]+"!=' instead" - } - if(split[1] === "") { - split[1] = "..*" - } - return new RegexTag( - split[0], - new RegExp("^" + split[1] + "$"), - true - ); - } - if (tag.indexOf("!~") >= 0) { - const split = Utils.SplitFirst(tag, "!~"); - if (split[1] === "*") { - split[1] = "..*" - } - return new RegexTag( - split[0], - split[1], - true - ); - } - if (tag.indexOf("~") >= 0) { - const split = Utils.SplitFirst(tag, "~"); - if (split[1] === "") { - throw "Detected a regextag with an empty regex; this is not allowed. Use '" + split[0] + "='instead (at " + context + ")" - } - if (split[1] === "*") { - split[1] = "..*" - } - return new RegexTag( - split[0], - split[1] - ); - } - if (tag.indexOf("=") >= 0) { - - - const split = Utils.SplitFirst(tag, "="); - if (split[1] == "*") { - throw `Error while parsing tag '${tag}' in ${context}: detected a wildcard on a normal value. Use a regex pattern instead` - } - return new Tag(split[0], split[1]) - } - throw `Error while parsing tag '${tag}' in ${context}: no key part and value part were found` - } - if (json.and !== undefined && json.or !== undefined) { - throw `Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined` + if (tag.indexOf("!~") >= 0) { + const split = Utils.SplitFirst(tag, "!~"); + if (split[1] === "*") { + throw `Don't use 'key!~*' - use 'key=' instead (empty string as value (in the tag ${tag} while parsing ${context})` + } + return new RegexTag( + split[0], + split[1], + true + ); + } + if (tag.indexOf("~~") >= 0) { + const split = Utils.SplitFirst(tag, "~~"); + if (split[1] === "*") { + split[1] = "..*" + } + return new RegexTag( + new RegExp("^"+split[0]+"$"), + new RegExp("^"+ split[1]+"$") + ); + } + if (tag.indexOf("!:=") >= 0) { + const split = Utils.SplitFirst(tag, "!:="); + return new SubstitutingTag(split[0], split[1], true); + } + if (tag.indexOf(":=") >= 0) { + const split = Utils.SplitFirst(tag, ":="); + return new SubstitutingTag(split[0], split[1]); } - if (json.and !== undefined) { - return new And(json.and.map(t => TagUtils.Tag(t, context))); + if (tag.indexOf("!=") >= 0) { + const split = Utils.SplitFirst(tag, "!="); + if (split[1] === "*") { + throw "At " + context + ": invalid tag " + tag + ". To indicate a missing tag, use '" + split[0] + "!=' instead" + } + if (split[1] === "") { + split[1] = "..*" + } + return new RegexTag( + split[0], + new RegExp("^" + split[1] + "$"), + true + ); } - if (json.or !== undefined) { - return new Or(json.or.map(t => TagUtils.Tag(t, context))); + if (tag.indexOf("!~") >= 0) { + const split = Utils.SplitFirst(tag, "!~"); + if (split[1] === "*") { + split[1] = "..*" + } + return new RegexTag( + split[0], + split[1], + true + ); } + if (tag.indexOf("~") >= 0) { + const split = Utils.SplitFirst(tag, "~"); + if (split[1] === "") { + throw "Detected a regextag with an empty regex; this is not allowed. Use '" + split[0] + "='instead (at " + context + ")" + } + if (split[1] === "*") { + split[1] = "..*" + } + return new RegexTag( + split[0], + split[1] + ); + } + if (tag.indexOf("=") >= 0) { + + + const split = Utils.SplitFirst(tag, "="); + if (split[1] == "*") { + throw `Error while parsing tag '${tag}' in ${context}: detected a wildcard on a normal value. Use a regex pattern instead` + } + return new Tag(split[0], split[1]) + } + throw `Error while parsing tag '${tag}' in ${context}: no key part and value part were found` } private static GetCount(key: string, value?: string) { diff --git a/Models/ThemeConfig/Conversion/PrepareTheme.ts b/Models/ThemeConfig/Conversion/PrepareTheme.ts index 7d2025db0d..658b622af9 100644 --- a/Models/ThemeConfig/Conversion/PrepareTheme.ts +++ b/Models/ThemeConfig/Conversion/PrepareTheme.ts @@ -417,11 +417,46 @@ class PreparePersonalTheme extends DesugaringStep { } +class WarnForUnsubstitutedLayersInTheme extends DesugaringStep{ + + constructor() { + super("Generates a warning if a theme uses an unsubstituted layer", ["layers"],"WarnForUnsubstitutedLayersInTheme"); + } + + convert(json: LayoutConfigJson, context: string): { result: LayoutConfigJson; errors?: string[]; warnings?: string[]; information?: string[] } { + if(json.hideFromOverview === true){ + return {result: json} + } + const warnings = [] + for (const layer of json.layers) { + if(typeof layer === "string"){ + continue + } + if(layer["builtin"] !== undefined){ + continue + } + if(layer["source"]["geojson"] !== undefined){ + // We turn a blind eye for import layers + continue + } + + const wrn = "The theme "+json.id+" has an inline layer: "+layer["id"]+". This is discouraged." + warnings.push(wrn) + } + return { + result: json, + warnings + }; + } + +} + export class PrepareTheme extends Fuse { constructor(state: DesugaringContext) { super( "Fully prepares and expands a theme", new PreparePersonalTheme(state), + // new WarnForUnsubstitutedLayersInTheme(), new OnEveryConcat("layers", new SubstituteLayer(state)), new SetDefault("socialImage", "assets/SocialImage.png", true), // We expand all tagrenderings first... diff --git a/Models/ThemeConfig/LayerConfig.ts b/Models/ThemeConfig/LayerConfig.ts index 82835571e1..405cf59be2 100644 --- a/Models/ThemeConfig/LayerConfig.ts +++ b/Models/ThemeConfig/LayerConfig.ts @@ -24,6 +24,9 @@ import {Utils} from "../../Utils"; import {TagsFilter} from "../../Logic/Tags/TagsFilter"; import Table from "../../UI/Base/Table"; import FilterConfigJson from "./Json/FilterConfigJson"; +import {And} from "../../Logic/Tags/And"; +import {Overpass} from "../../Logic/Osm/Overpass"; +import Constants from "../Constants"; export default class LayerConfig extends WithContextLoader { @@ -60,9 +63,9 @@ export default class LayerConfig extends WithContextLoader { public readonly filters: FilterConfig[]; public readonly filterIsSameAs: string; public readonly forceLoad: boolean; - - public readonly syncSelection: "no" | "local" | "theme-only" | "global" - + + public readonly syncSelection: "no" | "local" | "theme-only" | "global" + constructor( json: LayerConfigJson, context?: string, @@ -109,8 +112,8 @@ export default class LayerConfig extends WithContextLoader { this.source = new SourceConfig( { osmTags: osmTags, - geojsonSource: json.source["geoJson"], - geojsonSourceLevel: json.source["geoJsonZoomLevel"], + geojsonSource: json.source["geoJson"], + geojsonSourceLevel: json.source["geoJsonZoomLevel"], overpassScript: json.source["overpassScript"], isOsmCache: json.source["isOsmCache"], mercatorCrs: json.source["mercatorCrs"], @@ -236,10 +239,9 @@ export default class LayerConfig extends WithContextLoader { const hasCenterRendering = this.mapRendering.some(r => r.location.has("centroid") || r.location.has("start") || r.location.has("end")) if (this.lineRendering.length === 0 && this.mapRendering.length === 0) { - console.log(json.mapRendering) throw("The layer " + this.id + " does not have any maprenderings defined and will thus not show up on the map at all. If this is intentional, set maprenderings to 'null' instead of '[]'") } else if (!hasCenterRendering && this.lineRendering.length === 0 && !this.source.geojsonSource?.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")) { - throw "The layer " + this.id + " might not render ways. This might result in dropped information (at "+context+")" + throw "The layer " + this.id + " might not render ways. This might result in dropped information (at " + context + ")" } } @@ -251,10 +253,10 @@ export default class LayerConfig extends WithContextLoader { this.tagRenderings = (Utils.NoNull(json.tagRenderings) ?? []).map((tr, i) => new TagRenderingConfig(tr, this.id + ".tagRenderings[" + i + "]")) - if(json.filter !== undefined && json.filter !== null && json.filter["sameAs"] !== undefined){ + if (json.filter !== undefined && json.filter !== null && json.filter["sameAs"] !== undefined) { this.filterIsSameAs = json.filter["sameAs"] this.filters = [] - }else{ + } else { this.filters = (json.filter ?? []).map((option, i) => { return new FilterConfig(option, `${context}.filter-[${i}]`) }); @@ -316,8 +318,8 @@ export default class LayerConfig extends WithContextLoader { } return mapRendering.GetBaseIcon(this.GetBaseTags()) } - - public GetBaseTags(): any{ + + public GetBaseTags(): any { return TagUtils.changeAsProperties(this.source.osmTags.asChange({id: "node/-1"})) } @@ -367,7 +369,7 @@ export default class LayerConfig extends WithContextLoader { extraProps.push(new Combine(["This layer will automatically load ", new Link(dep.neededLayer, "./" + dep.neededLayer + ".md"), " into the layout as it depends on it: ", dep.reason, "(" + dep.context + ")"])) } - for (const revDep of Utils.Dedup( layerIsNeededBy?.get(this.id) ?? [])) { + for (const revDep of Utils.Dedup(layerIsNeededBy?.get(this.id) ?? [])) { extraProps.push(new Combine(["This layer is needed as dependency for layer", new Link(revDep, "#" + revDep)])) } @@ -402,7 +404,7 @@ export default class LayerConfig extends WithContextLoader { ]).SetClass("flex-col flex") } - const icon = this.mapRendering + const icon = this.mapRendering .filter(mr => mr.location.has("point")) .map(mr => mr.icon?.render?.txt) .find(i => i !== undefined) @@ -412,6 +414,15 @@ export default class LayerConfig extends WithContextLoader { iconImg = ` ` } + let overpassLink: BaseUIElement = undefined; + if (Constants.priviliged_layers.indexOf(this.id) < 0) { + try { + overpassLink = new Link("Execute on overpass", Overpass.AsOverpassTurboLink( new And(neededTags).optimize())) + } catch (e) { + console.error("Could not generate overpasslink for " + this.id) + } + } + return new Combine([ new Combine([ new Title(this.id, 1), @@ -427,7 +438,7 @@ export default class LayerConfig extends WithContextLoader { new Title("Basic tags for this layer", 2), "Elements must have the all of following tags to be shown on this layer:", new List(neededTags.map(t => t.asHumanString(true, false, {}))), - + overpassLink, new Title("Supported attributes", 2), quickOverview, ...this.tagRenderings.map(tr => tr.GenerateDocumentation())