From a3d26db84a7f9b8aeb50638df96fe3daf398b55f Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Thu, 9 Jan 2025 20:39:21 +0100 Subject: [PATCH] Logic: better support for tag optimization and simplifying expressions --- scripts/downloadNsiLogos.ts | 74 +++++- src/Logic/Tags/And.ts | 60 ++--- src/Logic/Tags/Or.ts | 1 + src/Logic/Tags/Tag.ts | 8 + src/Logic/Tags/TagUtils.ts | 75 +++--- src/Logic/Tags/TagsFilter.ts | 3 +- src/Logic/Web/NameSuggestionIndex.ts | 63 +++-- .../ThemeConfig/Conversion/ExpandFilter.ts | 220 ++++++++++++++++++ .../ThemeConfig/Conversion/PrepareLayer.ts | 172 +------------- .../ThemeConfig/Json/FilterConfigJson.ts | 9 + .../ThemeConfig/Json/LayerConfigJson.ts | 5 + 11 files changed, 430 insertions(+), 260 deletions(-) create mode 100644 src/Models/ThemeConfig/Conversion/ExpandFilter.ts diff --git a/scripts/downloadNsiLogos.ts b/scripts/downloadNsiLogos.ts index b058f8da4..94af62143 100644 --- a/scripts/downloadNsiLogos.ts +++ b/scripts/downloadNsiLogos.ts @@ -1,9 +1,14 @@ import Script from "./Script" import NameSuggestionIndex, { NSIItem } from "../src/Logic/Web/NameSuggestionIndex" import * as nsiWD from "../node_modules/name-suggestion-index/dist/wikidata.min.json" -import { existsSync, readFileSync, unlinkSync, writeFileSync } from "fs" +import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs" import ScriptUtils from "./ScriptUtils" import { Utils } from "../src/Utils" +import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson" +import FilterConfigJson, { FilterConfigOptionJson } from "../src/Models/ThemeConfig/Json/FilterConfigJson" +import { TagConfigJson } from "../src/Models/ThemeConfig/Json/TagConfigJson" +import { TagUtils } from "../src/Logic/Tags/TagUtils" +import { And } from "../src/Logic/Tags/And" class DownloadNsiLogos extends Script { constructor() { @@ -43,7 +48,7 @@ class DownloadNsiLogos extends Script { await ScriptUtils.DownloadFileTo(logos.facebook, path) // Validate const content = readFileSync(path, "utf8") - if (content.startsWith('{"error"')) { + if (content.startsWith("{\"error\"")) { unlinkSync(path) console.error("Attempted to fetch", logos.facebook, " but this gave an error") } else { @@ -86,12 +91,8 @@ class DownloadNsiLogos extends Script { return false } - async main(): Promise { - await this.downloadFor("operator") - await this.downloadFor("brand") - } - async downloadFor(type: "brand" | "operator"): Promise { + async downloadFor(type: string): Promise { const nsi = await NameSuggestionIndex.getNsiIndex() const items = nsi.allPossible(type) const basePath = "./public/assets/data/nsi/logos/" @@ -109,7 +110,7 @@ class DownloadNsiLogos extends Script { downloadCount++ } return downloaded - }) + }), ) for (let j = 0; j < results.length; j++) { let didDownload = results[j] @@ -124,6 +125,63 @@ class DownloadNsiLogos extends Script { } } } + + private async generateRendering(type: string) { + const nsi = await NameSuggestionIndex.getNsiIndex() + const items = nsi.allPossible(type) + const brandPrefix = [type, "name", "alt_name", "operator","brand"] + const filterOptions: FilterConfigOptionJson[] = items.map(item => { + let brandDetection: string[] = [] + let required: string[] = [] + const tags: Record = item.tags + for (const k in tags) { + if (brandPrefix.some(br => k === br || k.startsWith(br + ":"))) { + brandDetection.push(k + "=" + tags[k]) + } else { + required.push(k + "=" + tags[k]) + } + } + const osmTags = TagUtils.optimzeJson({ and: [...required, { or: brandDetection }] }) + return ({ + question: item.displayName, + icon: nsi.getIconUrl(item, type), + osmTags, + }) + }) + + const config: LayerConfigJson = { + "#dont-translate": "*", + id: "nsi_" + type, + source: "special:library", + description: { + en: "Exposes part of the NSI to reuse in other themes, e.g. for rendering", + }, + pointRendering: null, + filter: [ + { + id: type, + strict: true, + options: [{question: type}, ...filterOptions], + }, + ], + allowMove: false, + } + const path = "./assets/layers/nsi_" + type + mkdirSync(path, { recursive: true }) + writeFileSync(path + "/nsi_" + type + ".json", JSON.stringify(config, null, " ")) + console.log("Written", path) + } + + async main(): Promise { + const nsi = await NameSuggestionIndex.getNsiIndex() + const types = ["brand", "operator"] + for (const type of types) { + await this.generateRendering(type) + // await this.downloadFor(type) + } + } + + } new DownloadNsiLogos().run() diff --git a/src/Logic/Tags/And.ts b/src/Logic/Tags/And.ts index bf05b3100..6057ba478 100644 --- a/src/Logic/Tags/And.ts +++ b/src/Logic/Tags/And.ts @@ -130,39 +130,38 @@ export class And extends TagsFilter { * t1.shadows(t2) // => false * t2.shadows(t0) // => false * t2.shadows(t1) // => false + * + * + * const t1 = new And([new Tag("shop","clothes"), new Or([new Tag("brand","XYZ"),new Tag("brand:wikidata","Q1234")])]) + * const t2 = new And([new RegexTag("shop","mall",true), new Or([TagUtils.Tag("shop~*"), new Tag("craft","shoemaker")])]) + * t1.shadows(t2) // => true */ shadows(other: TagsFilter): boolean { - if (!(other instanceof And)) { - return false - } - + const phrases: TagsFilter[] = other instanceof And ? other.and : [other]; + // A phrase might be shadowed by a certain subsection. We keep track of this here + const shadowedOthers = phrases.map(() => false) for (const selfTag of this.and) { - let matchFound = false - for (const otherTag of other.and) { - matchFound = selfTag.shadows(otherTag) - if (matchFound) { - break + let shadowsSome = false; + let shadowsAll = true; + for (let i = 0; i < phrases.length; i++){ + const otherTag = phrases[i] + const doesShadow = selfTag.shadows(otherTag) + if(doesShadow){ + shadowedOthers[i] = true; } + shadowsSome ||= doesShadow; + shadowsAll &&= doesShadow; } - if (!matchFound) { - return false + // If A => X and A => Y, then + // A&B implies X&Y. We discovered an A that implies all needed values + if (shadowsAll) { + return true; + } + if (!shadowsSome) { + return false; } } - - for (const otherTag of other.and) { - let matchFound = false - for (const selfTag of this.and) { - matchFound = selfTag.shadows(otherTag) - if (matchFound) { - break - } - } - if (!matchFound) { - return false - } - } - - return true + return !shadowedOthers.some(v => !v); } usedKeys(): string[] { @@ -182,11 +181,13 @@ export class And extends TagsFilter { } /** - * IN some contexts, some expressions can be considered true, e.g. + * In some contexts, some expressions can be considered true, e.g. * (X=Y | (A=B & X=Y)) * ^---------^ * When the evaluation hits (A=B & X=Y), we know _for sure_ that X=Y does _not_ match, as it would have matched the first clause otherwise. - * This means that the entire 'AND' is considered FALSE + * This means that the entire 'AND' is considered FALSE in this case; but this is already handled by the first half. + * In other words: this long expression is equivalent to (A=B | X=Y). + * * * @return only phrases that should be kept. * @param knownExpression The expression which is known in the subexpression and for which calculations can be done @@ -204,13 +205,14 @@ export class And extends TagsFilter { * const expr = TagUtils.Tag({and: ["sport=climbing", {or:["club~*", "office~*"]}]} ) * expr.removePhraseConsideredKnown(new Tag("club","climbing"), false) // => expr */ - removePhraseConsideredKnown( + public removePhraseConsideredKnown( knownExpression: TagsFilter, value: boolean ): (TagsFilterClosed & OptimizedTag) | boolean { const newAnds: TagsFilter[] = [] for (const tag of this.and) { if (tag instanceof And) { + console.trace("Improper optimization") throw ( "Optimize expressions before using removePhraseConsideredKnown. Found an AND in an AND: " + this.asHumanString() diff --git a/src/Logic/Tags/Or.ts b/src/Logic/Tags/Or.ts index 1d967b8cd..dbcaed065 100644 --- a/src/Logic/Tags/Or.ts +++ b/src/Logic/Tags/Or.ts @@ -83,6 +83,7 @@ export class Or extends TagsFilter { return false } + shadows(other: TagsFilter): boolean { if (other instanceof Or) { for (const selfTag of this.or) { diff --git a/src/Logic/Tags/Tag.ts b/src/Logic/Tags/Tag.ts index e7f0f269f..51953da07 100644 --- a/src/Logic/Tags/Tag.ts +++ b/src/Logic/Tags/Tag.ts @@ -4,6 +4,8 @@ import { TagConfigJson } from "../../Models/ThemeConfig/Json/TagConfigJson" import { ExpressionSpecification } from "maplibre-gl" import { RegexTag } from "./RegexTag" import { OptimizedTag } from "./TagTypes" +import { Or } from "./Or" +import { And } from "./And" export class Tag extends TagsFilter { public key: string @@ -148,6 +150,12 @@ export class Tag extends TagsFilter { return other.matchesProperties({ [this.key]: this.value }) } } + if(other instanceof Or){ + return other.or.some(other => this.shadows(other)) + } + if(other instanceof And){ + return !other.and.some(other => !this.shadows(other)) + } return false } diff --git a/src/Logic/Tags/TagUtils.ts b/src/Logic/Tags/TagUtils.ts index d976d541b..a40ea8f48 100644 --- a/src/Logic/Tags/TagUtils.ts +++ b/src/Logic/Tags/TagUtils.ts @@ -133,11 +133,11 @@ export class TagUtils { "\n" + "```json\n" + "{\n" + - ' "mappings": [\n' + + " \"mappings\": [\n" + " {\n" + - ' "if":"key:={some_other_key}",\n' + - ' "then": "...",\n' + - ' "hideInAnswer": "some_other_key="\n' + + " \"if\":\"key:={some_other_key}\",\n" + + " \"then\": \"...\",\n" + + " \"hideInAnswer\": \"some_other_key=\"\n" + " }\n" + " ]\n" + "}\n" + @@ -175,10 +175,10 @@ export class TagUtils { "\n" + "```json\n" + "{\n" + - ' "osmTags": {\n' + - ' "or": [\n' + - ' "amenity=school",\n' + - ' "amenity=kindergarten"\n' + + " \"osmTags\": {\n" + + " \"or\": [\n" + + " \"amenity=school\",\n" + + " \"amenity=kindergarten\"\n" + " ]\n" + " }\n" + "}\n" + @@ -194,7 +194,7 @@ export class TagUtils { "If the schema-files note a type [`TagConfigJson`](https://github.com/pietervdvn/MapComplete/blob/develop/src/Models/ThemeConfig/Json/TagConfigJson.ts), you can use one of these values.\n" + "\n" + "In some cases, not every type of tags-filter can be used. For example, _rendering_ an option with a regex is\n" + - 'fine (`"if": "brand~[Bb]randname", "then":" The brand is Brandname"`); but this regex can not be used to write a value\n' + + "fine (`\"if\": \"brand~[Bb]randname\", \"then\":\" The brand is Brandname\"`); but this regex can not be used to write a value\n" + "into the database. The theme loader will however refuse to work with such inconsistencies and notify you of this while\n" + "you are building your theme.\n" + "\n" + @@ -205,18 +205,18 @@ export class TagUtils { "\n" + "```json\n" + "{\n" + - ' "and": [\n' + - ' "key=value",\n' + + " \"and\": [\n" + + " \"key=value\",\n" + " {\n" + - ' "or": [\n' + - ' "other_key=value",\n' + - ' "other_key=some_other_value"\n' + + " \"or\": [\n" + + " \"other_key=value\",\n" + + " \"other_key=some_other_value\"\n" + " ]\n" + " },\n" + - ' "key_which_should_be_missing=",\n' + - ' "key_which_should_have_a_value~*",\n' + - ' "key~.*some_regex_a*_b+_[a-z]?",\n' + - ' "height<1"\n' + + " \"key_which_should_be_missing=\",\n" + + " \"key_which_should_have_a_value~*\",\n" + + " \"key~.*some_regex_a*_b+_[a-z]?\",\n" + + " \"height<1\"\n" + " ]\n" + "}\n" + "```\n" + @@ -246,7 +246,7 @@ export class TagUtils { static asProperties( tags: TagsFilter | TagsFilter[], - baseproperties: Record = {} + baseproperties: Record = {}, ) { if (Array.isArray(tags)) { tags = new And(tags) @@ -274,11 +274,11 @@ export class TagUtils { static SplitKeysRegex(tagsFilters: UploadableTag[], allowRegex: false): Record static SplitKeysRegex( tagsFilters: UploadableTag[], - allowRegex: boolean + allowRegex: boolean, ): Record static SplitKeysRegex( tagsFilters: UploadableTag[], - allowRegex: boolean + allowRegex: boolean, ): Record { const keyValues: Record = {} tagsFilters = [...tagsFilters] // copy all, use as queue @@ -307,7 +307,7 @@ export class TagUtils { if (typeof key !== "string") { console.error( "Invalid type to flatten the multiAnswer: key is a regex too", - tagsFilter + tagsFilter, ) throw "Invalid type to FlattenMultiAnswer: key is a regex too" } @@ -508,7 +508,7 @@ export class TagUtils { public static Tag(json: TagConfigJson, context?: string | ConversionContext): TagsFilterClosed public static Tag( json: TagConfigJson, - context: string | ConversionContext = "" + context: string | ConversionContext = "", ): TagsFilterClosed { try { const ctx = typeof context === "string" ? context : context.path.join(".") @@ -540,7 +540,7 @@ export class TagUtils { throw `Error at ${context}: detected a non-uploadable tag at a location where this is not supported: ${t.asHumanString( false, false, - {} + {}, )}` }) @@ -661,7 +661,7 @@ export class TagUtils { */ public static removeShadowedElementsFrom( blacklist: TagsFilter[], - listToFilter: TagsFilter[] + listToFilter: TagsFilter[], ): TagsFilter[] { return listToFilter.filter((tf) => !blacklist.some((guard) => guard.shadows(tf))) } @@ -699,7 +699,7 @@ export class TagUtils { */ public static containsEquivalents( guards: ReadonlyArray, - listToFilter: ReadonlyArray + listToFilter: ReadonlyArray, ): boolean { return listToFilter.some((tf) => guards.some((guard) => guard.shadows(tf))) } @@ -743,7 +743,7 @@ export class TagUtils { values.push(i + "") } return values - }) + }), ) return Utils.NoNull(spec) } @@ -751,13 +751,13 @@ export class TagUtils { private static ParseTagUnsafe(json: TagConfigJson, context: string = ""): TagsFilterClosed { if (json === undefined) { throw new Error( - `Error while parsing a tag: 'json' is undefined in ${context}. Make sure all the tags are defined and at least one tag is present in a complex expression` + `Error while parsing a tag: 'json' is undefined in ${context}. Make sure all the tags are defined and at least one tag is present in a complex expression`, ) } if (typeof json != "string") { if (json["and"] !== undefined && json["or"] !== undefined) { throw `${context}: Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined. Did you override a value? Perhaps use \`"=parent": { ... }\` instead of "parent": {...}\` to trigger a replacement and not a fuse of values. The value is ${JSON.stringify( - json + json, )}` } if (json["and"] !== undefined) { @@ -839,13 +839,13 @@ export class TagUtils { return new RegexTag( withRegex.key, new RegExp(".+", "si" + withRegex.modifier), - withRegex.invert + withRegex.invert, ) } return new RegexTag( withRegex.key, new RegExp("^(" + value + ")$", "s" + withRegex.modifier), - withRegex.invert + withRegex.invert, ) } @@ -967,10 +967,19 @@ export class TagUtils { return ["", "## `" + mode + "` " + doc.name, "", doc.docs, "", ""].join("\n") }), "## " + - TagUtils.comparators.map((comparator) => "`" + comparator[0] + "`").join(" ") + - " Logical comparators", + TagUtils.comparators.map((comparator) => "`" + comparator[0] + "`").join(" ") + + " Logical comparators", TagUtils.numberAndDateComparisonDocs, TagUtils.logicalOperator, ].join("\n") } + + static fromProperties(tags: Record): TagConfigJson | boolean { + + const opt = new And(Object.keys(tags).map(k => new Tag(k, tags[k]))).optimize() + if (opt === true || opt === false) { + return opt + } + return opt.asJson() + } } diff --git a/src/Logic/Tags/TagsFilter.ts b/src/Logic/Tags/TagsFilter.ts index 48c88dbbb..e4c0deb00 100644 --- a/src/Logic/Tags/TagsFilter.ts +++ b/src/Logic/Tags/TagsFilter.ts @@ -9,7 +9,8 @@ export abstract class TagsFilter { /** * Indicates some form of equivalency: - * if `this.shadows(t)`, then `this.matches(properties)` implies that `t.matches(properties)` for all possible properties + * if `this.shadows(t)`, then `this.matches(properties)` implies that `t.matches(properties)` for all possible properties. + * In other words: 'this' is a _stronger_ condition then 't' */ abstract shadows(other: TagsFilter): boolean diff --git a/src/Logic/Web/NameSuggestionIndex.ts b/src/Logic/Web/NameSuggestionIndex.ts index 946a7b569..0edc6adfc 100644 --- a/src/Logic/Web/NameSuggestionIndex.ts +++ b/src/Logic/Web/NameSuggestionIndex.ts @@ -41,14 +41,14 @@ interface NSIEntry { * Represents a single brand/operator/flagpole/... */ export interface NSIItem { - displayName: string - id: string + readonly displayName: string + readonly id: string locationSet: { include: string[] exclude: string[] } - tags: Record - fromTemplate?: boolean + readonly tags: Readonly> + readonly fromTemplate?: boolean } export default class NameSuggestionIndex { @@ -77,7 +77,7 @@ export default class NameSuggestionIndex { } > >, - features: Readonly + features: Readonly, ) { this.nsiFile = nsiFile this.nsiWdFile = nsiWdFile @@ -92,10 +92,10 @@ export default class NameSuggestionIndex { } const [nsi, nsiWd, features] = await Promise.all( ["./assets/data/nsi/nsi.min.json", "./assets/data/nsi/wikidata.min.json", "./assets/data/nsi/featureCollection.min.json"].map((url) => - Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30) - ) + Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30), + ), ) - NameSuggestionIndex.inited = new NameSuggestionIndex(nsi, nsiWd["wikidata"], features) + NameSuggestionIndex.inited = new NameSuggestionIndex(nsi, nsiWd["wikidata"], features) return NameSuggestionIndex.inited } @@ -126,13 +126,13 @@ export default class NameSuggestionIndex { try { return Utils.downloadJsonCached>( `./assets/data/nsi/stats/${type}.${c.toUpperCase()}.json`, - 24 * 60 * 60 * 1000 + 24 * 60 * 60 * 1000, ) } catch (e) { console.error("Could not fetch " + type + " statistics due to", e) return undefined } - }) + }), ) stats = Utils.NoNull(stats) if (stats.length === 1) { @@ -173,17 +173,17 @@ export default class NameSuggestionIndex { public async generateMappings( type: string, tags: Record, - country: string[], + country?: string[], location?: [number, number], options?: { /** * If set, sort by frequency instead of alphabetically */ sortByFrequency: boolean - } + }, ): Promise { const mappings: (Mapping & { frequency: number })[] = [] - const frequencies = await NameSuggestionIndex.fetchFrequenciesFor(type, country) + const frequencies = country !== undefined ? await NameSuggestionIndex.fetchFrequenciesFor(type, country) : {} for (const key in tags) { if (key.startsWith("_")) { continue @@ -194,7 +194,7 @@ export default class NameSuggestionIndex { key, value, country.join(";"), - location + location, ) if (!actualBrands) { continue @@ -202,8 +202,7 @@ export default class NameSuggestionIndex { for (const nsiItem of actualBrands) { const tags = nsiItem.tags const frequency = frequencies[nsiItem.displayName] - const logos = this.nsiWdFile[nsiItem.tags[type + ":wikidata"]]?.logos - const iconUrl = logos?.facebook ?? logos?.wikidata + const iconUrl = this.getIconExternalUrl(nsiItem, type) const hasIcon = iconUrl !== undefined let icon = undefined if (hasIcon) { @@ -240,7 +239,7 @@ export default class NameSuggestionIndex { } public supportedTags( - type: "operator" | "brand" | "flag" | "transit" | string + type: "operator" | "brand" | "flag" | "transit" | string, ): Record { const tags: Record = {} const keys = Object.keys(this.nsiFile.nsi) @@ -263,7 +262,7 @@ export default class NameSuggestionIndex { * Returns a list of all brands/operators * @param type */ - public allPossible(type: "brand" | "operator"): NSIItem[] { + public allPossible(type: string): NSIItem[] { const options: NSIItem[] = [] const tags = this.supportedTags(type) for (const osmKey in tags) { @@ -285,10 +284,10 @@ export default class NameSuggestionIndex { type: string, tags: { key: string; value: string }[], country: string = undefined, - location: [number, number] = undefined + location: [number, number] = undefined, ): NSIItem[] { return tags.flatMap((tag) => - this.getSuggestionsForKV(type, tag.key, tag.value, country, location) + this.getSuggestionsForKV(type, tag.key, tag.value, country, location), ) } @@ -311,7 +310,7 @@ export default class NameSuggestionIndex { key: string, value: string, country: string = undefined, - location: [number, number] = undefined + location: [number, number] = undefined, ): NSIItem[] { const path = `${type}s/${key}/${value}` const entry = this.nsiFile.nsi[path] @@ -375,9 +374,29 @@ export default class NameSuggestionIndex { center: [number, number], options: { sortByFrequency: boolean - } + }, ): Promise { const nsi = await NameSuggestionIndex.getNsiIndex() return nsi.generateMappings(key, tags, country, center, options) } + + + /** + * Where can we find the URL on the world wide web? + * Probably facebook! Don't use in the website, might expose people + * @param nsiItem + * @param type + */ + private getIconExternalUrl(nsiItem: NSIItem, type: string): string { + const logos = this.nsiWdFile[nsiItem.tags[type + ":wikidata"]]?.logos + return logos?.facebook ?? logos?.wikidata + } + + public getIconUrl(nsiItem: NSIItem, type: string) { + let icon = "./assets/data/nsi/logos/" + nsiItem.id + if (this.isSvg(nsiItem, type)) { + icon = icon + ".svg" + } + return icon + } } diff --git a/src/Models/ThemeConfig/Conversion/ExpandFilter.ts b/src/Models/ThemeConfig/Conversion/ExpandFilter.ts new file mode 100644 index 000000000..4588525f8 --- /dev/null +++ b/src/Models/ThemeConfig/Conversion/ExpandFilter.ts @@ -0,0 +1,220 @@ +import { DesugaringContext, DesugaringStep } from "./Conversion" +import { LayerConfigJson } from "../Json/LayerConfigJson" +import FilterConfigJson, { FilterConfigOptionJson } from "../Json/FilterConfigJson" +import predifined_filters from "../../../../assets/layers/filters/filters.json" +import { TagRenderingConfigJson } from "../Json/TagRenderingConfigJson" +import { ConversionContext } from "./ConversionContext" +import { QuestionableTagRenderingConfigJson } from "../Json/QuestionableTagRenderingConfigJson" +import { Utils } from "../../../Utils" +import { TagUtils } from "../../../Logic/Tags/TagUtils" +import { Tag } from "../../../Logic/Tags/Tag" +import { RegexTag } from "../../../Logic/Tags/RegexTag" +import { Or } from "../../../Logic/Tags/Or" +import Translations from "../../../UI/i18n/Translations" +import { FlatTag, OptimizedTag, TagsFilterClosed } from "../../../Logic/Tags/TagTypes" +import { TagsFilter } from "../../../Logic/Tags/TagsFilter" +import { And } from "../../../Logic/Tags/And" + +export class PruneFilters extends DesugaringStep{ + constructor() { + super("Removes all filters which are impossible, e.g. because they conflict with the base tags", ["filter"],"PruneFilters") + } + + private prune(sourceTags:FlatTag, filter: FilterConfigJson, context: ConversionContext): FilterConfigJson{ + if(!filter.strict){ + return filter + } + const countBefore = filter.options.length + const newOptions: FilterConfigOptionJson[] = filter.options.filter(option => { + if(!option.osmTags){ + return true + } + const condition = TagUtils.Tag(option.osmTags).optimize() + return condition.shadows(sourceTags); + + }).map(option => { + if(!option.osmTags){ + return option + } + let basetags: TagsFilter = And.construct([TagUtils.Tag(option.osmTags)]).optimize() + if(basetags instanceof And){ + basetags = basetags.removePhraseConsideredKnown(sourceTags, true) + } + return {...option, osmTags: basetags.asJson()} + }) + const countAfter = newOptions.length + if(countAfter !== countBefore){ + context.enters("filter", filter.id ).info("Pruned "+(countBefore-countAfter)+" options away from filter (out of "+countBefore+")") + } + return {...filter, options: newOptions, strict: undefined} + + } + + public convert(json: LayerConfigJson, context: ConversionContext): LayerConfigJson { + if(!Array.isArray(json.filter) || typeof json.source === "string"){ + return json + } + if(!json.source["osmTags"]){ + return json + } + const sourceTags = TagUtils.Tag(json.source["osmTags"]) + return {...json, filter: json.filter?.map(obj => this.prune(sourceTags, obj, context))} + } +} +export class ExpandFilter extends DesugaringStep { + private static readonly predefinedFilters = ExpandFilter.load_filters() + private _state: DesugaringContext + + constructor(state: DesugaringContext) { + super( + [ + "Expands filters: replaces a shorthand by the value found in 'filters.json'.", + "If the string is formatted 'layername.filtername, it will be looked up into that layer instead. Note that pruning should still be done", + ].join(" "), + ["filter"], + "ExpandFilter", + ) + this._state = state + } + + private static load_filters(): Map { + const filters = new Map() + for (const filter of predifined_filters.filter) { + filters.set(filter.id, filter) + } + return filters + } + + public static buildFilterFromTagRendering( + tr: TagRenderingConfigJson, + context: ConversionContext, + ): FilterConfigJson { + if (!(tr.mappings?.length >= 1)) { + context.err( + "Found a matching tagRendering to base a filter on, but this tagRendering does not contain any mappings", + ) + } + const qtr = tr + const options = qtr.mappings.map((mapping) => { + let icon: string = mapping.icon?.["path"] ?? mapping.icon + let emoji: string = undefined + if (Utils.isEmoji(icon)) { + emoji = icon + icon = undefined + } + let osmTags = TagUtils.Tag(mapping.if) + if (qtr.multiAnswer && osmTags instanceof Tag) { + osmTags = new RegexTag( + osmTags.key, + new RegExp("^(.+;)?" + osmTags.value + "(;.+)$", "is"), + ) + } + if (mapping.alsoShowIf) { + osmTags = new Or([osmTags, TagUtils.Tag(mapping.alsoShowIf)]) + } + + return { + question: mapping.then, + osmTags: osmTags.asJson(), + searchTerms: mapping.searchTerms, + icon, + emoji, + } + }) + // Add default option + options.unshift({ + question: tr["question"] ?? Translations.t.general.filterPanel.allTypes, + osmTags: undefined, + searchTerms: undefined, + }) + return { + id: tr["id"], + options, + } + } + + convert(json: LayerConfigJson, context: ConversionContext): LayerConfigJson { + if (json?.filter === undefined || json?.filter === null) { + return json // Nothing to change here + } + + if (json.filter["sameAs"] !== undefined) { + return json // Nothing to change here + } + + const newFilters: FilterConfigJson[] = [] + const filters = <(FilterConfigJson | string)[]>json.filter + + /** + * Create filters based on builtin filters or create them based on the tagRendering + */ + for (let i = 0; i < filters.length; i++) { + const filter = filters[i] + if (filter === undefined) { + continue + } + if (typeof filter !== "string") { + newFilters.push(filter) + continue + } + + const matchingTr = ( + json.tagRenderings.find((tr) => !!tr && tr["id"] === filter) + ) + if (matchingTr) { + const filter = ExpandFilter.buildFilterFromTagRendering( + matchingTr, + context.enters("filter", i), + ) + newFilters.push(filter) + continue + } + + if (filter.indexOf(".") > 0) { + if (!(this._state.sharedLayers?.size > 0)) { + // This is a bootstrapping-run, we can safely ignore this + continue + } + const split = filter.split(".") + if (split.length > 2) { + context.err( + "invalid filter name: " + filter + ", expected `layername.filterid`", + ) + } + const layer = this._state.sharedLayers.get(split[0]) + if (layer === undefined) { + context.err("Layer '" + split[0] + "' not found") + } + const expectedId = split[1] + const expandedFilter = (<(FilterConfigJson | string)[]>layer.filter).find( + (f) => typeof f !== "string" && f.id === expectedId, + ) + if (expandedFilter === undefined) { + context.err("Did not find filter with name " + filter) + } else { + newFilters.push(expandedFilter) + } + continue + } + // Search for the filter: + const found = ExpandFilter.predefinedFilters.get(filter) + if (found === undefined) { + const suggestions = Utils.sortedByLevenshteinDistance( + filter, + Array.from(ExpandFilter.predefinedFilters.keys()), + (t) => t, + ) + context + .enter(filter) + .err( + "While searching for predefined filter " + + filter + + ": this filter is not found. Perhaps you meant one of: " + + suggestions, + ) + } + newFilters.push(found) + } + return { ...json, filter: newFilters } + } +} diff --git a/src/Models/ThemeConfig/Conversion/PrepareLayer.ts b/src/Models/ThemeConfig/Conversion/PrepareLayer.ts index 5d86d2883..fcc924f16 100644 --- a/src/Models/ThemeConfig/Conversion/PrepareLayer.ts +++ b/src/Models/ThemeConfig/Conversion/PrepareLayer.ts @@ -10,10 +10,7 @@ import { SetDefault, } from "./Conversion" import { LayerConfigJson } from "../Json/LayerConfigJson" -import { - MinimalTagRenderingConfigJson, - TagRenderingConfigJson, -} from "../Json/TagRenderingConfigJson" +import { MinimalTagRenderingConfigJson, TagRenderingConfigJson } from "../Json/TagRenderingConfigJson" import { Utils } from "../../../Utils" import RewritableConfigJson from "../Json/RewritableConfigJson" import SpecialVisualizations from "../../../UI/SpecialVisualizations" @@ -21,8 +18,7 @@ import Translations from "../../../UI/i18n/Translations" import { Translation } from "../../../UI/i18n/Translation" import tagrenderingconfigmeta from "../../../../src/assets/schemas/tagrenderingconfigmeta.json" import { AddContextToTranslations } from "./AddContextToTranslations" -import FilterConfigJson, { FilterConfigOptionJson } from "../Json/FilterConfigJson" -import predifined_filters from "../../../../assets/layers/filters/filters.json" +import FilterConfigJson from "../Json/FilterConfigJson" import { TagConfigJson } from "../Json/TagConfigJson" import PointRenderingConfigJson, { IconConfigJson } from "../Json/PointRenderingConfigJson" import ValidationUtils from "./ValidationUtils" @@ -33,9 +29,7 @@ import LineRenderingConfigJson from "../Json/LineRenderingConfigJson" import { ConversionContext } from "./ConversionContext" import { ExpandRewrite } from "./ExpandRewrite" import { TagUtils } from "../../../Logic/Tags/TagUtils" -import { Tag } from "../../../Logic/Tags/Tag" -import { RegexTag } from "../../../Logic/Tags/RegexTag" -import { Or } from "../../../Logic/Tags/Or" +import { ExpandFilter, PruneFilters } from "./ExpandFilter" class AddFiltersFromTagRenderings extends DesugaringStep { constructor() { @@ -108,163 +102,6 @@ class AddFiltersFromTagRenderings extends DesugaringStep { return { ...json, filter: filters } } } -class ExpandFilter extends DesugaringStep { - private static readonly predefinedFilters = ExpandFilter.load_filters() - private _state: DesugaringContext - - constructor(state: DesugaringContext) { - super( - [ - "Expands filters: replaces a shorthand by the value found in 'filters.json'.", - "If the string is formatted 'layername.filtername, it will be looked up into that layer instead.", - ].join(" "), - ["filter"], - "ExpandFilter" - ) - this._state = state - } - - private static load_filters(): Map { - const filters = new Map() - for (const filter of predifined_filters.filter) { - filters.set(filter.id, filter) - } - return filters - } - - public static buildFilterFromTagRendering( - tr: TagRenderingConfigJson, - context: ConversionContext - ): FilterConfigJson { - if (!(tr.mappings?.length >= 1)) { - context.err( - "Found a matching tagRendering to base a filter on, but this tagRendering does not contain any mappings" - ) - } - const qtr = tr - const options = qtr.mappings.map((mapping) => { - let icon: string = mapping.icon?.["path"] ?? mapping.icon - let emoji: string = undefined - if (Utils.isEmoji(icon)) { - emoji = icon - icon = undefined - } - let osmTags = TagUtils.Tag(mapping.if) - if (qtr.multiAnswer && osmTags instanceof Tag) { - osmTags = new RegexTag( - osmTags.key, - new RegExp("^(.+;)?" + osmTags.value + "(;.+)$", "is") - ) - } - if (mapping.alsoShowIf) { - osmTags = new Or([osmTags, TagUtils.Tag(mapping.alsoShowIf)]) - } - - return { - question: mapping.then, - osmTags: osmTags.asJson(), - searchTerms: mapping.searchTerms, - icon, - emoji, - } - }) - // Add default option - options.unshift({ - question: tr["question"] ?? Translations.t.general.filterPanel.allTypes, - osmTags: undefined, - searchTerms: undefined, - }) - return { - id: tr["id"], - options, - } - } - - convert(json: LayerConfigJson, context: ConversionContext): LayerConfigJson { - if (json?.filter === undefined || json?.filter === null) { - return json // Nothing to change here - } - - if (json.filter["sameAs"] !== undefined) { - return json // Nothing to change here - } - - const newFilters: FilterConfigJson[] = [] - const filters = <(FilterConfigJson | string)[]>json.filter - - /** - * Create filters based on builtin filters or create them based on the tagRendering - */ - for (let i = 0; i < filters.length; i++) { - const filter = filters[i] - if (filter === undefined) { - continue - } - if (typeof filter !== "string") { - newFilters.push(filter) - continue - } - - const matchingTr = ( - json.tagRenderings.find((tr) => !!tr && tr["id"] === filter) - ) - if (matchingTr) { - const filter = ExpandFilter.buildFilterFromTagRendering( - matchingTr, - context.enters("filter", i) - ) - newFilters.push(filter) - continue - } - - if (filter.indexOf(".") > 0) { - if (!(this._state.sharedLayers?.size > 0)) { - // This is a bootstrapping-run, we can safely ignore this - continue - } - const split = filter.split(".") - if (split.length > 2) { - context.err( - "invalid filter name: " + filter + ", expected `layername.filterid`" - ) - } - const layer = this._state.sharedLayers.get(split[0]) - if (layer === undefined) { - context.err("Layer '" + split[0] + "' not found") - } - const expectedId = split[1] - const expandedFilter = (<(FilterConfigJson | string)[]>layer.filter).find( - (f) => typeof f !== "string" && f.id === expectedId - ) - if (expandedFilter === undefined) { - context.err("Did not find filter with name " + filter) - } else { - newFilters.push(expandedFilter) - } - continue - } - // Search for the filter: - const found = ExpandFilter.predefinedFilters.get(filter) - if (found === undefined) { - const suggestions = Utils.sortedByLevenshteinDistance( - filter, - Array.from(ExpandFilter.predefinedFilters.keys()), - (t) => t - ) - context - .enter(filter) - .err( - "While searching for predefined filter " + - filter + - ": this filter is not found. Perhaps you meant one of: " + - suggestions - ) - } - newFilters.push(found) - } - return { ...json, filter: newFilters } - } -} class ExpandTagRendering extends Conversion< | string @@ -1481,7 +1318,8 @@ export class PrepareLayer extends Fuse { new Concat(new ExpandTagRendering(state, layer, { noHardcodedStrings: true })) ), new AddFiltersFromTagRenderings(), - new ExpandFilter(state) + new ExpandFilter(state), + new PruneFilters() ) } } diff --git a/src/Models/ThemeConfig/Json/FilterConfigJson.ts b/src/Models/ThemeConfig/Json/FilterConfigJson.ts index c12ff126e..ed1b71041 100644 --- a/src/Models/ThemeConfig/Json/FilterConfigJson.ts +++ b/src/Models/ThemeConfig/Json/FilterConfigJson.ts @@ -20,6 +20,15 @@ export default interface FilterConfigJson { * An id/name for this filter, used to set the URL parameters */ id: string + /** + * If set, the options will be pruned. Only items for which the filter match the layer source will be kept. + * + * For example, we import types of brands from the nsi. This contains a ton of items, e.g. + * [{question: "Brand X", osmTags: {"and": ["shop=clothes", "brand=Brand X]}, {osmTags: {"and": "shop=convenience", ...} ...} ] + * Of course, when making a layer about `shop=clothes`, we'll only want to keep the clothes shops. + * If set to strict and the source is `shop=clothes`, only those options which have shop=clothes will be returned + */ + strict?: boolean /** * The options for a filter * If there are multiple options these will be a list of radio buttons diff --git a/src/Models/ThemeConfig/Json/LayerConfigJson.ts b/src/Models/ThemeConfig/Json/LayerConfigJson.ts index 48ebd5de5..6b5ad7717 100644 --- a/src/Models/ThemeConfig/Json/LayerConfigJson.ts +++ b/src/Models/ThemeConfig/Json/LayerConfigJson.ts @@ -601,4 +601,9 @@ export interface LayerConfigJson { * group: hidden */ snapName?: Translatable + + /** + * group: hidden + */ + "#dont-translate": "*" }