First version of unit handling: canonicalizing on input

Customizations/JSON/LayoutConfig.ts

@@ -5,6 +5,7 @@ import {LayoutConfigJson} from "./LayoutConfigJson";
 import AllKnownLayers from "../AllKnownLayers";
 import SharedTagRenderings from "../SharedTagRenderings";
 import {Utils} from "../../Utils";
+import {Unit} from "./Unit";
 
 export default class LayoutConfig {
     public readonly id: string;
@@ -30,7 +31,6 @@ export default class LayoutConfig {
         maxZoom: number,
         minNeededElements: number
     };
-
     public readonly hideFromOverview: boolean;
     public lockLocation: boolean | [[number, number], [number, number]];
     public readonly enableUserBadge: boolean;
@@ -42,12 +42,12 @@ export default class LayoutConfig {
     public readonly enableGeolocation: boolean;
     public readonly enableBackgroundLayerSelection: boolean;
     public readonly enableShowAllQuestions: boolean;
-
     public readonly customCss?: string;
     /*
     How long is the cache valid, in seconds?
      */
     public readonly cacheTimeout?: number;
+    public readonly units: { appliesToKeys: Set<string>, applicableUnits: Unit[] }[] = []
     private readonly _official: boolean;
 
     constructor(json: LayoutConfigJson, official = true, context?: string) {
@@ -185,6 +185,51 @@ export default class LayoutConfig {
         this.enableShowAllQuestions = json.enableShowAllQuestions ?? false;
         this.customCss = json.customCss;
         this.cacheTimeout = json.cacheTimout ?? (60 * 24 * 60 * 60)
+
+
+        if ((json.units ?? []).length !== 0) {
+            for (let i1 = 0; i1 < json.units.length; i1++) {
+                let unit = json.units[i1];
+                const appliesTo = unit.appliesToKey
+
+                for (let i = 0; i < appliesTo.length; i++) {
+                    let key = appliesTo[i];
+                    if (key.trim() !== key) {
+                        throw `${context}.unit[${i1}].appliesToKey[${i}] is invalid: it starts or ends with whitespace`
+                    }
+                }
+
+                if ((unit.applicableUnits ?? []).length === 0) {
+                    throw  `${context}: define at least one applicable unit`
+                }
+                // Some keys do have unit handling
+
+                const defaultSet = unit.applicableUnits.filter(u => u.default === true)
+                // No default is defined - we pick the first as default
+                if(defaultSet.length === 0){
+                    unit.applicableUnits[0].default = true 
+                }
+              
+                // Check that there are not multiple defaults
+                if (defaultSet.length > 1) {
+                    throw `Multiple units are set as default: they have canonical values of ${defaultSet.map(u => u.canonicalDenomination).join(", ")}`
+                }
+                const applicable = unit.applicableUnits.map((u, i) => new Unit(u, `${context}.units[${i}]`))
+                this.units.push({
+                    appliesToKeys: new Set(appliesTo),
+                    applicableUnits: applicable
+                })
+            }
+
+            const seenKeys = new Set<string>()
+            for (const unit of this.units) {
+                const alreadySeen = Array.from(unit.appliesToKeys).filter(key => seenKeys.has(key));
+                if (alreadySeen.length > 0) {
+                    throw `${context}.units: multiple units define the same keys. The key(s) ${alreadySeen.join(",")} occur multiple times`
+                }
+                unit.appliesToKeys.forEach(key => seenKeys.add(key))
+            }
+        }
     }
 
     public CustomCodeSnippets(): string[] {

Customizations/JSON/LayoutConfigJson.ts

@@ -1,14 +1,15 @@
 import {LayerConfigJson} from "./LayerConfigJson";
 import {TagRenderingConfigJson} from "./TagRenderingConfigJson";
+import UnitConfigJson from "./UnitConfigJson";
 
 /**
  * Defines the entire theme.
- * 
+ *
  * A theme is the collection of the layers that are shown; the intro text, the icon, ...
  * It more or less defines the entire experience.
- * 
+ *
  * Most of the fields defined here are metadata about the theme, such as its name, description, supported languages, default starting location, ...
- * 
+ *
  * The main chunk of the json will however be the 'layers'-array, where the details of your layers are.
  *
  * General remark: a type (string | any) indicates either a fixed or a translatable string.
@@ -16,10 +17,10 @@ import {TagRenderingConfigJson} from "./TagRenderingConfigJson";
 export interface LayoutConfigJson {
     /**
      * The id of this layout.
-     * 
+     *
      * This is used as hashtag in the changeset message, which will read something like "Adding data with #mapcomplete for theme #<the theme id>"
      * Make sure it is something decent and descriptive, it should be a simple, lowercase string.
-     * 
+     *
      * On official themes, it'll become the name of the page, e.g.
      * 'cyclestreets' which become 'cyclestreets.html'
      */
@@ -29,7 +30,7 @@ export interface LayoutConfigJson {
      * Who helped to create this theme and should be attributed?
      */
     credits?: string;
-    
+
     /**
      * Who does maintian this preset?
      */
@@ -49,7 +50,7 @@ export interface LayoutConfigJson {
      * If the theme supports multiple languages, use a list: `["en","nl","fr"]` to allow the user to pick any of them
      */
     language: string | string[];
-    
+
     /**
      * The title, as shown in the welcome message and the more-screen
      */
@@ -60,7 +61,7 @@ export interface LayoutConfigJson {
      * Note that if this one is not defined, the first sentence of 'description' is used
      */
     shortDescription?: string | any;
-    
+
     /**
      * The description, as shown in the welcome message and the more-screen
      */
@@ -116,7 +117,7 @@ export interface LayoutConfigJson {
 
     /**
      * An override applied on all layers of the theme.
-     * 
+     *
      * E.g.: if there are two layers defined:
      * ```
      * "layers"[
@@ -124,7 +125,7 @@ export interface LayoutConfigJson {
      *  {"title", ..., "tagRenderings", [...], "osmSource":{"tags" ...}}
      * ]
      * ```
-     * 
+     *
      * and overrideAll is specified:
      * ```
      * "overrideAll": {
@@ -136,11 +137,11 @@ export interface LayoutConfigJson {
      *  {"title", ..., "tagRenderings", [...], "osmSource":{"tags" ..., "geoJsonSource":"xyz"}}
      * ]
      * ```
-     * 
+     *
      * If the overrideAll contains a list where the keys starts with a plus, the values will be appended (instead of discarding the old list)
      */
     overrideAll?: any;
-    
+
     /**
      * The id of the default background. BY default: vanilla OSM
      */
@@ -149,42 +150,107 @@ export interface LayoutConfigJson {
     /**
      * The number of seconds that a feature is allowed to stay in the cache.
      * The caching flow is as following:
-     * 
+     *
      * 1. The application is opened the first time
      * 2. An overpass query is run
      * 3. The result is saved to local storage
-     * 
+     *
      * On the next opening:
      *
      * 1. The application is opened
      * 2. Data is loaded from cache and displayed
      * 3. An overpass query is run
      * 4. All data (both from overpass ánd local storage) are saved again to local storage (except when to old)
-     * 
+     *
      * Default value: 60 days
      */
     cacheTimout?: number;
-    
+
-    
+
     /**
      * The layers to display.
-     * 
+     *
      * Every layer contains a description of which feature to display - the overpassTags which are queried.
      * Instead of running one query for every layer, the query is fused.
-     * 
+     *
      * Afterwards, every layer is given the list of features.
      * Every layer takes away the features that match with them*, and give the leftovers to the next layers.
-     * 
+     *
      * This implies that the _order_ of the layers is important in the case of features with the same tags;
      * as the later layers might never receive their feature.
-     * 
+     *
      * *layers can also remove 'leftover'-features if the leftovers overlap with a feature in the layer itself
-     * 
+     *
      * Note that builtin layers can be reused. Either put in the name of the layer to reuse, or use {builtin: "layername", override: ...}
      * The 'override'-object will be copied over the original values of the layer, which allows to change certain aspects of the layer
-     * 
+     *
      */
-    layers: (LayerConfigJson | string | {builtin: string, override: any})[],
+    layers: (LayerConfigJson | string | { builtin: string, override: any })[],
+
+    /**
+     * In some cases, a value is represented in a certain unit (such as meters for heigt/distance/..., km/h for speed, ...)
+     *
+     * Sometimes, multiple denominations are possible (e.g. km/h vs mile/h; megawatt vs kilowatt vs gigawatt for power generators, ...)
+     *
+     * This brings in some troubles, as there are multiple ways to write it (no denomitation, 'm' vs 'meter' 'metre', ...)
+     *
+     * Not only do we want to write consistent data to OSM, we also want to present this consistently to the user.
+     * This is handled by defining units.
+     *
+     * # Usage
+     *
+     * First of all, you define which keys have units applied, for example:
+     *
+     * ```
+     * units: [
+     *  appliesTo: ["maxspeed", "maxspeed:hgv", "maxspeed:bus"]
+     *  applicableUnits: [
+     *      ...
+     *  ]
+     * ]
+     * ```
+     *
+     * ApplicableUnits defines which is the canonical extension, how it is presented to the user, ...:
+     *
+     * ```
+     * applicableUnits: [
+     * {
+     *     canonicalDenomination: "km/h",
+     *     alternativeDenomination: ["km/u", "kmh", "kph"]
+     *     default: true,
+     *     human: {
+     *         en: "kilometer/hour",
+     *         nl: "kilometer/uur"
+     *     },
+     *     humanShort: {
+     *         en: "km/h",
+     *         nl: "km/u"
+     *     }
+     * },
+     * {
+     *     canoncialDenomination: "mph",
+     *     ... similar for miles an hour ...
+     * }
+     * ]
+     * ```
+     *
+     *
+     * If this is defined, then every key which the denominations apply to (`maxspeed`, `maxspeed:hgv` and `maxspeed:bus`) will be rewritten at the metatagging stage:
+     * every value will be parsed and the canonical extension will be added add presented to the other parts of the code.
+     *
+     * Also, if a freeform text field is used, an extra dropdown with applicable denominations will be given
+     *
+     */
+    
+    units?: {
+
+        /**
+         * Every key from this list will be normalized
+         */
+        appliesToKey: string[],
+
+        applicableUnits: UnitConfigJson[]
+    }[]
 
     /**
      * If defined, data will be clustered.
@@ -218,7 +284,7 @@ export interface LayoutConfigJson {
      * Off by default, which will enable panning to the entire world
      */
     lockLocation?: boolean | [[number, number], [number, number]];
-    
+
     enableUserBadge?: boolean;
     enableShareScreen?: boolean;
     enableMoreQuests?: boolean;

Customizations/JSON/TagRenderingConfig.ts

@@ -8,6 +8,7 @@ import {TagUtils} from "../../Logic/Tags/TagUtils";
 import {And} from "../../Logic/Tags/And";
 import {TagsFilter} from "../../Logic/Tags/TagsFilter";
 
+
 /***
  * The parsed version of TagRenderingConfigJSON
  * Identical data, but with some methods and validation
@@ -64,11 +65,16 @@ export default class TagRenderingConfig {
             this.condition = condition;
         }
         if (json.freeform) {
+
+
+
             this.freeform = {
                 key: json.freeform.key,
                 type: json.freeform.type ?? "string",
                 addExtraTags: json.freeform.addExtraTags?.map((tg, i) =>
-                    FromJSON.Tag(tg, `${context}.extratag[${i}]`)) ?? []
+                    FromJSON.Tag(tg, `${context}.extratag[${i}]`)) ?? [],
+              
+              
             }
             if (json.freeform["extraTags"] !== undefined) {
                 throw `Freeform.extraTags is defined. This should probably be 'freeform.addExtraTag' (at ${context})`
@@ -76,6 +82,9 @@ export default class TagRenderingConfig {
             if (this.freeform.key === undefined || this.freeform.key === "") {
                 throw `Freeform.key is undefined or the empty string - this is not allowed; either fill out something or remove the freeform block alltogether. Error in ${context}`
             }
+            
+            
+            
             if (ValidatedTextField.AllTypes[this.freeform.type] === undefined) {
                 const knownKeys = ValidatedTextField.tpList.map(tp => tp.name).join(", ");
                 throw `Freeform.key ${this.freeform.key} is an invalid type. Known keys are ${knownKeys}`
@@ -91,8 +100,8 @@ export default class TagRenderingConfig {
         this.multiAnswer = json.multiAnswer ?? false
         if (json.mappings) {
 
-            if(!Array.isArray(json.mappings)){
+            if (!Array.isArray(json.mappings)) {
-                throw "Tagrendering has a 'mappings'-object, but expected a list ("+context+")"
+                throw "Tagrendering has a 'mappings'-object, but expected a list (" + context + ")"
             }
 
             this.mappings = json.mappings.map((mapping, i) => {
@@ -104,15 +113,15 @@ export default class TagRenderingConfig {
                 if (mapping.ifnot !== undefined && !this.multiAnswer) {
                     throw `${context}.mapping[${i}]: Invalid mapping: ifnot defined, but the tagrendering is not a multianswer`
                 }
-                
+
-                if(mapping.if === undefined){
+                if (mapping.if === undefined) {
                     throw `${context}.mapping[${i}]: Invalid mapping: "if" is not defined, but the tagrendering is not a multianswer`
                 }
-                if(typeof mapping.if !== "string" && mapping.if["length"] !== undefined){
+                if (typeof mapping.if !== "string" && mapping.if["length"] !== undefined) {
                     throw `${context}.mapping[${i}]: Invalid mapping: "if" is defined as an array. Use {"and": <your conditions>} or {"or": <your conditions>} instead`
                 }
-                
+
-                
+
                 let hideInAnswer: boolean | TagsFilter = false;
                 if (typeof mapping.hideInAnswer === "boolean") {
                     hideInAnswer = mapping.hideInAnswer;
@@ -246,22 +255,22 @@ export default class TagRenderingConfig {
      * @param tags
      * @constructor
      */
-    public GetRenderValues(tags: any): Translation[]{
+    public GetRenderValues(tags: any): Translation[] {
-        if(!this.multiAnswer){
+        if (!this.multiAnswer) {
             return [this.GetRenderValue(tags)]
         }
 
         // A flag to check that the freeform key isn't matched multiple times 
         // If it is undefined, it is "used" already, or at least we don't have to check for it anymore
-        let freeformKeyUsed = this.freeform?.key === undefined; 
+        let freeformKeyUsed = this.freeform?.key === undefined;
         // We run over all the mappings first, to check if the mapping matches
         const applicableMappings: Translation[] = Utils.NoNull((this.mappings ?? [])?.map(mapping => {
             if (mapping.if === undefined) {
                 return mapping.then;
             }
             if (TagUtils.MatchesMultiAnswer(mapping.if, tags)) {
-                if(!freeformKeyUsed){
+                if (!freeformKeyUsed) {
-                    if(mapping.if.usedKeys().indexOf(this.freeform.key) >= 0){
+                    if (mapping.if.usedKeys().indexOf(this.freeform.key) >= 0) {
                         // This mapping matches the freeform key - we mark the freeform key to be ignored!
                         freeformKeyUsed = true;
                     }
@@ -270,8 +279,7 @@ export default class TagRenderingConfig {
             }
             return undefined;
         }))
-        
+
-        
 
         if (!freeformKeyUsed
             && tags[this.freeform.key] !== undefined) {
@@ -279,9 +287,10 @@ export default class TagRenderingConfig {
         }
         return applicableMappings
     }
-    
+
     /**
      * Gets the correct rendering value (or undefined if not known)
+     * Not compatible with multiAnswer - use GetRenderValueS instead in that case
      * @constructor
      */
     public GetRenderValue(tags: any): Translation {
@@ -308,14 +317,14 @@ export default class TagRenderingConfig {
     }
 
     public ExtractImages(isIcon: boolean): Set<string> {
-        
+
         const usedIcons = new Set<string>()
         this.render?.ExtractImages(isIcon)?.forEach(usedIcons.add, usedIcons)
 
         for (const mapping of this.mappings ?? []) {
             mapping.then.ExtractImages(isIcon).forEach(usedIcons.add, usedIcons)
         }
-        
+
         return usedIcons;
     }
 

Customizations/JSON/TagRenderingConfigJson.ts

@@ -45,6 +45,8 @@ export interface TagRenderingConfigJson {
          * Useful to add a 'fixme=freeform textfield used - to be checked'
          **/
         addExtraTags?: string[];
+
+        
     },
 
     /**

Customizations/JSON/Unit.ts

@@ -0,0 +1,86 @@
+import {Translation} from "../../UI/i18n/Translation";
+import UnitConfigJson from "./UnitConfigJson";
+import Translations from "../../UI/i18n/Translations";
+
+export class Unit {
+    public readonly human: Translation;
+    private readonly alternativeDenominations: string [];
+    private readonly canonical: string;
+    private readonly default: boolean;
+    private readonly prefix: boolean;
+
+    constructor(json: UnitConfigJson, context: string) {
+        context = `${context}.unit(${json.canonicalDenomination})`
+        this.canonical = json.canonicalDenomination.trim()
+        if ((this.canonical ?? "") === "") {
+            throw `${context}: this unit has no decent canonical value defined`
+        }
+
+        json.alternativeDenomination.forEach((v, i) => {
+            if (((v?.trim() ?? "") === "")) {
+                throw `${context}.alternativeDenomination.${i}: invalid alternative denomination: undefined, null or only whitespace`
+            }
+        })
+
+        this.alternativeDenominations = json.alternativeDenomination?.map(v => v.trim()) ?? []
+
+        this.default = json.default ?? false;
+
+        this.human = Translations.T(json.human, context + "human")
+
+        this.prefix = json.prefix ?? false;
+
+    }
+
+    public canonicalValue(value: string) {
+        const stripped = this.StrippedValue(value)
+        if(stripped === null){
+            return null;
+        }
+        return stripped + this.canonical
+    }
+
+    /**
+     * Returns the core value (without unit) if:
+     * - the value ends with the canonical or an alternative value (or begins with if prefix is set)
+     * - the value is a Number (without unit) and default is set
+     *
+     * Returns null if it doesn't match this unit
+     * @param value
+     * @constructor
+     */
+    private StrippedValue(value: string): string {
+
+        if (this.prefix) {
+            if (value.startsWith(this.canonical)) {
+                return value.substring(this.canonical.length).trim();
+            }
+            for (const alternativeValue of this.alternativeDenominations) {
+                if (value.startsWith(alternativeValue)) {
+                    return value.substring(alternativeValue.length).trim();
+                }
+            }
+        } else {
+            if (value.endsWith(this.canonical)) {
+                return value.substring(0, value.length - this.canonical.length).trim();
+            }
+            for (const alternativeValue of this.alternativeDenominations) {
+                if (value.endsWith(alternativeValue)) {
+                    return value.substring(0, value.length - alternativeValue.length).trim();
+                }
+            }
+        }
+
+
+        if (this.default) {
+            const parsed = Number(value.trim())
+            if (!isNaN(parsed)) {
+                return value.trim();
+            }
+        }
+
+        return null;
+    }
+
+
+}

Customizations/JSON/UnitConfigJson.ts

@@ -0,0 +1,36 @@
+export default interface UnitConfigJson{
+
+    /**
+     * The canonical value which will be added to the text.
+     * e.g. "m" for meters
+     * If the user inputs '42', the canonical value will be added and it'll become '42m'
+     */
+    canonicalDenomination: string,
+
+    /**
+     * A list of alternative values which can occur in the OSM database - used for parsing.
+     */
+    alternativeDenomination?: string[],
+
+    /**
+     * The value for humans in the dropdown. This should not use abbreviations and should be translated, e.g.
+     * {
+     *     "en": "meter",
+     *     "fr": "metre"
+     * }
+     */
+    human?:string | any
+
+    /**
+     * If set, then the canonical value will be prefixed instead, e.g. for '€'
+     * Note that if all values use 'prefix', the dropdown might move to before the text field
+     */
+    prefix?: boolean
+
+    /**
+     * The default interpretation - only one can be set.
+     * If none is set, the first unit will be considered the default interpretation of a value without a unit
+     */
+    default?: boolean
+
+}

Logic/SimpleMetaTagger.ts

@@ -75,6 +75,39 @@ export default class SimpleMetaTagger {
             feature.area = sqMeters;
         })
     );
+
+    private static canonicalize = new SimpleMetaTagger(
+        {
+            doc: "If 'units' is defined in the layoutConfig, then this metatagger will rewrite the specified keys to have the canonical form (e.g. `1meter` will be rewritten to `1m`)",
+            keys: []
+
+        },
+        (feature => {
+            const units = State.state.layoutToUse.data.units ?? [];
+            for (const key in feature.properties) {
+                if(!feature.properties.hasOwnProperty(key)){
+                    continue;
+                }
+                for (const unit of units) {
+                    if (!unit.appliesToKeys.has(key)) {
+                        continue;
+                    }
+                    const value = feature.properties[key]
+                    
+                    for (const applicableUnit of unit.applicableUnits) {
+                        const canonical = applicableUnit.canonicalValue(value)
+                        if (canonical == null) {
+                            continue
+                        }
+                        console.log("Rewritten ", key, " from", value, "into", canonical)
+                        feature.properties[key] = canonical;
+                    }
+                }
+
+            }
+        })
+    )
+
     private static lngth = new SimpleMetaTagger(
         {
             keys: ["_length", "_length:km"],
@@ -215,7 +248,7 @@ export default class SimpleMetaTagger {
             keys: ["_width:needed", "_width:needed:no_pedestrians", "_width:difference"],
             doc: "Legacy for a specific project calculating the needed width for safe traffic on a road. Only activated if 'width:carriageway' is present"
         },
-        (feature: any, index: number) => {
+        feature => {
 
             const properties = feature.properties;
             if (properties["width:carriageway"] === undefined) {
@@ -352,6 +385,7 @@ export default class SimpleMetaTagger {
         SimpleMetaTagger.latlon,
         SimpleMetaTagger.surfaceArea,
         SimpleMetaTagger.lngth,
+        SimpleMetaTagger.canonicalize,
         SimpleMetaTagger.country,
         SimpleMetaTagger.isOpen,
         SimpleMetaTagger.carriageWayWidth,

assets/themes/climbing/climbing.json

@@ -816,6 +816,20 @@
       "wayHandling": 0
     }
   ],
+  "units": [
+    {
+      "appliesToKey": ["climbing:length"],
+      "applicableUnits": [{
+        "canonicalDenomination": "m",
+        "alternativeDenomination": ["meter","meters"],
+        "human": {
+          "en": "meter",
+          "nl": "meter"
+        },
+        "default": true
+      }]
+    }
+  ],
   "roamingRenderings": [
     {
       "#": "Website",

scripts/generateLayerOverview.ts

@@ -214,7 +214,11 @@ class LayerOverviewUtils {
             const errors = layerErrorCount.concat(themeErrorCount).join("\n")
             console.log(errors)
             const msg = (`Found ${layerErrorCount.length} errors in the layers; ${themeErrorCount.length} errors in the themes`)
+            console.log ("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+
             console.log(msg)
+            console.log ("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+   
             if (process.argv.indexOf("--report") >= 0) {
                 console.log("Writing report!")
                 writeFileSync("layer_report.txt", errors)
@@ -227,4 +231,4 @@ class LayerOverviewUtils {
     }
 }
 
-new LayerOverviewUtils().main(process.argv)
+    new LayerOverviewUtils().main(process.argv)

test/TestAll.ts

@@ -33,7 +33,9 @@ const allTests = [
     new GeoOperationsSpec(),
     new ImageSearcherSpec(),
     new ThemeSpec(),
-    new UtilsSpec()]
+    new UtilsSpec(),
+    new UtilsSpec()
+]
 
 
 for (const test of allTests) {

test/Units.spec.ts

@@ -0,0 +1,33 @@
+import T from "./TestHelper";
+import {Unit} from "../Customizations/JSON/Unit";
+import {equal} from "assert";
+
+export default class UnitsSpec extends T {
+
+    constructor() {
+        super("Units", [
+            ["Simple canonicalize", () => {
+
+                const unit = new Unit({
+                    canonicalDenomination: "m",
+                    alternativeDenomination: ["meter"],
+                    'default': true,
+                    human: {
+                        en: "meter"
+                    }
+                }, "test")
+
+                equal(unit.canonicalValue("42m"), "42m")
+                equal(unit.canonicalValue("42"), "42m")
+                equal(unit.canonicalValue("42 m"), "42m")
+                equal(unit.canonicalValue("42 meter"), "42m")
+
+
+            }]
+
+
+        ]);
+
+    }
+
+}