Wikidata language picker

scripts/fetchLanguages.ts

@@ -6,19 +6,10 @@ import * as wds from "wikidata-sdk"
 import {Utils} from "../Utils";
 import ScriptUtils from "./ScriptUtils";
 import {existsSync, readFileSync, writeFileSync} from "fs";
-import * as used_languages from "../assets/generated/used_languages.json"
 import {QuestionableTagRenderingConfigJson} from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
 import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
-
-const languageRemap = {
-    // MapComplete (or weblate) on the left, language of wikimedia on the right
-    "nb":"nb_NO",
-    "zh-hant":"zh_Hant",
-    "zh-hans":"zh_Hans",
-    "pt-br":"pt_BR"
-}
-
-const usedLanguages : Set<string> = new Set(used_languages.languages)
+import WikidataUtils from "../Utils/WikidataUtils";
+import LanguageUtils from "../Utils/LanguageUtils";
 
 async function fetch(target: string){
     const regular = await fetchRegularLanguages()
@@ -75,32 +66,13 @@ async function fetchSpecial(id: number, code: string) {
     return bindings
 }
 
-function extract(data){
-    console.log("Got "+data.length+" entries")
-    const perId = new Map<string, Map<string, string>>();
-    for (const element of data) {
-        let id = element.code.value
-        id = languageRemap[id] ?? id
-        let labelLang = element.label["xml:lang"]
-        labelLang = languageRemap[labelLang] ?? labelLang
-        const value = element.label.value
-        if(!perId.has(id)){
-            perId.set(id, new Map<string, string>())
-        }
-        perId.get(id).set(labelLang, value)
-    }
-
-    console.log("Got "+perId.size+" languages")
-    return perId
-}
-
 function getNativeList(langs: Map<string, Map<string, string>>){
     const native = {}
     const keys: string[] = Array.from(langs.keys())
     keys.sort()
     for (const key of keys) {
         const translations: Map<string, string> = langs.get(key)
-        if(!usedLanguages.has(key)){
+        if(!LanguageUtils.usedLanguages.has(key)){
             continue
         }
         native[key] = translations.get(key)
@@ -143,17 +115,17 @@ async function main(wipeCache = false){
         console.log("Reusing the cached file")
     }
     const data = JSON.parse(readFileSync( cacheFile, "UTF8"))
-    const perId = extract(data)
+    const perId = WikidataUtils.extractLanguageData(data, WikidataUtils.languageRemapping)
     const nativeList = getNativeList(perId)
     writeFileSync("./assets/language_native.json", JSON.stringify(nativeList, null, "  "))
     
 
     const translations = Utils.MapToObj(perId, (value, key) => {
-        if(!usedLanguages.has(key)){
+        if(!LanguageUtils.usedLanguages.has(key)){
             return undefined // Remove unused languages
         }
         return Utils.MapToObj(value, (v, k ) => {
-            if(!usedLanguages.has(k)){
+            if(!LanguageUtils.usedLanguages.has(k)){
                 return undefined
             }
             return v

scripts/generateLayerOverview.ts

@@ -20,6 +20,7 @@ import {PrepareLayer} from "../Models/ThemeConfig/Conversion/PrepareLayer";
 import {PrepareTheme} from "../Models/ThemeConfig/Conversion/PrepareTheme";
 import {DesugaringContext} from "../Models/ThemeConfig/Conversion/Conversion";
 import {Utils} from "../Utils";
+import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
 
 // This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
 // It spits out an overview of those to be used to load them
@@ -216,7 +217,7 @@ class LayerOverviewUtils {
         writeFileSync("./assets/generated/known_layers.json", JSON.stringify({layers: Array.from(sharedLayers.values())}))
 
 
-        if (recompiledThemes.length > 0) {
+        if (recompiledThemes.length > 0 && !(recompiledThemes.length === 1 && recompiledThemes[0] === "mapcomplate-changes")) {
             // mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
             const iconsPerTheme =
                 Array.from(sharedThemes.values()).map(th => ({
@@ -231,6 +232,10 @@ class LayerOverviewUtils {
         }
 
         this.checkAllSvgs()
+        
+        if(AllKnownLayouts.getSharedLayersConfigs().size == 0){
+            throw "This was a bootstrapping-run. Run generate layeroverview again!"
+        }
 
         const green = s => '\x1b[92m' + s + '\x1b[0m'
         console.log(green("All done!"))
@@ -242,11 +247,11 @@ class LayerOverviewUtils {
         console.log("   ---------- VALIDATING BUILTIN LAYERS ---------")
 
         const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist);
-        const sharedLayers = new Map<string, LayerConfigJson>()
         const state: DesugaringContext = {
             tagRenderings: sharedTagRenderings,
-            sharedLayers
+            sharedLayers: AllKnownLayouts.getSharedLayersConfigs()
         }
+        const sharedLayers = new Map<string, LayerConfigJson>()
         const prepLayer = new PrepareLayer(state);
         const skippedLayers: string[] = []
         const recompiledLayers: string[] = []
@@ -258,6 +263,7 @@ class LayerOverviewUtils {
                     const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
                     sharedLayers.set(sharedLayer.id, sharedLayer)
                     skippedLayers.push(sharedLayer.id)
+                    console.log("Loaded "+sharedLayer.id)
                     continue;
                 }
 

scripts/thieves/readIdPresets.ts

@@ -1,16 +1,12 @@
 /***
  * Parses presets from the iD repository and extracts some usefull tags from them
  */
-import ScriptUtils from "./ScriptUtils";
+import ScriptUtils from "../ScriptUtils";
 import {existsSync, readFileSync, writeFileSync} from "fs";
-import * as known_languages from "../assets/language_native.json"
-import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
-import {
-    MappingConfigJson,
-
-    QuestionableTagRenderingConfigJson
-} from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
-import SmallLicense from "../Models/smallLicense";
+import * as known_languages from "../../assets/language_native.json"
+import {LayerConfigJson} from "../../Models/ThemeConfig/Json/LayerConfigJson";
+import {MappingConfigJson} from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
+import SmallLicense from "../../Models/smallLicense";
 
 interface IconThief {
     steal(iconName: string): boolean

scripts/thieves/stealLanguages.ts

@@ -0,0 +1,86 @@
+/*
+* Uses the languages in and to every translation from wikidata to generate a language question in wikidata/wikidata
+* */
+
+import WikidataUtils from "../../Utils/WikidataUtils";
+import {existsSync, mkdirSync, readFileSync, writeFileSync} from "fs";
+import {LayerConfigJson} from "../../Models/ThemeConfig/Json/LayerConfigJson";
+import {MappingConfigJson} from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
+import LanguageUtils from "../../Utils/LanguageUtils";
+
+function main(){
+    const sourcepath = "assets/generated/languages-wd.json";
+    console.log(`Converting language data file '${sourcepath}' into a tagMapping`)
+    const languages = WikidataUtils.extractLanguageData(JSON.parse(readFileSync(sourcepath, "utf8")), {})
+    const mappings : MappingConfigJson[] = []
+    const schoolmappings : MappingConfigJson[] = []
+
+    languages.forEach((l, code) => {
+        const then : Record<string, string>= {}
+        l.forEach((tr, lng) => {
+            const languageCodeWeblate = WikidataUtils.languageRemapping[lng] ?? lng;
+            if(!LanguageUtils.usedLanguages.has(languageCodeWeblate)){
+                return;
+            }
+            then[languageCodeWeblate] = tr
+        })
+        mappings.push(<MappingConfigJson>{
+            if: "language:" + code + "=yes",
+            ifnot: "language:" + code + "=",
+            searchTerms: {
+                "*": [code]
+            },
+            then
+        })
+
+       schoolmappings.push(<MappingConfigJson>{
+            if: "school:language=" + code,
+            then
+        })
+    })
+    
+   
+    const wikidataLayer = <LayerConfigJson>{
+        id: "wikidata",
+        description: "Various tagrenderings which are generated from Wikidata. Automatically generated with a script, don't edit manually",
+        "#dont-translate": "*",
+        "source": {
+            "osmTags": "id~*"
+        },
+        "mapRendering": null,
+        tagRenderings: [
+            {
+                id: "language",
+                // @ts-ignore
+                description: "Enables to pick *a single* 'language:<lng>=yes' within the mappings",
+                mappings,
+            },
+            {
+                builtin: "wikidata.language",
+                override: {
+                    id: "language-multi",
+                    // @ts-ignore
+                    description: "Enables to pick *multiple* 'language:<lng>=yes' within the mappings",
+                    multiAnswer: true
+                }
+               
+            },
+            {
+                id:"school-language",
+                // @ts-ignore
+                description: "Enables to pick a single 'school:language=<lng>' within the mappings",
+                multiAnswer: true,
+                mappings: schoolmappings
+            }
+        ]
+    }
+    const dir = "./assets/layers/wikidata/"
+    if(!existsSync(dir)){
+        mkdirSync(dir)
+    }
+    const path = dir + "wikidata.json"
+    writeFileSync(path, JSON.stringify(wikidataLayer, null, "  "))
+    console.log("Written "+path)
+}
+
+main()