Merge master

scripts/build.sh

@@ -1,4 +1,3 @@
-#! /bin/bash
 echo "Starting build"
 # The build script; we build the application step by step as building everything at once takes too much RAM
 # Should be run from the repository root

scripts/fetchLanguages.ts

@@ -7,6 +7,9 @@ import {Utils} from "../Utils";
 import ScriptUtils from "./ScriptUtils";
 import {existsSync, readFileSync, writeFileSync} from "fs";
 import * as used_languages from "../assets/generated/used_languages.json"
+import {QuestionableTagRenderingConfigJson} from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
+import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
+
 const languageRemap = {
     // MapComplete (or weblate) on the left, language of wikimedia on the right
     "nb":"nb_NO",
@@ -25,7 +28,6 @@ async function fetch(target: string){
 
 async function fetchRegularLanguages() {
 
-    ScriptUtils.fixUtils()
     console.log("Fetching languages")
 
     const sparql = 'SELECT ?lang ?label ?code \n' +
@@ -67,7 +69,6 @@ async function fetchSpecial(id: number, code: string) {
         '} '
     const url = wds.sparqlQuery(sparql)
 
-// request the generated URL with your favorite HTTP request library
     const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
     const bindings = result.results.bindings
     bindings.forEach(binding => binding["code"] = {value: code})
@@ -107,6 +108,32 @@ function getNativeList(langs: Map<string, Map<string, string>>){
     return native
 }
 
+async function getOfficialLanguagesPerCountry() : Promise<Map<string, string[]>>{
+    const lngs = new Map<string, string[]>();
+    const sparql = `SELECT ?country ?countryLabel ?countryCode ?language ?languageCode ?languageLabel
+    WHERE
+    {
+            ?country wdt:P31/wdt:P279* wd:Q3624078;
+        wdt:P297 ?countryCode;
+        wdt:P37 ?language.
+            ?language wdt:P218 ?languageCode.
+            SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+    }`
+    const url = wds.sparqlQuery(sparql)
+
+    const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
+    const bindings : {countryCode: {value: string}, languageCode: {value: string}}[]= result.results.bindings
+    for (const binding of bindings) {
+        const countryCode = binding.countryCode.value
+        const language = binding.languageCode.value
+        if(lngs.get(countryCode) === undefined){
+            lngs.set(countryCode, [])
+        }
+        lngs.get(countryCode).push(language)
+    }
+    return lngs;
+}
+
 async function main(wipeCache = false){
     const cacheFile = "./assets/generated/languages-wd.json"
     if(wipeCache || !existsSync(cacheFile)){
@@ -121,7 +148,7 @@ async function main(wipeCache = false){
     writeFileSync("./assets/language_native.json", JSON.stringify(nativeList, null, "  "))
     
 
-    const translations = Utils.MapToObj<Map<string, string>>(perId, (value, key) => {
+    const translations = Utils.MapToObj(perId, (value, key) => {
         if(!usedLanguages.has(key)){
             return undefined // Remove unused languages
         }
@@ -135,7 +162,44 @@ async function main(wipeCache = false){
     
     writeFileSync("./assets/language_translations.json", 
         JSON.stringify(translations, null, "  "))
+    
+    
+    let officialLanguages : Record<string, string[]>;
+    const officialLanguagesPath = "./assets/language_in_country.json"
+    if(existsSync("./assets/languages_in_country.json") && !wipeCache){
+        officialLanguages = JSON.parse(readFileSync(officialLanguagesPath, "utf8"))
+    }else {
+        officialLanguages = Utils.MapToObj(await getOfficialLanguagesPerCountry(), t => t)
+        writeFileSync(officialLanguagesPath, JSON.stringify(officialLanguages, null, "  "))
+    }
+    
+    const perLanguage = Utils.TransposeMap(officialLanguages);
+    console.log(JSON.stringify(perLanguage, null, " "))
+    const mappings: {if: string, then: Record<string, string>, hideInAnswer: string}[] = []
+    for (const language of Object.keys(perLanguage)) {
+        const countries = Utils.Dedup(perLanguage[language].map(c => c.toLowerCase()))
+        mappings.push({
+            if: "language="+language,
+            then: translations[language],
+            hideInAnswer : "_country="+countries.join("|")
+        })
+    }
+    
+    const tagRenderings =  <QuestionableTagRenderingConfigJson> {
+        id: "official-language",
+        mappings,
+        question: "What languages are spoken here?"
+    }
+    
+    writeFileSync("./assets/layers/language/language.json", JSON.stringify(<LayerConfigJson>{
+        id:"language",
+        description: "Various tagRenderings to help language tooling",
+        tagRenderings
+    }, null, "  "))
+    
 }
 
 const forceRefresh = process.argv[2] === "--force-refresh"
-main(forceRefresh).then(() => console.log("Done!"))
+ScriptUtils.fixUtils()
+main(forceRefresh).then(() => console.log("Done!"))
+

scripts/generateLayerOverview.ts

@@ -19,7 +19,6 @@ import {PrepareLayer} from "../Models/ThemeConfig/Conversion/PrepareLayer";
 import {PrepareTheme} from "../Models/ThemeConfig/Conversion/PrepareTheme";
 import {DesugaringContext} from "../Models/ThemeConfig/Conversion/Conversion";
 import {Utils} from "../Utils";
-import {And} from "../Logic/Tags/And";
 
 // This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
 // It spits out an overview of those to be used to load them
@@ -262,6 +261,7 @@ class LayerOverviewUtils {
             tagRenderings: this.getSharedTagRenderings(knownImagePaths),
             publicLayers
         }
+        const nonDefaultLanguages : {theme: string, language: string}[] = []
         for (const themeInfo of themeFiles) {
             let themeFile = themeInfo.parsed
             const themePath = themeInfo.path

scripts/generateStats.ts

@@ -66,8 +66,8 @@ async function main(includeTags = true) {
     writeFileSync("./assets/key_totals.json",
         JSON.stringify(
             {
-                keys: Utils.MapToObj(keyTotal),
-                tags: Utils.MapToObj(tagTotal, v => Utils.MapToObj(v))
+                keys: Utils.MapToObj(keyTotal, t => t),
+                tags: Utils.MapToObj(tagTotal, v => Utils.MapToObj(v, t => t))
             },
             null, "  "
         )

scripts/generateTranslations.ts

@@ -32,6 +32,10 @@ class TranslationPart {
     }
 
     addTranslationObject(translations: any, context?: string) {
+        if(translations["#"] === "no-translations"){
+            console.log("Ignoring object at ",context,"which has '#':'no-translations'")
+            return;
+        }
         for (const translationsKey in translations) {
             if (!translations.hasOwnProperty(translationsKey)) {
                 continue;
@@ -267,6 +271,9 @@ class TranslationPart {
  * @param tr
  */
 function isTranslation(tr: any): boolean {
+    if(tr["#"] === "no-translations") {
+        return false
+    }
     for (const key in tr) {
         if (typeof tr[key] !== "string") {
             return false;