midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Download official languages per country

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-06-24 16:49:03 +02:00
parent e1faf5595d
commit 5aaf57137c
4 changed files with 1660 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

72
scripts/fetchLanguages.ts
View file

@ -7,6 +7,9 @@ import {Utils} from "../Utils";
import ScriptUtils from "./ScriptUtils";
import {existsSync, readFileSync, writeFileSync} from "fs";
import * as used_languages from "../assets/generated/used_languages.json"
import {QuestionableTagRenderingConfigJson} from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
const languageRemap = {
// MapComplete (or weblate) on the left, language of wikimedia on the right
"nb":"nb_NO",
@ -25,7 +28,6 @@ async function fetch(target: string){
async function fetchRegularLanguages() {
ScriptUtils.fixUtils()
console.log("Fetching languages")
const sparql = 'SELECT ?lang ?label ?code \n' +
@ -67,7 +69,6 @@ async function fetchSpecial(id: number, code: string) {
'} '
const url = wds.sparqlQuery(sparql)
// request the generated URL with your favorite HTTP request library
const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
const bindings = result.results.bindings
bindings.forEach(binding => binding["code"] = {value: code})
@ -107,6 +108,32 @@ function getNativeList(langs: Map<string, Map<string, string>>){
return native
}
async function getOfficialLanguagesPerCountry() : Promise<Map<string, string[]>>{
const lngs = new Map<string, string[]>();
const sparql = `SELECT ?country ?countryLabel ?countryCode ?language ?languageCode ?languageLabel
WHERE
{
?country wdt:P31/wdt:P279* wd:Q3624078;
wdt:P297 ?countryCode;
wdt:P37 ?language.
?language wdt:P218 ?languageCode.
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
}`
const url = wds.sparqlQuery(sparql)
const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
const bindings : {countryCode: {value: string}, languageCode: {value: string}}[]= result.results.bindings
for (const binding of bindings) {
const countryCode = binding.countryCode.value
const language = binding.languageCode.value
if(lngs.get(countryCode) === undefined){
lngs.set(countryCode, [])
}
lngs.get(countryCode).push(language)
}
return lngs;
}
async function main(wipeCache = false){
const cacheFile = "./assets/generated/languages-wd.json"
if(wipeCache || !existsSync(cacheFile)){
@ -121,7 +148,7 @@ async function main(wipeCache = false){
writeFileSync("./assets/language_native.json", JSON.stringify(nativeList, null, " "))
const translations = Utils.MapToObj<Map<string, string>>(perId, (value, key) => {
const translations = Utils.MapToObj(perId, (value, key) => {
if(!usedLanguages.has(key)){
return undefined // Remove unused languages
}
@ -135,7 +162,44 @@ async function main(wipeCache = false){
writeFileSync("./assets/language_translations.json",
JSON.stringify(translations, null, " "))
let officialLanguages : Record<string, string[]>;
const officialLanguagesPath = "./assets/language_in_country.json"
if(existsSync("./assets/languages_in_country.json") && !wipeCache){
officialLanguages = JSON.parse(readFileSync(officialLanguagesPath, "utf8"))
}else {
officialLanguages = Utils.MapToObj(await getOfficialLanguagesPerCountry(), t => t)
writeFileSync(officialLanguagesPath, JSON.stringify(officialLanguages, null, " "))
}
const perLanguage = Utils.TransposeMap(officialLanguages);
console.log(JSON.stringify(perLanguage, null, " "))
const mappings: {if: string, then: Record<string, string>, hideInAnswer: string}[] = []
for (const language of Object.keys(perLanguage)) {
const countries = Utils.Dedup(perLanguage[language].map(c => c.toLowerCase()))
mappings.push({
if: "language="+language,
then: translations[language],
hideInAnswer : "_country="+countries.join("|")
})
}
const tagRenderings = <QuestionableTagRenderingConfigJson> {
id: "official-language",
mappings,
question: "What languages are spoken here?"
}
writeFileSync("./assets/layers/language/language.json", JSON.stringify(<LayerConfigJson>{
id:"language",
description: "Various tagRenderings to help language tooling",
tagRenderings
}, null, " "))
}
const forceRefresh = process.argv[2] === "--force-refresh"
main(forceRefresh).then(() => console.log("Done!"))
ScriptUtils.fixUtils()
main(forceRefresh).then(() => console.log("Done!"))