Scripts: create script to import layers from studio into official mapcomplete

This commit is contained in:
Pieter Vander Vennet 2025-07-05 04:39:30 +02:00
parent c7b905d1fb
commit db685dc05f
5 changed files with 186 additions and 81 deletions

View file

@ -2,7 +2,7 @@
* Script to download images from Wikimedia Commons, and save them together with license information. * Script to download images from Wikimedia Commons, and save them together with license information.
*/ */
import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs" import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { unescape } from "querystring" import { unescape } from "querystring"
import SmallLicense from "../src/Models/smallLicense" import SmallLicense from "../src/Models/smallLicense"
@ -123,7 +123,7 @@ const templateMapping = {
"Template:CC0": "CC0 1.0", "Template:CC0": "CC0 1.0",
} }
async function main(args: string[]) { export async function main(args: string[]) {
if (args.length < 2) { if (args.length < 2) {
console.log("Usage: downloadCommons.ts <output folder> <url> <?url> <?url> .. ") console.log("Usage: downloadCommons.ts <output folder> <url> <?url> <?url> .. ")
console.log( console.log(
@ -345,4 +345,4 @@ function removeLinks(text: string): string {
return text.replace(/<a.*?>(.*?)<\/a>/g, "$1") return text.replace(/<a.*?>(.*?)<\/a>/g, "$1")
} }
main(process.argv.slice(2)) // main(process.argv.slice(2))

View file

@ -0,0 +1,73 @@
import Script from "./Script"
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
import { DoesImageExist } from "../src/Models/ThemeConfig/Conversion/Validation"
import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext"
import Constants from "../src/Models/Constants"
import { main as downloadCommons } from "./downloadCommons"
import { WikimediaImageProvider } from "../src/Logic/ImageProviders/WikimediaImageProvider"
import { Utils } from "../src/Utils"
import { GenerateLicenseInfo } from "./generateLicenseInfo"
class ImportCustomTheme extends Script {
constructor() {
super("Given the path of a custom layer, will load the layer into mapcomplete as official")
}
async main(args: string[]) {
const path = args[0]
const layerconfig = <LayerConfigJson>JSON.parse(readFileSync(path, "utf-8"))
const id = layerconfig.id
const dirPath = "./assets/layers/" + id
if (!existsSync(dirPath)) {
mkdirSync(dirPath)
}
const imageFinder = new ExtractImages(true)
const theme: ThemeConfigJson = <ThemeConfigJson>{
layers: [layerconfig],
id: "dummy-theme-during-import",
title: {
en: "Dummy",
},
icon: "./assets/svg/plus.svg",
description: {
en: "Dummy",
},
}
const usedImagesAll: {
path: string;
context: string,
location: (string | number)[]
}[] = imageFinder.convert(theme, ConversionContext.construct([], ["checking images"]))
const usedImages = usedImagesAll.filter(img => !img.path.startsWith("./assets"))
.filter(img => Constants.defaultPinIcons.indexOf(img.path) < 0)
console.log(usedImages)
const wm = WikimediaImageProvider.singleton
for (const usedImage of usedImages) {
if (usedImage.path.indexOf("wikimedia") >= 0) {
const filename = WikimediaImageProvider.extractFileName(usedImage.path)
console.log("Canonical URL is", filename)
await downloadCommons([dirPath, "https://commons.wikimedia.org/wiki/File:"+filename])
console.log("Used image context:", usedImage.context)
const replaceAt = (usedImage.location)
.slice(2) // We drop ".layer.0" as we put this in a dummy theme
.map(
breadcrumb => isNaN(Number(breadcrumb)) ? breadcrumb : Number(breadcrumb),
)
console.log("Replacement target location is", replaceAt)
Utils.WalkPath(replaceAt, layerconfig, (_, path) => {
console.log("Found",path, "to replace with", filename,"origvalue:", _)
return `./assets/layers/${id}/${filename}`
})
}
}
writeFileSync("./assets/layers/"+id+"/"+id+".json", JSON.stringify(layerconfig, null, " "))
}
}
new ImportCustomTheme().run()
new GenerateLicenseInfo().run()

View file

@ -43,6 +43,7 @@ export class WikimediaImageProvider extends ImageProvider {
if (filename.startsWith("File:")) { if (filename.startsWith("File:")) {
filename = filename.substring(5) filename = filename.substring(5)
} }
return filename.trim().replace(/\s+/g, "_") return filename.trim().replace(/\s+/g, "_")
} }
@ -51,10 +52,14 @@ export class WikimediaImageProvider extends ImageProvider {
* WikimediaImageProvider.extractFileName("https://commons.wikimedia.org/wiki/File:Somefile.jpg") // => "Somefile.jpg" * WikimediaImageProvider.extractFileName("https://commons.wikimedia.org/wiki/File:Somefile.jpg") // => "Somefile.jpg"
* WikimediaImageProvider.extractFileName("https://commons.wikimedia.org/wiki/File:S%C3%A8vres%20-%20square_madame_de_Pompadour_-_bo%C3%AEte_%C3%A0_livres.jpg?uselang=en") // => "Sèvres_-_square_madame_de_Pompadour_-_boîte_à_livres.jpg" * WikimediaImageProvider.extractFileName("https://commons.wikimedia.org/wiki/File:S%C3%A8vres%20-%20square_madame_de_Pompadour_-_bo%C3%AEte_%C3%A0_livres.jpg?uselang=en") // => "Sèvres_-_square_madame_de_Pompadour_-_boîte_à_livres.jpg"
*/ */
private static extractFileName(url: string) { public static extractFileName(url: string) {
if (!url.startsWith("http")) { if (!url.startsWith("http")) {
return url return url
} }
const thumbMatch = url.match(this.thumbUrlRegex)
if (thumbMatch) {
return thumbMatch[1]
}
const path = decodeURIComponent(new URL(url).pathname) const path = decodeURIComponent(new URL(url).pathname)
return WikimediaImageProvider.makeCanonical(path.substring(path.lastIndexOf("/") + 1)) return WikimediaImageProvider.makeCanonical(path.substring(path.lastIndexOf("/") + 1))
} }
@ -76,16 +81,17 @@ export class WikimediaImageProvider extends ImageProvider {
return WikimediaImageProvider.commonsPrefixes.some((prefix) => value.startsWith(prefix)) return WikimediaImageProvider.commonsPrefixes.some((prefix) => value.startsWith(prefix))
} }
private static readonly thumbUrlRegex = /^https:\/\/upload.wikimedia.org\/.*\/([^/]+)\/.*-\1/
private static removeCommonsPrefix(value: string): string { private static removeCommonsPrefix(value: string): string {
if (value.startsWith("https://upload.wikimedia.org/")) { if (value.startsWith("https://upload.wikimedia.org/")) {
value = value.substring(value.lastIndexOf("/") + 1) value = value.substring(value.lastIndexOf("/") + 1)
value = decodeURIComponent(value) value = decodeURIComponent(value)
if (!value.startsWith("File:")) {
value = "File:" + value
}
return value
} }
if (value.startsWith("File:")) {
return value
}
for (const prefix of WikimediaImageProvider.commonsPrefixes) { for (const prefix of WikimediaImageProvider.commonsPrefixes) {
if (value.startsWith(prefix)) { if (value.startsWith(prefix)) {
let part = value.substr(prefix.length) let part = value.substr(prefix.length)
@ -139,6 +145,9 @@ export class WikimediaImageProvider extends ImageProvider {
* *
* const result = await WikimediaImageProvider.singleton.ExtractUrls("wikimedia_commons", "File:Sèvres_-_square_madame_de_Pompadour_-_boîte_à_livres.jpg") * const result = await WikimediaImageProvider.singleton.ExtractUrls("wikimedia_commons", "File:Sèvres_-_square_madame_de_Pompadour_-_boîte_à_livres.jpg")
* result[0].url_hd // => "https://commons.wikimedia.org/wiki/Special:FilePath/File%3AS%C3%A8vres_-_square_madame_de_Pompadour_-_bo%C3%AEte_%C3%A0_livres.jpg" * result[0].url_hd // => "https://commons.wikimedia.org/wiki/Special:FilePath/File%3AS%C3%A8vres_-_square_madame_de_Pompadour_-_bo%C3%AEte_%C3%A0_livres.jpg"
*
* const result = await WikimediaImageProvider.singleton.ExtractUrls("image", "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e2/Edinburgh_City_police_box_001.jpg/576px-Edinburgh_City_police_box_001.jpg")
* result[0].url_hd // => "https://commons.wikimedia.org/wiki/File:Edinburgh_City_police_box_001.jpg"
*/ */
public async ExtractUrls(key: string, value: string): undefined | Promise<ProvidedImage[]> { public async ExtractUrls(key: string, value: string): undefined | Promise<ProvidedImage[]> {
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value) const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)

View file

@ -10,7 +10,7 @@ import { ConversionContext } from "./ConversionContext"
export class ExtractImages extends Conversion< export class ExtractImages extends Conversion<
ThemeConfigJson, ThemeConfigJson,
{ path: string; context: string }[] { path: string; context: string; location: (string|number)[] }[]
> { > {
private static readonly layoutMetaPaths = metapaths.filter((mp) => { private static readonly layoutMetaPaths = metapaths.filter((mp) => {
const typeHint = mp.hints.typehint const typeHint = mp.hints.typehint
@ -24,11 +24,11 @@ export class ExtractImages extends Conversion<
) )
}) })
private static readonly tagRenderingMetaPaths = tagrenderingmetapaths private static readonly tagRenderingMetaPaths = tagrenderingmetapaths
private _isOfficial: boolean private readonly _isOfficial: boolean
private _sharedTagRenderings: Set<string> private _sharedTagRenderings: Set<string>
constructor(isOfficial: boolean, sharedTagRenderings: Set<string>) { constructor(isOfficial: boolean, sharedTagRenderings: Set<string> = new Set()) {
super("ExctractImages", "Extract all images from a layoutConfig using the meta paths.", []) super("ExtractImages", "Extract all images from a themeConfig using the meta paths.")
this._isOfficial = isOfficial this._isOfficial = isOfficial
this._sharedTagRenderings = sharedTagRenderings this._sharedTagRenderings = sharedTagRenderings
} }
@ -111,8 +111,8 @@ export class ExtractImages extends Conversion<
convert( convert(
json: ThemeConfigJson, json: ThemeConfigJson,
context: ConversionContext context: ConversionContext
): { path: string; context: string }[] { ): { path: string; context: string, location: (string | number)[] }[] {
const allFoundImages: { path: string; context: string }[] = [] const allFoundImages: { path: string; context: string, location: (string | number)[] }[] = []
for (const metapath of ExtractImages.layoutMetaPaths) { for (const metapath of ExtractImages.layoutMetaPaths) {
const mightBeTr = ExtractImages.mightBeTagRendering(<any>metapath) const mightBeTr = ExtractImages.mightBeTagRendering(<any>metapath)
@ -143,7 +143,7 @@ export class ExtractImages extends Conversion<
continue continue
} }
allFoundImages.push({ path: foundImage, context: context + "." + path }) allFoundImages.push({ path: foundImage, context: context + "." + path, location: path })
} else { } else {
// This is a tagRendering. // This is a tagRendering.
// Either every rendered value might be an icon // Either every rendered value might be an icon
@ -177,6 +177,7 @@ export class ExtractImages extends Conversion<
allFoundImages.push({ allFoundImages.push({
path: img.leaf, path: img.leaf,
context: context + "." + path, context: context + "." + path,
location: img.path
}) })
} }
} }
@ -191,6 +192,7 @@ export class ExtractImages extends Conversion<
.map((path) => ({ .map((path) => ({
path, path,
context: context + "." + path, context: context + "." + path,
location:img.path
})) }))
) )
} }
@ -211,12 +213,13 @@ export class ExtractImages extends Conversion<
allFoundImages.push({ allFoundImages.push({
context: context.path.join(".") + "." + foundElement.path.join("."), context: context.path.join(".") + "." + foundElement.path.join("."),
path: foundElement.leaf, path: foundElement.leaf,
location: foundElement.path
}) })
} }
} }
} }
const cleanedImages: { path: string; context: string }[] = [] const cleanedImages: { path: string; context: string, location: (string | number)[] }[] = []
for (const foundImage of allFoundImages) { for (const foundImage of allFoundImages) {
if (foundImage.path.startsWith("<") && foundImage.path.endsWith(">")) { if (foundImage.path.startsWith("<") && foundImage.path.endsWith(">")) {
@ -225,7 +228,8 @@ export class ExtractImages extends Conversion<
const images = Array.from(doc.getElementsByTagName("img")) const images = Array.from(doc.getElementsByTagName("img"))
const paths = images.map((i) => i.getAttribute("src")) const paths = images.map((i) => i.getAttribute("src"))
cleanedImages.push( cleanedImages.push(
...paths.map((path) => ({ path, context: foundImage.context + " (in html)" })) ...paths.map((path) => ({ path, context: foundImage.context + " (in html)",
location: foundImage.location }))
) )
continue continue
} }
@ -242,7 +246,7 @@ export class ExtractImages extends Conversion<
) )
) )
for (const path of allPaths) { for (const path of allPaths) {
cleanedImages.push({ path, context: foundImage.context }) cleanedImages.push({ path, context: foundImage.context , location:foundImage.location})
} }
} }
@ -255,9 +259,8 @@ export class FixImages extends DesugaringStep<ThemeConfigJson> {
constructor(knownImages: Set<string>) { constructor(knownImages: Set<string>) {
super( super(
"fixImages",
"Walks over the entire theme and replaces images to the relative URL. Only works if the ID of the theme is an URL", "Walks over the entire theme and replaces images to the relative URL. Only works if the ID of the theme is an URL",
[],
"fixImages"
) )
this._knownImages = knownImages this._knownImages = knownImages
} }

View file

@ -9,7 +9,7 @@ export class Utils {
public static runningFromConsole = typeof window === "undefined" public static runningFromConsole = typeof window === "undefined"
public static externalDownloadFunction: ( public static externalDownloadFunction: (
url: string, url: string,
headers?: any headers?: any,
) => Promise<{ content: string } | { redirect: string }> ) => Promise<{ content: string } | { redirect: string }>
public static Special_visualizations_tagsToApplyHelpText = `These can either be a tag to add, such as \`amenity=fast_food\` or can use a substitution, e.g. \`addr:housenumber=$number\`. public static Special_visualizations_tagsToApplyHelpText = `These can either be a tag to add, such as \`amenity=fast_food\` or can use a substitution, e.g. \`addr:housenumber=$number\`.
This new point will then have the tags \`amenity=fast_food\` and \`addr:housenumber\` with the value that was saved in \`number\` in the original feature. This new point will then have the tags \`amenity=fast_food\` and \`addr:housenumber\` with the value that was saved in \`number\` in the original feature.
@ -61,7 +61,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
if (Utils.runningFromConsole) { if (Utils.runningFromConsole) {
return return
} }
DOMPurify.addHook("afterSanitizeAttributes", function (node) { DOMPurify.addHook("afterSanitizeAttributes", function(node) {
// set all elements owning target to target=_blank + add noopener noreferrer // set all elements owning target to target=_blank + add noopener noreferrer
const target = node.getAttribute("target") const target = node.getAttribute("target")
if (target) { if (target) {
@ -83,7 +83,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
*/ */
public static ParseVisArgs<T = Record<string, string>>( public static ParseVisArgs<T = Record<string, string>>(
specs: { name: string; defaultValue?: string }[], specs: { name: string; defaultValue?: string }[],
args: string[] args: string[],
): T { ): T {
const parsed: Record<string, string> = {} const parsed: Record<string, string> = {}
if (args.length > specs.length) { if (args.length > specs.length) {
@ -238,7 +238,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
object: any, object: any,
name: string, name: string,
init: () => any, init: () => any,
whenDone?: () => void whenDone?: () => void,
) { ) {
Object.defineProperty(object, name, { Object.defineProperty(object, name, {
enumerable: false, enumerable: false,
@ -266,7 +266,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
object: any, object: any,
name: string, name: string,
init: () => Promise<any>, init: () => Promise<any>,
whenDone?: () => void whenDone?: () => void,
) { ) {
Object.defineProperty(object, name, { Object.defineProperty(object, name, {
enumerable: false, enumerable: false,
@ -334,7 +334,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
*/ */
public static DedupOnId<T = { id: string }>( public static DedupOnId<T = { id: string }>(
arr: T[], arr: T[],
toKey?: (t: T) => string | string[] toKey?: (t: T) => string | string[],
): T[] { ): T[] {
const uniq: T[] = [] const uniq: T[] = []
const seen = new Set<string>() const seen = new Set<string>()
@ -461,7 +461,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static SubstituteKeys( public static SubstituteKeys(
txt: string | undefined, txt: string | undefined,
tags: Record<string, any> | undefined, tags: Record<string, any> | undefined,
useLang?: string useLang?: string,
): string | undefined { ): string | undefined {
if (txt === undefined) { if (txt === undefined) {
return undefined return undefined
@ -493,7 +493,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
"SubstituteKeys received a BaseUIElement to substitute in - this is probably a bug and will be downcast to a string\nThe key is", "SubstituteKeys received a BaseUIElement to substitute in - this is probably a bug and will be downcast to a string\nThe key is",
key, key,
"\nThe value is", "\nThe value is",
v v,
) )
v = v.InnerConstructElement()?.textContent v = v.InnerConstructElement()?.textContent
} }
@ -614,7 +614,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
if (!Array.isArray(targetV)) { if (!Array.isArray(targetV)) {
throw new Error( throw new Error(
"Cannot concatenate: value to add is not an array: " + "Cannot concatenate: value to add is not an array: " +
JSON.stringify(targetV) JSON.stringify(targetV),
) )
} }
if (Array.isArray(sourceV)) { if (Array.isArray(sourceV)) {
@ -622,9 +622,9 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
} else { } else {
throw new Error( throw new Error(
"Could not merge concatenate " + "Could not merge concatenate " +
JSON.stringify(sourceV) + JSON.stringify(sourceV) +
" and " + " and " +
JSON.stringify(targetV) JSON.stringify(targetV),
) )
} }
} else { } else {
@ -660,16 +660,19 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
/** /**
* Walks the specified path into the object till the end. * Walks the specified path into the object till the end.
* *
* If a list is encountered, this is transparently walked recursively on every object. * If a list is encountered, the behaviour depends on the next breadcrumb:
* - if it is a string, the list is transparently walked recursively on every object.
* - it is is a number, that index will be taken
*
* If 'null' or 'undefined' is encountered, this method stops * If 'null' or 'undefined' is encountered, this method stops
* *
* The leaf objects are replaced in the object itself by the specified function. * The leaf objects are replaced in the object itself by the specified function.
*/ */
public static WalkPath( public static WalkPath(
path: string[], path: (string | number)[],
object: any, object: any,
replaceLeaf: (leaf: any, travelledPath: string[]) => any, replaceLeaf: (leaf: any, travelledPath: string[]) => any,
travelledPath: string[] = [] travelledPath: (string)[] = [],
): void { ): void {
if (object == null) { if (object == null) {
return return
@ -699,12 +702,28 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
return return
} }
if (Array.isArray(sub)) { if (Array.isArray(sub)) {
sub.forEach((el, i) => if (typeof path[1] === "number") {
Utils.WalkPath(path.slice(1), el, replaceLeaf, [...travelledPath, head, "" + i]) const i = path[1]
) if(path.length == 2){
// We found the leaf
const leaf = sub[i]
if (leaf !== undefined) {
sub[i] = replaceLeaf(leaf, [...travelledPath, ""+head,""+i])
if (sub[i] === undefined) {
delete sub[i]
}
}
}else{
Utils.WalkPath(path.slice(2), sub[i], replaceLeaf, [...travelledPath, "" + head, "" + i])
}
} else {
sub.forEach((el, i) =>
Utils.WalkPath(path.slice(1), el, replaceLeaf, [...travelledPath, "" + head, "" + i]),
)
}
return return
} }
Utils.WalkPath(path.slice(1), sub, replaceLeaf, [...travelledPath, head]) Utils.WalkPath(path.slice(1), sub, replaceLeaf, [...travelledPath, ""+head])
} }
/** /**
@ -717,7 +736,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
path: string[], path: string[],
object: any, object: any,
collectedList: { leaf: any; path: string[] }[] = [], collectedList: { leaf: any; path: string[] }[] = [],
travelledPath: string[] = [] travelledPath: string[] = [],
): { leaf: any; path: string[] }[] { ): { leaf: any; path: string[] }[] {
if (object === undefined || object === null) { if (object === undefined || object === null) {
return collectedList return collectedList
@ -747,7 +766,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
if (Array.isArray(sub)) { if (Array.isArray(sub)) {
sub.forEach((el, i) => sub.forEach((el, i) =>
Utils.CollectPath(path.slice(1), el, collectedList, [...travelledPath, "" + i]) Utils.CollectPath(path.slice(1), el, collectedList, [...travelledPath, "" + i]),
) )
return collectedList return collectedList
} }
@ -791,7 +810,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
json: any, json: any,
f: (v: object | number | string | boolean | undefined, path: string[]) => any, f: (v: object | number | string | boolean | undefined, path: string[]) => any,
isLeaf: (object) => boolean = undefined, isLeaf: (object) => boolean = undefined,
path: string[] = [] path: string[] = [],
) { ) {
if (json === undefined || json === null) { if (json === undefined || json === null) {
return f(json, path) return f(json, path)
@ -830,7 +849,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
json: any, json: any,
collect: (v: number | string | boolean | undefined, path: string[]) => any, collect: (v: number | string | boolean | undefined, path: string[]) => any,
isLeaf: (object) => boolean = undefined, isLeaf: (object) => boolean = undefined,
path = [] path = [],
): void { ): void {
if (json === undefined) { if (json === undefined) {
return return
@ -875,7 +894,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static async download( public static async download(
url: string, url: string,
headers?: Record<string, string> headers?: Record<string, string>,
): Promise<string | undefined> { ): Promise<string | undefined> {
const result = await Utils.downloadAdvanced(url, headers) const result = await Utils.downloadAdvanced(url, headers)
if (result["error"] !== undefined) { if (result["error"] !== undefined) {
@ -889,7 +908,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
headers?: Record<string, string>, headers?: Record<string, string>,
method: "POST" | "GET" | "PUT" | "UPDATE" | "DELETE" | "OPTIONS" = "GET", method: "POST" | "GET" | "PUT" | "UPDATE" | "DELETE" | "OPTIONS" = "GET",
content?: string, content?: string,
maxAttempts: number = 3 maxAttempts: number = 3,
): Promise< ): Promise<
| { content: string } | { content: string }
| { redirect: string } | { redirect: string }
@ -913,7 +932,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
console.log( console.log(
`Request to ${url} failed, Trying again in a moment. Attempt ${ `Request to ${url} failed, Trying again in a moment. Attempt ${
i + 1 i + 1
}/${maxAttempts}` }/${maxAttempts}`,
) )
await Utils.waitFor((i + 1) * 500) await Utils.waitFor((i + 1) * 500)
} }
@ -927,7 +946,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
url: string, url: string,
headers?: Record<string, string>, headers?: Record<string, string>,
method: "POST" | "GET" | "PUT" | "UPDATE" | "DELETE" | "OPTIONS" = "GET", method: "POST" | "GET" | "PUT" | "UPDATE" | "DELETE" | "OPTIONS" = "GET",
content?: string content?: string,
): Promise< ): Promise<
| { content: string } | { content: string }
| { redirect: string } | { redirect: string }
@ -970,12 +989,12 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
xhr.onerror = (ev: ProgressEvent<EventTarget>) => xhr.onerror = (ev: ProgressEvent<EventTarget>) =>
reject( reject(
"Could not get " + "Could not get " +
url + url +
", xhr status code is " + ", xhr status code is " +
xhr.status + xhr.status +
" (" + " (" +
xhr.statusText + xhr.statusText +
")" ")",
) )
}) })
} }
@ -983,7 +1002,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static upload( public static upload(
url: string, url: string,
data: string | Blob, data: string | Blob,
headers?: Record<string, string> headers?: Record<string, string>,
): Promise<string> { ): Promise<string> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest() const xhr = new XMLHttpRequest()
@ -1012,13 +1031,13 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
url: string, url: string,
maxCacheTimeMs: number, maxCacheTimeMs: number,
headers?: Record<string, string>, headers?: Record<string, string>,
dontCacheErrors: boolean = false dontCacheErrors: boolean = false,
): Promise<T> { ): Promise<T> {
const result = await Utils.downloadJsonCachedAdvanced( const result = await Utils.downloadJsonCachedAdvanced(
url, url,
maxCacheTimeMs, maxCacheTimeMs,
headers, headers,
dontCacheErrors dontCacheErrors,
) )
if (result["content"]) { if (result["content"]) {
return result["content"] return result["content"]
@ -1031,7 +1050,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
maxCacheTimeMs: number, maxCacheTimeMs: number,
headers?: Record<string, string>, headers?: Record<string, string>,
dontCacheErrors = false, dontCacheErrors = false,
maxAttempts = 3 maxAttempts = 3,
): Promise< ): Promise<
{ content: T } | { error: string; url: string; statuscode?: number; errContent?: object } { content: T } | { error: string; url: string; statuscode?: number; errContent?: object }
> { > {
@ -1043,10 +1062,10 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
} }
const promise = const promise =
/*NO AWAIT as we work with the promise directly */ Utils.downloadJsonAdvanced<T>( /*NO AWAIT as we work with the promise directly */ Utils.downloadJsonAdvanced<T>(
url, url,
headers, headers,
maxAttempts maxAttempts,
) )
Utils._download_cache.set(url, { promise, timestamp: new Date().getTime() }) Utils._download_cache.set(url, { promise, timestamp: new Date().getTime() })
try { try {
return await promise return await promise
@ -1060,12 +1079,12 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static async downloadJson<T = object | []>( public static async downloadJson<T = object | []>(
url: string, url: string,
headers?: Record<string, string> headers?: Record<string, string>,
): Promise<T> ): Promise<T>
public static async downloadJson<T>(url: string, headers?: Record<string, string>): Promise<T> public static async downloadJson<T>(url: string, headers?: Record<string, string>): Promise<T>
public static async downloadJson( public static async downloadJson(
url: string, url: string,
headers?: Record<string, string> headers?: Record<string, string>,
): Promise<object | []> { ): Promise<object | []> {
const result = await Utils.downloadJsonAdvanced(url, headers) const result = await Utils.downloadJsonAdvanced(url, headers)
if (result["content"]) { if (result["content"]) {
@ -1085,7 +1104,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static async downloadJsonAdvanced<T = object | []>( public static async downloadJsonAdvanced<T = object | []>(
url: string, url: string,
headers?: Record<string, string>, headers?: Record<string, string>,
maxAttempts = 3 maxAttempts = 3,
): Promise< ): Promise<
{ content: T } | { error: string; url: string; statuscode?: number; errContent?: object } { content: T } | { error: string; url: string; statuscode?: number; errContent?: object }
> { > {
@ -1098,7 +1117,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
Utils.Merge({ accept: "application/json" }, headers ?? {}), Utils.Merge({ accept: "application/json" }, headers ?? {}),
"GET", "GET",
undefined, undefined,
maxAttempts maxAttempts,
) )
if (result["error"] !== undefined) { if (result["error"] !== undefined) {
return <{ error: string; url: string; statuscode?: number }>result return <{ error: string; url: string; statuscode?: number }>result
@ -1121,7 +1140,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
"due to", "due to",
e, e,
"\n", "\n",
e.stack e.stack,
) )
return { error: "malformed", url } return { error: "malformed", url }
} }
@ -1142,7 +1161,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
| "{gpx=application/gpx+xml}" | "{gpx=application/gpx+xml}"
| "application/json" | "application/json"
| "image/png" | "image/png"
} },
) { ) {
const element = document.createElement("a") const element = document.createElement("a")
let file let file
@ -1263,19 +1282,19 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static sortedByLevenshteinDistance( public static sortedByLevenshteinDistance(
reference: string, reference: string,
ts: ReadonlyArray<string> ts: ReadonlyArray<string>,
): string[] ): string[]
public static sortedByLevenshteinDistance<T>( public static sortedByLevenshteinDistance<T>(
reference: string, reference: string,
ts: ReadonlyArray<T>, ts: ReadonlyArray<T>,
getName: (t: T) => string getName: (t: T) => string,
): T[] ): T[]
public static sortedByLevenshteinDistance<T>( public static sortedByLevenshteinDistance<T>(
reference: string, reference: string,
ts: ReadonlyArray<T>, ts: ReadonlyArray<T>,
getName?: (t: T) => string getName?: (t: T) => string,
): T[] { ): T[] {
getName ??= (str) => <string> str; getName ??= (str) => <string>str
const withDistance: [T, number][] = ts.map((t) => [ const withDistance: [T, number][] = ts.map((t) => [
t, t,
Utils.levenshteinDistance(getName(t), reference), Utils.levenshteinDistance(getName(t), reference),
@ -1300,7 +1319,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
track[j][i] = Math.min( track[j][i] = Math.min(
track[j][i - 1] + 1, // deletion track[j][i - 1] + 1, // deletion
track[j - 1][i] + 1, // insertion track[j - 1][i] + 1, // insertion
track[j - 1][i - 1] + indicator // substitution track[j - 1][i - 1] + indicator, // substitution
) )
} }
} }
@ -1310,11 +1329,11 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static MapToObj<V>(d: Map<string, V>): Record<string, V> public static MapToObj<V>(d: Map<string, V>): Record<string, V>
public static MapToObj<V, T>( public static MapToObj<V, T>(
d: Map<string, V>, d: Map<string, V>,
onValue: (t: V, key: string) => T onValue: (t: V, key: string) => T,
): Record<string, T> ): Record<string, T>
public static MapToObj<V, T>( public static MapToObj<V, T>(
d: Map<string, V>, d: Map<string, V>,
onValue: (t: V, key: string) => T = undefined onValue: (t: V, key: string) => T = undefined,
): Record<string, T> { ): Record<string, T> {
const o = {} const o = {}
const keys = Array.from(d.keys()) const keys = Array.from(d.keys())
@ -1332,7 +1351,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
* Utils.TransposeMap({"a" : ["b", "c"], "x" : ["b", "y"]}) // => {"b" : ["a", "x"], "c" : ["a"], "y" : ["x"]} * Utils.TransposeMap({"a" : ["b", "c"], "x" : ["b", "y"]}) // => {"b" : ["a", "x"], "c" : ["a"], "y" : ["x"]}
*/ */
public static TransposeMap<K extends string, V extends string>( public static TransposeMap<K extends string, V extends string>(
d: Record<K, V[]> d: Record<K, V[]>,
): Record<V, K[]> { ): Record<V, K[]> {
const newD: Record<V, K[]> = <any>{} const newD: Record<V, K[]> = <any>{}
@ -1355,7 +1374,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
* {"a": "b", "c":"d"} // => {"b":"a", "d":"c"} * {"a": "b", "c":"d"} // => {"b":"a", "d":"c"}
*/ */
public static transposeMapSimple<K extends string, V extends string>( public static transposeMapSimple<K extends string, V extends string>(
d: Record<K, V> d: Record<K, V>,
): Record<V, K> { ): Record<V, K> {
const inv = <Record<V, K>>{} const inv = <Record<V, K>>{}
for (const k in d) { for (const k in d) {
@ -1438,7 +1457,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
} }
public static asDict( public static asDict(
tags: { key: string; value: string | number }[] tags: { key: string; value: string | number }[],
): Map<string, string | number> { ): Map<string, string | number> {
const d = new Map<string, string | number>() const d = new Map<string, string | number>()
@ -1451,7 +1470,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
public static asRecord<K extends string | number | symbol, V>( public static asRecord<K extends string | number | symbol, V>(
keys: K[], keys: K[],
f: (k: K) => V f: (k: K) => V,
): Record<K, V> { ): Record<K, V> {
const results = <Record<K, V>>{} const results = <Record<K, V>>{}
for (const key of keys) { for (const key of keys) {
@ -1564,7 +1583,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
* *
*/ */
public static splitIntoSubstitutionParts( public static splitIntoSubstitutionParts(
template: string template: string,
): ({ message: string } | { subs: string })[] { ): ({ message: string } | { subs: string })[] {
const preparts = template.split("{") const preparts = template.split("{")
const spec: ({ message: string } | { subs: string })[] = [] const spec: ({ message: string } | { subs: string })[] = []
@ -1738,7 +1757,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
} }
private static findParentWithScrolling( private static findParentWithScrolling(
element: HTMLBaseElement | HTMLDivElement element: HTMLBaseElement | HTMLDivElement,
): HTMLBaseElement | HTMLDivElement { ): HTMLBaseElement | HTMLDivElement {
// Check if the element itself has scrolling // Check if the element itself has scrolling
if (element.scrollHeight > element.clientHeight) { if (element.scrollHeight > element.clientHeight) {
@ -1820,6 +1839,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
href = href.replaceAll(/ /g, "%20") href = href.replaceAll(/ /g, "%20")
return href return href
} }
private static emojiRegex = /[\p{Extended_Pictographic}🛰]/u private static emojiRegex = /[\p{Extended_Pictographic}🛰]/u
/** /**
@ -1855,6 +1875,6 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
for (const key of allKeys) { for (const key of allKeys) {
copy[key] = object[key] copy[key] = object[key]
} }
return <T> copy return <T>copy
} }
} }