Merge master

This commit is contained in:
Pieter Vander Vennet 2024-07-28 02:38:24 +02:00
commit 1b01f75905
186 changed files with 4169 additions and 2235 deletions

View file

@ -54,15 +54,15 @@ class ToSlideshowJson {
sections.push(currentSection)
currentSection = []
}
line = line.replace("src=\"../../public/", "src=\"./")
line = line.replace("src=\"../../", "src=\"./")
line = line.replace('src="../../public/', 'src="./')
line = line.replace('src="../../', 'src="./')
currentSection.push(line)
}
sections.push(currentSection)
writeFileSync(
this._target,
JSON.stringify({
sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0)
sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0),
})
)
}
@ -81,7 +81,7 @@ class WikiPageGenerator {
generate() {
let wikiPage =
"{|class=\"wikitable sortable\"\n" +
'{|class="wikitable sortable"\n' +
"! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" +
"|-"
@ -139,7 +139,7 @@ export class GenerateDocs extends Script {
}
this.WriteMarkdownFile("./Docs/Tags_format.md", TagUtils.generateDocs(), [
"src/Logic/Tags/TagUtils.ts"
"src/Logic/Tags/TagUtils.ts",
])
new ToSlideshowJson(
@ -165,24 +165,20 @@ export class GenerateDocs extends Script {
})
this.WriteMarkdownFile("./Docs/SpecialRenderings.md", SpecialVisualizations.HelpMessage(), [
"src/UI/SpecialVisualizations.ts"
"src/UI/SpecialVisualizations.ts",
])
this.WriteMarkdownFile(
"./Docs/CalculatedTags.md",
[
"# Metatags",
SimpleMetaTaggers.HelpText(),
ExtraFunctions.HelpText()
].join("\n"),
["# Metatags", SimpleMetaTaggers.HelpText(), ExtraFunctions.HelpText()].join("\n"),
["src/Logic/SimpleMetaTagger.ts", "src/Logic/ExtraFunctions.ts"]
)
this.WriteMarkdownFile("./Docs/SpecialInputElements.md", Validators.HelpText(), [
"src/UI/InputElement/Validators.ts"
"src/UI/InputElement/Validators.ts",
])
this.WriteMarkdownFile("./Docs/ChangesetMeta.md", Changes.getDocs(), [
"src/Logic/Osm/Changes.ts",
"src/Logic/Osm/ChangesetHandler.ts"
"src/Logic/Osm/ChangesetHandler.ts",
])
new WikiPageGenerator().generate()
@ -225,20 +221,21 @@ export class GenerateDocs extends Script {
const warnAutomated =
"[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)\n\n"
const generatedFrom =
[
"This document is autogenerated from",
autogenSource.map(s => `[${s}](https://github.com/pietervdvn/MapComplete/blob/develop/${s})`).join(", ")
].join(" ")
const generatedFrom = [
"This document is autogenerated from",
autogenSource
.map((s) => `[${s}](https://github.com/pietervdvn/MapComplete/blob/develop/${s})`)
.join(", "),
].join(" ")
writeFileSync(filename, warnAutomated + md + "\n\n" + generatedFrom + "\n")
}
private generateHotkeyDocs() {
new ThemeViewState(new LayoutConfig(<any>bookcases), new Set())
this.WriteMarkdownFile("./Docs/Hotkeys.md", Hotkeys.generateDocumentation(), ["src/UI/Base/Hotkeys.ts"])
this.WriteMarkdownFile("./Docs/Hotkeys.md", Hotkeys.generateDocumentation(), [
"src/UI/Base/Hotkeys.ts",
])
}
private generateBuiltinUnits() {
@ -271,7 +268,7 @@ export class GenerateDocs extends Script {
}
this.WriteMarkdownFile("./Docs/builtin_units.md", ["# Units", ...els].join("\n\n"), [
`assets/layers/unit/unit.json`
`assets/layers/unit/unit.json`,
])
}
@ -438,39 +435,41 @@ export class GenerateDocs extends Script {
}
private generateForTheme(theme: LayoutConfig): void {
const allLayers = AllSharedLayers.getSharedLayersConfigs()
const layersToShow = theme.layers
.filter((l) => !l.id.startsWith("note_import_") && l.id !== "favourite" && Constants.added_by_default.indexOf(<any>l.id) < 0)
const layersToInline = layersToShow.filter(l => !allLayers.has(l.id))
const layersToShow = theme.layers.filter(
(l) =>
!l.id.startsWith("note_import_") &&
l.id !== "favourite" &&
Constants.added_by_default.indexOf(<any>l.id) < 0
)
const layersToInline = layersToShow.filter((l) => !allLayers.has(l.id))
const el = [
["##",
[
"##",
theme.title,
"(",
`[${theme.id}](https://mapcomplete.org/${theme.id})`,
")"
")",
].join(" "),
"_This document details some technical information about this MapComplete theme, mostly about the attributes used in the theme. Various links point toward more information about the attributes, e.g. to the OpenStreetMap-wiki, to TagInfo or tools creating statistics_",
"The theme introduction reads:\n",
"> "+parse_html(theme.description.textFor("en")).textContent.replace(/\n/g, " "),
"> " + parse_html(theme.description.textFor("en")).textContent.replace(/\n/g, " "),
"",
"This theme contains the following layers:",
MarkdownUtils.list(
layersToShow
.map((l) => {
if (allLayers.has(l.id)) {
return (`[${l.id}](../Layers/${l.id}.md)`)
}
return (`[${l.id} (defined in this theme)](#${l.id.trim().replace(/ /g, "-")})`)
})
layersToShow.map((l) => {
if (allLayers.has(l.id)) {
return `[${l.id}](../Layers/${l.id}.md)`
}
return `[${l.id} (defined in this theme)](#${l.id.trim().replace(/ /g, "-")})`
})
),
"Available languages:",
MarkdownUtils.list(theme.language.filter((ln) => ln !== "_context")),
"# Layers defined in this theme configuration file",
"These layers can not be reused in different themes.",
...layersToInline.map(l => l.GenerateDocumentation(null))
...layersToInline.map((l) => l.GenerateDocumentation(null)),
].join("\n")
this.WriteMarkdownFile(
"./Docs/Themes/" + theme.id + ".md",
@ -530,7 +529,9 @@ export class GenerateDocs extends Script {
"# Special and other useful layers",
"MapComplete has a few data layers available in the theme which have special properties through builtin-hooks. Furthermore, there are some normal layers (which are built from normal Theme-config files) but are so general that they get a mention here.",
"# Priviliged layers",
MarkdownUtils.list(Constants.priviliged_layers.map((id) => "[" + id + "](#" + id + ")")),
MarkdownUtils.list(
Constants.priviliged_layers.map((id) => "[" + id + "](#" + id + ")")
),
...Utils.NoNull(
Constants.priviliged_layers.map((id) => AllSharedLayers.sharedLayers.get(id))
).map((l) =>
@ -548,9 +549,11 @@ export class GenerateDocs extends Script {
Array.from(AllSharedLayers.sharedLayers.keys()).map(
(id) => `[${id}](./Layers/${id}.md)`
)
)
),
].join("\n\n")
this.WriteMarkdownFile("./Docs/BuiltinLayers.md", el, ["src/Customizations/AllKnownLayouts.ts"])
this.WriteMarkdownFile("./Docs/BuiltinLayers.md", el, [
"src/Customizations/AllKnownLayouts.ts",
])
}
}

View file

@ -73,9 +73,13 @@ export class GenerateFavouritesLayer extends Script {
}
private addTagRenderings(proto: LayerConfigJson) {
const addedByDefault = (<{labels: string[], id: string}[]> questions.tagRenderings)
.filter(tr => tr?.["labels"]?.indexOf("added_by_default") > 0 || tr?.["labels"]?.indexOf("added_by_default_top") > 0 )
.map(tr => tr.id)
const addedByDefault = (<{ labels: string[]; id: string }[]>questions.tagRenderings)
.filter(
(tr) =>
tr?.["labels"]?.indexOf("added_by_default") > 0 ||
tr?.["labels"]?.indexOf("added_by_default_top") > 0
)
.map((tr) => tr.id)
const blacklistedIds = new Set([
"images",
"questions",
@ -87,7 +91,7 @@ export class GenerateFavouritesLayer extends Script {
"delete-button",
"all-tags",
"all_tags",
...addedByDefault
...addedByDefault,
])
const generatedTagRenderings: (string | QuestionableTagRenderingConfigJson)[] = []
@ -236,8 +240,8 @@ export class GenerateFavouritesLayer extends Script {
if (seenTitleIcons.has(titleIcon.id)) {
continue
}
if(titleIcon.id === undefined){
continue
if (titleIcon.id === undefined) {
continue
}
seenTitleIcons.add(titleIcon.id)
console.log("Adding title icon", titleIcon.id)

View file

@ -24,7 +24,7 @@ function genImages(dryrun = false) {
.replace(/\n/g, " ")
.replace(/\r/g, "")
.replace(/\\/g, "\\")
.replace(/"/g, "\\\"")
.replace(/"/g, '\\"')
.replaceAll(" ", " ")
let hasNonAsciiChars = Array.from(svg)
@ -38,20 +38,18 @@ function genImages(dryrun = false) {
const nameUC = name.toUpperCase().at(0) + name.substring(1)
const svelteCode =
"<script>\nexport let color = \"#000000\"\n</script>\n" +
'<script>\nexport let color = "#000000"\n</script>\n' +
svg
.replace(
"<svg ",
"<svg {...$$$$restProps} on:click on:mouseover on:mouseenter on:mouseleave on:keydown on:focus ",
"<svg {...$$$$restProps} on:click on:mouseover on:mouseenter on:mouseleave on:keydown on:focus "
)
.replace(/\\"/g, "\"")
.replace(/\\"/g, '"')
.replace(/(rgb\(0%,0%,0%\)|#000000|#000)/g, "{color}")
fs.writeFileSync("./src/assets/svg/" + nameUC + ".svelte", svelteCode, "utf8")
}
}
class GenerateIncludedImages extends Script {
constructor() {
super("Converts all images from assets/svg into svelte-classes.")
@ -59,7 +57,6 @@ class GenerateIncludedImages extends Script {
async main(args: string[]): Promise<void> {
genImages()
}
}

View file

@ -332,7 +332,6 @@ class LayerOverviewUtils extends Script {
return <QuestionableTagRenderingConfigJson[]>sharedQuestions.tagRenderings
}
return this.getSharedTagRenderings(
doesImageExist,
dict,
@ -807,7 +806,11 @@ class LayerOverviewUtils extends Script {
ScriptUtils.ReadSvgSync(themeFile.icon, (svg) => {
const width: string = svg["$"].width
if (width === undefined) {
throw "The logo at " + themeFile.icon + " does not have a defined width"
throw (
"The logo at " +
themeFile.icon +
" does not have a defined width"
)
}
const height: string = svg["$"].height
const err = themeFile.hideFromOverview ? console.warn : console.error
@ -818,9 +821,12 @@ class LayerOverviewUtils extends Script {
err(e)
}
if (width?.endsWith("%")) {
throw "The logo at " + themeFile.icon + " has a relative width; this is not supported"
throw (
"The logo at " +
themeFile.icon +
" has a relative width; this is not supported"
)
}
const w = parseInt(width)

View file

@ -457,27 +457,26 @@ class GenerateLayouts extends Script {
let ogImage = layout.socialImage
let twitterImage = ogImage
if (ogImage === LayoutConfig.defaultSocialImage && layout.official) {
try{
try {
ogImage = (await this.createSocialImage(layout, "")) ?? layout.socialImage
twitterImage = (await this.createSocialImage(layout, "Wide")) ?? layout.socialImage
}catch (e) {
} catch (e) {
console.error("Could not generate image:", e)
}
}
if (twitterImage.endsWith(".svg")) {
try{
// svgs are badly supported as social image, we use a generated svg instead
twitterImage = await this.createIcon(twitterImage, 512, alreadyWritten)
}catch (e) {
try {
// svgs are badly supported as social image, we use a generated svg instead
twitterImage = await this.createIcon(twitterImage, 512, alreadyWritten)
} catch (e) {
console.error("Could not generate image:", e)
}
}
if (ogImage.endsWith(".svg")) {
try{
try {
ogImage = await this.createIcon(ogImage, 512, alreadyWritten)
}catch (e) {
} catch (e) {
console.error("Could not generate image:", e)
}
}

View file

@ -9,7 +9,6 @@ import { ImmutableStore } from "../src/Logic/UIEventSource"
import { Utils } from "../src/Utils"
class HandleErrors extends Script {
constructor() {
super("Inspects the errors made on a given day. Argument: path to errors")
}
@ -30,9 +29,10 @@ class HandleErrors extends Script {
continue
}
try {
const parsed: {
ip: string, index: number, date: string,
ip: string
index: number
date: string
message: {
stacktrace: string
message: string
@ -51,18 +51,25 @@ class HandleErrors extends Script {
}
console.log(e.username, e.layout, e.message, parsed.date)
for (const pendingChange of e.pendingChanges) {
console.log("\t https://osm.org/" + pendingChange.type + "/" + pendingChange.id, pendingChange.meta.changeType, pendingChange.doDelete ? "DELETE" : "")
console.log(
"\t https://osm.org/" + pendingChange.type + "/" + pendingChange.id,
pendingChange.meta.changeType,
pendingChange.doDelete ? "DELETE" : ""
)
}
const neededIds = Changes.GetNeededIds(e.pendingChanges)
// We _do not_ pass in the Changes object itself - we want the data from OSM directly in order to apply the changes
const osmObjects: { id: string, osmObj: OsmObject | "deleted" }[] = await Promise.all<{
id: string;
osmObj: OsmObject | "deleted"
}>(
neededIds.map(async id => ({ id, osmObj: await downloader.DownloadObjectAsync(id) }))
)
const osmObjects: { id: string; osmObj: OsmObject | "deleted" }[] =
await Promise.all<{
id: string
osmObj: OsmObject | "deleted"
}>(
neededIds.map(async (id) => ({
id,
osmObj: await downloader.DownloadObjectAsync(id),
}))
)
const objects = osmObjects
.filter((obj) => obj.osmObj !== "deleted")
@ -70,40 +77,53 @@ class HandleErrors extends Script {
const { toUpload, refused } = Changes.fragmentChanges(e.pendingChanges, objects)
const changes: {
newObjects: OsmObject[]
modifiedObjects: OsmObject[]
deletedObjects: OsmObject[]
} = new Changes({
dryRun: new ImmutableStore(true),
osmConnection
osmConnection,
}).CreateChangesetObjects(toUpload, objects)
const changeset = Changes.createChangesetFor("", changes)
const path = "error_changeset_" + parsed.index + "_" + e.layout + "_" + e.username + ".osc"
if(changeset === "<osmChange version='0.6' generator='Mapcomplete 0.44.7'></osmChange>"){
console.log("Changes for "+parsed.index+": empty changeset, not creating a file for it")
}else if (createdChangesets.has(changeset)) {
console.log("Changeset " + parsed.index + " is identical to previously seen changeset, not writing to file")
const path =
"error_changeset_" + parsed.index + "_" + e.layout + "_" + e.username + ".osc"
if (
changeset ===
"<osmChange version='0.6' generator='Mapcomplete 0.44.7'></osmChange>"
) {
console.log(
"Changes for " +
parsed.index +
": empty changeset, not creating a file for it"
)
} else if (createdChangesets.has(changeset)) {
console.log(
"Changeset " +
parsed.index +
" is identical to previously seen changeset, not writing to file"
)
} else {
writeFileSync(path, changeset, "utf8")
createdChangesets.add(changeset)
}
const refusedContent = JSON.stringify(refused)
if (refusedFiles.has(refusedContent)) {
console.log("Refused changes for " + parsed.index + " is identical to previously seen changeset, not writing to file")
console.log(
"Refused changes for " +
parsed.index +
" is identical to previously seen changeset, not writing to file"
)
} else {
writeFileSync(path + ".refused.json", refusedContent, "utf8")
refusedFiles.add(refusedContent)
}
console.log("Written", path, "with " + e.pendingChanges.length + " changes")
} catch (e) {
console.log("Parsing line failed:", e)
}
}
}
}

View file

@ -6,7 +6,6 @@ import ScriptUtils from "./ScriptUtils"
import { IncomingMessage } from "node:http"
class ServerErrorReport extends Script {
private errorReport = 0
constructor() {
@ -14,7 +13,8 @@ class ServerErrorReport extends Script {
}
private getFilename(logDirectory: string, d: Date): string {
return logDirectory +
return (
logDirectory +
"/" +
d.getUTCFullYear() +
"_" +
@ -22,11 +22,18 @@ class ServerErrorReport extends Script {
"_" +
d.getUTCDate() +
".lines.json"
)
}
public reportError(path: string, queryParams: URLSearchParams, req: IncomingMessage, body: string | undefined, logDirectory: string): string {
public reportError(
path: string,
queryParams: URLSearchParams,
req: IncomingMessage,
body: string | undefined,
logDirectory: string
): string {
if (!body) {
throw "{\"error\": \"No body; use a post request\"}"
throw '{"error": "No body; use a post request"}'
}
console.log(body)
const ip = <string>req.headers["x-forwarded-for"]
@ -39,14 +46,13 @@ class ServerErrorReport extends Script {
const d = new Date()
const file = this.getFilename(logDirectory, d)
const date = d.toISOString()
const contents =
"\n" + JSON.stringify({ ip, index: this.errorReport, date, message: body })
const contents = "\n" + JSON.stringify({ ip, index: this.errorReport, date, message: body })
if (!existsSync(file)) {
writeFileSync(file, contents)
} else {
appendFileSync(file, contents)
}
this. errorReport++
this.errorReport++
return `{"status":"ok", "nr": ${this.errorReport}}`
}
@ -58,8 +64,8 @@ class ServerErrorReport extends Script {
console.log("Created this directory")
}
if (!existsSync(logDirectory+"/csp")) {
mkdirSync(logDirectory+"/csp")
if (!existsSync(logDirectory + "/csp")) {
mkdirSync(logDirectory + "/csp")
console.log("Created this directory")
}
@ -75,23 +81,33 @@ class ServerErrorReport extends Script {
errorsToday = contents.split("\n").length
}
return JSON.stringify({
"online": true,
"errors_today": errorsToday,
online: true,
errors_today: errorsToday,
})
},
},
{
mustMatch: "report",
mimetype: "application/json",
handle: async (path: string, queryParams: URLSearchParams, req: IncomingMessage, body: string | undefined) => {
handle: async (
path: string,
queryParams: URLSearchParams,
req: IncomingMessage,
body: string | undefined
) => {
return this.reportError(path, queryParams, req, body, logDirectory)
},
},
{
mustMatch: "csp",
mimetype: "application/json",
handle: async (path: string, queryParams: URLSearchParams, req: IncomingMessage, body: string | undefined) => {
return this.reportError(path, queryParams, req, body, logDirectory+"/csp")
handle: async (
path: string,
queryParams: URLSearchParams,
req: IncomingMessage,
body: string | undefined
) => {
return this.reportError(path, queryParams, req, body, logDirectory + "/csp")
},
},
])

View file

@ -54,16 +54,18 @@ class ServerLdScrape extends Script {
const start = new Date()
const cache: Record<string, { date: Date; contents: any }> = {}
new Server(port, {}, [
{mustMatch: "status",
mimetype: "application/json",
handle: async () => {
return JSON.stringify({
online: true,
cached_entries: Object.keys(cache).length,
booted: start,
uptime: Math.floor((new Date().getTime() - start.getTime()) / 1000)
})
}},
{
mustMatch: "status",
mimetype: "application/json",
handle: async () => {
return JSON.stringify({
online: true,
cached_entries: Object.keys(cache).length,
booted: start,
uptime: Math.floor((new Date().getTime() - start.getTime()) / 1000),
})
},
},
{
mustMatch: "extractgraph",
mimetype: "application/ld+json",