loviuz/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

merge develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-04-09 17:18:30 +02:00
commit 3e4708b0b9
506 changed files with 7945 additions and 74587 deletions
Download patch file Download diff file Expand all files Collapse all files

93
scripts/GenerateSeries.ts
View file

@ -2,10 +2,7 @@ import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../src/Utils"
import Script from "./Script"
import { GeoOperations } from "../src/Logic/GeoOperations"
import { Feature, Polygon } from "geojson"
import { Tiles } from "../src/Models/TileRange"
import { BBox } from "../src/Logic/BBox"
class StatsDownloader {
private readonly urlTemplate =
@ -205,13 +202,6 @@ class GenerateSeries extends Script {
const targetDir = args[0] ?? "../../git/MapComplete-data"
await this.downloadStatistics(targetDir + "/changeset-metadata")
this.generateCenterPoints(
targetDir + "/changeset-metadata",
targetDir + "/mapcomplete-changes/",
{
zoomlevel: 8,
}
)
}
private async downloadStatistics(targetDir: string) {
@ -229,94 +219,19 @@ class GenerateSeries extends Script {
day = Number(process.argv[4])
}
do {
let downloaded = false
while (!downloaded) {
try {
await new StatsDownloader(targetDir).DownloadStats(year, month, day)
break
downloaded = true
} catch (e) {
console.log(e)
}
} while (true)
}
const allFiles = readdirSync(targetDir).filter((p) => p.endsWith(".json"))
writeFileSync(targetDir + "/file-overview.json", JSON.stringify(allFiles))
}
private generateCenterPoints(
sourceDir: string,
targetDir: string,
options: {
zoomlevel: number
}
) {
const allPaths = readdirSync(sourceDir).filter(
(p) => p.startsWith("stats.") && p.endsWith(".json")
)
let allFeatures: ChangeSetData[] = allPaths.flatMap(
(path) => JSON.parse(readFileSync(sourceDir + "/" + path, "utf-8")).features
)
allFeatures = allFeatures.filter(
(f) =>
f?.properties !== undefined &&
(f.properties.editor === null ||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
)
allFeatures = allFeatures.filter(
(f) => f.geometry !== null && f.properties.metadata?.theme !== "EMPTY CS"
)
allFeatures = allFeatures.filter(
(f) =>
f?.properties !== undefined &&
(f.properties.editor === null ||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
)
allFeatures = allFeatures.filter(
(f) => f.properties.metadata?.theme !== "EMPTY CS" && f.geometry.coordinates.length > 0
)
const centerpointsAll = allFeatures.map((f) => {
const centerpoint = GeoOperations.centerpoint(f)
const c = centerpoint.geometry.coordinates
// OsmCha doesn't adhere to the Geojson standard and uses `lat` `lon` as coordinates instead of `lon`, `lat`
centerpoint.geometry.coordinates = [c[1], c[0]]
return centerpoint
})
const centerpoints = centerpointsAll.filter((p) => {
const bbox = BBox.get(p)
if (bbox.minLat === -90 && bbox.maxLat === -90) {
// Due to some bug somewhere, those invalid bboxes might appear if the latitude is < 90
// This crashes the 'spreadIntoBBoxes
// As workaround, we simply ignore them for now
return false
}
return true
})
console.log("Found", centerpoints.length, " changesets in total")
const perBbox = GeoOperations.spreadIntoBboxes(centerpoints, options.zoomlevel)
for (const [tileNumber, features] of perBbox) {
const [z, x, y] = Tiles.tile_from_index(tileNumber)
const path = `${targetDir}/tile_${z}_${x}_${y}.geojson`
features.forEach((f) => {
delete f.bbox
})
writeFileSync(
path,
JSON.stringify(
{
type: "FeatureCollection",
features: features,
},
null,
" "
)
)
ScriptUtils.erasableLog("Written ", path, "which has ", features.length, "features")
}
}
}
new GenerateSeries().run()

2
scripts/downloadEli.ts
View file

@ -25,7 +25,7 @@ class DownloadEli extends Script {
const props = layer.properties
if (props.type === "bing") {
// A lot of work to implement - see https://github.com/pietervdvn/MapComplete/issues/648
// A lot of work to implement - see https://source.mapcomplete.org/MapComplete/MapComplete/issues/648
try {
const bing = await BingRasterLayer.get()
if (bing === "error") {

12
scripts/generateDocs.ts
View file

@ -31,6 +31,7 @@ import MarkdownUtils from "../src/Utils/MarkdownUtils"
import { parse as parse_html } from "node-html-parser"
import { AvailableRasterLayers } from "../src/Models/RasterLayers"
import { ImmutableStore } from "../src/Logic/UIEventSource"
import * as unitUsage from "../Docs/Schemas/UnitConfigJson.schema.json"
/**
* Converts a markdown-file into a .json file, which a walkthrough/slideshow element can use
@ -268,7 +269,12 @@ export class GenerateDocs extends Script {
private generateBuiltinUnits() {
const layer = new LayerConfig(<LayerConfigJson>unit, "units", true)
const els: string[] = ["## " + layer.id]
const els: string[] = [
"# Units",
"## How to use",
unitUsage.description,
"Units ",
"## " + layer.id]
for (const unit of layer.units) {
els.push("### " + unit.quantity)
@ -295,8 +301,8 @@ export class GenerateDocs extends Script {
}
}
this.WriteMarkdownFile("./Docs/builtin_units.md", ["# Units", ...els].join("\n\n"), [
`assets/layers/unit/unit.json`,
this.WriteMarkdownFile("./Docs/builtin_units.md", els.join("\n\n"), [
`assets/layers/unit/unit.json`, `src/Models/ThemeConfig/Json/UnitConfigJson.ts`
])
}

2
scripts/generateLayouts.ts
View file

@ -426,7 +426,7 @@ class GenerateLayouts extends Script {
const csp: Record<string, string> = {
"default-src": "'self'",
"child-src": "'self' blob: ",
"img-src": "* data: blob:", // maplibre depends on 'data:' to load
"img-src": "* data: blob:", // maplibre depends on 'data:' to load; 'blob:' is needed for both 360° stuff and local storage
"report-to": "https://report.mapcomplete.org/csp",
"worker-src": "'self' blob:", // Vite somehow loads the worker via a 'blob'
"style-src": "'self' 'unsafe-inline'", // unsafe-inline is needed to change the default background pin colours

2
scripts/generateTaginfoProjectFiles.ts
View file

@ -74,7 +74,7 @@ function generateLayerUsage(layer: LayerConfig): TagInfoPrototype[] {
const usesImageUpload = (tr.render?.txt?.indexOf("image_upload") ?? -2) > 0
if (usesImageCarousel || usesImageUpload) {
const descrNoUpload = `Images are displayed based on the keys image, image:0, image:1,..., panoramax, panoramax:0, panoramx:1, ... , wikidata, wikipedia, wikimedia_commons and mapillary`
const descrNoUpload = `Images are displayed based on the keys image, image:0, image:1,..., panoramax, panoramax:0, panoramx:1, ... , wikidata, wikipedia, wikimedia_commons and mapillary`
const descrUpload = `${descrNoUpload} Furthermore, this layer shows images based on the keys panoramax, image, wikidata, wikipedia, wikimedia_commons and mapillary`
const shownText = (usesImageUpload ? descrUpload : descrNoUpload) + condition

2
scripts/hetzner/deployHetzner.sh
View file

@ -7,7 +7,7 @@
# apt install unzip
# npm i -g csp-logger
# wget https://github.com/pietervdvn/latlon2country/raw/main/tiles.zip
# wget https://source.mapcomplete.org/MapComplete/latlon2country/raw/branch/main/tiles.zip
# unzip tiles.zip
cp config.json config.json.bu &&

2
scripts/nsiLogos.ts
View file

@ -74,7 +74,7 @@ class NsiLogos extends Script {
ttl--
const dloaded = await Utils.downloadAdvanced(url, {
"User-Agent":
"MapComplete NSI scraper/0.1 (https://github.com/pietervdvn/MapComplete; pietervdvn@posteo.net)",
"MapComplete NSI scraper/0.1 (https://source.mapcomplete.org/MapComplete/MapComplete; pietervdvn@posteo.net)",
})
const redirect: string | undefined = dloaded["redirect"]
if (redirect) {

117
scripts/repairPanoramax.ts Normal file
View file

@ -0,0 +1,117 @@
import Script from "./Script"
import { Overpass } from "../src/Logic/Osm/Overpass"
import { Or } from "../src/Logic/Tags/Or"
import { RegexTag } from "../src/Logic/Tags/RegexTag"
import { Utils } from "../src/Utils"
import Constants from "../src/Models/Constants"
import { ImmutableStore } from "../src/Logic/UIEventSource"
import { BBox } from "../src/Logic/BBox"
import ChangeTagAction from "../src/Logic/Osm/Actions/ChangeTagAction"
import { Tag } from "../src/Logic/Tags/Tag"
import { Panoramax } from "panoramax-js/dist"
import { Changes } from "../src/Logic/Osm/Changes"
import OsmChangeAction from "../src/Logic/Osm/Actions/OsmChangeAction"
import { writeFileSync } from "fs"
import { Feature } from "geojson"
class RepairPanoramax extends Script {
private static readonly europe: Feature = {
"type": "Feature",
"properties": {},
"geometry": {
"coordinates": [
[
[
-20.091159690050006,
25.773375277790038
],
[
46.12276429398841,
25.773375277790038
],
[
46.12276429398841,
65.41389761819318
],
[
-20.091159690050006,
65.41389761819318
],
[
-20.091159690050006,
25.773375277790038
]
]
],
"type": "Polygon"
}
}
constructor() {
super("See https://source.mapcomplete.org/MapComplete/MapComplete/issues/2372\n" +
"We accidentally added the full image URL instead of the hash due to a bug. This scripts rewrites all")
}
async main(args: string[]): Promise<void> {
const keys = ["panoramax", ...Utils.TimesT(10, i => "panoramax:" + i)]
const overpass = new Overpass(
new Or(
keys.map(k => new RegexTag(k, /^https:\/\/panoramax.mapcomplete.org\/.*/))
),
[],
Constants.defaultOverpassUrls[0],
new ImmutableStore(500)
)
const [wrongFeatures] = await overpass.queryGeoJson(BBox.global)
const allChanges: OsmChangeAction[] = []
for (const f of wrongFeatures.features) {
for (const key of keys) {
const v = f.properties[key]
if (!v) {
continue
}
const pre = "https://panoramax.mapcomplete.org/api/pictures/"
const pre1 = "https://panoramax.mapcomplete.org/permanent/"
let correct: string
if (v.startsWith(pre)) {
correct = v.substring(pre.length)
correct = correct.substring(0, correct.indexOf("/"))
} else if (v.startsWith(pre1)) {
const stripped = v.substring(pre1.length)
correct = stripped.replace(/\//g, "").replace(/\.jpg$/, "")
correct = correct.substring(0, 8) + "-" + correct.substring(8)
} else if (Panoramax.isId(v)) {
// This is a valid ID that came on an object also having an _invalid_ id
// We can safely skip this
continue
} else {
throw "Unknown url " + v
}
console.log(key, correct, " old: ", v)
if (!Panoramax.isId(correct)) {
throw "Constructed an invalid id:" + correct
}
const change = new ChangeTagAction(
f.properties.id,
new Tag(key, correct),
f.properties,
{
theme: "fix",
changeType: "fix"
}
)
allChanges.push(change)
}
}
const xml = await Changes.createChangesetXMLForJosm(allChanges)
console.log(xml)
const path = "repairPanoramax.osc"
writeFileSync(path, xml)
console.log("Written XML to", path)
}
}
new RepairPanoramax().run()

2
scripts/serverLdScrape.ts
View file

@ -19,7 +19,7 @@ class ServerLdScrape extends Script {
accept: "application/html",
},
/* {
"User-Agent": "MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
"User-Agent": "MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://source.mapcomplete.org/MapComplete",
"accept": "application/html"
},
{