midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-08-26 01:30:48 +02:00
commit 125139a672
313 changed files with 2392 additions and 19940 deletions
Download patch file Download diff file Expand all files Collapse all files

51
scripts/generateDocs.ts
View file

@ -29,6 +29,7 @@ import { Changes } from "../src/Logic/Osm/Changes"
import TableOfContents from "../src/UI/Base/TableOfContents"
import MarkdownUtils from "../src/Utils/MarkdownUtils"
import { parse as parse_html } from "node-html-parser"
import { AvailableRasterLayers } from "../src/Models/RasterLayers"
/**
* Converts a markdown-file into a .json file, which a walkthrough/slideshow element can use
@ -54,15 +55,15 @@ class ToSlideshowJson {
sections.push(currentSection)
currentSection = []
}
line = line.replace('src="../../public/', 'src="./')
line = line.replace('src="../../', 'src="./')
line = line.replace("src=\"../../public/", "src=\"./")
line = line.replace("src=\"../../", "src=\"./")
currentSection.push(line)
}
sections.push(currentSection)
writeFileSync(
this._target,
JSON.stringify({
sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0),
sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0)
})
)
}
@ -81,7 +82,7 @@ class WikiPageGenerator {
generate() {
let wikiPage =
'{|class="wikitable sortable"\n' +
"{|class=\"wikitable sortable\"\n" +
"! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" +
"|-"
@ -139,7 +140,7 @@ export class GenerateDocs extends Script {
}
this.WriteMarkdownFile("./Docs/Tags_format.md", TagUtils.generateDocs(), [
"src/Logic/Tags/TagUtils.ts",
"src/Logic/Tags/TagUtils.ts"
])
new ToSlideshowJson(
@ -165,7 +166,7 @@ export class GenerateDocs extends Script {
})
this.WriteMarkdownFile("./Docs/SpecialRenderings.md", SpecialVisualizations.HelpMessage(), [
"src/UI/SpecialVisualizations.ts",
"src/UI/SpecialVisualizations.ts"
])
this.WriteMarkdownFile(
"./Docs/CalculatedTags.md",
@ -173,13 +174,31 @@ export class GenerateDocs extends Script {
["src/Logic/SimpleMetaTagger.ts", "src/Logic/ExtraFunctions.ts"]
)
this.WriteMarkdownFile("./Docs/SpecialInputElements.md", Validators.HelpText(), [
"src/UI/InputElement/Validators.ts",
"src/UI/InputElement/Validators.ts"
])
this.WriteMarkdownFile("./Docs/ChangesetMeta.md", Changes.getDocs(), [
"src/Logic/Osm/Changes.ts",
"src/Logic/Osm/ChangesetHandler.ts",
"src/Logic/Osm/ChangesetHandler.ts"
])
const eli = await AvailableRasterLayers.editorLayerIndex()
this.WriteMarkdownFile("./Docs/ELI-overview.md",
[
"# Layers in the Editor Layer Index",
"This table gives a summary of ids, names and other metainformation. [See the online, interactive map here](https://osmlab.github.io/editor-layer-index/) or [visit the repository](https://github.com/osmlab/editor-layer-index)",
MarkdownUtils.table(
["id", "name", "category", "Best", "attribution"],
eli.map(f => [f.properties.id, f.properties.name, f.properties.category, f.properties.best ? "⭐" : "",
f.properties.attribution?.html ?? f.properties.attribution?.text
])
)
].join("\n\n"), [
"./public/assets/data/editor-layer-index.json"
]
)
new WikiPageGenerator().generate()
console.log("Generated docs")
@ -212,7 +231,7 @@ export class GenerateDocs extends Script {
md = TableOfContents.insertTocIntoMd(md)
}
md.replace(/\n\n\n+/g, "\n\n")
md = md.replace(/\n\n\n+/g, "\n\n")
if (!md.endsWith("\n")) {
md += "\n"
@ -225,7 +244,7 @@ export class GenerateDocs extends Script {
"This document is autogenerated from",
autogenSource
.map((s) => `[${s}](https://github.com/pietervdvn/MapComplete/blob/develop/${s})`)
.join(", "),
.join(", ")
].join(" ")
writeFileSync(filename, warnAutomated + md + "\n\n" + generatedFrom + "\n")
@ -234,7 +253,7 @@ export class GenerateDocs extends Script {
private generateHotkeyDocs() {
new ThemeViewState(new LayoutConfig(<any>bookcases), new Set())
this.WriteMarkdownFile("./Docs/Hotkeys.md", Hotkeys.generateDocumentation(), [
"src/UI/Base/Hotkeys.ts",
"src/UI/Base/Hotkeys.ts"
])
}
@ -268,7 +287,7 @@ export class GenerateDocs extends Script {
}
this.WriteMarkdownFile("./Docs/builtin_units.md", ["# Units", ...els].join("\n\n"), [
`assets/layers/unit/unit.json`,
`assets/layers/unit/unit.json`
])
}
@ -449,7 +468,7 @@ export class GenerateDocs extends Script {
theme.title,
"(",
`[${theme.id}](https://mapcomplete.org/${theme.id})`,
")",
")"
].join(" "),
"_This document details some technical information about this MapComplete theme, mostly about the attributes used in the theme. Various links point toward more information about the attributes, e.g. to the OpenStreetMap-wiki, to TagInfo or tools creating statistics_",
@ -469,7 +488,7 @@ export class GenerateDocs extends Script {
MarkdownUtils.list(theme.language.filter((ln) => ln !== "_context")),
"# Layers defined in this theme configuration file",
"These layers can not be reused in different themes.",
...layersToInline.map((l) => l.GenerateDocumentation(null)),
...layersToInline.map((l) => l.GenerateDocumentation(null))
].join("\n")
this.WriteMarkdownFile(
"./Docs/Themes/" + theme.id + ".md",
@ -549,10 +568,10 @@ export class GenerateDocs extends Script {
Array.from(AllSharedLayers.sharedLayers.keys()).map(
(id) => `[${id}](./Layers/${id}.md)`
)
),
)
].join("\n\n")
this.WriteMarkdownFile("./Docs/BuiltinLayers.md", el, [
"src/Customizations/AllKnownLayouts.ts",
"src/Customizations/AllKnownLayouts.ts"
])
}
}

10
scripts/generateSunnyUnlabeled.ts
View file

@ -7,14 +7,20 @@ export class GenerateSunnyUnlabeled extends Script {
super("Generates 'sunny-unlabeled.json' based on sunny.json")
}
async main(args: string[]): Promise<void> {
generateUnlabeled() {
const unlabeled = { "#": "AUTOMATICALLY GENERATED! Do not edit.", ...sunny }
unlabeled.name = unlabeled.name + "-unlabeled"
unlabeled.layers = sunny.layers.filter(
(l) => l.type !== "symbol" || !l.layout["text-field"]
(l) => l.type !== "symbol" || !l.layout["text-field"],
)
writeFileSync("public/assets/sunny-unlabeled.json", JSON.stringify(unlabeled, null, " "))
}
async main(args: string[]): Promise<void> {
this.generateUnlabeled()
}
}
new GenerateSunnyUnlabeled().run()

3
scripts/handleErrors.ts
View file

@ -76,7 +76,7 @@ class HandleErrors extends Script {
deletedObjects: OsmObject[]
} = changesObj.CreateChangesetObjects(toUpload, objects, true)
const changeset = Changes.createChangesetFor("", changes)
const changeset = Changes.buildChangesetXML("", changes)
const path =
"error_changeset_" + parsed.index + "_" + e.layout + "_" + e.username + ".osc"
if (
@ -163,7 +163,6 @@ ${changeset}`
for (const parsed of all) {
try {
await this.handleError(parsed, changesObj, downloader, createdChangesets, refusedFiles)
} catch (e) {
console.error("ERROR: could not handle ", parsed, " due to", e)

2
scripts/importscripts/cleanRepair.ts
View file

@ -313,7 +313,7 @@ export default class CleanRepair extends Script {
const changedObjects = changes.CreateChangesetObjects(changesToMake, objects)
const osc = Changes.createChangesetFor("", changedObjects)
const osc = Changes.buildChangesetXML("", changedObjects)
writeFileSync("Cleanup.osc", osc, "utf8")
}

16
scripts/osm2pgsql/createNewDatabase.ts Normal file
View file

@ -0,0 +1,16 @@
import Script from "../Script"
import { OsmPoiDatabase } from "./osmPoiDatabase"
class CreateNewDatabase extends Script {
constructor() {
super("Creates a new version of the database. Usage: `createNewDatabase -- YYYY-MM-DD` which will create database `osm-poi.YYYY-MM-DD`")
}
async main(args: string[]): Promise<void> {
const db = new OsmPoiDatabase("postgresql://user:password@localhost:5444")
await db.createNew(args[0])
}
}
new CreateNewDatabase().run()

16
scripts/osm2pgsql/deleteOldDbs.ts Normal file
View file

@ -0,0 +1,16 @@
import Script from "../Script"
import { OsmPoiDatabase } from "./osmPoiDatabase"
class DeleteOldDbs extends Script {
constructor() {
super("Drops all but the newest `osm-poi.*`")
}
async main(args: string[]): Promise<void> {
const db = new OsmPoiDatabase("postgresql://user:password@localhost:5444")
await db.deleteAllButLatest()
}
}
new DeleteOldDbs().run()

240
scripts/osm2pgsql/osmPoiDatabase.ts Normal file
View file

@ -0,0 +1,240 @@
import { Client } from "pg"
/**
* Just the OSM2PGSL default database
*/
export interface PoiDatabaseMeta {
attributes
current_timestamp
db_format
flat_node_file
import_timestamp
output
prefix
replication_base_url
replication_sequence_number
replication_timestamp
style
updatable
version
}
/**
* Connects with a Postgis database, gives back how much items there are within the given BBOX
*/
export class OsmPoiDatabase {
private static readonly prefixes: ReadonlyArray<string> = ["pois", "lines", "polygons"]
private _client: Client
private isConnected = false
private supportedLayers: Set<string> = undefined
private supportedLayersDate: Date = undefined
private metaCache: PoiDatabaseMeta = undefined
private metaCacheDate: Date = undefined
private readonly _connectionString: string
private static readonly _prefix = "osm-poi" as const
constructor(connectionString: string) {
this._connectionString = connectionString
}
private getMetaClient() {
return new Client(this._connectionString + "/postgres")
}
private async connectIfNeeded() {
if (this.isConnected) {
return
}
this.metaCache = undefined
await this.connectToLatest()
this._client.once("end", () => {
this.isConnected = false
})
this._client.once("error", () => {
if (this.isConnected) {
this.isConnected = false
try {
this._client.end()
} catch (e) {
console.error("Could not disconnect")
}
}
})
this.isConnected = true
}
async findSuitableDatabases(): Promise<string[]> {
const metaclient = this.getMetaClient()
await metaclient.connect()
try {
const meta = await metaclient.query("SELECT datname FROM pg_database")
let latest: string = undefined
let latestDate: Date = new Date(0)
const dbs: string[] = []
for (const row of meta.rows) {
const name: string = row["datname"]
if (!name.startsWith(OsmPoiDatabase._prefix)) {
continue
}
const nm = name.substring(OsmPoiDatabase._prefix.length + 1)
dbs.push(nm)
}
dbs.sort()
return dbs
} finally {
await metaclient.end()
}
}
async searchLatest() {
const dbs = await this.findSuitableDatabases()
let latest: string = undefined
let latestDate: Date = undefined
for (const name of dbs) {
const date = new Date(name)
if (latestDate.getTime() < date.getTime()) {
latest = name
latestDate = date
}
}
if (latest === undefined) {
throw "No suitable database found"
}
console.log("Latest database is:", latest)
return latest
}
async createNew(date: string) {
const dbname = `${OsmPoiDatabase._prefix}.${date}`
console.log("Attempting to create a new database with name", dbname)
const metaclient = this.getMetaClient()
await metaclient.connect()
try {
await metaclient.query(`CREATE DATABASE "${dbname}"`)
console.log("Database created - installing extensions")
const client = new Client(this._connectionString + "/" + dbname)
try {
await client.connect()
await client.query(`CREATE EXTENSION IF NOT EXISTS postgis`)
console.log("Created database", dbname, "with postgis")
} finally {
await client.end()
}
} finally {
await metaclient.end()
}
}
async deleteAllButLatest(){
const dbs = await this.findSuitableDatabases()
for (let i = 0; i < dbs.length - 1; i++) {
await this.deleteDatabase(dbs[i])
}
}
/**
* DANGEROUS
* Drops de database with the given name
* @param date
*/
async deleteDatabase(date: string) {
const metaclient = this.getMetaClient()
await metaclient.connect()
try {
await metaclient.query(`DROP DATABASE "${OsmPoiDatabase._prefix}.${date}"`)
console.log(`Dropped database ${OsmPoiDatabase._prefix}.${date}`)
} finally {
await metaclient.end()
}
}
async connectToLatest() {
const latest = await this.searchLatest()
this._client = new Client(this._connectionString + "/" + latest)
await this._client.connect()
}
async getCount(
layer: string,
bbox: [[number, number], [number, number]] = undefined,
): Promise<{ count: number; lat: number; lon: number }> {
await this.connectIfNeeded()
let total: number = 0
let latSum = 0
let lonSum = 0
for (const prefix of OsmPoiDatabase.prefixes) {
let query =
"SELECT COUNT(*), ST_AsText(ST_Centroid(ST_Collect(geom))) FROM " +
prefix +
"_" +
layer
if (bbox) {
query += ` WHERE ST_MakeEnvelope (${bbox[0][0]}, ${bbox[0][1]}, ${bbox[1][0]}, ${bbox[1][1]}, 4326) ~ geom`
}
const result = await this._client.query(query)
const count = Number(result.rows[0].count)
let point = result.rows[0].st_astext
if (count === 0) {
continue
}
total += count
if (!point) {
continue
}
point = point.substring(6, point.length - 1)
const [lon, lat] = point.split(" ")
latSum += lat * count
lonSum += lon * count
}
return { count: total, lat: latSum / total, lon: lonSum / total }
}
disconnect() {
this._client.end()
}
async getLayers(): Promise<Set<string>> {
if (
this.supportedLayers !== undefined &&
new Date().getTime() - this.supportedLayersDate.getTime() < 1000 * 60 * 60 * 24
) {
return this.supportedLayers
}
const q =
"SELECT table_name \n" +
"FROM information_schema.tables \n" +
"WHERE table_schema = 'public' AND table_name LIKE 'lines_%';"
const result = await this._client.query(q)
const layers = result.rows.map((r) => r.table_name.substring("lines_".length))
this.supportedLayers = new Set(layers)
this.supportedLayersDate = new Date()
return this.supportedLayers
}
async getMeta(): Promise<PoiDatabaseMeta> {
const now = new Date()
if (this.metaCache !== undefined) {
const diffSec = (this.metaCacheDate.getTime() - now.getTime()) / 1000
if (diffSec < 120) {
return this.metaCache
}
}
await this.connectIfNeeded()
const result = await this._client.query("SELECT * FROM public.osm2pgsql_properties")
const meta = {}
for (const { property, value } of result.rows) {
meta[property] = value
}
this.metaCacheDate = now
this.metaCache = <PoiDatabaseMeta>meta
return this.metaCache
}
}

120
scripts/osm2pgsql/tilecountServer.ts
View file

@ -1,124 +1,8 @@
import { Client } from "pg"
import { Tiles } from "../../src/Models/TileRange"
import { Server } from "../server"
import Script from "../Script"
import { OsmPoiDatabase } from "./osmPoiDatabase"
/**
* Just the OSM2PGSL default database
*/
interface PoiDatabaseMeta {
attributes
current_timestamp
db_format
flat_node_file
import_timestamp
output
prefix
replication_base_url
replication_sequence_number
replication_timestamp
style
updatable
version
}
/**
* Connects with a Postgis database, gives back how much items there are within the given BBOX
*/
class OsmPoiDatabase {
private static readonly prefixes: ReadonlyArray<string> = ["pois", "lines", "polygons"]
private readonly _client: Client
private isConnected = false
private supportedLayers: Set<string> = undefined
private supportedLayersDate: Date = undefined
private metaCache: PoiDatabaseMeta = undefined
private metaCacheDate: Date = undefined
constructor(connectionString: string) {
this._client = new Client(connectionString)
}
async getCount(
layer: string,
bbox: [[number, number], [number, number]] = undefined
): Promise<{ count: number; lat: number; lon: number }> {
if (!this.isConnected) {
await this._client.connect()
this.isConnected = true
}
let total: number = 0
let latSum = 0
let lonSum = 0
for (const prefix of OsmPoiDatabase.prefixes) {
let query =
"SELECT COUNT(*), ST_AsText(ST_Centroid(ST_Collect(geom))) FROM " +
prefix +
"_" +
layer
if (bbox) {
query += ` WHERE ST_MakeEnvelope (${bbox[0][0]}, ${bbox[0][1]}, ${bbox[1][0]}, ${bbox[1][1]}, 4326) ~ geom`
}
const result = await this._client.query(query)
const count = Number(result.rows[0].count)
let point = result.rows[0].st_astext
if (count === 0) {
continue
}
total += count
if (!point) {
continue
}
point = point.substring(6, point.length - 1)
const [lon, lat] = point.split(" ")
latSum += lat * count
lonSum += lon * count
}
return { count: total, lat: latSum / total, lon: lonSum / total }
}
disconnect() {
this._client.end()
}
async getLayers(): Promise<Set<string>> {
if (
this.supportedLayers !== undefined &&
new Date().getTime() - this.supportedLayersDate.getTime() < 1000 * 60 * 60 * 24
) {
return this.supportedLayers
}
const q =
"SELECT table_name \n" +
"FROM information_schema.tables \n" +
"WHERE table_schema = 'public' AND table_name LIKE 'lines_%';"
const result = await this._client.query(q)
const layers = result.rows.map((r) => r.table_name.substring("lines_".length))
this.supportedLayers = new Set(layers)
this.supportedLayersDate = new Date()
return this.supportedLayers
}
async getMeta(): Promise<PoiDatabaseMeta> {
const now = new Date()
if (this.metaCache !== undefined) {
const diffSec = (this.metaCacheDate.getTime() - now.getTime()) / 1000
if (diffSec < 120) {
return this.metaCache
}
}
const result = await this._client.query("SELECT * FROM public.osm2pgsql_properties")
const meta = {}
for (const { property, value } of result.rows) {
meta[property] = value
}
this.metaCacheDate = now
this.metaCache = <PoiDatabaseMeta>meta
return this.metaCache
}
}
class CachedSqlCount {
private readonly _cache: Record<
@ -169,7 +53,7 @@ class TileCountServer extends Script {
}
async main(args: string[]): Promise<void> {
const connectionString = args[0] ?? "postgresql://user:password@localhost:5444/osm-poi"
const connectionString = args[0] ?? "postgresql://user:password@localhost:5444"
const port = Number(args[1] ?? 2345)
console.log("Connecting to database", connectionString)
const tcs = new OsmPoiDatabase(connectionString)

29
scripts/osm2pgsql/update.sh
View file

@ -1,8 +1,27 @@
#! /bin/bash
npm run init
npm run generate
npm run refresh:layeroverview
# Full database update. DOwnload latest from planet.osm.org, build update script, setup and seed it
nvm use
npm run init # contains a 'npm run generate, which builds the layers'
npm run generate:buildDbScript
mv build_db.sh ~/data/
transmission-cli https://planet.osm.org/pbf/planet-latest.osm.pbf.torrent -f ./on_data_downloaded.sh &>nohup_transmission.log
mv build_db.lua ~/data/
TIMESTAMP=$(osmium fileinfo ~/data/planet-latest.osm.pbf -g header.option.timestamp)
DATE=$(echo $TIMESTAMP | sed "s/T.*//")
echo $DATE
npm run create:database -- -- ${DATE/T.*//}
cd ~/data || exit
rm planet-latest.osm.pbf
wget https://planet.osm.org/pbf/planet-latest.osm.pbf
rm seeddb.log
osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi planet-latest.osm.pbf >> seeddb.log
# To see the progress
# tail -f seeddb.log
npm run delete:database:old
# Restart tileserver
export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi.${DATE}
nohup ./pg_tileserv >> pg_tileserv.log &