forked from MapComplete/MapComplete
Scripts: improve documentation and try to speed up things
This commit is contained in:
parent
e512b40117
commit
fe39817d14
3 changed files with 53 additions and 52 deletions
|
|
@ -5,7 +5,7 @@ on:
|
|||
|
||||
|
||||
jobs:
|
||||
daily_data_maintenance:
|
||||
update_database:
|
||||
runs-on: [ osm-cache ]
|
||||
steps:
|
||||
- uses: https://source.mapcomplete.org/actions/checkout@v4
|
||||
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
run: npm run build:dbscript # output: build_db.lua
|
||||
|
||||
|
||||
- name: Print planet file date
|
||||
- name: Create and fill database
|
||||
shell: bash
|
||||
run: |
|
||||
# LAIN has a weekly updated planet file in /data/planet-latest.osm.pbf
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
# Setting up a synced OSM-server for quick layer access
|
||||
|
||||
## Requirements
|
||||
|
||||
Debian machine (e.g. Ubuntu)
|
||||
|
||||
`apt install osmium-tool docker.io osm2pgsql`
|
||||
|
||||
## Setting up the SQL-server (only once):
|
||||
|
||||
`sudo docker run --name some-postgis -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -d -p 5444:5432 -v ~/data/pgsql/:/var/lib/postgresql/data postgis/postgis`
|
||||
|
||||
Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot.
|
||||
|
||||
- Open PGAdmin, open the PGSQL-tool (CLI-button at the top); alternativly: psql --host=172.17.0.1 --port=5444
|
||||
--user=user
|
||||
- Open PGAdmin, open the PGSQL-tool (CLI-button at the top); alternativly:
|
||||
`psql --host=172.17.0.1 --port=5444 --user=user`
|
||||
- Run `show config_file;` to get the config file location (in docker). This is probably
|
||||
`/var/lib/postgresql/data/postgresql.conf`
|
||||
- In a terminal, run `sudo docker exec -i <docker-container-id> bash` (run `sudo docker ps` to get the container id)
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ class GenerateLayouts extends Script {
|
|||
private eliUrlsCached: string[]
|
||||
private date = new Date().toISOString()
|
||||
private branchName: string = undefined
|
||||
|
||||
private alreadyWritten: Set<string> = new Set<string>()
|
||||
constructor() {
|
||||
super("Generates an '<theme>.html' and 'index_<theme>.ts' for every theme")
|
||||
}
|
||||
|
|
@ -70,7 +70,7 @@ class GenerateLayouts extends Script {
|
|||
})
|
||||
}
|
||||
|
||||
async createIcon(iconPath: string, size: number, alreadyWritten: string[]) {
|
||||
async createIcon(iconPath: string, size: number) {
|
||||
let name = iconPath.split(".").slice(0, -1).join(".") // drop svg suffix
|
||||
if (name.startsWith("./")) {
|
||||
name = name.substring(2)
|
||||
|
|
@ -78,10 +78,10 @@ class GenerateLayouts extends Script {
|
|||
|
||||
const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png`
|
||||
const targetpath = `public/${newname}`
|
||||
if (alreadyWritten.indexOf(newname) >= 0) {
|
||||
if (this.alreadyWritten.has(newname)) {
|
||||
return newname
|
||||
}
|
||||
alreadyWritten.push(newname)
|
||||
this.alreadyWritten.add(newname)
|
||||
if (existsSync(targetpath)) {
|
||||
return newname
|
||||
}
|
||||
|
|
@ -163,8 +163,7 @@ class GenerateLayouts extends Script {
|
|||
}
|
||||
|
||||
async createManifest(
|
||||
layout: ThemeConfig,
|
||||
alreadyWritten: string[]
|
||||
layout: ThemeConfig
|
||||
): Promise<{
|
||||
manifest: any
|
||||
whiteIcons: string[]
|
||||
|
|
@ -199,8 +198,8 @@ class GenerateLayouts extends Script {
|
|||
|
||||
const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512]
|
||||
for (const size of sizes) {
|
||||
const name = await this.createIcon(path, size, alreadyWritten)
|
||||
const whiteIcon = await this.createIcon(whiteBackgroundPath, size, alreadyWritten)
|
||||
const name = await this.createIcon(path, size)
|
||||
const whiteIcon = await this.createIcon(whiteBackgroundPath, size)
|
||||
whiteIcons.push(whiteIcon)
|
||||
icons.push({
|
||||
src: name,
|
||||
|
|
@ -447,8 +446,7 @@ class GenerateLayouts extends Script {
|
|||
async createLandingPage(
|
||||
layout: ThemeConfig,
|
||||
layoutJson: ThemeConfigJson,
|
||||
whiteIcons,
|
||||
alreadyWritten
|
||||
whiteIcons
|
||||
) {
|
||||
Locale.language.setData(layout.language[0])
|
||||
const targetLanguage = layout.language[0]
|
||||
|
|
@ -471,7 +469,7 @@ class GenerateLayouts extends Script {
|
|||
if (twitterImage.endsWith(".svg")) {
|
||||
try {
|
||||
// svgs are badly supported as social image, we use a generated svg instead
|
||||
twitterImage = await this.createIcon(twitterImage, 512, alreadyWritten)
|
||||
twitterImage = await this.createIcon(twitterImage, 512)
|
||||
} catch (e) {
|
||||
console.error("Could not generate image:", e)
|
||||
}
|
||||
|
|
@ -479,7 +477,7 @@ class GenerateLayouts extends Script {
|
|||
|
||||
if (ogImage.endsWith(".svg")) {
|
||||
try {
|
||||
ogImage = await this.createIcon(ogImage, 512, alreadyWritten)
|
||||
ogImage = await this.createIcon(ogImage, 512)
|
||||
} catch (e) {
|
||||
console.error("Could not generate image:", e)
|
||||
}
|
||||
|
|
@ -611,64 +609,62 @@ class GenerateLayouts extends Script {
|
|||
}
|
||||
}
|
||||
|
||||
private async createThemeFor(layoutConfigJson: ThemeConfigJson) {
|
||||
const layout = new ThemeConfig(layoutConfigJson, true)
|
||||
const layoutName = layout.id
|
||||
|
||||
const err = (err) => {
|
||||
if (err !== null) {
|
||||
console.log("Could not write manifest for ", layoutName, " because ", err)
|
||||
}
|
||||
}
|
||||
const { manifest, whiteIcons } = await this.createManifest(layout)
|
||||
const manif = JSON.stringify(manifest, undefined, 2)
|
||||
const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
|
||||
writeFile("public/" + manifestLocation, manif, err)
|
||||
|
||||
// Create a landing page for the given theme
|
||||
const landing = await this.createLandingPage(
|
||||
layout,
|
||||
layoutConfigJson,
|
||||
whiteIcons
|
||||
)
|
||||
|
||||
writeFile(this.enc(layout.id) + ".html", landing, err)
|
||||
await this.createIndexFor(layout)
|
||||
}
|
||||
|
||||
async main(): Promise<void> {
|
||||
const alreadyWritten = []
|
||||
this.createDir("./public/assets/")
|
||||
this.createDir("./public/assets/generated")
|
||||
this.createDir("./public/assets/generated/images")
|
||||
|
||||
const blacklist = [
|
||||
const blacklist = new Set([
|
||||
"",
|
||||
"test",
|
||||
".",
|
||||
"..",
|
||||
"manifest",
|
||||
"index",
|
||||
"inspector",
|
||||
"land",
|
||||
"preferences",
|
||||
"account",
|
||||
"openstreetmap",
|
||||
"custom",
|
||||
"theme",
|
||||
]
|
||||
])
|
||||
const args = process.argv
|
||||
const theme = args[2]
|
||||
if (theme !== undefined) {
|
||||
console.warn("Only generating layout " + theme)
|
||||
}
|
||||
const paths = ScriptUtils.readDirRecSync("./public/assets/generated/themes/", 1)
|
||||
for (const i in paths) {
|
||||
const layoutConfigJson = <ThemeConfigJson>JSON.parse(readFileSync(paths[i], "utf8"))
|
||||
if (theme !== undefined && layoutConfigJson.id !== theme) {
|
||||
continue
|
||||
}
|
||||
const layout = new ThemeConfig(layoutConfigJson, true)
|
||||
const layoutName = layout.id
|
||||
if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) {
|
||||
console.log(`Skipping a layout with name ${layoutName}, it is on the blacklist`)
|
||||
continue
|
||||
}
|
||||
const err = (err) => {
|
||||
if (err !== null) {
|
||||
console.log("Could not write manifest for ", layoutName, " because ", err)
|
||||
}
|
||||
}
|
||||
const { manifest, whiteIcons } = await this.createManifest(layout, alreadyWritten)
|
||||
const manif = JSON.stringify(manifest, undefined, 2)
|
||||
const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
|
||||
writeFile("public/" + manifestLocation, manif, err)
|
||||
const configs = paths.map(path => <ThemeConfigJson>JSON.parse(readFileSync(path, "utf8")))
|
||||
const configsFiltered = configs
|
||||
.filter(config => (theme !== undefined && config.id === theme) || !blacklist.has(config.id))
|
||||
|
||||
// Create a landing page for the given theme
|
||||
const landing = await this.createLandingPage(
|
||||
layout,
|
||||
layoutConfigJson,
|
||||
whiteIcons,
|
||||
alreadyWritten
|
||||
)
|
||||
|
||||
writeFile(this.enc(layout.id) + ".html", landing, err)
|
||||
await this.createIndexFor(layout)
|
||||
}
|
||||
await Promise.all(configsFiltered.map(config => this.createThemeFor(config)))
|
||||
|
||||
const { manifest } = await this.createManifest(
|
||||
new ThemeConfig({
|
||||
|
|
@ -681,8 +677,7 @@ class GenerateLayouts extends Script {
|
|||
startZoom: 0,
|
||||
title: { en: "MapComplete" },
|
||||
description: { en: "A thematic map viewer and editor based on OpenStreetMap" },
|
||||
}),
|
||||
alreadyWritten
|
||||
})
|
||||
)
|
||||
|
||||
const manif = JSON.stringify(manifest, undefined, 2)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue