Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Scripts: add counts to 'generateBuildDbScript'; add cutoff

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-07-04 21:06:27 +02:00
parent 4627f77c4f
commit b365f6e602
2 changed files with 81 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

30
scripts/osm2pgsql/generateBuildDbScript.ts
View file

@ -8,6 +8,7 @@ import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation"
import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts"
import { OsmObject } from "../../src/Logic/Osm/OsmObject"
import TagInfo from "../../src/Logic/Web/TagInfo"
class LuaSnippets {
public static helpers = [
@ -192,24 +193,35 @@ class GenerateBuildDbScript extends Script {
const allLayers = new ValidateThemeEnsemble().convertStrict(
AllKnownLayouts.allKnownLayouts.values()
)
const aed = allLayers.get("defibrillator")
allLayers.clear()
allLayers.set("defibrillator", aed)
// const aed = allLayers.get("defibrillator")
// allLayers.clear()
// allLayers.set("defibrillator", aed)
if (allLayers.size === 0) {
throw "No layers found at all"
}
const notCountedCutoff = 100*1000*1000
const notCounted: string[] = []
const allNeededLayers: Map<string, { tags: TagsFilter; foundInTheme: string[] }> = new Map<
string,
{ tags: TagsFilter; foundInTheme: string[] }
>()
const tagInfo = new TagInfo()
const layerTotals = new Map<string, number>();
for (const key of allLayers.keys()) {
const layer = allLayers.get(key)
if (layer.isCounted) {
allNeededLayers.set(key, layer)
} else {
if (!layer.isCounted) {
notCounted.push(key)
continue
}
let total = await tagInfo.getCountEstimateFor(layer.tags)
console.log("Keys " + layer.tags.asHumanString() + " might have up to " + total + " items")
layerTotals.set(key, total)
if (total > notCountedCutoff) {
notCounted.push(key)
console.log("NOT indexing layer " + key + " as it exceeds the cutoff of", notCountedCutoff)
continue
}
allNeededLayers.set(key, layer)
}
const generators: GenerateLayerLua[] = []
@ -228,7 +240,7 @@ class GenerateBuildDbScript extends Script {
fs.writeFileSync(path, script, "utf-8")
console.log("Written", path)
console.log(
"Following layers are _not_ indexed as they are not counted:",
"Following layers are _not_ indexed as they are marked to be not indexed or exceeded the count limit:",
notCounted.join(", ")
)
console.log(
@ -236,6 +248,10 @@ class GenerateBuildDbScript extends Script {
" layers will be created with 3 tables each. Make sure to set 'max_connections' to at least " +
(10 + 3 * allNeededLayers.size)
)
console.log("Layer totals (including non-indexed):")
for (const key of layerTotals.keys()) {
console.log(key,",\t",layerTotals.get(key), ",\t", allLayers.get(key).isCounted)
}
}
private earlyAbort() {