diff --git a/scripts/osm2pgsql/generateBuildDbScript.ts b/scripts/osm2pgsql/generateBuildDbScript.ts index 21555fd816..9c44a39c4e 100644 --- a/scripts/osm2pgsql/generateBuildDbScript.ts +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -8,6 +8,7 @@ import { RegexTag } from "../../src/Logic/Tags/RegexTag" import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation" import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts" import { OsmObject } from "../../src/Logic/Osm/OsmObject" +import TagInfo from "../../src/Logic/Web/TagInfo" class LuaSnippets { public static helpers = [ @@ -192,24 +193,35 @@ class GenerateBuildDbScript extends Script { const allLayers = new ValidateThemeEnsemble().convertStrict( AllKnownLayouts.allKnownLayouts.values() ) - const aed = allLayers.get("defibrillator") - allLayers.clear() - allLayers.set("defibrillator", aed) + // const aed = allLayers.get("defibrillator") + // allLayers.clear() + // allLayers.set("defibrillator", aed) if (allLayers.size === 0) { throw "No layers found at all" } + const notCountedCutoff = 100*1000*1000 const notCounted: string[] = [] const allNeededLayers: Map = new Map< string, { tags: TagsFilter; foundInTheme: string[] } >() + const tagInfo = new TagInfo() + const layerTotals = new Map(); for (const key of allLayers.keys()) { const layer = allLayers.get(key) - if (layer.isCounted) { - allNeededLayers.set(key, layer) - } else { + if (!layer.isCounted) { notCounted.push(key) + continue } + let total = await tagInfo.getCountEstimateFor(layer.tags) + console.log("Keys " + layer.tags.asHumanString() + " might have up to " + total + " items") + layerTotals.set(key, total) + if (total > notCountedCutoff) { + notCounted.push(key) + console.log("NOT indexing layer " + key + " as it exceeds the cutoff of", notCountedCutoff) + continue + } + allNeededLayers.set(key, layer) } const generators: GenerateLayerLua[] = [] @@ -228,7 +240,7 @@ class GenerateBuildDbScript extends Script { fs.writeFileSync(path, script, "utf-8") console.log("Written", path) console.log( - "Following layers are _not_ indexed as they are not counted:", + "Following layers are _not_ indexed as they are marked to be not indexed or exceeded the count limit:", notCounted.join(", ") ) console.log( @@ -236,6 +248,10 @@ class GenerateBuildDbScript extends Script { " layers will be created with 3 tables each. Make sure to set 'max_connections' to at least " + (10 + 3 * allNeededLayers.size) ) + console.log("Layer totals (including non-indexed):") + for (const key of layerTotals.keys()) { + console.log(key,",\t",layerTotals.get(key), ",\t", allLayers.get(key).isCounted) + } } private earlyAbort() { diff --git a/src/Logic/Web/TagInfo.ts b/src/Logic/Web/TagInfo.ts index 2587ed1dcb..78a1ad27e6 100644 --- a/src/Logic/Web/TagInfo.ts +++ b/src/Logic/Web/TagInfo.ts @@ -1,6 +1,7 @@ import { Utils } from "../../Utils" import type { FeatureCollection } from "geojson" -import ScriptUtils from "../../../scripts/ScriptUtils" +import { Tag } from "../Tags/Tag" +import { TagsFilter } from "../Tags/TagsFilter" export interface TagInfoStats { /** @@ -36,6 +37,62 @@ export default class TagInfo { this._backend = backend } + public async getCountEstimateFor(filter: TagsFilter): Promise { + if (filter["and"]) { + const subtags = filter["and"] + let minCount = Infinity + for (const subtag of subtags) { + const count = await this.getCountEstimateFor(subtag) + if(count === undefined){ + continue + } + // The objects should meet _all_ requirements, only the smallest group will thus have influence + if (count < minCount) { + minCount = count + } + } + return minCount + } + + if (filter["or"]) { + const subtags = filter["or"] + let total = 0 + for (const subtag of subtags) { + const count = await this.getCountEstimateFor(subtag) + // THe groups might be completely disjoint, we sum + if(count === undefined){ + continue + } + total += count + } + return total + } + + + const t = filter + if (t.key && t.value) { + if (t.isNegative()) { + return undefined + } + if(typeof t.key !== "string"){ + // This is a regexkey, skip + return undefined + } + let v = undefined + if (typeof t.value === "string") { + v = t.value + } + const stats = await this.getStats(t.key, v) + return stats.data.find(item => item.type === "all").count + } + if(t.key && t.value === ""){ + return undefined + } + console.log("Invalid filter") + throw "Cannot handle " + JSON.stringify(filter.asJson()) + + } + public async getStats(key: string, value?: string): Promise { let url: string if (value) {