Scripts: add counts to 'generateBuildDbScript'; add cutoff

scripts/osm2pgsql/generateBuildDbScript.ts

@@ -8,6 +8,7 @@ import { RegexTag } from "../../src/Logic/Tags/RegexTag"
 import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation"
 import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts"
 import { OsmObject } from "../../src/Logic/Osm/OsmObject"
+import TagInfo from "../../src/Logic/Web/TagInfo"
 
 class LuaSnippets {
     public static helpers = [
@@ -192,24 +193,35 @@ class GenerateBuildDbScript extends Script {
         const allLayers = new ValidateThemeEnsemble().convertStrict(
             AllKnownLayouts.allKnownLayouts.values()
         )
-        const aed = allLayers.get("defibrillator")
-        allLayers.clear()
-        allLayers.set("defibrillator", aed)
+        // const aed = allLayers.get("defibrillator")
+        // allLayers.clear()
+        // allLayers.set("defibrillator", aed)
         if (allLayers.size === 0) {
             throw "No layers found at all"
         }
+        const notCountedCutoff = 100*1000*1000
         const notCounted: string[] = []
         const allNeededLayers: Map<string, { tags: TagsFilter; foundInTheme: string[] }> = new Map<
             string,
             { tags: TagsFilter; foundInTheme: string[] }
         >()
+        const tagInfo = new TagInfo()
+        const layerTotals = new Map<string, number>();
         for (const key of allLayers.keys()) {
             const layer = allLayers.get(key)
-            if (layer.isCounted) {
-                allNeededLayers.set(key, layer)
-            } else {
+            if (!layer.isCounted) {
                 notCounted.push(key)
+                continue
             }
+            let total = await tagInfo.getCountEstimateFor(layer.tags)
+            console.log("Keys " + layer.tags.asHumanString() + " might have up to " + total + " items")
+            layerTotals.set(key, total)
+            if (total > notCountedCutoff) {
+                notCounted.push(key)
+                console.log("NOT indexing layer " + key + " as it exceeds the cutoff of", notCountedCutoff)
+                continue
+            }
+            allNeededLayers.set(key, layer)
         }
         const generators: GenerateLayerLua[] = []
 
@@ -228,7 +240,7 @@ class GenerateBuildDbScript extends Script {
         fs.writeFileSync(path, script, "utf-8")
         console.log("Written", path)
         console.log(
-            "Following layers are _not_ indexed as they are not counted:",
+            "Following layers are _not_ indexed as they are marked to be not indexed or exceeded the count limit:",
             notCounted.join(", ")
         )
         console.log(
@@ -236,6 +248,10 @@ class GenerateBuildDbScript extends Script {
                 " layers will be created with 3 tables each. Make sure to set 'max_connections' to at least  " +
                 (10 + 3 * allNeededLayers.size)
         )
+        console.log("Layer totals (including non-indexed):")
+        for (const key of layerTotals.keys()) {
+            console.log(key,",\t",layerTotals.get(key), ",\t", allLayers.get(key).isCounted)
+        }
     }
 
     private earlyAbort() {

src/Logic/Web/TagInfo.ts

@@ -1,6 +1,7 @@
 import { Utils } from "../../Utils"
 import type { FeatureCollection } from "geojson"
-import ScriptUtils from "../../../scripts/ScriptUtils"
+import { Tag } from "../Tags/Tag"
+import { TagsFilter } from "../Tags/TagsFilter"
 
 export interface TagInfoStats {
     /**
@@ -36,6 +37,62 @@ export default class TagInfo {
         this._backend = backend
     }
 
+    public async getCountEstimateFor(filter: TagsFilter): Promise<number | undefined> {
+        if (filter["and"]) {
+            const subtags = <TagsFilter[]>filter["and"]
+            let minCount = Infinity
+            for (const subtag of subtags) {
+                const count = await this.getCountEstimateFor(subtag)
+                if(count === undefined){
+                    continue
+                }
+                // The objects should meet _all_ requirements, only the smallest group will thus have influence
+                if (count < minCount) {
+                    minCount = count
+                }
+            }
+            return minCount
+        }
+
+        if (filter["or"]) {
+            const subtags = <TagsFilter[]>filter["or"]
+            let total = 0
+            for (const subtag of subtags) {
+                const count = await this.getCountEstimateFor(subtag)
+                // THe groups might be completely disjoint, we sum
+                if(count === undefined){
+                    continue
+                }
+                total += count
+            }
+            return total
+        }
+
+
+        const t = <Tag>filter
+        if (t.key && t.value) {
+            if (t.isNegative()) {
+                return undefined
+            }
+            if(typeof t.key !== "string"){
+                // This is a regexkey, skip
+                return undefined
+            }
+            let v = undefined
+            if (typeof t.value === "string") {
+                v = t.value
+            }
+            const stats = await this.getStats(t.key, v)
+            return stats.data.find(item => item.type === "all").count
+        }
+        if(t.key && t.value === ""){
+            return undefined
+        }
+        console.log("Invalid filter")
+        throw "Cannot handle " + JSON.stringify(filter.asJson())
+
+    }
+
     public async getStats(key: string, value?: string): Promise<TagInfoStats> {
         let url: string
         if (value) {