First working version of fully automatic uploader

Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts

@@ -119,7 +119,11 @@ export default class SaveTileToLocalStorageActor {
     }
 
     private SetIdb(tileIndex, data) {
-        IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data)
+        try{
+            IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data)
+        }catch(e){
+            console.error("Could not save tile to indexed-db: ", e, "tileIndex is:", tileIndex, "for layer", this._layer.id)
+        }
     }
 
     private GetIdb(tileIndex) {

Logic/FeatureSource/FeaturePipeline.ts

@@ -171,6 +171,7 @@ export default class FeaturePipeline {
                     state.currentBounds, state.locationControl,
                     (tileIndex, freshness) => self.freshnesses.get(id).addTileLoad(tileIndex, freshness),
                     (tile) => {
+                        console.debug("Loaded tile ", id, tile.tileIndex, "from local cache")
                         new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
                         hierarchy.registerTile(tile);
                         tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@@ -247,6 +248,7 @@ export default class FeaturePipeline {
                 })
         })
 
+        
         if (state.layoutToUse.trackAllNodes) {
             const fullNodeDb = new FullNodeDatabaseSource(
                 state.filteredLayers.data.filter(l => l.layerDef.id === "type_node")[0],
@@ -289,6 +291,10 @@ export default class FeaturePipeline {
         // A NewGeometryFromChangesFeatureSource does not split per layer, so we do this next
         new PerLayerFeatureSourceSplitter(state.filteredLayers,
             (perLayer) => {
+                if(perLayer.features.data.length === 0){
+                    return
+                }
+            
                 // We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this
                 perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer)
                 // AT last, we always apply the metatags whenever possible
@@ -309,7 +315,7 @@ export default class FeaturePipeline {
 
         this.runningQuery = updater.runningQuery.map(
             overpass => {
-                console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,",
+                console.log("FeaturePipeline: runningQuery state changed: Overpass", overpass ? "is querying," : "is idle,",
                     "osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs " + neededTilesFromOsm.data?.length + " tiles (already got " + osmFeatureSource.downloadedTiles.size + " tiles )" : "is idle")
                 return overpass || osmFeatureSource.isRunning.data;
             }, [osmFeatureSource.isRunning]
@@ -355,7 +361,15 @@ export default class FeaturePipeline {
             if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) {
                 continue;
             }
-            const freshness = this.freshnesses.get(flayer.layerDef.id).freshnessFor(z, x, y)
+            if(flayer.layerDef.maxAgeOfCache === 0){
+                return undefined;
+            }
+            const freshnessCalc = this.freshnesses.get(flayer.layerDef.id)
+            if(freshnessCalc === undefined){
+                console.warn("No freshness tracker found for ", flayer.layerDef.id)
+                return undefined
+            }
+            const freshness = freshnessCalc.freshnessFor(z, x, y)
             if (freshness === undefined) {
                 // SOmething is undefined --> we return undefined as we have to download
                 return undefined
@@ -409,11 +423,13 @@ export default class FeaturePipeline {
         const minzoom = Math.min(...state.layoutToUse.layers.map(layer => layer.minzoom))
         const overpassIsActive = state.currentBounds.map(bbox => {
             if (bbox === undefined) {
+                console.debug("Disabling overpass source: no bbox")
                 return false
             }
             let zoom = state.locationControl.data.zoom
             if (zoom < minzoom) {
                 // We are zoomed out over the zoomlevel of any layer
+                console.debug("Disabling overpass source: zoom < minzoom")
                 return false;
             }
 
@@ -456,31 +472,26 @@ export default class FeaturePipeline {
         if(src === undefined){
             throw "Src is undefined"
         }
-        window.setTimeout(
+        const layerDef = src.layer.layerDef;
-            () => {
+        MetaTagging.addMetatags(
-                const layerDef = src.layer.layerDef;
+            src.features.data,
-                MetaTagging.addMetatags(
+            {
-                    src.features.data,
+                memberships: this.relationTracker,
-                    {
+                getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox),
-                        memberships: this.relationTracker,
+                getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id)
-                        getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox),
-                        getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id)
-                    },
-                    layerDef,
-                    state,
-                    {
-                        includeDates: true,
-                        // We assume that the non-dated metatags are already set by the cache generator
-                        includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer
-                    }
-                )
             },
-            15
+            layerDef,
+            state,
+            {
+                includeDates: true,
+                // We assume that the non-dated metatags are already set by the cache generator
+                includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer
+            }
         )
 
     }
 
-    private updateAllMetaTagging() {
+    public updateAllMetaTagging() {
         const self = this;
         console.debug("Updating the meta tagging of all tiles as new data got loaded")
         this.perLayerHierarchy.forEach(hierarchy => {

Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts

@@ -36,21 +36,15 @@ export default class PerLayerFeatureSourceSplitter {
             const featuresPerLayer = new Map<string, { feature, freshness } []>();
             const noLayerFound = []
 
-            function addTo(layer: FilteredLayer, feature: { feature, freshness }) {
+            for (const layer of layers.data) {
-                const id = layer.layerDef.id
+                featuresPerLayer.set(layer.layerDef.id, [])
-                const list = featuresPerLayer.get(id)
-                if (list !== undefined) {
-                    list.push(feature)
-                } else {
-                    featuresPerLayer.set(id, [feature])
-                }
             }
 
             for (const f of features) {
                 for (const layer of layers.data) {
                     if (layer.layerDef.source.osmTags.matchesProperties(f.feature.properties)) {
                         // We have found our matching layer!
-                        addTo(layer, f)
+                        featuresPerLayer.set(layer.layerDef.id, [f])
                         if (!layer.layerDef.passAllFeatures) {
                             // If not 'passAllFeatures', we are done for this feature
                             break;

Logic/FeatureSource/Sources/GeoJsonSource.ts

@@ -126,7 +126,7 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
 
                 eventSource.setData(eventSource.data.concat(newFeatures))
 
-            }).catch(msg => console.error("Could not load geojson layer", url, "due to", msg))
+            }).catch(msg => console.debug("Could not load geojson layer", url, "due to", msg))
     }
 
 }

Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts

@@ -5,13 +5,14 @@ import Loc from "../../../Models/Loc";
 import DynamicTileSource from "./DynamicTileSource";
 import {Utils} from "../../../Utils";
 import GeoJsonSource from "../Sources/GeoJsonSource";
+import {BBox} from "../../BBox";
 
 export default class DynamicGeoJsonTileSource extends DynamicTileSource {
     constructor(layer: FilteredLayer,
                 registerLayer: (layer: FeatureSourceForLayer & Tiled) => void,
                 state: {
                     locationControl: UIEventSource<Loc>
-                    leafletMap: any
+                    currentBounds: UIEventSource<BBox>
                 }) {
         const source = layer.layerDef.source
         if (source.geojsonZoomLevel === undefined) {
@@ -29,7 +30,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
                 .replace("{x}_{y}.geojson", "overview.json")
                 .replace("{layer}", layer.layerDef.id)
 
-            Utils.downloadJson(whitelistUrl).then(
+            Utils.downloadJsonCached(whitelistUrl, 1000*60*60).then(
                 json => {
                     const data = new Map<number, Set<number>>();
                     for (const x in json) {

Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts

@@ -4,6 +4,7 @@ import {UIEventSource} from "../../UIEventSource";
 import Loc from "../../../Models/Loc";
 import TileHierarchy from "./TileHierarchy";
 import {Tiles} from "../../../Models/TileRange";
+import {BBox} from "../../BBox";
 
 /***
  * A tiled source which dynamically loads the required tiles at a fixed zoom level
@@ -17,8 +18,8 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
         zoomlevel: number,
         constructTile: (zxy: [number, number, number]) => (FeatureSourceForLayer & Tiled),
         state: {
+            currentBounds: UIEventSource<BBox>;
             locationControl: UIEventSource<Loc>
-            leafletMap: any
         }
     ) {
         const self = this;
@@ -37,7 +38,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
                 }
 
                 // Yup, this is cheating to just get the bounds here
-                const bounds = state.leafletMap.data?.getBounds()
+                const bounds = state.currentBounds.data
                 if (bounds === undefined) {
                     // We'll retry later
                     return undefined
@@ -50,7 +51,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
                 }
                 return needed
             }
-            , [layer.isDisplayed, state.leafletMap]).stabilized(250);
+            , [layer.isDisplayed, state.currentBounds]).stabilized(250);
 
         neededTiles.addCallbackAndRunD(neededIndexes => {
             console.log("Tiled geojson source ", layer.layerDef.id, " needs", neededIndexes)

Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts

@@ -63,10 +63,10 @@ export default class OsmFeatureSource {
             try {
 
                 for (const neededTile of neededTiles) {
-                    console.log("Tile download", Tiles.tile_from_index(neededTile).join("/"), "started")
+                    console.log("Tile download from OSM", Tiles.tile_from_index(neededTile).join("/"), "started")
                     self.downloadedTiles.add(neededTile)
                     self.LoadTile(...Tiles.tile_from_index(neededTile)).then(_ => {
-                        console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded")
+                        console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded from OSM")
                     })
                 }
             } catch (e) {

Logic/Osm/Changes.ts

@@ -1,5 +1,4 @@
 import {OsmNode, OsmObject, OsmRelation, OsmWay} from "./OsmObject";
-import State from "../../State";
 import {UIEventSource} from "../UIEventSource";
 import Constants from "../../Models/Constants";
 import OsmChangeAction from "./Actions/OsmChangeAction";
@@ -13,6 +12,7 @@ import {ElementStorage} from "../ElementStorage";
 import {GeoLocationPointProperties} from "../Actors/GeoLocationHandler";
 import {GeoOperations} from "../GeoOperations";
 import {ChangesetTag} from "./ChangesetHandler";
+import {OsmConnection} from "./OsmConnection";
 
 /**
  * Handles all changes made to OSM.
@@ -33,14 +33,23 @@ export class Changes {
     private readonly previouslyCreated: OsmObject[] = []
     private readonly _leftRightSensitive: boolean;
 
-    private _state: { allElements: ElementStorage; historicalUserLocations: FeatureSource }
+    public readonly state: { allElements: ElementStorage; historicalUserLocations: FeatureSource; osmConnection: OsmConnection }
     
-    constructor(leftRightSensitive: boolean = false) {
+    public readonly extraComment:UIEventSource<string> = new UIEventSource(undefined)
+
+    constructor(
+        state?: {
+            allElements: ElementStorage,
+            historicalUserLocations: FeatureSource,
+            osmConnection: OsmConnection
+        },
+        leftRightSensitive: boolean = false) {
         this._leftRightSensitive = leftRightSensitive;
         // We keep track of all changes just as well
         this.allChanges.setData([...this.pendingChanges.data])
         // If a pending change contains a negative ID, we save that
         this._nextId = Math.min(-1, ...this.pendingChanges.data?.map(pch => pch.id) ?? [])
+        this.state = state;
 
         // Note: a changeset might be reused which was opened just before and might have already used some ids
         // This doesn't matter however, as the '-1' is per piecewise upload, not global per changeset
@@ -120,7 +129,7 @@ export class Changes {
 
     private calculateDistanceToChanges(change: OsmChangeAction, changeDescriptions: ChangeDescription[]) {
 
-        if (this._state === undefined) {
+        if (this.state === undefined) {
             // No state loaded -> we can't calculate...
             return;
         }
@@ -129,7 +138,7 @@ export class Changes {
             return;
         }
         const now = new Date()
-        const recentLocationPoints = this._state.historicalUserLocations.features.data.map(ff => ff.feature)
+        const recentLocationPoints = this.state.historicalUserLocations.features.data.map(ff => ff.feature)
             .filter(feat => feat.geometry.type === "Point")
             .filter(feat => {
                 const visitTime = new Date((<GeoLocationPointProperties>feat.properties).date)
@@ -149,7 +158,7 @@ export class Changes {
 
         const changedObjectCoordinates: [number, number][] = []
 
-        const feature = this._state.allElements.ContainingFeatures.get(change.mainObjectId)
+        const feature = this.state.allElements.ContainingFeatures.get(change.mainObjectId)
         if (feature !== undefined) {
             changedObjectCoordinates.push(GeoOperations.centerpointCoordinates(feature))
         }
@@ -189,12 +198,6 @@ export class Changes {
         this.allChanges.ping()
     }
 
-    public useLocationHistory(state: {
-        allElements: ElementStorage,
-        historicalUserLocations: FeatureSource
-    }) {
-        this._state = state
-    }
 
     public registerIdRewrites(mappings: Map<string, string>): void {
         CreateNewNodeAction.registerIdRewrites(mappings)
@@ -281,9 +284,14 @@ export class Changes {
 
         // This method is only called with changedescriptions for this theme
         const theme = pending[0].meta.theme
+        let comment = "Adding data with #MapComplete for theme #" + theme
+        if(this.extraComment.data !== undefined){
+            comment+="\n\n"+this.extraComment.data
+        }
+        
         const metatags: ChangesetTag[] = [{
             key: "comment",
-            value: "Adding data with #MapComplete for theme #" + theme
+            value: comment
         },
             {
                 key: "theme",
@@ -294,7 +302,7 @@ export class Changes {
             ...perBinMessage
         ]
 
-        await State.state.osmConnection.changesetHandler.UploadChangeset(
+        await this.state.osmConnection.changesetHandler.UploadChangeset(
             (csId) => Changes.createChangesetFor("" + csId, changes),
             metatags
         )

Logic/Osm/ChangesetHandler.ts

@@ -1,9 +1,7 @@
 import escapeHtml from "escape-html";
-// @ts-ignore
+import UserDetails, {OsmConnection} from "./OsmConnection";
-import {OsmConnection, UserDetails} from "./OsmConnection";
 import {UIEventSource} from "../UIEventSource";
 import {ElementStorage} from "../ElementStorage";
-import State from "../../State";
 import Locale from "../../UI/i18n/Locale";
 import Constants from "../../Models/Constants";
 import {Changes} from "./Changes";
@@ -287,8 +285,8 @@ export class ChangesetHandler {
                 ["language", Locale.language.data],
                 ["host", window.location.host],
                 ["path", path],
-                ["source", State.state.currentUserLocation.features.data.length > 0 ? "survey" : undefined],
+                ["source", self.changes.state["currentUserLocation"]?.features?.data?.length > 0 ? "survey" : undefined],
-                ["imagery", State.state.backgroundLayer.data.id],
+                ["imagery", self.changes.state["backgroundLayer"]?.data?.id],
                 ...changesetTags.map(cstag => [cstag.key, cstag.value])
             ]
                 .filter(kv => (kv[1] ?? "") !== "")

Logic/Osm/OsmConnection.ts

@@ -70,7 +70,8 @@ export class OsmConnection {
                     // Used to keep multiple changesets open and to write to the correct changeset
                     layoutName: string,
                     singlePage?: boolean,
-                    osmConfiguration?: "osm" | "osm-test"
+                    osmConfiguration?: "osm" | "osm-test",
+                    attemptLogin?: true | boolean
                 }
     ) {
         this.fakeUser = options.fakeUser ?? false;
@@ -117,7 +118,7 @@ export class OsmConnection {
             options.oauth_token.setData(undefined);
 
         }
-        if (this.auth.authenticated()) {
+        if (this.auth.authenticated() && (options.attemptLogin !== false)) {
             this.AttemptLogin(); // Also updates the user badge
         } else {
             console.log("Not authenticated");

Logic/State/ElementsState.ts

@@ -49,7 +49,8 @@ export default class ElementsState extends FeatureSwitchState {
     constructor(layoutToUse: LayoutConfig) {
         super(layoutToUse);
 
-        this.changes = new Changes(layoutToUse?.isLeftRightSensitive() ?? false)
+        // @ts-ignore
+        this.changes = new Changes(this,layoutToUse?.isLeftRightSensitive() ?? false)
         {
             // -- Location control initialization
             const zoom = UIEventSource.asFloat(

Logic/State/FeaturePipelineState.ts

@@ -9,6 +9,7 @@ import MapState from "./MapState";
 import SelectedFeatureHandler from "../Actors/SelectedFeatureHandler";
 import Hash from "../Web/Hash";
 import {BBox} from "../BBox";
+import {FeatureSourceForLayer} from "../FeatureSource/FeatureSource";
 
 export default class FeaturePipelineState extends MapState {
 

Logic/State/MapState.ts

@@ -82,8 +82,8 @@ export default class MapState extends UserRelatedState {
     public overlayToggles: { config: TilesourceConfig, isDisplayed: UIEventSource<boolean> }[]
 
 
-    constructor(layoutToUse: LayoutConfig) {
+    constructor(layoutToUse: LayoutConfig, options?: {attemptLogin: true | boolean}) {
-        super(layoutToUse);
+        super(layoutToUse, options);
 
         this.availableBackgroundLayers = AvailableBaseLayers.AvailableLayersAt(this.locationControl);
 
@@ -265,7 +265,6 @@ export default class MapState extends UserRelatedState {
 
         let gpsLayerDef: FilteredLayer = this.filteredLayers.data.filter(l => l.layerDef.id === "gps_location_history")[0]
         this.historicalUserLocations = new SimpleFeatureSource(gpsLayerDef, Tiles.tile_index(0, 0, 0), features);
-        this.changes.useLocationHistory(this)
 
 
         const asLine = features.map(allPoints => {

Logic/State/UserRelatedState.ts

@@ -36,7 +36,7 @@ export default class UserRelatedState extends ElementsState {
     public installedThemes: UIEventSource<{ layout: LayoutConfig; definition: string }[]>;
 
 
-    constructor(layoutToUse: LayoutConfig) {
+    constructor(layoutToUse: LayoutConfig, options:{attemptLogin : true | boolean}) {
         super(layoutToUse);
 
         this.osmConnection = new OsmConnection({
@@ -50,7 +50,8 @@ export default class UserRelatedState extends ElementsState {
                 "Used to complete the login"
             ),
             layoutName: layoutToUse?.id,
-            osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data
+            osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data,
+            attemptLogin: options?.attemptLogin
         })
 
         this.mangroveIdentity = new MangroveIdentity(

Logic/UIEventSource.ts

@@ -289,6 +289,7 @@ export class UIEventSource<T> {
 
         const stack = new Error().stack.split("\n");
         const callee = stack[1]
+        
         const newSource = new UIEventSource<J>(
             f(this.data),
             "map(" + this.tag + ")@"+callee
@@ -298,7 +299,7 @@ export class UIEventSource<T> {
             newSource.setData(f(self.data));
         }
 
-        this.addCallbackAndRun(update);
+        this.addCallback(update);
         for (const extraSource of extraSources) {
             extraSource?.addCallback(update);
         }

Models/Constants.ts

@@ -2,7 +2,7 @@ import {Utils} from "../Utils";
 
 export default class Constants {
 
-    public static vNumber = "0.13.0-alpha-6";
+    public static vNumber = "0.13.0-alpha-7";
     public static ImgurApiKey = '7070e7167f0a25a'
     public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85"
 

UI/AutomatonGui.ts

@@ -5,7 +5,6 @@ import Title from "./Base/Title";
 import Toggle from "./Input/Toggle";
 import {SubtleButton} from "./Base/SubtleButton";
 import LayoutConfig from "../Models/ThemeConfig/LayoutConfig";
-import UserRelatedState from "../Logic/State/UserRelatedState";
 import ValidatedTextField from "./Input/ValidatedTextField";
 import {Utils} from "../Utils";
 import {UIEventSource} from "../Logic/UIEventSource";
@@ -16,11 +15,19 @@ import {LocalStorageSource} from "../Logic/Web/LocalStorageSource";
 import {DropDown} from "./Input/DropDown";
 import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
 import MinimapImplementation from "./Base/MinimapImplementation";
-import State from "../State";
 import {OsmConnection} from "../Logic/Osm/OsmConnection";
-import FeaturePipelineState from "../Logic/State/FeaturePipelineState";
+import {BBox} from "../Logic/BBox";
+import MapState from "../Logic/State/MapState";
+import FeaturePipeline from "../Logic/FeatureSource/FeaturePipeline";
+import LayerConfig from "../Models/ThemeConfig/LayerConfig";
+import TagRenderingConfig from "../Models/ThemeConfig/TagRenderingConfig";
+import FeatureSource from "../Logic/FeatureSource/FeatureSource";
+import List from "./Base/List";
+import {QueryParameters} from "../Logic/Web/QueryParameters";
+import {SubstitutedTranslation} from "./SubstitutedTranslation";
+import {AutoAction} from "./Popup/AutoApplyButton";
 
-export default class AutomatonGui extends Combine {
+class AutomatonGui extends Combine {
 
     constructor() {
 
@@ -28,7 +35,8 @@ export default class AutomatonGui extends Combine {
             allElements: undefined,
             changes: undefined,
             layoutName: "automaton",
-            singlePage: true
+            singlePage: false,
+            oauth_token: QueryParameters.GetQueryParameter("oauth_token", "OAuth token")
         });
 
         super([
@@ -39,35 +47,174 @@ export default class AutomatonGui extends Combine {
             ]).SetClass("flex"),
             new Toggle(
                 AutomatonGui.GenerateMainPanel(),
-                new SubtleButton(Svg.osm_logo_svg(), "Login to get started"),
+                new SubtleButton(Svg.osm_logo_svg(), "Login to get started").onClick(() => osmConnection.AttemptLogin()),
                 osmConnection.isLoggedIn
             )])
     }
 
-    private static AutomationPanel(layoutToUse: LayoutConfig, tiles: UIEventSource<number[]>): BaseUIElement {
+    private static startedTiles = new Set<number>()
-        const handledTiles = new UIEventSource(0)
 
-        const state = new FeaturePipelineState(layoutToUse)
+    private static TileHandler(layoutToUse: LayoutConfig, tileIndex: number, targetLayer: string, targetAction: TagRenderingConfig, extraCommentText: UIEventSource<string>, whenDone: ((result: string) => void)): BaseUIElement {
 
+        if (AutomatonGui.startedTiles.has(tileIndex)) {
+            throw "Already started tile " + tileIndex
+        }
+        AutomatonGui.startedTiles.add(tileIndex)
 
-        const nextTile = tiles.map(indices => {
+        const state = new MapState(layoutToUse, {attemptLogin: false})
-            if (indices === undefined) {
+        extraCommentText.syncWith( state.changes.extraComment)
-                return "No tiles loaded - can not automate";
+        const [z, x, y] = Tiles.tile_from_index(tileIndex)
-            }
+        state.locationControl.setData({
-            const currentTile = handledTiles.data
+            zoom: z,
-            const tileIndex = indices[currentTile]
+            lon: x,
-            if (tileIndex === undefined) {
+            lat: y
-                return "All done!";
+        })
+        state.currentBounds.setData(
+            BBox.fromTileIndex(tileIndex)
+        )
+
+        let targetTiles: UIEventSource<FeatureSource[]> = new UIEventSource<FeatureSource[]>([])
+        const pipeline = new FeaturePipeline((tile => {
+            const layerId = tile.layer.layerDef.id
+            if (layerId === targetLayer) {
+                targetTiles.data.push(tile)
+                targetTiles.ping()
             }
+        }), state)
 
+        state.locationControl.ping();
+        state.currentBounds.ping();
+        const stateToShow = new UIEventSource("")
 
-            return "" + tileIndex
+        pipeline.runningQuery.map(
-        }, [handledTiles])
+            async isRunning => {
+                if (targetTiles.data.length === 0) {
+                    stateToShow.setData("No data loaded yet...")
+                    return;
+                }
+                if (isRunning) {
+                    stateToShow.setData("Waiting for all layers to be loaded... Has " + targetTiles.data.length + " tiles already")
+                    return;
+                }
+                if (targetTiles.data.length === 0) {
+                    stateToShow.setData("No features found to apply the action")
+                    whenDone("empty")
+                    return true;
+                }
+                stateToShow.setData("Applying metatags")
+                pipeline.updateAllMetaTagging()
+                stateToShow.setData("Gathering applicable elements")
+
+                let handled = 0
+                let inspected = 0
+                for (const targetTile of targetTiles.data) {
+
+                    for (const ffs of targetTile.features.data) {
+                        inspected++
+                        if (inspected % 10 === 0) {
+                            stateToShow.setData("Inspected " + inspected + " features, updated " + handled + " features")
+                        }
+                        const feature = ffs.feature
+                        const rendering = targetAction.GetRenderValue(feature.properties).txt
+                        const actions = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering)
+                            .map(obj => obj.special))
+                        for (const action of actions) {
+                            const auto = <AutoAction>action.func
+                            if (auto.supportsAutoAction !== true) {
+                                continue
+                            }
+
+                            await auto.applyActionOn({
+                                layoutToUse: state.layoutToUse,
+                                changes: state.changes
+                            }, state.allElements.getEventSourceById(feature.properties.id), action.args)
+                            handled++
+                        }
+                    }
+                }
+                stateToShow.setData("Done! Inspected " + inspected + " features, updated " + handled + " features")
+
+                if (inspected === 0) {
+                    whenDone("empty")
+                    return;
+                }
+
+                if (handled === 0) {
+                    window.setTimeout(() => whenDone("no-action"), 1000)
+                }else{
+                    state.changes.flushChanges("handled tile automatically, time to flush!")
+                    whenDone("fixed")
+                }
+
+            }, [targetTiles])
 
         return new Combine([
-            new VariableUiElement(handledTiles.map(i => "" + i)),
+            new Title("Performing action for tile " + tileIndex, 1),
-            new VariableUiElement(nextTile)
+            new VariableUiElement(stateToShow)]).SetClass("flex flex-col")
-        ])
+    }
+
+    private static AutomationPanel(layoutToUse: LayoutConfig, indices: number[], extraCommentText: UIEventSource<string>, tagRenderingToAutomate: { layer: LayerConfig, tagRendering: TagRenderingConfig }): BaseUIElement {
+        const layerId = tagRenderingToAutomate.layer.id
+        const trId = tagRenderingToAutomate.tagRendering.id
+        const tileState = LocalStorageSource.GetParsed("automation-tile_state-" + layerId + "-" + trId, {})
+
+        if (indices === undefined) {
+            return new FixedUiElement("No tiles loaded - can not automate")
+        }
+
+
+        const nextTileToHandle = tileState.map(handledTiles => {
+            for (const index of indices) {
+                if (handledTiles[index] !== undefined) {
+                    // Already handled
+                    continue
+                }
+                return index
+            }
+            return undefined
+        })
+        nextTileToHandle.addCallback(t => console.warn("Next tile to handle is", t))
+
+        const neededTimes = new UIEventSource<number[]>([])
+        const automaton = new VariableUiElement(nextTileToHandle.map(tileIndex => {
+            if (tileIndex === undefined) {
+                return new FixedUiElement("All done!").SetClass("thanks")
+            }
+            console.warn("Triggered map on nextTileToHandle",tileIndex)
+            const start = new Date()
+            return AutomatonGui.TileHandler(layoutToUse, tileIndex, layerId, tagRenderingToAutomate.tagRendering, extraCommentText,(result) => {
+                const end = new Date()
+                const timeNeeded = (end.getTime() - start.getTime()) / 1000;
+                neededTimes.data.push(timeNeeded)
+                neededTimes.ping()
+                tileState.data[tileIndex] = result
+                tileState.ping();
+            });
+        }))
+
+
+        const statistics = new VariableUiElement(tileState.map(states => {
+            let total = 0
+            const perResult = new Map<string, number>()
+            for (const key in states) {
+                total++
+                const result = states[key]
+                perResult.set(result, (perResult.get(result) ?? 0) + 1)
+            }
+
+            let sum = 0
+            neededTimes.data.forEach(v => {
+                sum = sum + v
+            })
+            let timePerTile = sum / neededTimes.data.length
+
+            return new Combine(["Handled " + total + "/" + indices.length + " tiles: ",
+                new List(Array.from(perResult.keys()).map(key => key + ": " + perResult.get(key))),
+                "Handling one tile needs " + (Math.floor(timePerTile * 100) / 100) + "s on average. Estimated time left: " + Math.floor((indices.length - total) * timePerTile) + "s"
+            ]).SetClass("flex flex-col")
+        }))
+
+        return new Combine([statistics, automaton]).SetClass("flex flex-col")
     }
 
     private static GenerateMainPanel(): BaseUIElement {
@@ -85,18 +232,22 @@ export default class AutomatonGui extends Combine {
         tilepath.SetClass("w-full")
         LocalStorageSource.Get("automation-tile_path").syncWith(tilepath.GetValue(), true)
 
-        const tilesToRunOver = tilepath.GetValue().bind(path => {
+        
+        let tilesToRunOver = tilepath.GetValue().bind(path => {
             if (path === undefined) {
                 return undefined
             }
-            return UIEventSource.FromPromiseWithErr(Utils.downloadJson(path))
+            return UIEventSource.FromPromiseWithErr(Utils.downloadJsonCached(path,1000*60*60))
         })
         
+        const targetZoom = 14
+
         const tilesPerIndex = tilesToRunOver.map(tiles => {
+            
             if (tiles === undefined || tiles["error"] !== undefined) {
                 return undefined
             }
-            let indexes = [];
+            let indexes : number[] = [];
             const tilesS = tiles["success"]
             const z = Number(tilesS["zoom"])
             for (const key in tilesS) {
@@ -107,13 +258,31 @@ export default class AutomatonGui extends Combine {
                 const ys = tilesS[key]
                 indexes.push(...ys.map(y => Tiles.tile_index(z, x, y)))
             }
-            return indexes
+
+            console.log("Got ", indexes.length, "indexes")
+            let rezoomed = new Set<number>()
+            for (const index of indexes) {
+                let [z, x, y] = Tiles.tile_from_index(index)
+                while (z > targetZoom) {
+                    z--
+                    x = Math.floor(x / 2)
+                    y = Math.floor(y / 2)
+                }
+                rezoomed.add(Tiles.tile_index(z, x, y))
+            }
+
+
+            return Array.from(rezoomed)
         })
 
+        const extraComment = ValidatedTextField.InputForType("text")
+        LocalStorageSource.Get("automaton-extra-comment").syncWith(extraComment.GetValue())
+
         return new Combine([
             themeSelect,
             "Specify the path to a tile overview. This is a hosted .json of the format {x : [y0, y1, y2], x1: [y0, ...]} where x is a string and y are numbers",
             tilepath,
+            extraComment,
             new VariableUiElement(tilesToRunOver.map(t => {
                 if (t === undefined) {
                     return "No path given or still loading..."
@@ -128,10 +297,43 @@ export default class AutomatonGui extends Combine {
                 if (layoutToUse === undefined) {
                     return new FixedUiElement("Select a valid layout")
                 }
+                if (tilesPerIndex.data === undefined || tilesPerIndex.data.length === 0) {
+                    return "No tiles given"
+                }
 
-                return AutomatonGui.AutomationPanel(layoutToUse, tilesPerIndex)
+                const automatableTagRenderings: { layer: LayerConfig, tagRendering: TagRenderingConfig }[] = []
+                for (const layer of layoutToUse.layers) {
+                    for (const tagRendering of layer.tagRenderings) {
+                        if (tagRendering.group === "auto") {
+                            automatableTagRenderings.push({layer, tagRendering: tagRendering})
+                        }
+                    }
+                }
+                console.log("Automatable tag renderings:", automatableTagRenderings)
+                if (automatableTagRenderings.length === 0) {
+                    return new FixedUiElement('This theme does not have any tagRendering with "group": "auto" set').SetClass("alert")
+                }
+                const pickAuto = new DropDown("Pick the action to automate",
+                    [
+                        {
+                            value: undefined,
+                            shown: "Pick an option"
+                        },
+                        ...automatableTagRenderings.map(config => (
+                            {
+                                shown: config.layer.id + " - " + config.tagRendering.id,
+                                value: config
+                            }
+                        ))
+                    ]
+                )
 
-            }))
+
+                return new Combine([
+                    pickAuto,
+                    new VariableUiElement(pickAuto.GetValue().map(auto => auto === undefined ? undefined : AutomatonGui.AutomationPanel(layoutToUse, tilesPerIndex.data, extraComment.GetValue(), auto)))])
+
+            }, [tilesPerIndex])).SetClass("flex flex-col")
 
 
         ]).SetClass("flex flex-col")

UI/Popup/AutoApplyButton.ts

@@ -17,11 +17,16 @@ import {VariableUiElement} from "../Base/VariableUIElement";
 import Loading from "../Base/Loading";
 import {OsmConnection} from "../../Logic/Osm/OsmConnection";
 import Translations from "../i18n/Translations";
+import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
+import {Changes} from "../../Logic/Osm/Changes";
 
 export interface AutoAction extends SpecialVisualization {
     supportsAutoAction: boolean
 
-    applyActionOn(state: FeaturePipelineState, tagSource: UIEventSource<any>, argument: string[]): Promise<void>
+    applyActionOn(state: {
+        layoutToUse: LayoutConfig,
+        changes: Changes
+    }, tagSource: UIEventSource<any>, argument: string[]): Promise<void>
 }
 
 export default class AutoApplyButton implements SpecialVisualization {

UI/Popup/TagApplyButton.ts

@@ -12,6 +12,8 @@ import Toggle from "../Input/Toggle";
 import {Utils} from "../../Utils";
 import {Tag} from "../../Logic/Tags/Tag";
 import FeaturePipelineState from "../../Logic/State/FeaturePipelineState";
+import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
+import {Changes} from "../../Logic/Osm/Changes";
 
 export default class TagApplyButton implements AutoAction {
     public readonly funcName = "tag_apply";
@@ -79,7 +81,10 @@ export default class TagApplyButton implements AutoAction {
 
     public readonly example = "`{tag_apply(survey_date=$_now:date, Surveyed today!)}`, `{tag_apply(addr:street=$addr:street, Apply the address, apply_icon.svg, _closest_osm_id)";
 
-    async applyActionOn(state: FeaturePipelineState, tags: UIEventSource<any>, args: string[]) : Promise<void>{
+    async applyActionOn(state: {
+        layoutToUse: LayoutConfig,
+        changes: Changes
+    }, tags: UIEventSource<any>, args: string[]) : Promise<void>{
         const tagsToApply = TagApplyButton.generateTagsToApply(args[0], tags)
         const targetIdKey = args[3]
 

assets/themes/grb_import/missing_streets.json

@@ -76,6 +76,10 @@
     {
       "builtin": "crab_address",
       "override": {
+        "source": {
+          "geoJson": "http://127.0.0.1:8080/tile_{z}_{x}_{y}.geojson",
+          "geoJsonZoomLevel": 18
+        },
         "mapRendering": [
           {
             "iconSize": "5,5,center",
@@ -101,8 +105,8 @@
         "_embedded_crab_addresses:=Array.from(new Set(feat.overlapWith('crab_address').map(ff => ff.feat.properties).filter(p => p._HNRLABEL.toLowerCase() === (feat.properties['addr:housenumber'] + (feat.properties['addr:unit']??'')).toLowerCase()).map(p => p.STRAATNM)))",
         "_singular_import:=feat.get('_embedded_crab_addresses')?.length == 1",
         "_name_to_apply:=feat.get('_embedded_crab_addresses')[0]",
-        "_nearby_street_names:=feat.closestn('named_streets',5,'name', 500).map(ff => ff.feat.properties.name)",
+        "_nearby_street_names:=feat.closestn('named_streets',5,'name', 1000).map(ff => [ff.feat.properties.name, ff.feat.properties['alt_name'], ff.feat.properties['name:nl']])",
-        "_spelling_is_correct:= feat.get('_nearby_street_names').indexOf(feat.properties['_name_to_apply']) >= 0"
+        "_spelling_is_correct:= [].concat(...feat.get('_nearby_street_names')).indexOf(feat.properties['_name_to_apply']) >= 0"
       ],
       "mapRendering": [
         {
@@ -134,6 +138,7 @@
       "tagRenderings": [
         {
           "id": "apply_streetname",
+          "group": "auto",
           "render": "{tag_apply(addr:street=$_name_to_apply ,Apply the CRAB-street onto this building)}",
           "mappings": [
             {
@@ -146,8 +151,7 @@
             }
           ]
         }
-      ],
+      ]
-      "passAllFeatures": true
     }
   ],
   "hideFromOverview": true

scripts/generateTileOverview.ts

@@ -2,7 +2,6 @@
  * Generates an overview for which tiles exist and which don't
  */
 import ScriptUtils from "./ScriptUtils";
-import {Tiles} from "../Models/TileRange";
 import {writeFileSync} from "fs";
 
 function main(args: string[]) {
@@ -28,10 +27,10 @@ function main(args: string[]) {
         
         const x = match[2]
         const y = match[3]
-        if(!indices[x] !== undefined){
+        if(indices[x] === undefined){
             indices[x] = []
         }
-        indices[x] .push(Number(y))
+        indices[x].push(Number(y))
     }
     indices["zoom"] = zoomLevel;
     const match = files[0].match("\(.*\)_\([0-9]*\)_\([0-9]*\)_\([0-9]*\).geojson")