From e922768f993c81d3a92cd8666ee565d42521bfe4 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Mon, 13 Dec 2021 02:05:34 +0100 Subject: [PATCH] First working version of fully automatic uploader --- .../Actors/SaveTileToLocalStorageActor.ts | 6 +- Logic/FeatureSource/FeaturePipeline.ts | 55 ++-- .../PerLayerFeatureSourceSplitter.ts | 12 +- Logic/FeatureSource/Sources/GeoJsonSource.ts | 2 +- .../DynamicGeoJsonTileSource.ts | 5 +- .../TiledFeatureSource/DynamicTileSource.ts | 7 +- .../TiledFeatureSource/OsmFeatureSource.ts | 4 +- Logic/Osm/Changes.ts | 36 ++- Logic/Osm/ChangesetHandler.ts | 8 +- Logic/Osm/OsmConnection.ts | 5 +- Logic/State/ElementsState.ts | 3 +- Logic/State/FeaturePipelineState.ts | 1 + Logic/State/MapState.ts | 5 +- Logic/State/UserRelatedState.ts | 5 +- Logic/UIEventSource.ts | 3 +- Models/Constants.ts | 2 +- UI/AutomatonGui.ts | 258 ++++++++++++++++-- UI/Popup/AutoApplyButton.ts | 7 +- UI/Popup/TagApplyButton.ts | 7 +- assets/themes/grb_import/missing_streets.json | 12 +- scripts/generateTileOverview.ts | 5 +- 21 files changed, 342 insertions(+), 106 deletions(-) diff --git a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts index c1da5ab371..5b884ac87b 100644 --- a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts +++ b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts @@ -119,7 +119,11 @@ export default class SaveTileToLocalStorageActor { } private SetIdb(tileIndex, data) { - IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data) + try{ + IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data) + }catch(e){ + console.error("Could not save tile to indexed-db: ", e, "tileIndex is:", tileIndex, "for layer", this._layer.id) + } } private GetIdb(tileIndex) { diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index 43099c7134..895bc155b8 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -171,6 +171,7 @@ export default class FeaturePipeline { state.currentBounds, state.locationControl, (tileIndex, freshness) => self.freshnesses.get(id).addTileLoad(tileIndex, freshness), (tile) => { + console.debug("Loaded tile ", id, tile.tileIndex, "from local cache") new RegisteringAllFromFeatureSourceActor(tile, state.allElements) hierarchy.registerTile(tile); tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) @@ -247,6 +248,7 @@ export default class FeaturePipeline { }) }) + if (state.layoutToUse.trackAllNodes) { const fullNodeDb = new FullNodeDatabaseSource( state.filteredLayers.data.filter(l => l.layerDef.id === "type_node")[0], @@ -289,6 +291,10 @@ export default class FeaturePipeline { // A NewGeometryFromChangesFeatureSource does not split per layer, so we do this next new PerLayerFeatureSourceSplitter(state.filteredLayers, (perLayer) => { + if(perLayer.features.data.length === 0){ + return + } + // We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer) // AT last, we always apply the metatags whenever possible @@ -309,7 +315,7 @@ export default class FeaturePipeline { this.runningQuery = updater.runningQuery.map( overpass => { - console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,", + console.log("FeaturePipeline: runningQuery state changed: Overpass", overpass ? "is querying," : "is idle,", "osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs " + neededTilesFromOsm.data?.length + " tiles (already got " + osmFeatureSource.downloadedTiles.size + " tiles )" : "is idle") return overpass || osmFeatureSource.isRunning.data; }, [osmFeatureSource.isRunning] @@ -355,7 +361,15 @@ export default class FeaturePipeline { if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) { continue; } - const freshness = this.freshnesses.get(flayer.layerDef.id).freshnessFor(z, x, y) + if(flayer.layerDef.maxAgeOfCache === 0){ + return undefined; + } + const freshnessCalc = this.freshnesses.get(flayer.layerDef.id) + if(freshnessCalc === undefined){ + console.warn("No freshness tracker found for ", flayer.layerDef.id) + return undefined + } + const freshness = freshnessCalc.freshnessFor(z, x, y) if (freshness === undefined) { // SOmething is undefined --> we return undefined as we have to download return undefined @@ -409,11 +423,13 @@ export default class FeaturePipeline { const minzoom = Math.min(...state.layoutToUse.layers.map(layer => layer.minzoom)) const overpassIsActive = state.currentBounds.map(bbox => { if (bbox === undefined) { + console.debug("Disabling overpass source: no bbox") return false } let zoom = state.locationControl.data.zoom if (zoom < minzoom) { // We are zoomed out over the zoomlevel of any layer + console.debug("Disabling overpass source: zoom < minzoom") return false; } @@ -456,31 +472,26 @@ export default class FeaturePipeline { if(src === undefined){ throw "Src is undefined" } - window.setTimeout( - () => { - const layerDef = src.layer.layerDef; - MetaTagging.addMetatags( - src.features.data, - { - memberships: this.relationTracker, - getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox), - getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id) - }, - layerDef, - state, - { - includeDates: true, - // We assume that the non-dated metatags are already set by the cache generator - includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer - } - ) + const layerDef = src.layer.layerDef; + MetaTagging.addMetatags( + src.features.data, + { + memberships: this.relationTracker, + getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox), + getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id) }, - 15 + layerDef, + state, + { + includeDates: true, + // We assume that the non-dated metatags are already set by the cache generator + includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer + } ) } - private updateAllMetaTagging() { + public updateAllMetaTagging() { const self = this; console.debug("Updating the meta tagging of all tiles as new data got loaded") this.perLayerHierarchy.forEach(hierarchy => { diff --git a/Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts b/Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts index 3e6f5f2a7f..c9c58fff37 100644 --- a/Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts +++ b/Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts @@ -36,21 +36,15 @@ export default class PerLayerFeatureSourceSplitter { const featuresPerLayer = new Map(); const noLayerFound = [] - function addTo(layer: FilteredLayer, feature: { feature, freshness }) { - const id = layer.layerDef.id - const list = featuresPerLayer.get(id) - if (list !== undefined) { - list.push(feature) - } else { - featuresPerLayer.set(id, [feature]) - } + for (const layer of layers.data) { + featuresPerLayer.set(layer.layerDef.id, []) } for (const f of features) { for (const layer of layers.data) { if (layer.layerDef.source.osmTags.matchesProperties(f.feature.properties)) { // We have found our matching layer! - addTo(layer, f) + featuresPerLayer.set(layer.layerDef.id, [f]) if (!layer.layerDef.passAllFeatures) { // If not 'passAllFeatures', we are done for this feature break; diff --git a/Logic/FeatureSource/Sources/GeoJsonSource.ts b/Logic/FeatureSource/Sources/GeoJsonSource.ts index e22835808c..a229bb4932 100644 --- a/Logic/FeatureSource/Sources/GeoJsonSource.ts +++ b/Logic/FeatureSource/Sources/GeoJsonSource.ts @@ -126,7 +126,7 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled { eventSource.setData(eventSource.data.concat(newFeatures)) - }).catch(msg => console.error("Could not load geojson layer", url, "due to", msg)) + }).catch(msg => console.debug("Could not load geojson layer", url, "due to", msg)) } } diff --git a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts index 21aeec1c1d..eac489ca36 100644 --- a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts @@ -5,13 +5,14 @@ import Loc from "../../../Models/Loc"; import DynamicTileSource from "./DynamicTileSource"; import {Utils} from "../../../Utils"; import GeoJsonSource from "../Sources/GeoJsonSource"; +import {BBox} from "../../BBox"; export default class DynamicGeoJsonTileSource extends DynamicTileSource { constructor(layer: FilteredLayer, registerLayer: (layer: FeatureSourceForLayer & Tiled) => void, state: { locationControl: UIEventSource - leafletMap: any + currentBounds: UIEventSource }) { const source = layer.layerDef.source if (source.geojsonZoomLevel === undefined) { @@ -29,7 +30,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { .replace("{x}_{y}.geojson", "overview.json") .replace("{layer}", layer.layerDef.id) - Utils.downloadJson(whitelistUrl).then( + Utils.downloadJsonCached(whitelistUrl, 1000*60*60).then( json => { const data = new Map>(); for (const x in json) { diff --git a/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts b/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts index dcc415f31e..bcc7b71865 100644 --- a/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts @@ -4,6 +4,7 @@ import {UIEventSource} from "../../UIEventSource"; import Loc from "../../../Models/Loc"; import TileHierarchy from "./TileHierarchy"; import {Tiles} from "../../../Models/TileRange"; +import {BBox} from "../../BBox"; /*** * A tiled source which dynamically loads the required tiles at a fixed zoom level @@ -17,8 +18,8 @@ export default class DynamicTileSource implements TileHierarchy (FeatureSourceForLayer & Tiled), state: { + currentBounds: UIEventSource; locationControl: UIEventSource - leafletMap: any } ) { const self = this; @@ -37,7 +38,7 @@ export default class DynamicTileSource implements TileHierarchy { console.log("Tiled geojson source ", layer.layerDef.id, " needs", neededIndexes) diff --git a/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts b/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts index b7e11e8345..35fa4bacd9 100644 --- a/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts @@ -63,10 +63,10 @@ export default class OsmFeatureSource { try { for (const neededTile of neededTiles) { - console.log("Tile download", Tiles.tile_from_index(neededTile).join("/"), "started") + console.log("Tile download from OSM", Tiles.tile_from_index(neededTile).join("/"), "started") self.downloadedTiles.add(neededTile) self.LoadTile(...Tiles.tile_from_index(neededTile)).then(_ => { - console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded") + console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded from OSM") }) } } catch (e) { diff --git a/Logic/Osm/Changes.ts b/Logic/Osm/Changes.ts index 1a5de57816..9fded4d20b 100644 --- a/Logic/Osm/Changes.ts +++ b/Logic/Osm/Changes.ts @@ -1,5 +1,4 @@ import {OsmNode, OsmObject, OsmRelation, OsmWay} from "./OsmObject"; -import State from "../../State"; import {UIEventSource} from "../UIEventSource"; import Constants from "../../Models/Constants"; import OsmChangeAction from "./Actions/OsmChangeAction"; @@ -13,6 +12,7 @@ import {ElementStorage} from "../ElementStorage"; import {GeoLocationPointProperties} from "../Actors/GeoLocationHandler"; import {GeoOperations} from "../GeoOperations"; import {ChangesetTag} from "./ChangesetHandler"; +import {OsmConnection} from "./OsmConnection"; /** * Handles all changes made to OSM. @@ -33,14 +33,23 @@ export class Changes { private readonly previouslyCreated: OsmObject[] = [] private readonly _leftRightSensitive: boolean; - private _state: { allElements: ElementStorage; historicalUserLocations: FeatureSource } + public readonly state: { allElements: ElementStorage; historicalUserLocations: FeatureSource; osmConnection: OsmConnection } + + public readonly extraComment:UIEventSource = new UIEventSource(undefined) - constructor(leftRightSensitive: boolean = false) { + constructor( + state?: { + allElements: ElementStorage, + historicalUserLocations: FeatureSource, + osmConnection: OsmConnection + }, + leftRightSensitive: boolean = false) { this._leftRightSensitive = leftRightSensitive; // We keep track of all changes just as well this.allChanges.setData([...this.pendingChanges.data]) // If a pending change contains a negative ID, we save that this._nextId = Math.min(-1, ...this.pendingChanges.data?.map(pch => pch.id) ?? []) + this.state = state; // Note: a changeset might be reused which was opened just before and might have already used some ids // This doesn't matter however, as the '-1' is per piecewise upload, not global per changeset @@ -120,7 +129,7 @@ export class Changes { private calculateDistanceToChanges(change: OsmChangeAction, changeDescriptions: ChangeDescription[]) { - if (this._state === undefined) { + if (this.state === undefined) { // No state loaded -> we can't calculate... return; } @@ -129,7 +138,7 @@ export class Changes { return; } const now = new Date() - const recentLocationPoints = this._state.historicalUserLocations.features.data.map(ff => ff.feature) + const recentLocationPoints = this.state.historicalUserLocations.features.data.map(ff => ff.feature) .filter(feat => feat.geometry.type === "Point") .filter(feat => { const visitTime = new Date((feat.properties).date) @@ -149,7 +158,7 @@ export class Changes { const changedObjectCoordinates: [number, number][] = [] - const feature = this._state.allElements.ContainingFeatures.get(change.mainObjectId) + const feature = this.state.allElements.ContainingFeatures.get(change.mainObjectId) if (feature !== undefined) { changedObjectCoordinates.push(GeoOperations.centerpointCoordinates(feature)) } @@ -189,12 +198,6 @@ export class Changes { this.allChanges.ping() } - public useLocationHistory(state: { - allElements: ElementStorage, - historicalUserLocations: FeatureSource - }) { - this._state = state - } public registerIdRewrites(mappings: Map): void { CreateNewNodeAction.registerIdRewrites(mappings) @@ -281,9 +284,14 @@ export class Changes { // This method is only called with changedescriptions for this theme const theme = pending[0].meta.theme + let comment = "Adding data with #MapComplete for theme #" + theme + if(this.extraComment.data !== undefined){ + comment+="\n\n"+this.extraComment.data + } + const metatags: ChangesetTag[] = [{ key: "comment", - value: "Adding data with #MapComplete for theme #" + theme + value: comment }, { key: "theme", @@ -294,7 +302,7 @@ export class Changes { ...perBinMessage ] - await State.state.osmConnection.changesetHandler.UploadChangeset( + await this.state.osmConnection.changesetHandler.UploadChangeset( (csId) => Changes.createChangesetFor("" + csId, changes), metatags ) diff --git a/Logic/Osm/ChangesetHandler.ts b/Logic/Osm/ChangesetHandler.ts index 866c61faac..50e899f383 100644 --- a/Logic/Osm/ChangesetHandler.ts +++ b/Logic/Osm/ChangesetHandler.ts @@ -1,9 +1,7 @@ import escapeHtml from "escape-html"; -// @ts-ignore -import {OsmConnection, UserDetails} from "./OsmConnection"; +import UserDetails, {OsmConnection} from "./OsmConnection"; import {UIEventSource} from "../UIEventSource"; import {ElementStorage} from "../ElementStorage"; -import State from "../../State"; import Locale from "../../UI/i18n/Locale"; import Constants from "../../Models/Constants"; import {Changes} from "./Changes"; @@ -287,8 +285,8 @@ export class ChangesetHandler { ["language", Locale.language.data], ["host", window.location.host], ["path", path], - ["source", State.state.currentUserLocation.features.data.length > 0 ? "survey" : undefined], - ["imagery", State.state.backgroundLayer.data.id], + ["source", self.changes.state["currentUserLocation"]?.features?.data?.length > 0 ? "survey" : undefined], + ["imagery", self.changes.state["backgroundLayer"]?.data?.id], ...changesetTags.map(cstag => [cstag.key, cstag.value]) ] .filter(kv => (kv[1] ?? "") !== "") diff --git a/Logic/Osm/OsmConnection.ts b/Logic/Osm/OsmConnection.ts index 3ef0593c44..cbf6051272 100644 --- a/Logic/Osm/OsmConnection.ts +++ b/Logic/Osm/OsmConnection.ts @@ -70,7 +70,8 @@ export class OsmConnection { // Used to keep multiple changesets open and to write to the correct changeset layoutName: string, singlePage?: boolean, - osmConfiguration?: "osm" | "osm-test" + osmConfiguration?: "osm" | "osm-test", + attemptLogin?: true | boolean } ) { this.fakeUser = options.fakeUser ?? false; @@ -117,7 +118,7 @@ export class OsmConnection { options.oauth_token.setData(undefined); } - if (this.auth.authenticated()) { + if (this.auth.authenticated() && (options.attemptLogin !== false)) { this.AttemptLogin(); // Also updates the user badge } else { console.log("Not authenticated"); diff --git a/Logic/State/ElementsState.ts b/Logic/State/ElementsState.ts index b0df7cdb8c..90abe4d037 100644 --- a/Logic/State/ElementsState.ts +++ b/Logic/State/ElementsState.ts @@ -49,7 +49,8 @@ export default class ElementsState extends FeatureSwitchState { constructor(layoutToUse: LayoutConfig) { super(layoutToUse); - this.changes = new Changes(layoutToUse?.isLeftRightSensitive() ?? false) + // @ts-ignore + this.changes = new Changes(this,layoutToUse?.isLeftRightSensitive() ?? false) { // -- Location control initialization const zoom = UIEventSource.asFloat( diff --git a/Logic/State/FeaturePipelineState.ts b/Logic/State/FeaturePipelineState.ts index 7322fb59f6..5f1e6d426f 100644 --- a/Logic/State/FeaturePipelineState.ts +++ b/Logic/State/FeaturePipelineState.ts @@ -9,6 +9,7 @@ import MapState from "./MapState"; import SelectedFeatureHandler from "../Actors/SelectedFeatureHandler"; import Hash from "../Web/Hash"; import {BBox} from "../BBox"; +import {FeatureSourceForLayer} from "../FeatureSource/FeatureSource"; export default class FeaturePipelineState extends MapState { diff --git a/Logic/State/MapState.ts b/Logic/State/MapState.ts index 7bcdbada6f..a5b13e0987 100644 --- a/Logic/State/MapState.ts +++ b/Logic/State/MapState.ts @@ -82,8 +82,8 @@ export default class MapState extends UserRelatedState { public overlayToggles: { config: TilesourceConfig, isDisplayed: UIEventSource }[] - constructor(layoutToUse: LayoutConfig) { - super(layoutToUse); + constructor(layoutToUse: LayoutConfig, options?: {attemptLogin: true | boolean}) { + super(layoutToUse, options); this.availableBackgroundLayers = AvailableBaseLayers.AvailableLayersAt(this.locationControl); @@ -265,7 +265,6 @@ export default class MapState extends UserRelatedState { let gpsLayerDef: FilteredLayer = this.filteredLayers.data.filter(l => l.layerDef.id === "gps_location_history")[0] this.historicalUserLocations = new SimpleFeatureSource(gpsLayerDef, Tiles.tile_index(0, 0, 0), features); - this.changes.useLocationHistory(this) const asLine = features.map(allPoints => { diff --git a/Logic/State/UserRelatedState.ts b/Logic/State/UserRelatedState.ts index cd8a6f1f65..1e70803529 100644 --- a/Logic/State/UserRelatedState.ts +++ b/Logic/State/UserRelatedState.ts @@ -36,7 +36,7 @@ export default class UserRelatedState extends ElementsState { public installedThemes: UIEventSource<{ layout: LayoutConfig; definition: string }[]>; - constructor(layoutToUse: LayoutConfig) { + constructor(layoutToUse: LayoutConfig, options:{attemptLogin : true | boolean}) { super(layoutToUse); this.osmConnection = new OsmConnection({ @@ -50,7 +50,8 @@ export default class UserRelatedState extends ElementsState { "Used to complete the login" ), layoutName: layoutToUse?.id, - osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data + osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data, + attemptLogin: options?.attemptLogin }) this.mangroveIdentity = new MangroveIdentity( diff --git a/Logic/UIEventSource.ts b/Logic/UIEventSource.ts index f29ef5b67a..3aebda8532 100644 --- a/Logic/UIEventSource.ts +++ b/Logic/UIEventSource.ts @@ -289,6 +289,7 @@ export class UIEventSource { const stack = new Error().stack.split("\n"); const callee = stack[1] + const newSource = new UIEventSource( f(this.data), "map(" + this.tag + ")@"+callee @@ -298,7 +299,7 @@ export class UIEventSource { newSource.setData(f(self.data)); } - this.addCallbackAndRun(update); + this.addCallback(update); for (const extraSource of extraSources) { extraSource?.addCallback(update); } diff --git a/Models/Constants.ts b/Models/Constants.ts index d87e2873f7..d9c923e34f 100644 --- a/Models/Constants.ts +++ b/Models/Constants.ts @@ -2,7 +2,7 @@ import {Utils} from "../Utils"; export default class Constants { - public static vNumber = "0.13.0-alpha-6"; + public static vNumber = "0.13.0-alpha-7"; public static ImgurApiKey = '7070e7167f0a25a' public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85" diff --git a/UI/AutomatonGui.ts b/UI/AutomatonGui.ts index eb82d7a359..1e5a42658e 100644 --- a/UI/AutomatonGui.ts +++ b/UI/AutomatonGui.ts @@ -5,7 +5,6 @@ import Title from "./Base/Title"; import Toggle from "./Input/Toggle"; import {SubtleButton} from "./Base/SubtleButton"; import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"; -import UserRelatedState from "../Logic/State/UserRelatedState"; import ValidatedTextField from "./Input/ValidatedTextField"; import {Utils} from "../Utils"; import {UIEventSource} from "../Logic/UIEventSource"; @@ -16,11 +15,19 @@ import {LocalStorageSource} from "../Logic/Web/LocalStorageSource"; import {DropDown} from "./Input/DropDown"; import {AllKnownLayouts} from "../Customizations/AllKnownLayouts"; import MinimapImplementation from "./Base/MinimapImplementation"; -import State from "../State"; import {OsmConnection} from "../Logic/Osm/OsmConnection"; -import FeaturePipelineState from "../Logic/State/FeaturePipelineState"; +import {BBox} from "../Logic/BBox"; +import MapState from "../Logic/State/MapState"; +import FeaturePipeline from "../Logic/FeatureSource/FeaturePipeline"; +import LayerConfig from "../Models/ThemeConfig/LayerConfig"; +import TagRenderingConfig from "../Models/ThemeConfig/TagRenderingConfig"; +import FeatureSource from "../Logic/FeatureSource/FeatureSource"; +import List from "./Base/List"; +import {QueryParameters} from "../Logic/Web/QueryParameters"; +import {SubstitutedTranslation} from "./SubstitutedTranslation"; +import {AutoAction} from "./Popup/AutoApplyButton"; -export default class AutomatonGui extends Combine { +class AutomatonGui extends Combine { constructor() { @@ -28,7 +35,8 @@ export default class AutomatonGui extends Combine { allElements: undefined, changes: undefined, layoutName: "automaton", - singlePage: true + singlePage: false, + oauth_token: QueryParameters.GetQueryParameter("oauth_token", "OAuth token") }); super([ @@ -39,35 +47,174 @@ export default class AutomatonGui extends Combine { ]).SetClass("flex"), new Toggle( AutomatonGui.GenerateMainPanel(), - new SubtleButton(Svg.osm_logo_svg(), "Login to get started"), + new SubtleButton(Svg.osm_logo_svg(), "Login to get started").onClick(() => osmConnection.AttemptLogin()), osmConnection.isLoggedIn )]) } - private static AutomationPanel(layoutToUse: LayoutConfig, tiles: UIEventSource): BaseUIElement { - const handledTiles = new UIEventSource(0) + private static startedTiles = new Set() - const state = new FeaturePipelineState(layoutToUse) + private static TileHandler(layoutToUse: LayoutConfig, tileIndex: number, targetLayer: string, targetAction: TagRenderingConfig, extraCommentText: UIEventSource, whenDone: ((result: string) => void)): BaseUIElement { + if (AutomatonGui.startedTiles.has(tileIndex)) { + throw "Already started tile " + tileIndex + } + AutomatonGui.startedTiles.add(tileIndex) - const nextTile = tiles.map(indices => { - if (indices === undefined) { - return "No tiles loaded - can not automate"; - } - const currentTile = handledTiles.data - const tileIndex = indices[currentTile] - if (tileIndex === undefined) { - return "All done!"; + const state = new MapState(layoutToUse, {attemptLogin: false}) + extraCommentText.syncWith( state.changes.extraComment) + const [z, x, y] = Tiles.tile_from_index(tileIndex) + state.locationControl.setData({ + zoom: z, + lon: x, + lat: y + }) + state.currentBounds.setData( + BBox.fromTileIndex(tileIndex) + ) + + let targetTiles: UIEventSource = new UIEventSource([]) + const pipeline = new FeaturePipeline((tile => { + const layerId = tile.layer.layerDef.id + if (layerId === targetLayer) { + targetTiles.data.push(tile) + targetTiles.ping() } + }), state) + state.locationControl.ping(); + state.currentBounds.ping(); + const stateToShow = new UIEventSource("") - return "" + tileIndex - }, [handledTiles]) + pipeline.runningQuery.map( + async isRunning => { + if (targetTiles.data.length === 0) { + stateToShow.setData("No data loaded yet...") + return; + } + if (isRunning) { + stateToShow.setData("Waiting for all layers to be loaded... Has " + targetTiles.data.length + " tiles already") + return; + } + if (targetTiles.data.length === 0) { + stateToShow.setData("No features found to apply the action") + whenDone("empty") + return true; + } + stateToShow.setData("Applying metatags") + pipeline.updateAllMetaTagging() + stateToShow.setData("Gathering applicable elements") + + let handled = 0 + let inspected = 0 + for (const targetTile of targetTiles.data) { + + for (const ffs of targetTile.features.data) { + inspected++ + if (inspected % 10 === 0) { + stateToShow.setData("Inspected " + inspected + " features, updated " + handled + " features") + } + const feature = ffs.feature + const rendering = targetAction.GetRenderValue(feature.properties).txt + const actions = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering) + .map(obj => obj.special)) + for (const action of actions) { + const auto = action.func + if (auto.supportsAutoAction !== true) { + continue + } + + await auto.applyActionOn({ + layoutToUse: state.layoutToUse, + changes: state.changes + }, state.allElements.getEventSourceById(feature.properties.id), action.args) + handled++ + } + } + } + stateToShow.setData("Done! Inspected " + inspected + " features, updated " + handled + " features") + + if (inspected === 0) { + whenDone("empty") + return; + } + + if (handled === 0) { + window.setTimeout(() => whenDone("no-action"), 1000) + }else{ + state.changes.flushChanges("handled tile automatically, time to flush!") + whenDone("fixed") + } + + }, [targetTiles]) return new Combine([ - new VariableUiElement(handledTiles.map(i => "" + i)), - new VariableUiElement(nextTile) - ]) + new Title("Performing action for tile " + tileIndex, 1), + new VariableUiElement(stateToShow)]).SetClass("flex flex-col") + } + + private static AutomationPanel(layoutToUse: LayoutConfig, indices: number[], extraCommentText: UIEventSource, tagRenderingToAutomate: { layer: LayerConfig, tagRendering: TagRenderingConfig }): BaseUIElement { + const layerId = tagRenderingToAutomate.layer.id + const trId = tagRenderingToAutomate.tagRendering.id + const tileState = LocalStorageSource.GetParsed("automation-tile_state-" + layerId + "-" + trId, {}) + + if (indices === undefined) { + return new FixedUiElement("No tiles loaded - can not automate") + } + + + const nextTileToHandle = tileState.map(handledTiles => { + for (const index of indices) { + if (handledTiles[index] !== undefined) { + // Already handled + continue + } + return index + } + return undefined + }) + nextTileToHandle.addCallback(t => console.warn("Next tile to handle is", t)) + + const neededTimes = new UIEventSource([]) + const automaton = new VariableUiElement(nextTileToHandle.map(tileIndex => { + if (tileIndex === undefined) { + return new FixedUiElement("All done!").SetClass("thanks") + } + console.warn("Triggered map on nextTileToHandle",tileIndex) + const start = new Date() + return AutomatonGui.TileHandler(layoutToUse, tileIndex, layerId, tagRenderingToAutomate.tagRendering, extraCommentText,(result) => { + const end = new Date() + const timeNeeded = (end.getTime() - start.getTime()) / 1000; + neededTimes.data.push(timeNeeded) + neededTimes.ping() + tileState.data[tileIndex] = result + tileState.ping(); + }); + })) + + + const statistics = new VariableUiElement(tileState.map(states => { + let total = 0 + const perResult = new Map() + for (const key in states) { + total++ + const result = states[key] + perResult.set(result, (perResult.get(result) ?? 0) + 1) + } + + let sum = 0 + neededTimes.data.forEach(v => { + sum = sum + v + }) + let timePerTile = sum / neededTimes.data.length + + return new Combine(["Handled " + total + "/" + indices.length + " tiles: ", + new List(Array.from(perResult.keys()).map(key => key + ": " + perResult.get(key))), + "Handling one tile needs " + (Math.floor(timePerTile * 100) / 100) + "s on average. Estimated time left: " + Math.floor((indices.length - total) * timePerTile) + "s" + ]).SetClass("flex flex-col") + })) + + return new Combine([statistics, automaton]).SetClass("flex flex-col") } private static GenerateMainPanel(): BaseUIElement { @@ -85,18 +232,22 @@ export default class AutomatonGui extends Combine { tilepath.SetClass("w-full") LocalStorageSource.Get("automation-tile_path").syncWith(tilepath.GetValue(), true) - const tilesToRunOver = tilepath.GetValue().bind(path => { + + let tilesToRunOver = tilepath.GetValue().bind(path => { if (path === undefined) { return undefined } - return UIEventSource.FromPromiseWithErr(Utils.downloadJson(path)) + return UIEventSource.FromPromiseWithErr(Utils.downloadJsonCached(path,1000*60*60)) }) + + const targetZoom = 14 const tilesPerIndex = tilesToRunOver.map(tiles => { + if (tiles === undefined || tiles["error"] !== undefined) { return undefined } - let indexes = []; + let indexes : number[] = []; const tilesS = tiles["success"] const z = Number(tilesS["zoom"]) for (const key in tilesS) { @@ -107,13 +258,31 @@ export default class AutomatonGui extends Combine { const ys = tilesS[key] indexes.push(...ys.map(y => Tiles.tile_index(z, x, y))) } - return indexes + + console.log("Got ", indexes.length, "indexes") + let rezoomed = new Set() + for (const index of indexes) { + let [z, x, y] = Tiles.tile_from_index(index) + while (z > targetZoom) { + z-- + x = Math.floor(x / 2) + y = Math.floor(y / 2) + } + rezoomed.add(Tiles.tile_index(z, x, y)) + } + + + return Array.from(rezoomed) }) + const extraComment = ValidatedTextField.InputForType("text") + LocalStorageSource.Get("automaton-extra-comment").syncWith(extraComment.GetValue()) + return new Combine([ themeSelect, "Specify the path to a tile overview. This is a hosted .json of the format {x : [y0, y1, y2], x1: [y0, ...]} where x is a string and y are numbers", tilepath, + extraComment, new VariableUiElement(tilesToRunOver.map(t => { if (t === undefined) { return "No path given or still loading..." @@ -128,10 +297,43 @@ export default class AutomatonGui extends Combine { if (layoutToUse === undefined) { return new FixedUiElement("Select a valid layout") } + if (tilesPerIndex.data === undefined || tilesPerIndex.data.length === 0) { + return "No tiles given" + } - return AutomatonGui.AutomationPanel(layoutToUse, tilesPerIndex) + const automatableTagRenderings: { layer: LayerConfig, tagRendering: TagRenderingConfig }[] = [] + for (const layer of layoutToUse.layers) { + for (const tagRendering of layer.tagRenderings) { + if (tagRendering.group === "auto") { + automatableTagRenderings.push({layer, tagRendering: tagRendering}) + } + } + } + console.log("Automatable tag renderings:", automatableTagRenderings) + if (automatableTagRenderings.length === 0) { + return new FixedUiElement('This theme does not have any tagRendering with "group": "auto" set').SetClass("alert") + } + const pickAuto = new DropDown("Pick the action to automate", + [ + { + value: undefined, + shown: "Pick an option" + }, + ...automatableTagRenderings.map(config => ( + { + shown: config.layer.id + " - " + config.tagRendering.id, + value: config + } + )) + ] + ) - })) + + return new Combine([ + pickAuto, + new VariableUiElement(pickAuto.GetValue().map(auto => auto === undefined ? undefined : AutomatonGui.AutomationPanel(layoutToUse, tilesPerIndex.data, extraComment.GetValue(), auto)))]) + + }, [tilesPerIndex])).SetClass("flex flex-col") ]).SetClass("flex flex-col") diff --git a/UI/Popup/AutoApplyButton.ts b/UI/Popup/AutoApplyButton.ts index 034485f52e..bf545e79b1 100644 --- a/UI/Popup/AutoApplyButton.ts +++ b/UI/Popup/AutoApplyButton.ts @@ -17,11 +17,16 @@ import {VariableUiElement} from "../Base/VariableUIElement"; import Loading from "../Base/Loading"; import {OsmConnection} from "../../Logic/Osm/OsmConnection"; import Translations from "../i18n/Translations"; +import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig"; +import {Changes} from "../../Logic/Osm/Changes"; export interface AutoAction extends SpecialVisualization { supportsAutoAction: boolean - applyActionOn(state: FeaturePipelineState, tagSource: UIEventSource, argument: string[]): Promise + applyActionOn(state: { + layoutToUse: LayoutConfig, + changes: Changes + }, tagSource: UIEventSource, argument: string[]): Promise } export default class AutoApplyButton implements SpecialVisualization { diff --git a/UI/Popup/TagApplyButton.ts b/UI/Popup/TagApplyButton.ts index fd4751fbfc..e09a6b179d 100644 --- a/UI/Popup/TagApplyButton.ts +++ b/UI/Popup/TagApplyButton.ts @@ -12,6 +12,8 @@ import Toggle from "../Input/Toggle"; import {Utils} from "../../Utils"; import {Tag} from "../../Logic/Tags/Tag"; import FeaturePipelineState from "../../Logic/State/FeaturePipelineState"; +import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig"; +import {Changes} from "../../Logic/Osm/Changes"; export default class TagApplyButton implements AutoAction { public readonly funcName = "tag_apply"; @@ -79,7 +81,10 @@ export default class TagApplyButton implements AutoAction { public readonly example = "`{tag_apply(survey_date=$_now:date, Surveyed today!)}`, `{tag_apply(addr:street=$addr:street, Apply the address, apply_icon.svg, _closest_osm_id)"; - async applyActionOn(state: FeaturePipelineState, tags: UIEventSource, args: string[]) : Promise{ + async applyActionOn(state: { + layoutToUse: LayoutConfig, + changes: Changes + }, tags: UIEventSource, args: string[]) : Promise{ const tagsToApply = TagApplyButton.generateTagsToApply(args[0], tags) const targetIdKey = args[3] diff --git a/assets/themes/grb_import/missing_streets.json b/assets/themes/grb_import/missing_streets.json index e25efdeeeb..7630d59036 100644 --- a/assets/themes/grb_import/missing_streets.json +++ b/assets/themes/grb_import/missing_streets.json @@ -76,6 +76,10 @@ { "builtin": "crab_address", "override": { + "source": { + "geoJson": "http://127.0.0.1:8080/tile_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 18 + }, "mapRendering": [ { "iconSize": "5,5,center", @@ -101,8 +105,8 @@ "_embedded_crab_addresses:=Array.from(new Set(feat.overlapWith('crab_address').map(ff => ff.feat.properties).filter(p => p._HNRLABEL.toLowerCase() === (feat.properties['addr:housenumber'] + (feat.properties['addr:unit']??'')).toLowerCase()).map(p => p.STRAATNM)))", "_singular_import:=feat.get('_embedded_crab_addresses')?.length == 1", "_name_to_apply:=feat.get('_embedded_crab_addresses')[0]", - "_nearby_street_names:=feat.closestn('named_streets',5,'name', 500).map(ff => ff.feat.properties.name)", - "_spelling_is_correct:= feat.get('_nearby_street_names').indexOf(feat.properties['_name_to_apply']) >= 0" + "_nearby_street_names:=feat.closestn('named_streets',5,'name', 1000).map(ff => [ff.feat.properties.name, ff.feat.properties['alt_name'], ff.feat.properties['name:nl']])", + "_spelling_is_correct:= [].concat(...feat.get('_nearby_street_names')).indexOf(feat.properties['_name_to_apply']) >= 0" ], "mapRendering": [ { @@ -134,6 +138,7 @@ "tagRenderings": [ { "id": "apply_streetname", + "group": "auto", "render": "{tag_apply(addr:street=$_name_to_apply ,Apply the CRAB-street onto this building)}", "mappings": [ { @@ -146,8 +151,7 @@ } ] } - ], - "passAllFeatures": true + ] } ], "hideFromOverview": true diff --git a/scripts/generateTileOverview.ts b/scripts/generateTileOverview.ts index 76158d04fa..d93aae640b 100644 --- a/scripts/generateTileOverview.ts +++ b/scripts/generateTileOverview.ts @@ -2,7 +2,6 @@ * Generates an overview for which tiles exist and which don't */ import ScriptUtils from "./ScriptUtils"; -import {Tiles} from "../Models/TileRange"; import {writeFileSync} from "fs"; function main(args: string[]) { @@ -28,10 +27,10 @@ function main(args: string[]) { const x = match[2] const y = match[3] - if(!indices[x] !== undefined){ + if(indices[x] === undefined){ indices[x] = [] } - indices[x] .push(Number(y)) + indices[x].push(Number(y)) } indices["zoom"] = zoomLevel; const match = files[0].match("\(.*\)_\([0-9]*\)_\([0-9]*\)_\([0-9]*\).geojson")