contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

First working version of fully automatic uploader

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-12-13 02:05:34 +01:00
parent 04dc373b1e
commit e922768f99
21 changed files with 342 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts
View file

@ -119,7 +119,11 @@ export default class SaveTileToLocalStorageActor {
}
private SetIdb(tileIndex, data) {
IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data)
try{
IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data)
}catch(e){
console.error("Could not save tile to indexed-db: ", e, "tileIndex is:", tileIndex, "for layer", this._layer.id)
}
}
private GetIdb(tileIndex) {

55
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -171,6 +171,7 @@ export default class FeaturePipeline {
state.currentBounds, state.locationControl,
(tileIndex, freshness) => self.freshnesses.get(id).addTileLoad(tileIndex, freshness),
(tile) => {
console.debug("Loaded tile ", id, tile.tileIndex, "from local cache")
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
hierarchy.registerTile(tile);
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@ -247,6 +248,7 @@ export default class FeaturePipeline {
})
})
if (state.layoutToUse.trackAllNodes) {
const fullNodeDb = new FullNodeDatabaseSource(
state.filteredLayers.data.filter(l => l.layerDef.id === "type_node")[0],
@ -289,6 +291,10 @@ export default class FeaturePipeline {
// A NewGeometryFromChangesFeatureSource does not split per layer, so we do this next
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(perLayer) => {
if(perLayer.features.data.length === 0){
return
}
// We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this
perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer)
// AT last, we always apply the metatags whenever possible
@ -309,7 +315,7 @@ export default class FeaturePipeline {
this.runningQuery = updater.runningQuery.map(
overpass => {
console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,",
console.log("FeaturePipeline: runningQuery state changed: Overpass", overpass ? "is querying," : "is idle,",
"osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs " + neededTilesFromOsm.data?.length + " tiles (already got " + osmFeatureSource.downloadedTiles.size + " tiles )" : "is idle")
return overpass || osmFeatureSource.isRunning.data;
}, [osmFeatureSource.isRunning]
@ -355,7 +361,15 @@ export default class FeaturePipeline {
if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) {
continue;
}
const freshness = this.freshnesses.get(flayer.layerDef.id).freshnessFor(z, x, y)
if(flayer.layerDef.maxAgeOfCache === 0){
return undefined;
}
const freshnessCalc = this.freshnesses.get(flayer.layerDef.id)
if(freshnessCalc === undefined){
console.warn("No freshness tracker found for ", flayer.layerDef.id)
return undefined
}
const freshness = freshnessCalc.freshnessFor(z, x, y)
if (freshness === undefined) {
// SOmething is undefined --> we return undefined as we have to download
return undefined
@ -409,11 +423,13 @@ export default class FeaturePipeline {
const minzoom = Math.min(...state.layoutToUse.layers.map(layer => layer.minzoom))
const overpassIsActive = state.currentBounds.map(bbox => {
if (bbox === undefined) {
console.debug("Disabling overpass source: no bbox")
return false
}
let zoom = state.locationControl.data.zoom
if (zoom < minzoom) {
// We are zoomed out over the zoomlevel of any layer
console.debug("Disabling overpass source: zoom < minzoom")
return false;
}
@ -456,31 +472,26 @@ export default class FeaturePipeline {
if(src === undefined){
throw "Src is undefined"
}
window.setTimeout(
() => {
const layerDef = src.layer.layerDef;
MetaTagging.addMetatags(
src.features.data,
{
memberships: this.relationTracker,
getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox),
getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id)
},
layerDef,
state,
{
includeDates: true,
// We assume that the non-dated metatags are already set by the cache generator
includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer
}
)
const layerDef = src.layer.layerDef;
MetaTagging.addMetatags(
src.features.data,
{
memberships: this.relationTracker,
getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox),
getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id)
},
15
layerDef,
state,
{
includeDates: true,
// We assume that the non-dated metatags are already set by the cache generator
includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer
}
)
}
private updateAllMetaTagging() {
public updateAllMetaTagging() {
const self = this;
console.debug("Updating the meta tagging of all tiles as new data got loaded")
this.perLayerHierarchy.forEach(hierarchy => {

12
Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts
View file

@ -36,21 +36,15 @@ export default class PerLayerFeatureSourceSplitter {
const featuresPerLayer = new Map<string, { feature, freshness } []>();
const noLayerFound = []
function addTo(layer: FilteredLayer, feature: { feature, freshness }) {
const id = layer.layerDef.id
const list = featuresPerLayer.get(id)
if (list !== undefined) {
list.push(feature)
} else {
featuresPerLayer.set(id, [feature])
}
for (const layer of layers.data) {
featuresPerLayer.set(layer.layerDef.id, [])
}
for (const f of features) {
for (const layer of layers.data) {
if (layer.layerDef.source.osmTags.matchesProperties(f.feature.properties)) {
// We have found our matching layer!
addTo(layer, f)
featuresPerLayer.set(layer.layerDef.id, [f])
if (!layer.layerDef.passAllFeatures) {
// If not 'passAllFeatures', we are done for this feature
break;

2
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -126,7 +126,7 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
eventSource.setData(eventSource.data.concat(newFeatures))
}).catch(msg => console.error("Could not load geojson layer", url, "due to", msg))
}).catch(msg => console.debug("Could not load geojson layer", url, "due to", msg))
}
}

5
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -5,13 +5,14 @@ import Loc from "../../../Models/Loc";
import DynamicTileSource from "./DynamicTileSource";
import {Utils} from "../../../Utils";
import GeoJsonSource from "../Sources/GeoJsonSource";
import {BBox} from "../../BBox";
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
constructor(layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer & Tiled) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
currentBounds: UIEventSource<BBox>
}) {
const source = layer.layerDef.source
if (source.geojsonZoomLevel === undefined) {
@ -29,7 +30,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}", layer.layerDef.id)
Utils.downloadJson(whitelistUrl).then(
Utils.downloadJsonCached(whitelistUrl, 1000*60*60).then(
json => {
const data = new Map<number, Set<number>>();
for (const x in json) {

7
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -4,6 +4,7 @@ import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
/***
* A tiled source which dynamically loads the required tiles at a fixed zoom level
@ -17,8 +18,8 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
zoomlevel: number,
constructTile: (zxy: [number, number, number]) => (FeatureSourceForLayer & Tiled),
state: {
currentBounds: UIEventSource<BBox>;
locationControl: UIEventSource<Loc>
leafletMap: any
}
) {
const self = this;
@ -37,7 +38,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
}
// Yup, this is cheating to just get the bounds here
const bounds = state.leafletMap.data?.getBounds()
const bounds = state.currentBounds.data
if (bounds === undefined) {
// We'll retry later
return undefined
@ -50,7 +51,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
}
return needed
}
, [layer.isDisplayed, state.leafletMap]).stabilized(250);
, [layer.isDisplayed, state.currentBounds]).stabilized(250);
neededTiles.addCallbackAndRunD(neededIndexes => {
console.log("Tiled geojson source ", layer.layerDef.id, " needs", neededIndexes)

4
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -63,10 +63,10 @@ export default class OsmFeatureSource {
try {
for (const neededTile of neededTiles) {
console.log("Tile download", Tiles.tile_from_index(neededTile).join("/"), "started")
console.log("Tile download from OSM", Tiles.tile_from_index(neededTile).join("/"), "started")
self.downloadedTiles.add(neededTile)
self.LoadTile(...Tiles.tile_from_index(neededTile)).then(_ => {
console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded")
console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded from OSM")
})
}
} catch (e) {

36
Logic/Osm/Changes.ts
View file

@ -1,5 +1,4 @@
import {OsmNode, OsmObject, OsmRelation, OsmWay} from "./OsmObject";
import State from "../../State";
import {UIEventSource} from "../UIEventSource";
import Constants from "../../Models/Constants";
import OsmChangeAction from "./Actions/OsmChangeAction";
@ -13,6 +12,7 @@ import {ElementStorage} from "../ElementStorage";
import {GeoLocationPointProperties} from "../Actors/GeoLocationHandler";
import {GeoOperations} from "../GeoOperations";
import {ChangesetTag} from "./ChangesetHandler";
import {OsmConnection} from "./OsmConnection";
/**
* Handles all changes made to OSM.
@ -33,14 +33,23 @@ export class Changes {
private readonly previouslyCreated: OsmObject[] = []
private readonly _leftRightSensitive: boolean;
private _state: { allElements: ElementStorage; historicalUserLocations: FeatureSource }
public readonly state: { allElements: ElementStorage; historicalUserLocations: FeatureSource; osmConnection: OsmConnection }
constructor(leftRightSensitive: boolean = false) {
public readonly extraComment:UIEventSource<string> = new UIEventSource(undefined)
constructor(
state?: {
allElements: ElementStorage,
historicalUserLocations: FeatureSource,
osmConnection: OsmConnection
},
leftRightSensitive: boolean = false) {
this._leftRightSensitive = leftRightSensitive;
// We keep track of all changes just as well
this.allChanges.setData([...this.pendingChanges.data])
// If a pending change contains a negative ID, we save that
this._nextId = Math.min(-1, ...this.pendingChanges.data?.map(pch => pch.id) ?? [])
this.state = state;
// Note: a changeset might be reused which was opened just before and might have already used some ids
// This doesn't matter however, as the '-1' is per piecewise upload, not global per changeset
@ -120,7 +129,7 @@ export class Changes {
private calculateDistanceToChanges(change: OsmChangeAction, changeDescriptions: ChangeDescription[]) {
if (this._state === undefined) {
if (this.state === undefined) {
// No state loaded -> we can't calculate...
return;
}
@ -129,7 +138,7 @@ export class Changes {
return;
}
const now = new Date()
const recentLocationPoints = this._state.historicalUserLocations.features.data.map(ff => ff.feature)
const recentLocationPoints = this.state.historicalUserLocations.features.data.map(ff => ff.feature)
.filter(feat => feat.geometry.type === "Point")
.filter(feat => {
const visitTime = new Date((<GeoLocationPointProperties>feat.properties).date)
@ -149,7 +158,7 @@ export class Changes {
const changedObjectCoordinates: [number, number][] = []
const feature = this._state.allElements.ContainingFeatures.get(change.mainObjectId)
const feature = this.state.allElements.ContainingFeatures.get(change.mainObjectId)
if (feature !== undefined) {
changedObjectCoordinates.push(GeoOperations.centerpointCoordinates(feature))
}
@ -189,12 +198,6 @@ export class Changes {
this.allChanges.ping()
}
public useLocationHistory(state: {
allElements: ElementStorage,
historicalUserLocations: FeatureSource
}) {
this._state = state
}
public registerIdRewrites(mappings: Map<string, string>): void {
CreateNewNodeAction.registerIdRewrites(mappings)
@ -281,9 +284,14 @@ export class Changes {
// This method is only called with changedescriptions for this theme
const theme = pending[0].meta.theme
let comment = "Adding data with #MapComplete for theme #" + theme
if(this.extraComment.data !== undefined){
comment+="\n\n"+this.extraComment.data
}
const metatags: ChangesetTag[] = [{
key: "comment",
value: "Adding data with #MapComplete for theme #" + theme
value: comment
},
{
key: "theme",
@ -294,7 +302,7 @@ export class Changes {
...perBinMessage
]
await State.state.osmConnection.changesetHandler.UploadChangeset(
await this.state.osmConnection.changesetHandler.UploadChangeset(
(csId) => Changes.createChangesetFor("" + csId, changes),
metatags
)

8
Logic/Osm/ChangesetHandler.ts
View file

@ -1,9 +1,7 @@
import escapeHtml from "escape-html";
// @ts-ignore
import {OsmConnection, UserDetails} from "./OsmConnection";
import UserDetails, {OsmConnection} from "./OsmConnection";
import {UIEventSource} from "../UIEventSource";
import {ElementStorage} from "../ElementStorage";
import State from "../../State";
import Locale from "../../UI/i18n/Locale";
import Constants from "../../Models/Constants";
import {Changes} from "./Changes";
@ -287,8 +285,8 @@ export class ChangesetHandler {
["language", Locale.language.data],
["host", window.location.host],
["path", path],
["source", State.state.currentUserLocation.features.data.length > 0 ? "survey" : undefined],
["imagery", State.state.backgroundLayer.data.id],
["source", self.changes.state["currentUserLocation"]?.features?.data?.length > 0 ? "survey" : undefined],
["imagery", self.changes.state["backgroundLayer"]?.data?.id],
...changesetTags.map(cstag => [cstag.key, cstag.value])
]
.filter(kv => (kv[1] ?? "") !== "")

5
Logic/Osm/OsmConnection.ts
View file

@ -70,7 +70,8 @@ export class OsmConnection {
// Used to keep multiple changesets open and to write to the correct changeset
layoutName: string,
singlePage?: boolean,
osmConfiguration?: "osm" | "osm-test"
osmConfiguration?: "osm" | "osm-test",
attemptLogin?: true | boolean
}
) {
this.fakeUser = options.fakeUser ?? false;
@ -117,7 +118,7 @@ export class OsmConnection {
options.oauth_token.setData(undefined);
}
if (this.auth.authenticated()) {
if (this.auth.authenticated() && (options.attemptLogin !== false)) {
this.AttemptLogin(); // Also updates the user badge
} else {
console.log("Not authenticated");

3
Logic/State/ElementsState.ts
View file

@ -49,7 +49,8 @@ export default class ElementsState extends FeatureSwitchState {
constructor(layoutToUse: LayoutConfig) {
super(layoutToUse);
this.changes = new Changes(layoutToUse?.isLeftRightSensitive() ?? false)
// @ts-ignore
this.changes = new Changes(this,layoutToUse?.isLeftRightSensitive() ?? false)
{
// -- Location control initialization
const zoom = UIEventSource.asFloat(

1
Logic/State/FeaturePipelineState.ts
View file

@ -9,6 +9,7 @@ import MapState from "./MapState";
import SelectedFeatureHandler from "../Actors/SelectedFeatureHandler";
import Hash from "../Web/Hash";
import {BBox} from "../BBox";
import {FeatureSourceForLayer} from "../FeatureSource/FeatureSource";
export default class FeaturePipelineState extends MapState {

5
Logic/State/MapState.ts
View file

@ -82,8 +82,8 @@ export default class MapState extends UserRelatedState {
public overlayToggles: { config: TilesourceConfig, isDisplayed: UIEventSource<boolean> }[]
constructor(layoutToUse: LayoutConfig) {
super(layoutToUse);
constructor(layoutToUse: LayoutConfig, options?: {attemptLogin: true | boolean}) {
super(layoutToUse, options);
this.availableBackgroundLayers = AvailableBaseLayers.AvailableLayersAt(this.locationControl);
@ -265,7 +265,6 @@ export default class MapState extends UserRelatedState {
let gpsLayerDef: FilteredLayer = this.filteredLayers.data.filter(l => l.layerDef.id === "gps_location_history")[0]
this.historicalUserLocations = new SimpleFeatureSource(gpsLayerDef, Tiles.tile_index(0, 0, 0), features);
this.changes.useLocationHistory(this)
const asLine = features.map(allPoints => {

5
Logic/State/UserRelatedState.ts
View file

@ -36,7 +36,7 @@ export default class UserRelatedState extends ElementsState {
public installedThemes: UIEventSource<{ layout: LayoutConfig; definition: string }[]>;
constructor(layoutToUse: LayoutConfig) {
constructor(layoutToUse: LayoutConfig, options:{attemptLogin : true | boolean}) {
super(layoutToUse);
this.osmConnection = new OsmConnection({
@ -50,7 +50,8 @@ export default class UserRelatedState extends ElementsState {
"Used to complete the login"
),
layoutName: layoutToUse?.id,
osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data
osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data,
attemptLogin: options?.attemptLogin
})
this.mangroveIdentity = new MangroveIdentity(

3
Logic/UIEventSource.ts
View file

@ -289,6 +289,7 @@ export class UIEventSource<T> {
const stack = new Error().stack.split("\n");
const callee = stack[1]
const newSource = new UIEventSource<J>(
f(this.data),
"map(" + this.tag + ")@"+callee
@ -298,7 +299,7 @@ export class UIEventSource<T> {
newSource.setData(f(self.data));
}
this.addCallbackAndRun(update);
this.addCallback(update);
for (const extraSource of extraSources) {
extraSource?.addCallback(update);
}

2
Models/Constants.ts
View file

@ -2,7 +2,7 @@ import {Utils} from "../Utils";
export default class Constants {
public static vNumber = "0.13.0-alpha-6";
public static vNumber = "0.13.0-alpha-7";
public static ImgurApiKey = '7070e7167f0a25a'
public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85"

258
UI/AutomatonGui.ts
View file

@ -5,7 +5,6 @@ import Title from "./Base/Title";
import Toggle from "./Input/Toggle";
import {SubtleButton} from "./Base/SubtleButton";
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig";
import UserRelatedState from "../Logic/State/UserRelatedState";
import ValidatedTextField from "./Input/ValidatedTextField";
import {Utils} from "../Utils";
import {UIEventSource} from "../Logic/UIEventSource";
@ -16,11 +15,19 @@ import {LocalStorageSource} from "../Logic/Web/LocalStorageSource";
import {DropDown} from "./Input/DropDown";
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
import MinimapImplementation from "./Base/MinimapImplementation";
import State from "../State";
import {OsmConnection} from "../Logic/Osm/OsmConnection";
import FeaturePipelineState from "../Logic/State/FeaturePipelineState";
import {BBox} from "../Logic/BBox";
import MapState from "../Logic/State/MapState";
import FeaturePipeline from "../Logic/FeatureSource/FeaturePipeline";
import LayerConfig from "../Models/ThemeConfig/LayerConfig";
import TagRenderingConfig from "../Models/ThemeConfig/TagRenderingConfig";
import FeatureSource from "../Logic/FeatureSource/FeatureSource";
import List from "./Base/List";
import {QueryParameters} from "../Logic/Web/QueryParameters";
import {SubstitutedTranslation} from "./SubstitutedTranslation";
import {AutoAction} from "./Popup/AutoApplyButton";
export default class AutomatonGui extends Combine {
class AutomatonGui extends Combine {
constructor() {
@ -28,7 +35,8 @@ export default class AutomatonGui extends Combine {
allElements: undefined,
changes: undefined,
layoutName: "automaton",
singlePage: true
singlePage: false,
oauth_token: QueryParameters.GetQueryParameter("oauth_token", "OAuth token")
});
super([
@ -39,35 +47,174 @@ export default class AutomatonGui extends Combine {
]).SetClass("flex"),
new Toggle(
AutomatonGui.GenerateMainPanel(),
new SubtleButton(Svg.osm_logo_svg(), "Login to get started"),
new SubtleButton(Svg.osm_logo_svg(), "Login to get started").onClick(() => osmConnection.AttemptLogin()),
osmConnection.isLoggedIn
)])
}
private static AutomationPanel(layoutToUse: LayoutConfig, tiles: UIEventSource<number[]>): BaseUIElement {
const handledTiles = new UIEventSource(0)
private static startedTiles = new Set<number>()
const state = new FeaturePipelineState(layoutToUse)
private static TileHandler(layoutToUse: LayoutConfig, tileIndex: number, targetLayer: string, targetAction: TagRenderingConfig, extraCommentText: UIEventSource<string>, whenDone: ((result: string) => void)): BaseUIElement {
if (AutomatonGui.startedTiles.has(tileIndex)) {
throw "Already started tile " + tileIndex
}
AutomatonGui.startedTiles.add(tileIndex)
const nextTile = tiles.map(indices => {
if (indices === undefined) {
return "No tiles loaded - can not automate";
}
const currentTile = handledTiles.data
const tileIndex = indices[currentTile]
if (tileIndex === undefined) {
return "All done!";
const state = new MapState(layoutToUse, {attemptLogin: false})
extraCommentText.syncWith( state.changes.extraComment)
const [z, x, y] = Tiles.tile_from_index(tileIndex)
state.locationControl.setData({
zoom: z,
lon: x,
lat: y
})
state.currentBounds.setData(
BBox.fromTileIndex(tileIndex)
)
let targetTiles: UIEventSource<FeatureSource[]> = new UIEventSource<FeatureSource[]>([])
const pipeline = new FeaturePipeline((tile => {
const layerId = tile.layer.layerDef.id
if (layerId === targetLayer) {
targetTiles.data.push(tile)
targetTiles.ping()
}
}), state)
state.locationControl.ping();
state.currentBounds.ping();
const stateToShow = new UIEventSource("")
return "" + tileIndex
}, [handledTiles])
pipeline.runningQuery.map(
async isRunning => {
if (targetTiles.data.length === 0) {
stateToShow.setData("No data loaded yet...")
return;
}
if (isRunning) {
stateToShow.setData("Waiting for all layers to be loaded... Has " + targetTiles.data.length + " tiles already")
return;
}
if (targetTiles.data.length === 0) {
stateToShow.setData("No features found to apply the action")
whenDone("empty")
return true;
}
stateToShow.setData("Applying metatags")
pipeline.updateAllMetaTagging()
stateToShow.setData("Gathering applicable elements")
let handled = 0
let inspected = 0
for (const targetTile of targetTiles.data) {
for (const ffs of targetTile.features.data) {
inspected++
if (inspected % 10 === 0) {
stateToShow.setData("Inspected " + inspected + " features, updated " + handled + " features")
}
const feature = ffs.feature
const rendering = targetAction.GetRenderValue(feature.properties).txt
const actions = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering)
.map(obj => obj.special))
for (const action of actions) {
const auto = <AutoAction>action.func
if (auto.supportsAutoAction !== true) {
continue
}
await auto.applyActionOn({
layoutToUse: state.layoutToUse,
changes: state.changes
}, state.allElements.getEventSourceById(feature.properties.id), action.args)
handled++
}
}
}
stateToShow.setData("Done! Inspected " + inspected + " features, updated " + handled + " features")
if (inspected === 0) {
whenDone("empty")
return;
}
if (handled === 0) {
window.setTimeout(() => whenDone("no-action"), 1000)
}else{
state.changes.flushChanges("handled tile automatically, time to flush!")
whenDone("fixed")
}
}, [targetTiles])
return new Combine([
new VariableUiElement(handledTiles.map(i => "" + i)),
new VariableUiElement(nextTile)
])
new Title("Performing action for tile " + tileIndex, 1),
new VariableUiElement(stateToShow)]).SetClass("flex flex-col")
}
private static AutomationPanel(layoutToUse: LayoutConfig, indices: number[], extraCommentText: UIEventSource<string>, tagRenderingToAutomate: { layer: LayerConfig, tagRendering: TagRenderingConfig }): BaseUIElement {
const layerId = tagRenderingToAutomate.layer.id
const trId = tagRenderingToAutomate.tagRendering.id
const tileState = LocalStorageSource.GetParsed("automation-tile_state-" + layerId + "-" + trId, {})
if (indices === undefined) {
return new FixedUiElement("No tiles loaded - can not automate")
}
const nextTileToHandle = tileState.map(handledTiles => {
for (const index of indices) {
if (handledTiles[index] !== undefined) {
// Already handled
continue
}
return index
}
return undefined
})
nextTileToHandle.addCallback(t => console.warn("Next tile to handle is", t))
const neededTimes = new UIEventSource<number[]>([])
const automaton = new VariableUiElement(nextTileToHandle.map(tileIndex => {
if (tileIndex === undefined) {
return new FixedUiElement("All done!").SetClass("thanks")
}
console.warn("Triggered map on nextTileToHandle",tileIndex)
const start = new Date()
return AutomatonGui.TileHandler(layoutToUse, tileIndex, layerId, tagRenderingToAutomate.tagRendering, extraCommentText,(result) => {
const end = new Date()
const timeNeeded = (end.getTime() - start.getTime()) / 1000;
neededTimes.data.push(timeNeeded)
neededTimes.ping()
tileState.data[tileIndex] = result
tileState.ping();
});
}))
const statistics = new VariableUiElement(tileState.map(states => {
let total = 0
const perResult = new Map<string, number>()
for (const key in states) {
total++
const result = states[key]
perResult.set(result, (perResult.get(result) ?? 0) + 1)
}
let sum = 0
neededTimes.data.forEach(v => {
sum = sum + v
})
let timePerTile = sum / neededTimes.data.length
return new Combine(["Handled " + total + "/" + indices.length + " tiles: ",
new List(Array.from(perResult.keys()).map(key => key + ": " + perResult.get(key))),
"Handling one tile needs " + (Math.floor(timePerTile * 100) / 100) + "s on average. Estimated time left: " + Math.floor((indices.length - total) * timePerTile) + "s"
]).SetClass("flex flex-col")
}))
return new Combine([statistics, automaton]).SetClass("flex flex-col")
}
private static GenerateMainPanel(): BaseUIElement {
@ -85,18 +232,22 @@ export default class AutomatonGui extends Combine {
tilepath.SetClass("w-full")
LocalStorageSource.Get("automation-tile_path").syncWith(tilepath.GetValue(), true)
const tilesToRunOver = tilepath.GetValue().bind(path => {
let tilesToRunOver = tilepath.GetValue().bind(path => {
if (path === undefined) {
return undefined
}
return UIEventSource.FromPromiseWithErr(Utils.downloadJson(path))
return UIEventSource.FromPromiseWithErr(Utils.downloadJsonCached(path,1000*60*60))
})
const targetZoom = 14
const tilesPerIndex = tilesToRunOver.map(tiles => {
if (tiles === undefined || tiles["error"] !== undefined) {
return undefined
}
let indexes = [];
let indexes : number[] = [];
const tilesS = tiles["success"]
const z = Number(tilesS["zoom"])
for (const key in tilesS) {
@ -107,13 +258,31 @@ export default class AutomatonGui extends Combine {
const ys = tilesS[key]
indexes.push(...ys.map(y => Tiles.tile_index(z, x, y)))
}
return indexes
console.log("Got ", indexes.length, "indexes")
let rezoomed = new Set<number>()
for (const index of indexes) {
let [z, x, y] = Tiles.tile_from_index(index)
while (z > targetZoom) {
z--
x = Math.floor(x / 2)
y = Math.floor(y / 2)
}
rezoomed.add(Tiles.tile_index(z, x, y))
}
return Array.from(rezoomed)
})
const extraComment = ValidatedTextField.InputForType("text")
LocalStorageSource.Get("automaton-extra-comment").syncWith(extraComment.GetValue())
return new Combine([
themeSelect,
"Specify the path to a tile overview. This is a hosted .json of the format {x : [y0, y1, y2], x1: [y0, ...]} where x is a string and y are numbers",
tilepath,
extraComment,
new VariableUiElement(tilesToRunOver.map(t => {
if (t === undefined) {
return "No path given or still loading..."
@ -128,10 +297,43 @@ export default class AutomatonGui extends Combine {
if (layoutToUse === undefined) {
return new FixedUiElement("Select a valid layout")
}
if (tilesPerIndex.data === undefined || tilesPerIndex.data.length === 0) {
return "No tiles given"
}
return AutomatonGui.AutomationPanel(layoutToUse, tilesPerIndex)
const automatableTagRenderings: { layer: LayerConfig, tagRendering: TagRenderingConfig }[] = []
for (const layer of layoutToUse.layers) {
for (const tagRendering of layer.tagRenderings) {
if (tagRendering.group === "auto") {
automatableTagRenderings.push({layer, tagRendering: tagRendering})
}
}
}
console.log("Automatable tag renderings:", automatableTagRenderings)
if (automatableTagRenderings.length === 0) {
return new FixedUiElement('This theme does not have any tagRendering with "group": "auto" set').SetClass("alert")
}
const pickAuto = new DropDown("Pick the action to automate",
[
{
value: undefined,
shown: "Pick an option"
},
...automatableTagRenderings.map(config => (
{
shown: config.layer.id + " - " + config.tagRendering.id,
value: config
}
))
]
)
}))
return new Combine([
pickAuto,
new VariableUiElement(pickAuto.GetValue().map(auto => auto === undefined ? undefined : AutomatonGui.AutomationPanel(layoutToUse, tilesPerIndex.data, extraComment.GetValue(), auto)))])
}, [tilesPerIndex])).SetClass("flex flex-col")
]).SetClass("flex flex-col")

7
UI/Popup/AutoApplyButton.ts
View file

@ -17,11 +17,16 @@ import {VariableUiElement} from "../Base/VariableUIElement";
import Loading from "../Base/Loading";
import {OsmConnection} from "../../Logic/Osm/OsmConnection";
import Translations from "../i18n/Translations";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
import {Changes} from "../../Logic/Osm/Changes";
export interface AutoAction extends SpecialVisualization {
supportsAutoAction: boolean
applyActionOn(state: FeaturePipelineState, tagSource: UIEventSource<any>, argument: string[]): Promise<void>
applyActionOn(state: {
layoutToUse: LayoutConfig,
changes: Changes
}, tagSource: UIEventSource<any>, argument: string[]): Promise<void>
}
export default class AutoApplyButton implements SpecialVisualization {

7
UI/Popup/TagApplyButton.ts
View file

@ -12,6 +12,8 @@ import Toggle from "../Input/Toggle";
import {Utils} from "../../Utils";
import {Tag} from "../../Logic/Tags/Tag";
import FeaturePipelineState from "../../Logic/State/FeaturePipelineState";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
import {Changes} from "../../Logic/Osm/Changes";
export default class TagApplyButton implements AutoAction {
public readonly funcName = "tag_apply";
@ -79,7 +81,10 @@ export default class TagApplyButton implements AutoAction {
public readonly example = "`{tag_apply(survey_date=$_now:date, Surveyed today!)}`, `{tag_apply(addr:street=$addr:street, Apply the address, apply_icon.svg, _closest_osm_id)";
async applyActionOn(state: FeaturePipelineState, tags: UIEventSource<any>, args: string[]) : Promise<void>{
async applyActionOn(state: {
layoutToUse: LayoutConfig,
changes: Changes
}, tags: UIEventSource<any>, args: string[]) : Promise<void>{
const tagsToApply = TagApplyButton.generateTagsToApply(args[0], tags)
const targetIdKey = args[3]

12
assets/themes/grb_import/missing_streets.json
View file

@ -76,6 +76,10 @@
{
"builtin": "crab_address",
"override": {
"source": {
"geoJson": "http://127.0.0.1:8080/tile_{z}_{x}_{y}.geojson",
"geoJsonZoomLevel": 18
},
"mapRendering": [
{
"iconSize": "5,5,center",
@ -101,8 +105,8 @@
"_embedded_crab_addresses:=Array.from(new Set(feat.overlapWith('crab_address').map(ff => ff.feat.properties).filter(p => p._HNRLABEL.toLowerCase() === (feat.properties['addr:housenumber'] + (feat.properties['addr:unit']??'')).toLowerCase()).map(p => p.STRAATNM)))",
"_singular_import:=feat.get('_embedded_crab_addresses')?.length == 1",
"_name_to_apply:=feat.get('_embedded_crab_addresses')[0]",
"_nearby_street_names:=feat.closestn('named_streets',5,'name', 500).map(ff => ff.feat.properties.name)",
"_spelling_is_correct:= feat.get('_nearby_street_names').indexOf(feat.properties['_name_to_apply']) >= 0"
"_nearby_street_names:=feat.closestn('named_streets',5,'name', 1000).map(ff => [ff.feat.properties.name, ff.feat.properties['alt_name'], ff.feat.properties['name:nl']])",
"_spelling_is_correct:= [].concat(...feat.get('_nearby_street_names')).indexOf(feat.properties['_name_to_apply']) >= 0"
],
"mapRendering": [
{
@ -134,6 +138,7 @@
"tagRenderings": [
{
"id": "apply_streetname",
"group": "auto",
"render": "{tag_apply(addr:street=$_name_to_apply ,Apply the CRAB-street onto this building)}",
"mappings": [
{
@ -146,8 +151,7 @@
}
]
}
],
"passAllFeatures": true
]
}
],
"hideFromOverview": true

5
scripts/generateTileOverview.ts
View file

@ -2,7 +2,6 @@
* Generates an overview for which tiles exist and which don't
*/
import ScriptUtils from "./ScriptUtils";
import {Tiles} from "../Models/TileRange";
import {writeFileSync} from "fs";
function main(args: string[]) {
@ -28,10 +27,10 @@ function main(args: string[]) {
const x = match[2]
const y = match[3]
if(!indices[x] !== undefined){
if(indices[x] === undefined){
indices[x] = []
}
indices[x] .push(Number(y))
indices[x].push(Number(y))
}
indices["zoom"] = zoomLevel;
const match = files[0].match("\(.*\)_\([0-9]*\)_\([0-9]*\)_\([0-9]*\).geojson")