midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

First working version of fully automatic uploader

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-12-13 02:05:34 +01:00
parent 04dc373b1e
commit e922768f99
21 changed files with 342 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts
View file

@ -119,7 +119,11 @@ export default class SaveTileToLocalStorageActor {
}
private SetIdb(tileIndex, data) {
IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data)
try{
IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data)
}catch(e){
console.error("Could not save tile to indexed-db: ", e, "tileIndex is:", tileIndex, "for layer", this._layer.id)
}
}
private GetIdb(tileIndex) {

55
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -171,6 +171,7 @@ export default class FeaturePipeline {
state.currentBounds, state.locationControl,
(tileIndex, freshness) => self.freshnesses.get(id).addTileLoad(tileIndex, freshness),
(tile) => {
console.debug("Loaded tile ", id, tile.tileIndex, "from local cache")
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
hierarchy.registerTile(tile);
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@ -247,6 +248,7 @@ export default class FeaturePipeline {
})
})
if (state.layoutToUse.trackAllNodes) {
const fullNodeDb = new FullNodeDatabaseSource(
state.filteredLayers.data.filter(l => l.layerDef.id === "type_node")[0],
@ -289,6 +291,10 @@ export default class FeaturePipeline {
// A NewGeometryFromChangesFeatureSource does not split per layer, so we do this next
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(perLayer) => {
if(perLayer.features.data.length === 0){
return
}
// We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this
perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer)
// AT last, we always apply the metatags whenever possible
@ -309,7 +315,7 @@ export default class FeaturePipeline {
this.runningQuery = updater.runningQuery.map(
overpass => {
console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,",
console.log("FeaturePipeline: runningQuery state changed: Overpass", overpass ? "is querying," : "is idle,",
"osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs " + neededTilesFromOsm.data?.length + " tiles (already got " + osmFeatureSource.downloadedTiles.size + " tiles )" : "is idle")
return overpass || osmFeatureSource.isRunning.data;
}, [osmFeatureSource.isRunning]
@ -355,7 +361,15 @@ export default class FeaturePipeline {
if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) {
continue;
}
const freshness = this.freshnesses.get(flayer.layerDef.id).freshnessFor(z, x, y)
if(flayer.layerDef.maxAgeOfCache === 0){
return undefined;
}
const freshnessCalc = this.freshnesses.get(flayer.layerDef.id)
if(freshnessCalc === undefined){
console.warn("No freshness tracker found for ", flayer.layerDef.id)
return undefined
}
const freshness = freshnessCalc.freshnessFor(z, x, y)
if (freshness === undefined) {
// SOmething is undefined --> we return undefined as we have to download
return undefined
@ -409,11 +423,13 @@ export default class FeaturePipeline {
const minzoom = Math.min(...state.layoutToUse.layers.map(layer => layer.minzoom))
const overpassIsActive = state.currentBounds.map(bbox => {
if (bbox === undefined) {
console.debug("Disabling overpass source: no bbox")
return false
}
let zoom = state.locationControl.data.zoom
if (zoom < minzoom) {
// We are zoomed out over the zoomlevel of any layer
console.debug("Disabling overpass source: zoom < minzoom")
return false;
}
@ -456,31 +472,26 @@ export default class FeaturePipeline {
if(src === undefined){
throw "Src is undefined"
}
window.setTimeout(
() => {
const layerDef = src.layer.layerDef;
MetaTagging.addMetatags(
src.features.data,
{
memberships: this.relationTracker,
getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox),
getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id)
},
layerDef,
state,
{
includeDates: true,
// We assume that the non-dated metatags are already set by the cache generator
includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer
}
)
const layerDef = src.layer.layerDef;
MetaTagging.addMetatags(
src.features.data,
{
memberships: this.relationTracker,
getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox),
getFeatureById: (id: string) => self.state.allElements.ContainingFeatures.get(id)
},
15
layerDef,
state,
{
includeDates: true,
// We assume that the non-dated metatags are already set by the cache generator
includeNonDates: layerDef.source.geojsonSource === undefined || !layerDef.source.isOsmCacheLayer
}
)
}
private updateAllMetaTagging() {
public updateAllMetaTagging() {
const self = this;
console.debug("Updating the meta tagging of all tiles as new data got loaded")
this.perLayerHierarchy.forEach(hierarchy => {

12
Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts
View file

@ -36,21 +36,15 @@ export default class PerLayerFeatureSourceSplitter {
const featuresPerLayer = new Map<string, { feature, freshness } []>();
const noLayerFound = []
function addTo(layer: FilteredLayer, feature: { feature, freshness }) {
const id = layer.layerDef.id
const list = featuresPerLayer.get(id)
if (list !== undefined) {
list.push(feature)
} else {
featuresPerLayer.set(id, [feature])
}
for (const layer of layers.data) {
featuresPerLayer.set(layer.layerDef.id, [])
}
for (const f of features) {
for (const layer of layers.data) {
if (layer.layerDef.source.osmTags.matchesProperties(f.feature.properties)) {
// We have found our matching layer!
addTo(layer, f)
featuresPerLayer.set(layer.layerDef.id, [f])
if (!layer.layerDef.passAllFeatures) {
// If not 'passAllFeatures', we are done for this feature
break;

2
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -126,7 +126,7 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
eventSource.setData(eventSource.data.concat(newFeatures))
}).catch(msg => console.error("Could not load geojson layer", url, "due to", msg))
}).catch(msg => console.debug("Could not load geojson layer", url, "due to", msg))
}
}

5
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -5,13 +5,14 @@ import Loc from "../../../Models/Loc";
import DynamicTileSource from "./DynamicTileSource";
import {Utils} from "../../../Utils";
import GeoJsonSource from "../Sources/GeoJsonSource";
import {BBox} from "../../BBox";
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
constructor(layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer & Tiled) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
currentBounds: UIEventSource<BBox>
}) {
const source = layer.layerDef.source
if (source.geojsonZoomLevel === undefined) {
@ -29,7 +30,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}", layer.layerDef.id)
Utils.downloadJson(whitelistUrl).then(
Utils.downloadJsonCached(whitelistUrl, 1000*60*60).then(
json => {
const data = new Map<number, Set<number>>();
for (const x in json) {

7
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -4,6 +4,7 @@ import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
/***
* A tiled source which dynamically loads the required tiles at a fixed zoom level
@ -17,8 +18,8 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
zoomlevel: number,
constructTile: (zxy: [number, number, number]) => (FeatureSourceForLayer & Tiled),
state: {
currentBounds: UIEventSource<BBox>;
locationControl: UIEventSource<Loc>
leafletMap: any
}
) {
const self = this;
@ -37,7 +38,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
}
// Yup, this is cheating to just get the bounds here
const bounds = state.leafletMap.data?.getBounds()
const bounds = state.currentBounds.data
if (bounds === undefined) {
// We'll retry later
return undefined
@ -50,7 +51,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
}
return needed
}
, [layer.isDisplayed, state.leafletMap]).stabilized(250);
, [layer.isDisplayed, state.currentBounds]).stabilized(250);
neededTiles.addCallbackAndRunD(neededIndexes => {
console.log("Tiled geojson source ", layer.layerDef.id, " needs", neededIndexes)

4
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -63,10 +63,10 @@ export default class OsmFeatureSource {
try {
for (const neededTile of neededTiles) {
console.log("Tile download", Tiles.tile_from_index(neededTile).join("/"), "started")
console.log("Tile download from OSM", Tiles.tile_from_index(neededTile).join("/"), "started")
self.downloadedTiles.add(neededTile)
self.LoadTile(...Tiles.tile_from_index(neededTile)).then(_ => {
console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded")
console.debug("Tile ", Tiles.tile_from_index(neededTile).join("/"), "loaded from OSM")
})
}
} catch (e) {

36
Logic/Osm/Changes.ts
View file

@ -1,5 +1,4 @@
import {OsmNode, OsmObject, OsmRelation, OsmWay} from "./OsmObject";
import State from "../../State";
import {UIEventSource} from "../UIEventSource";
import Constants from "../../Models/Constants";
import OsmChangeAction from "./Actions/OsmChangeAction";
@ -13,6 +12,7 @@ import {ElementStorage} from "../ElementStorage";
import {GeoLocationPointProperties} from "../Actors/GeoLocationHandler";
import {GeoOperations} from "../GeoOperations";
import {ChangesetTag} from "./ChangesetHandler";
import {OsmConnection} from "./OsmConnection";
/**
* Handles all changes made to OSM.
@ -33,14 +33,23 @@ export class Changes {
private readonly previouslyCreated: OsmObject[] = []
private readonly _leftRightSensitive: boolean;
private _state: { allElements: ElementStorage; historicalUserLocations: FeatureSource }
public readonly state: { allElements: ElementStorage; historicalUserLocations: FeatureSource; osmConnection: OsmConnection }
public readonly extraComment:UIEventSource<string> = new UIEventSource(undefined)
constructor(leftRightSensitive: boolean = false) {
constructor(
state?: {
allElements: ElementStorage,
historicalUserLocations: FeatureSource,
osmConnection: OsmConnection
},
leftRightSensitive: boolean = false) {
this._leftRightSensitive = leftRightSensitive;
// We keep track of all changes just as well
this.allChanges.setData([...this.pendingChanges.data])
// If a pending change contains a negative ID, we save that
this._nextId = Math.min(-1, ...this.pendingChanges.data?.map(pch => pch.id) ?? [])
this.state = state;
// Note: a changeset might be reused which was opened just before and might have already used some ids
// This doesn't matter however, as the '-1' is per piecewise upload, not global per changeset
@ -120,7 +129,7 @@ export class Changes {
private calculateDistanceToChanges(change: OsmChangeAction, changeDescriptions: ChangeDescription[]) {
if (this._state === undefined) {
if (this.state === undefined) {
// No state loaded -> we can't calculate...
return;
}
@ -129,7 +138,7 @@ export class Changes {
return;
}
const now = new Date()
const recentLocationPoints = this._state.historicalUserLocations.features.data.map(ff => ff.feature)
const recentLocationPoints = this.state.historicalUserLocations.features.data.map(ff => ff.feature)
.filter(feat => feat.geometry.type === "Point")
.filter(feat => {
const visitTime = new Date((<GeoLocationPointProperties>feat.properties).date)
@ -149,7 +158,7 @@ export class Changes {
const changedObjectCoordinates: [number, number][] = []
const feature = this._state.allElements.ContainingFeatures.get(change.mainObjectId)
const feature = this.state.allElements.ContainingFeatures.get(change.mainObjectId)
if (feature !== undefined) {
changedObjectCoordinates.push(GeoOperations.centerpointCoordinates(feature))
}
@ -189,12 +198,6 @@ export class Changes {
this.allChanges.ping()
}
public useLocationHistory(state: {
allElements: ElementStorage,
historicalUserLocations: FeatureSource
}) {
this._state = state
}
public registerIdRewrites(mappings: Map<string, string>): void {
CreateNewNodeAction.registerIdRewrites(mappings)
@ -281,9 +284,14 @@ export class Changes {
// This method is only called with changedescriptions for this theme
const theme = pending[0].meta.theme
let comment = "Adding data with #MapComplete for theme #" + theme
if(this.extraComment.data !== undefined){
comment+="\n\n"+this.extraComment.data
}
const metatags: ChangesetTag[] = [{
key: "comment",
value: "Adding data with #MapComplete for theme #" + theme
value: comment
},
{
key: "theme",
@ -294,7 +302,7 @@ export class Changes {
...perBinMessage
]
await State.state.osmConnection.changesetHandler.UploadChangeset(
await this.state.osmConnection.changesetHandler.UploadChangeset(
(csId) => Changes.createChangesetFor("" + csId, changes),
metatags
)

8
Logic/Osm/ChangesetHandler.ts
View file

@ -1,9 +1,7 @@
import escapeHtml from "escape-html";
// @ts-ignore
import {OsmConnection, UserDetails} from "./OsmConnection";
import UserDetails, {OsmConnection} from "./OsmConnection";
import {UIEventSource} from "../UIEventSource";
import {ElementStorage} from "../ElementStorage";
import State from "../../State";
import Locale from "../../UI/i18n/Locale";
import Constants from "../../Models/Constants";
import {Changes} from "./Changes";
@ -287,8 +285,8 @@ export class ChangesetHandler {
["language", Locale.language.data],
["host", window.location.host],
["path", path],
["source", State.state.currentUserLocation.features.data.length > 0 ? "survey" : undefined],
["imagery", State.state.backgroundLayer.data.id],
["source", self.changes.state["currentUserLocation"]?.features?.data?.length > 0 ? "survey" : undefined],
["imagery", self.changes.state["backgroundLayer"]?.data?.id],
...changesetTags.map(cstag => [cstag.key, cstag.value])
]
.filter(kv => (kv[1] ?? "") !== "")

5
Logic/Osm/OsmConnection.ts
View file

@ -70,7 +70,8 @@ export class OsmConnection {
// Used to keep multiple changesets open and to write to the correct changeset
layoutName: string,
singlePage?: boolean,
osmConfiguration?: "osm" | "osm-test"
osmConfiguration?: "osm" | "osm-test",
attemptLogin?: true | boolean
}
) {
this.fakeUser = options.fakeUser ?? false;
@ -117,7 +118,7 @@ export class OsmConnection {
options.oauth_token.setData(undefined);
}
if (this.auth.authenticated()) {
if (this.auth.authenticated() && (options.attemptLogin !== false)) {
this.AttemptLogin(); // Also updates the user badge
} else {
console.log("Not authenticated");

3
Logic/State/ElementsState.ts
View file

@ -49,7 +49,8 @@ export default class ElementsState extends FeatureSwitchState {
constructor(layoutToUse: LayoutConfig) {
super(layoutToUse);
this.changes = new Changes(layoutToUse?.isLeftRightSensitive() ?? false)
// @ts-ignore
this.changes = new Changes(this,layoutToUse?.isLeftRightSensitive() ?? false)
{
// -- Location control initialization
const zoom = UIEventSource.asFloat(

1
Logic/State/FeaturePipelineState.ts
View file

@ -9,6 +9,7 @@ import MapState from "./MapState";
import SelectedFeatureHandler from "../Actors/SelectedFeatureHandler";
import Hash from "../Web/Hash";
import {BBox} from "../BBox";
import {FeatureSourceForLayer} from "../FeatureSource/FeatureSource";
export default class FeaturePipelineState extends MapState {

5
Logic/State/MapState.ts
View file

@ -82,8 +82,8 @@ export default class MapState extends UserRelatedState {
public overlayToggles: { config: TilesourceConfig, isDisplayed: UIEventSource<boolean> }[]
constructor(layoutToUse: LayoutConfig) {
super(layoutToUse);
constructor(layoutToUse: LayoutConfig, options?: {attemptLogin: true | boolean}) {
super(layoutToUse, options);
this.availableBackgroundLayers = AvailableBaseLayers.AvailableLayersAt(this.locationControl);
@ -265,7 +265,6 @@ export default class MapState extends UserRelatedState {
let gpsLayerDef: FilteredLayer = this.filteredLayers.data.filter(l => l.layerDef.id === "gps_location_history")[0]
this.historicalUserLocations = new SimpleFeatureSource(gpsLayerDef, Tiles.tile_index(0, 0, 0), features);
this.changes.useLocationHistory(this)
const asLine = features.map(allPoints => {

5
Logic/State/UserRelatedState.ts
View file

@ -36,7 +36,7 @@ export default class UserRelatedState extends ElementsState {
public installedThemes: UIEventSource<{ layout: LayoutConfig; definition: string }[]>;
constructor(layoutToUse: LayoutConfig) {
constructor(layoutToUse: LayoutConfig, options:{attemptLogin : true | boolean}) {
super(layoutToUse);
this.osmConnection = new OsmConnection({
@ -50,7 +50,8 @@ export default class UserRelatedState extends ElementsState {
"Used to complete the login"
),
layoutName: layoutToUse?.id,
osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data
osmConfiguration: <'osm' | 'osm-test'>this.featureSwitchApiURL.data,
attemptLogin: options?.attemptLogin
})
this.mangroveIdentity = new MangroveIdentity(

3
Logic/UIEventSource.ts
View file

@ -289,6 +289,7 @@ export class UIEventSource<T> {
const stack = new Error().stack.split("\n");
const callee = stack[1]
const newSource = new UIEventSource<J>(
f(this.data),
"map(" + this.tag + ")@"+callee
@ -298,7 +299,7 @@ export class UIEventSource<T> {
newSource.setData(f(self.data));
}
this.addCallbackAndRun(update);
this.addCallback(update);
for (const extraSource of extraSources) {
extraSource?.addCallback(update);
}