midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-07-15 00:39:11 +02:00
commit 1010b159e5
87 changed files with 2292 additions and 718 deletions
Download patch file Download diff file Expand all files Collapse all files

143
Logic/Osm/Changes.ts
View file

@ -6,31 +6,38 @@ import Constants from "../../Models/Constants";
import FeatureSource from "../FeatureSource/FeatureSource";
import {TagsFilter} from "../Tags/TagsFilter";
import {Tag} from "../Tags/Tag";
import {OsmConnection} from "./OsmConnection";
import {LocalStorageSource} from "../Web/LocalStorageSource";
/**
* Handles all changes made to OSM.
* Needs an authenticator via OsmConnection
*/
export class Changes implements FeatureSource{
export class Changes implements FeatureSource {
private static _nextId = -1; // Newly assigned ID's are negative
public readonly name = "Newly added features"
/**
* The newly created points, as a FeatureSource
*/
public features = new UIEventSource<{feature: any, freshness: Date}[]>([]);
private static _nextId = -1; // Newly assigned ID's are negative
public features = new UIEventSource<{ feature: any, freshness: Date }[]>([]);
/**
* All the pending changes
*/
public readonly pending: UIEventSource<{ elementId: string, key: string, value: string }[]> =
new UIEventSource<{elementId: string; key: string; value: string}[]>([]);
public readonly pending = LocalStorageSource.GetParsed<{ elementId: string, key: string, value: string }[]>("pending-changes", [])
/**
* All the pending new objects to upload
*/
private readonly newObjects = LocalStorageSource.GetParsed<{ id: number, lat: number, lon: number }[]>("newObjects", [])
private readonly isUploading = new UIEventSource(false);
/**
* Adds a change to the pending changes
*/
private static checkChange(kv: {k: string, v: string}): { k: string, v: string } {
private static checkChange(kv: { k: string, v: string }): { k: string, v: string } {
const key = kv.k;
const value = kv.v;
if (key === undefined || key === null) {
@ -49,8 +56,7 @@ export class Changes implements FeatureSource{
return {k: key.trim(), v: value.trim()};
}
addTag(elementId: string, tagsFilter: TagsFilter,
tags?: UIEventSource<any>) {
const eventSource = tags ?? State.state?.allElements.getEventSourceById(elementId);
@ -59,7 +65,7 @@ export class Changes implements FeatureSource{
if (changes.length == 0) {
return;
}
for (const change of changes) {
if (elementTags[change.k] !== change.v) {
elementTags[change.k] = change.v;
@ -76,15 +82,14 @@ export class Changes implements FeatureSource{
* Uploads all the pending changes in one go.
* Triggered by the 'PendingChangeUploader'-actor in Actors
*/
public flushChanges(flushreason: string = undefined){
if(this.pending.data.length === 0){
public flushChanges(flushreason: string = undefined) {
if (this.pending.data.length === 0) {
return;
}
if(flushreason !== undefined){
if (flushreason !== undefined) {
console.log(flushreason)
}
this.uploadAll([], this.pending.data);
this.pending.setData([]);
this.uploadAll();
}
/**
@ -101,17 +106,13 @@ export class Changes implements FeatureSource{
*/
public createElement(basicTags: Tag[], lat: number, lon: number) {
console.log("Creating a new element with ", basicTags)
const osmNode = new OsmNode(this.getNewID());
const id = "node/" + osmNode.id;
osmNode.lat = lat;
osmNode.lon = lon;
const properties = {id: id};
const properties = {id: osmNode.id};
const geojson = {
"type": "Feature",
"properties": properties,
"id": id,
"id": properties.id,
"geometry": {
"type": "Point",
"coordinates": [
@ -121,46 +122,54 @@ export class Changes implements FeatureSource{
}
}
const changes = Changes.createTagChangeList(basicTags, properties, id);
// The basictags are COPIED, the id is included in the properties
// The tags are not yet written into the OsmObject, but this is applied onto a
const changes = [];
for (const kv of basicTags) {
if (typeof kv.value !== "string") {
throw "Invalid value: don't use a regex in a preset"
}
properties[kv.key] = kv.value;
changes.push({elementId:properties.id, key: kv.key, value: kv.value})
}
console.log("New feature added and pinged")
this.features.data.push({feature:geojson, freshness: new Date()});
this.features.data.push({feature: geojson, freshness: new Date()});
this.features.ping();
State.state.allElements.addOrGetElement(geojson).ping();
this.uploadAll([osmNode], changes);
if (State.state.osmConnection.userDetails.data.backend !== OsmConnection.oauth_configs.osm.url) {
properties["_backend"] = State.state.osmConnection.userDetails.data.backend
}
this.newObjects.data.push({id: osmNode.id, lat: lat, lon: lon})
this.pending.data.push(...changes)
this.pending.ping();
this.newObjects.ping();
return geojson;
}
private static createTagChangeList(basicTags: Tag[], properties: { id: string }, id: string) {
// The basictags are COPIED, the id is included in the properties
// The tags are not yet written into the OsmObject, but this is applied onto a
const changes = [];
for (const kv of basicTags) {
properties[kv.key] = kv.value;
if (typeof kv.value !== "string") {
throw "Invalid value: don't use a regex in a preset"
}
changes.push({elementId: id, key: kv.key, value: kv.value})
}
return changes;
}
private uploadChangesWithLatestVersions(
knownElements: OsmObject[], newElements: OsmObject[], pending: { elementId: string; key: string; value: string }[]) {
knownElements: OsmObject[]) {
const knownById = new Map<string, OsmObject>();
knownElements.forEach(knownElement => {
console.log("Setting ",knownElement.type + knownElement.id, knownElement)
knownById.set(knownElement.type + "/" + knownElement.id, knownElement)
})
const newElements: OsmNode [] = this.newObjects.data.map(spec => {
const newElement = new OsmNode(spec.id);
newElement.lat = spec.lat;
newElement.lon = spec.lon;
return newElement
})
// Here, inside the continuation, we know that all 'neededIds' are loaded in 'knownElements', which maps the ids onto the elements
// We apply the changes on them
for (const change of pending) {
for (const change of this.pending.data) {
if (parseInt(change.elementId.split("/")[1]) < 0) {
// This is a new element - we should apply this on one of the new elements
for (const newElement of newElements) {
@ -169,8 +178,6 @@ export class Changes implements FeatureSource{
}
}
} else {
console.log(knownById, change.elementId)
knownById.get(change.elementId).addTag(change.key, change.value);
}
}
@ -184,17 +191,35 @@ export class Changes implements FeatureSource{
}
}
if (changedElements.length == 0 && newElements.length == 0) {
console.log("No changes in any object");
console.log("No changes in any object - clearing");
this.pending.setData([])
this.newObjects.setData([])
return;
}
if (this.isUploading.data) {
return;
}
this.isUploading.setData(true)
console.log("Beginning upload...");
// At last, we build the changeset and upload
const self = this;
State.state.osmConnection.UploadChangeset(
State.state.layoutToUse.data,
State.state.allElements,
(csId) => Changes.createChangesetFor(csId,changedElements, newElements )
(csId) => Changes.createChangesetFor(csId,changedElements, newElements ),
() => {
// When done
console.log("Upload successfull!")
self.newObjects.setData([])
self.pending.setData([]);
self.isUploading.setData(false)
},
() => self.isUploading.setData(false) // Failed - mark to try again
);
};
@ -220,26 +245,21 @@ export class Changes implements FeatureSource{
creations +
"</create>";
}
if (modifications.length > 0) {
changes +=
"\n<modify>\n" +
modifications +
"\n</modify>";
}
changes += "</osmChange>";
return changes;
}
private uploadAll(
newElements: OsmObject[],
pending: { elementId: string; key: string; value: string }[]
) {
private uploadAll() {
const self = this;
const pending = this.pending.data;
let neededIds: string[] = [];
for (const change of pending) {
const id = change.elementId;
@ -252,8 +272,7 @@ export class Changes implements FeatureSource{
neededIds = Utils.Dedup(neededIds);
OsmObject.DownloadAll(neededIds).addCallbackAndRunD(knownElements => {
console.log("KnownElements:", knownElements)
self.uploadChangesWithLatestVersions(knownElements, newElements, pending)
self.uploadChangesWithLatestVersions(knownElements)
})
}

74
Logic/Osm/ChangesetHandler.ts
View file

@ -27,7 +27,7 @@ export class ChangesetHandler {
}
}
private static parseUploadChangesetResponse(response: XMLDocument, allElements: ElementStorage) {
private static parseUploadChangesetResponse(response: XMLDocument, allElements: ElementStorage): void {
const nodes = response.getElementsByTagName("node");
// @ts-ignore
for (const node of nodes) {
@ -69,7 +69,9 @@ export class ChangesetHandler {
public UploadChangeset(
layout: LayoutConfig,
allElements: ElementStorage,
generateChangeXML: (csid: string) => string) {
generateChangeXML: (csid: string) => string,
whenDone: (csId: string) => void,
onFail: () => void) {
if (this.userDetails.data.csCount == 0) {
// The user became a contributor!
@ -80,6 +82,7 @@ export class ChangesetHandler {
if (this._dryRun) {
const changesetXML = generateChangeXML("123456");
console.log(changesetXML);
whenDone("123456")
return;
}
@ -93,12 +96,14 @@ export class ChangesetHandler {
console.log(changeset);
self.AddChange(csId, changeset,
allElements,
() => {
},
whenDone,
(e) => {
console.error("UPLOADING FAILED!", e)
onFail()
}
)
}, {
onFail: onFail
})
} else {
// There still exists an open changeset (or at least we hope so)
@ -107,15 +112,13 @@ export class ChangesetHandler {
csId,
generateChangeXML(csId),
allElements,
() => {
},
whenDone,
(e) => {
console.warn("Could not upload, changeset is probably closed: ", e);
// Mark the CS as closed...
this.currentChangeset.setData("");
// ... and try again. As the cs is closed, no recursive loop can exist
self.UploadChangeset(layout, allElements, generateChangeXML);
self.UploadChangeset(layout, allElements, generateChangeXML, whenDone, onFail);
}
)
@ -161,18 +164,22 @@ export class ChangesetHandler {
const self = this;
this.OpenChangeset(layout, (csId: string) => {
// The cs is open - let us actually upload!
const changes = generateChangeXML(csId)
// The cs is open - let us actually upload!
const changes = generateChangeXML(csId)
self.AddChange(csId, changes, allElements, (csId) => {
console.log("Successfully deleted ", object.id)
self.CloseChangeset(csId, continuation)
}, (csId) => {
alert("Deletion failed... Should not happend")
// FAILED
self.CloseChangeset(csId, continuation)
})
}, true, reason)
self.AddChange(csId, changes, allElements, (csId) => {
console.log("Successfully deleted ", object.id)
self.CloseChangeset(csId, continuation)
}, (csId) => {
alert("Deletion failed... Should not happend")
// FAILED
self.CloseChangeset(csId, continuation)
})
}, {
isDeletionCS: true,
deletionReason: reason
}
)
}
private CloseChangeset(changesetId: string = undefined, continuation: (() => void) = () => {
@ -204,15 +211,20 @@ export class ChangesetHandler {
private OpenChangeset(
layout: LayoutConfig,
continuation: (changesetId: string) => void,
isDeletionCS: boolean = false,
deletionReason: string = undefined) {
options?: {
isDeletionCS?: boolean,
deletionReason?: string,
onFail?: () => void
}
) {
options = options ?? {}
options.isDeletionCS = options.isDeletionCS ?? false
const commentExtra = layout.changesetmessage !== undefined ? " - " + layout.changesetmessage : "";
let comment = `Adding data with #MapComplete for theme #${layout.id}${commentExtra}`
if (isDeletionCS) {
if (options.isDeletionCS) {
comment = `Deleting a point with #MapComplete for theme #${layout.id}${commentExtra}`
if (deletionReason) {
comment += ": " + deletionReason;
if (options.deletionReason) {
comment += ": " + options.deletionReason;
}
}
@ -221,7 +233,7 @@ export class ChangesetHandler {
const metadata = [
["created_by", `MapComplete ${Constants.vNumber}`],
["comment", comment],
["deletion", isDeletionCS ? "yes" : undefined],
["deletion", options.isDeletionCS ? "yes" : undefined],
["theme", layout.id],
["language", Locale.language.data],
["host", window.location.host],
@ -244,7 +256,9 @@ export class ChangesetHandler {
}, function (err, response) {
if (response === undefined) {
console.log("err", err);
alert("Could not upload change (opening failed). Please file a bug report")
if(options.onFail){
options.onFail()
}
return;
} else {
continuation(response);
@ -265,7 +279,7 @@ export class ChangesetHandler {
private AddChange(changesetId: string,
changesetXML: string,
allElements: ElementStorage,
continuation: ((changesetId: string, idMapping: any) => void),
continuation: ((changesetId: string) => void),
onFail: ((changesetId: string, reason: string) => void) = undefined) {
this.auth.xhr({
method: 'POST',
@ -280,9 +294,9 @@ export class ChangesetHandler {
}
return;
}
const mapping = ChangesetHandler.parseUploadChangesetResponse(response, allElements);
ChangesetHandler.parseUploadChangesetResponse(response, allElements);
console.log("Uploaded changeset ", changesetId);
continuation(changesetId, mapping);
continuation(changesetId);
});
}

10
Logic/Osm/OsmConnection.ts
View file

@ -30,7 +30,7 @@ export default class UserDetails {
export class OsmConnection {
public static readonly _oauth_configs = {
public static readonly oauth_configs = {
"osm": {
oauth_consumer_key: 'hivV7ec2o49Two8g9h8Is1VIiVOgxQ1iYexCbvem',
oauth_secret: 'wDBRTCem0vxD7txrg1y6p5r8nvmz8tAhET7zDASI',
@ -66,7 +66,7 @@ export class OsmConnection {
osmConfiguration: "osm" | "osm-test" = 'osm'
) {
this._singlePage = singlePage;
this._oauth_config = OsmConnection._oauth_configs[osmConfiguration] ?? OsmConnection._oauth_configs.osm;
this._oauth_config = OsmConnection.oauth_configs[osmConfiguration] ?? OsmConnection.oauth_configs.osm;
console.debug("Using backend", this._oauth_config.url)
OsmObject.SetBackendUrl(this._oauth_config.url + "/")
this._iframeMode = Utils.runningFromConsole ? false : window !== window.top;
@ -110,8 +110,10 @@ export class OsmConnection {
public UploadChangeset(
layout: LayoutConfig,
allElements: ElementStorage,
generateChangeXML: (csid: string) => string) {
this.changesetHandler.UploadChangeset(layout, allElements, generateChangeXML);
generateChangeXML: (csid: string) => string,
whenDone: (csId: string) => void,
onFail: () => {}) {
this.changesetHandler.UploadChangeset(layout, allElements, generateChangeXML, whenDone, onFail);
}
public GetPreference(key: string, prefix: string = "mapcomplete-"): UIEventSource<string> {

39
Logic/Osm/OsmObject.ts
View file

@ -5,7 +5,8 @@ import {UIEventSource} from "../UIEventSource";
export abstract class OsmObject {
protected static backendURL = "https://www.openstreetmap.org/"
private static defaultBackend = "https://www.openstreetmap.org/"
protected static backendURL = OsmObject.defaultBackend;
private static polygonFeatures = OsmObject.constructPolygonFeatures()
private static objectCache = new Map<string, UIEventSource<OsmObject>>();
private static referencingWaysCache = new Map<string, UIEventSource<OsmWay[]>>();
@ -36,15 +37,22 @@ export abstract class OsmObject {
this.backendURL = url;
}
static DownloadObject(id): UIEventSource<OsmObject> {
static DownloadObject(id: string, forceRefresh: boolean = false): UIEventSource<OsmObject> {
let src: UIEventSource<OsmObject>;
if (OsmObject.objectCache.has(id)) {
return OsmObject.objectCache.get(id)
src = OsmObject.objectCache.get(id)
if (forceRefresh) {
src.setData(undefined)
} else {
return src;
}
} else {
src = new UIEventSource<OsmObject>(undefined)
}
const splitted = id.split("/");
const type = splitted[0];
const idN = splitted[1];
const idN = Number(splitted[1]);
const src = new UIEventSource<OsmObject>(undefined)
OsmObject.objectCache.set(id, src);
const newContinuation = (element: OsmObject) => {
src.setData(element)
@ -160,11 +168,11 @@ export abstract class OsmObject {
})
}
public static DownloadAll(neededIds): UIEventSource<OsmObject[]> {
public static DownloadAll(neededIds, forceRefresh = true): UIEventSource<OsmObject[]> {
// local function which downloads all the objects one by one
// this is one big loop, running one download, then rerunning the entire function
const allSources: UIEventSource<OsmObject> [] = neededIds.map(id => OsmObject.DownloadObject(id))
const allSources: UIEventSource<OsmObject> [] = neededIds.map(id => OsmObject.DownloadObject(id, forceRefresh))
const allCompleted = new UIEventSource(undefined).map(_ => {
return !allSources.some(uiEventSource => uiEventSource.data === undefined)
}, allSources)
@ -172,7 +180,7 @@ export abstract class OsmObject {
if (completed) {
return allSources.map(src => src.data)
}
return []
return undefined
});
}
@ -286,6 +294,7 @@ export abstract class OsmObject {
self.LoadData(element)
self.SaveExtraData(element, nodes);
const meta = {
"_last_edit:contributor": element.user,
"_last_edit:contributor:uid": element.uid,
@ -294,6 +303,11 @@ export abstract class OsmObject {
"_version_number": element.version
}
if (OsmObject.backendURL !== OsmObject.defaultBackend) {
self.tags["_backend"] = OsmObject.backendURL
meta["_backend"] = OsmObject.backendURL;
}
continuation(self, meta);
}
);
@ -348,7 +362,7 @@ export class OsmNode extends OsmObject {
lat: number;
lon: number;
constructor(id) {
constructor(id: number) {
super("node", id);
}
@ -401,7 +415,7 @@ export class OsmWay extends OsmObject {
lat: number;
lon: number;
constructor(id) {
constructor(id: number) {
super("way", id);
}
@ -467,11 +481,10 @@ export class OsmWay extends OsmObject {
export class OsmRelation extends OsmObject {
members;
public members;
constructor(id) {
constructor(id: number) {
super("relation", id);
}
centerpoint(): [number, number] {