midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge branch 'alpha' into develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-10-13 17:23:02 +02:00
commit e0e1bfbe00
91 changed files with 4982 additions and 1093 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Logic/ExtraFunction.ts
View file

@ -119,7 +119,7 @@ export class ExtraFunction {
{
name: "closest",
doc: "Given either a list of geojson features or a single layer name, gives the single object which is nearest to the feature. In the case of ways/polygons, only the centerpoint is considered. Returns a single geojson feature or undefined if nothing is found (or not yet laoded)",
args: ["list of features"]
args: ["list of features or a layer name or '*' to get all features"]
},
(params, feature) => {
return (features) => ExtraFunction.GetClosestNFeatures(params, feature, features)?.[0]?.feat
@ -132,7 +132,7 @@ export class ExtraFunction {
doc: "Given either a list of geojson features or a single layer name, gives the n closest objects which are nearest to the feature (excluding the feature itself). In the case of ways/polygons, only the centerpoint is considered. " +
"Returns a list of `{feat: geojson, distance:number}` the empty list if nothing is found (or not yet loaded)\n\n" +
"If a 'unique tag key' is given, the tag with this key will only appear once (e.g. if 'name' is given, all features will have a different name)",
args: ["list of features or layer name", "amount of features", "unique tag key (optional)", "maxDistanceInMeters (optional)"]
args: ["list of features or layer name or '*' to get all features", "amount of features", "unique tag key (optional)", "maxDistanceInMeters (optional)"]
},
(params, feature) => {

3
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -402,6 +402,9 @@ export default class FeaturePipeline {
}
public GetFeaturesWithin(layerId: string, bbox: BBox): any[][] {
if(layerId === "*"){
return this.GetAllFeaturesWithin(bbox)
}
const requestedHierarchy = this.perLayerHierarchy.get(layerId)
if (requestedHierarchy === undefined) {
console.warn("Layer ", layerId, "is not defined. Try one of ", Array.from(this.perLayerHierarchy.keys()))

24
Logic/ImageProviders/AllImageProviders.ts
View file

@ -16,35 +16,35 @@ export default class AllImageProviders {
Mapillary.singleton,
WikidataImageProvider.singleton,
WikimediaImageProvider.singleton,
new GenericImageProvider([].concat(...Imgur.defaultValuePrefix, WikimediaImageProvider.commonsPrefix, ...Mapillary.valuePrefixes))]
new GenericImageProvider(
[].concat(...Imgur.defaultValuePrefix, ...WikimediaImageProvider.commonsPrefixes, ...Mapillary.valuePrefixes)
)
]
private static _cache: Map<string, UIEventSource<ProvidedImage[]>> = new Map<string, UIEventSource<ProvidedImage[]>>()
public static LoadImagesFor(tags: UIEventSource<any>, imagePrefix?: string): UIEventSource<ProvidedImage[]> {
const id = tags.data.id
if (id === undefined) {
public static LoadImagesFor(tags: UIEventSource<any>, tagKey?: string): UIEventSource<ProvidedImage[]> {
if (tags.data.id === undefined) {
return undefined;
}
const cached = this._cache.get(tags.data.id)
const cacheKey = tags.data.id+tagKey
const cached = this._cache.get(cacheKey)
if (cached !== undefined) {
return cached
}
const source = new UIEventSource([])
this._cache.set(id, source)
this._cache.set(cacheKey, source)
const allSources = []
for (const imageProvider of AllImageProviders.ImageAttributionSource) {
let prefixes = imageProvider.defaultKeyPrefixes
if(imagePrefix !== undefined){
prefixes = [...prefixes]
if(prefixes.indexOf("image") >= 0){
prefixes.splice(prefixes.indexOf("image"), 1)
}
prefixes.push(imagePrefix)
if(tagKey !== undefined){
prefixes = [tagKey]
}
const singleSource = imageProvider.GetRelevantUrls(tags, {

9
Logic/ImageProviders/GenericImageProvider.ts
View file

@ -20,7 +20,14 @@ export default class GenericImageProvider extends ImageProvider {
if (this._valuePrefixBlacklist.some(prefix => value.startsWith(prefix))) {
return []
}
try{
new URL(value)
}catch (_){
// Not a valid URL
return []
}
return [Promise.resolve({
key: key,
url: value,

69
Logic/ImageProviders/ImageProvider.ts
View file

@ -1,23 +1,26 @@
import {UIEventSource} from "../UIEventSource";
import BaseUIElement from "../../UI/BaseUIElement";
import {LicenseInfo} from "./LicenseInfo";
import {Utils} from "../../Utils";
export interface ProvidedImage {
url: string, key: string, provider: ImageProvider
url: string,
key: string,
provider: ImageProvider
}
export default abstract class ImageProvider {
public abstract readonly defaultKeyPrefixes : string[] = ["mapillary", "image"]
public abstract readonly defaultKeyPrefixes: string[] = ["mapillary", "image"]
private _cache = new Map<string, UIEventSource<LicenseInfo>>()
GetAttributionFor(url: string): UIEventSource<LicenseInfo> {
const cached = this._cache.get(url);
if (cached !== undefined) {
return cached;
}
const src =UIEventSource.FromPromise(this.DownloadAttribution(url))
const src = UIEventSource.FromPromise(this.DownloadAttribution(url))
this._cache.set(url, src)
return src;
}
@ -31,42 +34,48 @@ export default abstract class ImageProvider {
*/
public GetRelevantUrls(allTags: UIEventSource<any>, options?: {
prefixes?: string[]
}):UIEventSource<ProvidedImage[]> {
}): UIEventSource<ProvidedImage[]> {
const prefixes = options?.prefixes ?? this.defaultKeyPrefixes
if(prefixes === undefined){
throw "The image provider"+this.constructor.name+" doesn't define `defaultKeyPrefixes`"
if (prefixes === undefined) {
throw "The image provider" + this.constructor.name + " doesn't define `defaultKeyPrefixes`"
}
const relevantUrls = new UIEventSource<{ url: string; key: string; provider: ImageProvider }[]>([])
const seenValues = new Set<string>()
const self =this
allTags.addCallbackAndRunD(tags => {
for (const key in tags) {
if(!prefixes.some(prefix => key.startsWith(prefix))){
if (!prefixes.some(prefix => key.startsWith(prefix))) {
continue
}
const value = tags[key]
if(seenValues.has(value)){
continue
}
seenValues.add(value)
this.ExtractUrls(key, value).then(promises => {
for (const promise of promises ?? []) {
if(promise === undefined){
continue
}
promise.then(providedImage => {
if(providedImage === undefined){
return
}
relevantUrls.data.push(providedImage)
relevantUrls.ping()
})
const values = Utils.NoEmpty(tags[key]?.split(";")?.map(v => v.trim()) ?? [])
for (const value of values) {
if (seenValues.has(value)) {
continue
}
})
seenValues.add(value)
this.ExtractUrls(key, value).then(promises => {
for (const promise of promises ?? []) {
if (promise === undefined) {
continue
}
promise.then(providedImage => {
if (providedImage === undefined) {
return
}
relevantUrls.data.push(providedImage)
relevantUrls.ping()
})
}
})
}
}
})
return relevantUrls
}
public abstract ExtractUrls(key: string, value: string) : Promise<Promise<ProvidedImage>[]>;
public abstract ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]>;
}

1
Logic/ImageProviders/LicenseInfo.ts
View file

@ -1,4 +1,5 @@
export class LicenseInfo {
title: string = ""
artist: string = "";
license: string = "";
licenseShortName: string = "";

10
Logic/ImageProviders/WikidataImageProvider.ts
View file

@ -34,9 +34,17 @@ export class WikidataImageProvider extends ImageProvider {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, img)
allImages.push(...promises)
}
// P373 is 'commons category'
for (let cat of Array.from(entity.claims.get("P373") ?? [])) {
if(!cat.startsWith("Category:")){
cat = "Category:"+cat
}
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, cat)
allImages.push(...promises)
}
const commons = entity.commons
if (commons !== undefined) {
if (commons !== undefined && (commons.startsWith("Category:") || commons.startsWith("File:"))) {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
allImages.push(...promises)
}

131
Logic/ImageProviders/WikimediaImageProvider.ts
View file

@ -4,6 +4,7 @@ import Svg from "../../Svg";
import Link from "../../UI/Base/Link";
import {Utils} from "../../Utils";
import {LicenseInfo} from "./LicenseInfo";
import Wikimedia from "../Web/Wikimedia";
/**
* This module provides endpoints for wikimedia and others
@ -14,56 +15,12 @@ export class WikimediaImageProvider extends ImageProvider {
private readonly commons_key = "wikimedia_commons"
public readonly defaultKeyPrefixes = [this.commons_key,"image"]
public static readonly singleton = new WikimediaImageProvider();
public static readonly commonsPrefix = "https://commons.wikimedia.org/wiki/"
public static readonly commonsPrefixes = ["https://commons.wikimedia.org/wiki/", "https://upload.wikimedia.org", "File:"]
private constructor() {
super();
}
/**
* Recursively walks a wikimedia commons category in order to search for (image) files
* Returns (a promise of) a list of URLS
* @param categoryName The name of the wikimedia category
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
private static async GetImagesInCategory(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
}
if (maxLoad - imageOverview.length <= 0) {
console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
}
// We do have a continue token - let's load the next page
const recursive = await this.GetImagesInCategory(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
imageOverview.push(...recursive)
return imageOverview
}
private static ExtractFileName(url: string) {
if (!url.startsWith("http")) {
return url;
@ -87,7 +44,7 @@ export class WikimediaImageProvider extends ImageProvider {
}
private PrepareUrl(value: string): string {
private static PrepareUrl(value: string): string {
if (value.toLowerCase().startsWith("https://commons.wikimedia.org/wiki/")) {
return value;
@ -108,12 +65,26 @@ export class WikimediaImageProvider extends ImageProvider {
"&format=json&origin=*";
const data = await Utils.downloadJson(url)
const licenseInfo = new LicenseInfo();
const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata;
const pageInfo = data.query.pages[-1]
if(pageInfo === undefined){
return undefined;
}
const license = (pageInfo.imageinfo ?? [])[0]?.extmetadata;
if (license === undefined) {
console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!")
console.warn("The file", filename ,"has no usable metedata or license attached... Please fix the license info file yourself!")
return undefined;
}
let title = pageInfo.title
if(title.startsWith("File:")){
title= title.substr("File:".length)
}
if(title.endsWith(".jpg") || title.endsWith(".png")){
title = title.substring(0, title.length - 4)
}
licenseInfo.title = title
licenseInfo.artist = license.Artist?.value;
licenseInfo.license = license.License?.value;
licenseInfo.copyrighted = license.Copyrighted?.value;
@ -126,41 +97,71 @@ export class WikimediaImageProvider extends ImageProvider {
}
private async UrlForImage(image: string): Promise<ProvidedImage> {
private UrlForImage(image: string): ProvidedImage {
if (!image.startsWith("File:")) {
image = "File:" + image
}
return {url: this.PrepareUrl(image), key: undefined, provider: this}
return {url: WikimediaImageProvider.PrepareUrl(image), key: undefined, provider: this}
}
private static startsWithCommonsPrefix(value: string): boolean{
return WikimediaImageProvider.commonsPrefixes.some(prefix => value.startsWith(prefix))
}
private static removeCommonsPrefix(value: string): string{
if(value.startsWith("https://upload.wikimedia.org/")){
value = value.substring(value.lastIndexOf("/") + 1)
value = decodeURIComponent(value)
if(!value.startsWith("File:")){
value = "File:"+value
}
return value;
}
for (const prefix of WikimediaImageProvider.commonsPrefixes) {
if(value.startsWith(prefix)){
let part = value.substr(prefix.length)
if(prefix.startsWith("http")){
part = decodeURIComponent(part)
}
return part
}
}
return value;
}
public PrepUrl(value: string): ProvidedImage {
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("File:")) {
return this.UrlForImage(value)
}
// We do a last effort and assume this is a file
return this.UrlForImage("File:" + value)
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
if(key !== undefined && key !== this.commons_key && !value.startsWith(WikimediaImageProvider.commonsPrefix)){
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
if(key !== undefined && key !== this.commons_key && !hasCommonsPrefix){
return []
}
if (value.startsWith(WikimediaImageProvider.commonsPrefix)) {
value = value.substring(WikimediaImageProvider.commonsPrefix.length)
} else if (value.startsWith("https://upload.wikimedia.org")) {
const result: ProvidedImage = {
key: undefined,
url: value,
provider: this
}
return [Promise.resolve(result)]
}
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("Category:")) {
const urls = await WikimediaImageProvider.GetImagesInCategory(value)
return urls.map(image => this.UrlForImage(image))
const urls = await Wikimedia.GetCategoryContents(value)
return urls.filter(url => url.startsWith("File:")).map(image => Promise.resolve(this.UrlForImage(image)))
}
if (value.startsWith("File:")) {
return [this.UrlForImage(value)]
return [Promise.resolve(this.UrlForImage(value))]
}
if (value.startsWith("http")) {
// PRobably an error
return []
}
// We do a last effort and assume this is a file
return [this.UrlForImage("File:" + value)]
return [Promise.resolve(this.UrlForImage("File:" + value))]
}

4
Logic/MetaTagging.ts
View file

@ -64,12 +64,12 @@ export default class MetaTagging {
if(metatag.isLazy){
somethingChanged = true;
metatag.applyMetaTagsOnFeature(feature, freshness)
metatag.applyMetaTagsOnFeature(feature, freshness, layer)
}else{
const newValueAdded = metatag.applyMetaTagsOnFeature(feature, freshness)
const newValueAdded = metatag.applyMetaTagsOnFeature(feature, freshness, layer)
/* Note that the expression:
* `somethingChanged = newValueAdded || metatag.applyMetaTagsOnFeature(feature, freshness)`
* Is WRONG

18
Logic/Osm/Actions/ChangeDescription.ts
View file

@ -5,6 +5,24 @@ import {OsmNode, OsmRelation, OsmWay} from "../OsmObject";
*/
export interface ChangeDescription {
/**
* Metadata to be included in the changeset
*/
meta: {
/*
* The theme with which this changeset was made
*/
theme: string,
/**
* The type of the change
*/
changeType: "answer" | "create" | "split" | "delete" | string
/**
* THe motivation for the change, e.g. 'deleted because does not exist anymore'
*/
specialMotivation?: string
},
/**
* Identifier of the object
*/

13
Logic/Osm/Actions/ChangeTagAction.ts
View file

@ -7,12 +7,17 @@ export default class ChangeTagAction extends OsmChangeAction {
private readonly _elementId: string;
private readonly _tagsFilter: TagsFilter;
private readonly _currentTags: any;
private readonly _meta: {theme: string, changeType: string};
constructor(elementId: string, tagsFilter: TagsFilter, currentTags: any) {
constructor(elementId: string, tagsFilter: TagsFilter, currentTags: any, meta: {
theme: string,
changeType: "answer" | "soft-delete" | "add-image"
}) {
super();
this._elementId = elementId;
this._tagsFilter = tagsFilter;
this._currentTags = currentTags;
this._meta = meta;
}
/**
@ -43,10 +48,10 @@ export default class ChangeTagAction extends OsmChangeAction {
const type = typeId[0]
const id = Number(typeId [1])
return [{
// @ts-ignore
type: type,
type: <"node"|"way"|"relation"> type,
id: id,
tags: changedTags
tags: changedTags,
meta: this._meta
}]
}
}

21
Logic/Osm/Actions/CreateNewNodeAction.ts
View file

@ -14,8 +14,14 @@ export default class CreateNewNodeAction extends OsmChangeAction {
private readonly _lon: number;
private readonly _snapOnto: OsmWay;
private readonly _reusePointDistance: number;
private meta: { changeType: "create" | "import"; theme: string };
constructor(basicTags: Tag[], lat: number, lon: number, options?: { snapOnto: OsmWay, reusePointWithinMeters?: number }) {
constructor(basicTags: Tag[],
lat: number, lon: number,
options: {
snapOnto?: OsmWay,
reusePointWithinMeters?: number,
theme: string, changeType: "create" | "import" }) {
super()
this._basicTags = basicTags;
this._lat = lat;
@ -25,6 +31,10 @@ export default class CreateNewNodeAction extends OsmChangeAction {
}
this._snapOnto = options?.snapOnto;
this._reusePointDistance = options?.reusePointWithinMeters ?? 1
this.meta = {
theme: options.theme,
changeType: options.changeType
}
}
async CreateChangeDescriptions(changes: Changes): Promise<ChangeDescription[]> {
@ -47,7 +57,8 @@ export default class CreateNewNodeAction extends OsmChangeAction {
changes: {
lat: this._lat,
lon: this._lon
}
},
meta: this.meta
}
if (this._snapOnto === undefined) {
return [newPointChange]
@ -78,7 +89,8 @@ export default class CreateNewNodeAction extends OsmChangeAction {
return [{
tags: new And(this._basicTags).asChange(properties),
type: "node",
id: reusedPointId
id: reusedPointId,
meta: this.meta
}]
}
@ -99,7 +111,8 @@ export default class CreateNewNodeAction extends OsmChangeAction {
changes: {
coordinates: locations,
nodes: ids
}
},
meta:this.meta
}
]
}

253
Logic/Osm/Actions/DeleteAction.ts
View file

@ -1,225 +1,62 @@
import {UIEventSource} from "../../UIEventSource";
import {Translation} from "../../../UI/i18n/Translation";
import State from "../../../State";
import {OsmObject} from "../OsmObject";
import Translations from "../../../UI/i18n/Translations";
import Constants from "../../../Models/Constants";
import OsmChangeAction from "./OsmChangeAction";
import {Changes} from "../Changes";
import {ChangeDescription} from "./ChangeDescription";
import ChangeTagAction from "./ChangeTagAction";
import {TagsFilter} from "../../Tags/TagsFilter";
import {And} from "../../Tags/And";
import {Tag} from "../../Tags/Tag";
export default class DeleteAction {
export default class DeleteAction extends OsmChangeAction {
public readonly canBeDeleted: UIEventSource<{ canBeDeleted?: boolean, reason: Translation }>;
public readonly isDeleted = new UIEventSource<boolean>(false);
private readonly _softDeletionTags: TagsFilter;
private readonly meta: {
theme: string,
specialMotivation: string,
changeType: "deletion"
};
private readonly _id: string;
private readonly _allowDeletionAtChangesetCount: number;
private _hardDelete: boolean;
constructor(id: string, allowDeletionAtChangesetCount?: number) {
constructor(id: string,
softDeletionTags: TagsFilter,
meta: {
theme: string,
specialMotivation: string
},
hardDelete: boolean) {
super()
this._id = id;
this._allowDeletionAtChangesetCount = allowDeletionAtChangesetCount ?? Number.MAX_VALUE;
this._hardDelete = hardDelete;
this.meta = {...meta, changeType: "deletion"};
this._softDeletionTags = new And([softDeletionTags,
new Tag("fixme", `A mapcomplete user marked this feature to be deleted (${meta.specialMotivation})`)
]);
this.canBeDeleted = new UIEventSource<{ canBeDeleted?: boolean; reason: Translation }>({
canBeDeleted: undefined,
reason: Translations.t.delete.loading
})
this.CheckDeleteability(false)
}
protected async CreateChangeDescriptions(changes: Changes): Promise<ChangeDescription[]> {
/**
* Does actually delete the feature; returns the event source 'this.isDeleted'
* If deletion is not allowed, triggers the callback instead
*/
public DoDelete(reason: string, onNotAllowed: () => void): void {
const isDeleted = this.isDeleted
const self = this;
let deletionStarted = false;
this.canBeDeleted.addCallbackAndRun(
canBeDeleted => {
if (isDeleted.data || deletionStarted) {
// Already deleted...
return;
const osmObject = await OsmObject.DownloadObjectAsync(this._id)
if (this._hardDelete) {
return [{
meta: this.meta,
doDelete: true,
type: osmObject.type,
id: osmObject.id,
}]
} else {
return await new ChangeTagAction(
this._id, this._softDeletionTags, osmObject.tags,
{
theme: State.state?.layoutToUse?.id ?? "unkown",
changeType: "soft-delete"
}
if (canBeDeleted.canBeDeleted === false) {
// We aren't allowed to delete
deletionStarted = true;
onNotAllowed();
isDeleted.setData(true);
return;
}
if (!canBeDeleted) {
// We are not allowed to delete (yet), this might change in the future though
return;
}
deletionStarted = true;
OsmObject.DownloadObject(self._id).addCallbackAndRun(obj => {
if (obj === undefined) {
return;
}
State.state.osmConnection.changesetHandler.DeleteElement(
obj,
State.state.layoutToUse,
reason,
State.state.allElements,
() => {
isDeleted.setData(true)
}
)
})
}
)
}
/**
* Checks if the currently logged in user can delete the current point.
* State is written into this._canBeDeleted
* @constructor
* @private
*/
public CheckDeleteability(useTheInternet: boolean): void {
const t = Translations.t.delete;
const id = this._id;
const state = this.canBeDeleted
if (!id.startsWith("node")) {
this.canBeDeleted.setData({
canBeDeleted: false,
reason: t.isntAPoint
})
return;
).CreateChangeDescriptions(changes)
}
// Does the currently logged in user have enough experience to delete this point?
const deletingPointsOfOtherAllowed = State.state.osmConnection.userDetails.map(ud => {
if (ud === undefined) {
return undefined;
}
if (!ud.loggedIn) {
return false;
}
return ud.csCount >= Math.min(Constants.userJourney.deletePointsOfOthersUnlock, this._allowDeletionAtChangesetCount);
})
const previousEditors = new UIEventSource<number[]>(undefined)
const allByMyself = previousEditors.map(previous => {
if (previous === null || previous === undefined) {
// Not yet downloaded
return null;
}
const userId = State.state.osmConnection.userDetails.data.uid;
return !previous.some(editor => editor !== userId)
}, [State.state.osmConnection.userDetails])
// User allowed OR only edited by self?
const deletetionAllowed = deletingPointsOfOtherAllowed.map(isAllowed => {
if (isAllowed === undefined) {
// No logged in user => definitively not allowed to delete!
return false;
}
if (isAllowed === true) {
return true;
}
// At this point, the logged in user is not allowed to delete points created/edited by _others_
// however, we query OSM and if it turns out the current point has only be edited by the current user, deletion is allowed after all!
if (allByMyself.data === null && useTheInternet) {
// We kickoff the download here as it hasn't yet been downloaded. Note that this is mapped onto 'all by myself' above
OsmObject.DownloadHistory(id).map(versions => versions.map(version => version.tags["_last_edit:contributor:uid"])).syncWith(previousEditors)
}
if (allByMyself.data === true) {
// Yay! We can download!
return true;
}
if (allByMyself.data === false) {
// Nope, downloading not allowed...
return false;
}
// At this point, we don't have enough information yet to decide if the user is allowed to delete the current point...
return undefined;
}, [allByMyself])
const hasRelations: UIEventSource<boolean> = new UIEventSource<boolean>(null)
const hasWays: UIEventSource<boolean> = new UIEventSource<boolean>(null)
deletetionAllowed.addCallbackAndRunD(deletetionAllowed => {
if (deletetionAllowed === false) {
// Nope, we are not allowed to delete
state.setData({
canBeDeleted: false,
reason: t.notEnoughExperience
})
return true; // unregister this caller!
}
if (!useTheInternet) {
return;
}
// All right! We have arrived at a point that we should query OSM again to check that the point isn't a part of ways or relations
OsmObject.DownloadReferencingRelations(id).then(rels => {
hasRelations.setData(rels.length > 0)
})
OsmObject.DownloadReferencingWays(id).then(ways => {
hasWays.setData(ways.length > 0)
})
return true; // unregister to only run once
})
const hasWaysOrRelations = hasRelations.map(hasRelationsData => {
if (hasRelationsData === true) {
return true;
}
if (hasWays.data === true) {
return true;
}
if (hasWays.data === null || hasRelationsData === null) {
return null;
}
if (hasWays.data === false && hasRelationsData === false) {
return false;
}
return null;
}, [hasWays])
hasWaysOrRelations.addCallbackAndRun(
waysOrRelations => {
if (waysOrRelations == null) {
// Not yet loaded - we still wait a little bit
return;
}
if (waysOrRelations) {
// not deleteble by mapcomplete
state.setData({
canBeDeleted: false,
reason: t.partOfOthers
})
} else {
// alright, this point can be safely deleted!
state.setData({
canBeDeleted: true,
reason: allByMyself.data === true ? t.onlyEditedByLoggedInUser : t.safeDelete
})
}
}
)
}
}

16
Logic/Osm/Actions/RelationSplitHandler.ts
View file

@ -16,14 +16,16 @@ export interface RelationSplitInput {
*/
export default class RelationSplitHandler extends OsmChangeAction {
private readonly _input: RelationSplitInput;
private readonly _theme: string;
constructor(input: RelationSplitInput) {
constructor(input: RelationSplitInput, theme: string) {
super()
this._input = input;
this._theme = theme;
}
async CreateChangeDescriptions(changes: Changes): Promise<ChangeDescription[]> {
return new InPlaceReplacedmentRTSH(this._input).CreateChangeDescriptions(changes)
return new InPlaceReplacedmentRTSH(this._input, this._theme).CreateChangeDescriptions(changes)
}
@ -39,10 +41,12 @@ export default class RelationSplitHandler extends OsmChangeAction {
*/
export class InPlaceReplacedmentRTSH extends OsmChangeAction {
private readonly _input: RelationSplitInput;
private readonly _theme: string;
constructor(input: RelationSplitInput) {
constructor(input: RelationSplitInput, theme: string) {
super();
this._input = input;
this._theme = theme;
}
/**
@ -137,7 +141,11 @@ export class InPlaceReplacedmentRTSH extends OsmChangeAction {
return [{
id: relation.id,
type: "relation",
changes: {members: newMembers}
changes: {members: newMembers},
meta:{
changeType: "relation-fix",
theme: this._theme
}
}];
}

19
Logic/Osm/Actions/SplitAction.ts
View file

@ -14,16 +14,19 @@ interface SplitInfo {
export default class SplitAction extends OsmChangeAction {
private readonly wayId: string;
private readonly _splitPointsCoordinates: [number, number] []// lon, lat
private _meta: { theme: string, changeType: "split" };
/**
*
* @param wayId
* @param splitPointCoordinates: lon, lat
* @param meta
*/
constructor(wayId: string, splitPointCoordinates: [number, number][]) {
constructor(wayId: string, splitPointCoordinates: [number, number][], meta: {theme: string}) {
super()
this.wayId = wayId;
this._splitPointsCoordinates = splitPointCoordinates
this._meta = {...meta, changeType: "split"};
}
private static SegmentSplitInfo(splitInfo: SplitInfo[]): SplitInfo[][] {
@ -89,7 +92,8 @@ export default class SplitAction extends OsmChangeAction {
changes: {
lon: element.lngLat[0],
lat: element.lngLat[1]
}
},
meta: this._meta
})
}
@ -110,7 +114,8 @@ export default class SplitAction extends OsmChangeAction {
changes: {
coordinates: wayPart.map(p => p.lngLat),
nodes: nodeIds
}
},
meta: this._meta
})
allWayIdsInOrder.push(originalElement.id)
allWaysNodesInOrder.push(nodeIds)
@ -135,7 +140,8 @@ export default class SplitAction extends OsmChangeAction {
changes: {
coordinates: wayPart.map(p => p.lngLat),
nodes: nodeIds
}
},
meta: this._meta
})
allWayIdsInOrder.push(id)
@ -152,8 +158,8 @@ export default class SplitAction extends OsmChangeAction {
allWayIdsInOrder: allWayIdsInOrder,
originalNodes: originalNodes,
allWaysNodesInOrder: allWaysNodesInOrder,
originalWayId: originalElement.id
}).CreateChangeDescriptions(changes)
originalWayId: originalElement.id,
}, this._meta.theme).CreateChangeDescriptions(changes)
changeDescription.push(...changDescrs)
}
@ -240,7 +246,6 @@ export default class SplitAction extends OsmChangeAction {
closest = prevPoint
}
// Ok, we have a closest point!
if(closest.originalIndex === 0 || closest.originalIndex === originalPoints.length){
// We can not split on the first or last points...
continue

109
Logic/Osm/Changes.ts
View file

@ -64,9 +64,9 @@ export class Changes {
if (deletedElements.length > 0) {
changes +=
"\n<deleted>\n" +
"\n<delete>\n" +
deletedElements.map(e => e.ChangesetXML(csId)).join("\n") +
"\n</deleted>"
"\n</delete>"
}
changes += "</osmChange>";
@ -99,7 +99,7 @@ export class Changes {
}
this.isUploading.setData(true)
this.flushChangesAsync(flushreason)
this.flushChangesAsync()
.then(_ => {
this.isUploading.setData(false)
console.log("Changes flushed!");
@ -110,39 +110,94 @@ export class Changes {
})
}
private async flushChangesAsync(flushreason: string = undefined): Promise<void> {
/**
* UPload the selected changes to OSM.
* Returns 'true' if successfull and if they can be removed
* @param pending
* @private
*/
private async flushSelectChanges(pending: ChangeDescription[]): Promise<boolean>{
const self = this;
const neededIds = Changes.GetNeededIds(pending)
const osmObjects = await Promise.all(neededIds.map(id => OsmObject.DownloadObjectAsync(id)));
console.log("Got the fresh objects!", osmObjects, "pending: ", pending)
const changes: {
newObjects: OsmObject[],
modifiedObjects: OsmObject[]
deletedObjects: OsmObject[]
} = self.CreateChangesetObjects(pending, osmObjects)
if (changes.newObjects.length + changes.deletedObjects.length + changes.modifiedObjects.length === 0) {
console.log("No changes to be made")
return true
}
const meta = pending[0].meta
const perType = Array.from(Utils.Hist(pending.map(descr => descr.meta.changeType)), ([key, count]) => ({
key: key,
value: count,
aggregate: true
}))
const motivations = pending.filter(descr => descr.meta.specialMotivation !== undefined)
.map(descr => ({
key: descr.meta.changeType+":"+descr.type+"/"+descr.id,
value: descr.meta.specialMotivation
}))
const metatags = [{
key: "comment",
value: "Adding data with #MapComplete for theme #"+meta.theme
},
{
key:"theme",
value:meta.theme
},
...perType,
...motivations
]
await State.state.osmConnection.changesetHandler.UploadChangeset(
(csId) => Changes.createChangesetFor(""+csId, changes),
metatags
)
console.log("Upload successfull!")
return true;
}
private async flushChangesAsync(): Promise<void> {
const self = this;
try {
console.log("Beginning upload... " + flushreason ?? "");
// At last, we build the changeset and upload
const pending = self.pendingChanges.data;
const neededIds = Changes.GetNeededIds(pending)
const osmObjects = await Promise.all(neededIds.map(id => OsmObject.DownloadObjectAsync(id)));
console.log("Got the fresh objects!", osmObjects, "pending: ", pending)
const changes: {
newObjects: OsmObject[],
modifiedObjects: OsmObject[]
deletedObjects: OsmObject[]
} = self.CreateChangesetObjects(pending, osmObjects)
if (changes.newObjects.length + changes.deletedObjects.length + changes.modifiedObjects.length === 0) {
console.log("No changes to be made")
self.pendingChanges.setData([])
self.isUploading.setData(false)
const pendingPerTheme = new Map<string, ChangeDescription[]>()
for (const changeDescription of pending) {
const theme = changeDescription.meta.theme
if(!pendingPerTheme.has(theme)){
pendingPerTheme.set(theme, [])
}
pendingPerTheme.get(theme).push(changeDescription)
}
const successes = await Promise.all(Array.from(pendingPerTheme, ([key , value]) => value)
.map(async pendingChanges => {
try{
return await self.flushSelectChanges(pendingChanges);
}catch(e){
console.error("Could not upload some changes:",e)
return false
}
}))
if(!successes.some(s => s == false)){
// All changes successfull, we clear the data!
this.pendingChanges.setData([]);
}
await State.state.osmConnection.UploadChangeset(
State.state.layoutToUse,
State.state.allElements,
(csId) => Changes.createChangesetFor(csId, changes),
)
console.log("Upload successfull!")
this.pendingChanges.setData([]);
this.isUploading.setData(false)
} catch (e) {
console.error("Could not handle changes - probably an old, pending changeset in localstorage with an invalid format; erasing those", e)
self.pendingChanges.setData([])
}finally {
self.isUploading.setData(false)
}

226
Logic/Osm/ChangesetHandler.ts
View file

@ -6,29 +6,47 @@ import {ElementStorage} from "../ElementStorage";
import State from "../../State";
import Locale from "../../UI/i18n/Locale";
import Constants from "../../Models/Constants";
import {OsmObject} from "./OsmObject";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
import {Changes} from "./Changes";
import {Utils} from "../../Utils";
export interface ChangesetTag {
key: string,
value: string | number,
aggregate?: boolean
}
export class ChangesetHandler {
public readonly currentChangeset: UIEventSource<string>;
public readonly currentChangeset: UIEventSource<number>;
private readonly allElements: ElementStorage;
private osmConnection: OsmConnection;
private readonly changes: Changes;
private readonly _dryRun: boolean;
private readonly userDetails: UIEventSource<UserDetails>;
private readonly auth: any;
private readonly backend: string;
constructor(layoutName: string, dryRun: boolean, osmConnection: OsmConnection,
constructor(layoutName: string, dryRun: boolean,
osmConnection: OsmConnection,
allElements: ElementStorage,
changes: Changes,
auth) {
this.osmConnection = osmConnection;
this.allElements = allElements;
this.changes = changes;
this._dryRun = dryRun;
this.userDetails = osmConnection.userDetails;
this.backend = osmConnection._oauth_config.url
this.auth = auth;
this.currentChangeset = osmConnection.GetPreference("current-open-changeset-" + layoutName);
this.currentChangeset = osmConnection.GetPreference("current-open-changeset-" + layoutName).map(
str => {
const n = Number(str);
if (isNaN(n)) {
return undefined
}
return n
}, [], n => "" + n
);
if (dryRun) {
console.log("DRYRUN ENABLED");
@ -39,7 +57,7 @@ export class ChangesetHandler {
const oldId = parseInt(node.attributes.old_id.value);
if (node.attributes.new_id === undefined) {
// We just removed this point!
const element =this. allElements.getEventSourceById("node/" + oldId);
const element = this.allElements.getEventSourceById("node/" + oldId);
element.data._deleted = "yes"
element.ping();
return;
@ -56,6 +74,10 @@ export class ChangesetHandler {
}
console.log("Rewriting id: ", type + "/" + oldId, "-->", type + "/" + newId);
const element = this.allElements.getEventSourceById("node/" + oldId);
if(element === undefined){
// Element to rewrite not found, probably a node or relation that is not rendered
return undefined
}
element.data.id = type + "/" + newId;
this.allElements.addElementById(type + "/" + newId, element);
this.allElements.ContainingFeatures.set(type + "/" + newId, this.allElements.ContainingFeatures.get(type + "/" + oldId))
@ -83,7 +105,7 @@ export class ChangesetHandler {
}
}
this.changes.registerIdRewrites(mappings)
}
/**
@ -97,102 +119,96 @@ export class ChangesetHandler {
*
*/
public async UploadChangeset(
layout: LayoutConfig,
generateChangeXML: (csid: string) => string): Promise<void> {
generateChangeXML: (csid: number) => string,
extraMetaTags: ChangesetTag[]): Promise<void> {
if (!extraMetaTags.some(tag => tag.key === "comment") || !extraMetaTags.some(tag => tag.key === "theme")) {
throw "The meta tags should at least contain a `comment` and a `theme`"
}
if (this.userDetails.data.csCount == 0) {
// The user became a contributor!
this.userDetails.data.csCount = 1;
this.userDetails.ping();
}
if (this._dryRun) {
const changesetXML = generateChangeXML("123456");
const changesetXML = generateChangeXML(123456);
console.log(changesetXML);
return;
}
if (this.currentChangeset.data === undefined || this.currentChangeset.data === "") {
if (this.currentChangeset.data === undefined) {
// We have to open a new changeset
try {
const csId = await this.OpenChangeset(layout)
const csId = await this.OpenChangeset(extraMetaTags)
this.currentChangeset.setData(csId);
const changeset = generateChangeXML(csId);
console.log("Current changeset is:", changeset);
await this.AddChange(csId, changeset)
} catch (e) {
console.error("Could not open/upload changeset due to ", e)
this.currentChangeset.setData("")
this.currentChangeset.setData(undefined)
}
} else {
// There still exists an open changeset (or at least we hope so)
// Let's check!
const csId = this.currentChangeset.data;
try {
const oldChangesetMeta = await this.GetChangesetMeta(csId)
if (!oldChangesetMeta.open) {
// Mark the CS as closed...
this.currentChangeset.setData(undefined);
// ... and try again. As the cs is closed, no recursive loop can exist
await this.UploadChangeset(generateChangeXML, extraMetaTags)
return;
}
const extraTagsById = new Map<string, ChangesetTag>()
for (const extraMetaTag of extraMetaTags) {
extraTagsById.set(extraMetaTag.key, extraMetaTag)
}
const oldCsTags = oldChangesetMeta.tags
for (const key in oldCsTags) {
const newMetaTag = extraTagsById.get(key)
if (newMetaTag === undefined) {
extraMetaTags.push({
key: key,
value: oldCsTags[key]
})
} else if (newMetaTag.aggregate) {
let n = Number(newMetaTag.value)
if (isNaN(n)) {
n = 0
}
let o = Number(oldCsTags[key])
if (isNaN(o)) {
o = 0
}
// We _update_ the tag itself, as it'll be updated in 'extraMetaTags' straight away
newMetaTag.value = "" + (n + o)
} else {
// The old value is overwritten, thus we drop
}
}
await this.UpdateTags(csId, extraMetaTags.map(csTag => <[string, string]>[csTag.key, csTag.value]))
await this.AddChange(
csId,
generateChangeXML(csId))
} catch (e) {
console.warn("Could not upload, changeset is probably closed: ", e);
// Mark the CS as closed...
this.currentChangeset.setData("");
// ... and try again. As the cs is closed, no recursive loop can exist
await this.UploadChangeset(layout, generateChangeXML)
this.currentChangeset.setData(undefined);
}
}
}
/**
* Deletes the element with the given ID from the OSM database.
* DOES NOT PERFORM ANY SAFETY CHECKS!
*
* For the deletion of an element, a new, separate changeset is created with a slightly changed comment and some extra flags set.
* The CS will be closed afterwards.
*
* If dryrun is specified, will not actually delete the point but print the CS-XML to console instead
*
*/
public DeleteElement(object: OsmObject,
layout: LayoutConfig,
reason: string,
allElements: ElementStorage,
continuation: () => void) {
return this.DeleteElementAsync(object, layout, reason, allElements).then(continuation)
}
public async DeleteElementAsync(object: OsmObject,
layout: LayoutConfig,
reason: string,
allElements: ElementStorage): Promise<void> {
function generateChangeXML(csId: string) {
let [lat, lon] = object.centerpoint();
let changes = `<osmChange version='0.6' generator='Mapcomplete ${Constants.vNumber}'>`;
changes +=
`<delete><${object.type} id="${object.id}" version="${object.version}" changeset="${csId}" lat="${lat}" lon="${lon}" /></delete>`;
changes += "</osmChange>";
return changes;
}
if (this._dryRun) {
const changesetXML = generateChangeXML("123456");
console.log(changesetXML);
return;
}
const csId = await this.OpenChangeset(layout, {
isDeletionCS: true,
deletionReason: reason
})
// The cs is open - let us actually upload!
const changes = generateChangeXML(csId)
await this.AddChange(csId, changes)
await this.CloseChangeset(csId)
}
private async CloseChangeset(changesetId: string = undefined): Promise<void> {
private async CloseChangeset(changesetId: number = undefined): Promise<void> {
const self = this
return new Promise<void>(function (resolve, reject) {
if (changesetId === undefined) {
@ -202,7 +218,7 @@ export class ChangesetHandler {
return;
}
console.log("closing changeset", changesetId);
self.currentChangeset.setData("");
self.currentChangeset.setData(undefined);
self.auth.xhr({
method: 'PUT',
path: '/api/0.6/changeset/' + changesetId + '/close',
@ -217,39 +233,63 @@ export class ChangesetHandler {
})
}
private OpenChangeset(
layout: LayoutConfig,
options?: {
isDeletionCS?: boolean,
deletionReason?: string,
}
): Promise<string> {
private async GetChangesetMeta(csId: number): Promise<{
id: number,
open: boolean,
uid: number,
changes_count: number,
tags: any
}> {
const url = `${this.backend}/api/0.6/changeset/${csId}`
const csData = await Utils.downloadJson(url)
return csData.elements[0]
}
private async UpdateTags(
csId: number,
tags: [string, string][]) {
const self = this;
return new Promise<string>(function (resolve, reject) {
options = options ?? {}
options.isDeletionCS = options.isDeletionCS ?? false
const commentExtra = layout.changesetmessage !== undefined ? " - " + layout.changesetmessage : "";
let comment = `Adding data with #MapComplete for theme #${layout.id}${commentExtra}`
if (options.isDeletionCS) {
comment = `Deleting a point with #MapComplete for theme #${layout.id}${commentExtra}`
if (options.deletionReason) {
comment += ": " + options.deletionReason;
tags = Utils.NoNull(tags).filter(([k, v]) => k !== undefined && v !== undefined && k !== "" && v !== "")
const metadata = tags.map(kv => `<tag k="${kv[0]}" v="${escapeHtml(kv[1])}"/>`)
self.auth.xhr({
method: 'PUT',
path: '/api/0.6/changeset/' + csId,
options: {header: {'Content-Type': 'text/xml'}},
content: [`<osm><changeset>`,
metadata,
`</changeset></osm>`].join("")
}, function (err, response) {
if (response === undefined) {
console.log("err", err);
reject(err)
} else {
resolve(response);
}
}
});
})
}
private OpenChangeset(
changesetTags: ChangesetTag[]
): Promise<number> {
const self = this;
return new Promise<number>(function (resolve, reject) {
let path = window.location.pathname;
path = path.substr(1, path.lastIndexOf("/"));
const metadata = [
["created_by", `MapComplete ${Constants.vNumber}`],
["comment", comment],
["deletion", options.isDeletionCS ? "yes" : undefined],
["theme", layout.id],
["language", Locale.language.data],
["host", window.location.host],
["path", path],
["source", State.state.currentGPSLocation.data !== undefined ? "survey" : undefined],
["imagery", State.state.backgroundLayer.data.id],
["theme-creator", layout.maintainer]
...changesetTags.map(cstag => [cstag.key, cstag.value])
]
.filter(kv => (kv[1] ?? "") !== "")
.map(kv => `<tag k="${kv[0]}" v="${escapeHtml(kv[1])}"/>`)
@ -268,7 +308,7 @@ export class ChangesetHandler {
console.log("err", err);
reject(err)
} else {
resolve(response);
resolve(Number(response));
}
});
})
@ -278,8 +318,8 @@ export class ChangesetHandler {
/**
* Upload a changesetXML
*/
private AddChange(changesetId: string,
changesetXML: string): Promise<string> {
private AddChange(changesetId: number,
changesetXML: string): Promise<number> {
const self = this;
return new Promise(function (resolve, reject) {
self.auth.xhr({

7
Logic/Osm/OsmConnection.ts
View file

@ -124,13 +124,6 @@ export class OsmConnection {
}
}
public UploadChangeset(
layout: LayoutConfig,
allElements: ElementStorage,
generateChangeXML: (csid: string) => string): Promise<void> {
return this.changesetHandler.UploadChangeset(layout, generateChangeXML);
}
public GetPreference(key: string, prefix: string = "mapcomplete-"): UIEventSource<string> {
return this.preferencesHandler.GetPreference(key, prefix);
}

3
Logic/Osm/OsmObject.ts
View file

@ -11,7 +11,7 @@ export abstract class OsmObject {
private static polygonFeatures = OsmObject.constructPolygonFeatures()
private static objectCache = new Map<string, UIEventSource<OsmObject>>();
private static historyCache = new Map<string, UIEventSource<OsmObject[]>>();
type: string;
type: "node" | "way" | "relation";
id: number;
/**
* The OSM tags as simple object
@ -23,6 +23,7 @@ export abstract class OsmObject {
protected constructor(type: string, id: number) {
this.id = id;
// @ts-ignore
this.type = type;
this.tags = {
id: `${this.type}/${id}`

20
Logic/SimpleMetaTagger.ts
View file

@ -6,6 +6,7 @@ import Combine from "../UI/Base/Combine";
import BaseUIElement from "../UI/BaseUIElement";
import Title from "../UI/Base/Title";
import {FixedUiElement} from "../UI/Base/FixedUiElement";
import LayerConfig from "../Models/ThemeConfig/LayerConfig";
const cardinalDirections = {
@ -62,6 +63,20 @@ export default class SimpleMetaTagger {
return true;
})
);
private static layerInfo = new SimpleMetaTagger(
{
doc: "The layer-id to which this feature belongs. Note that this might be return any applicable if `passAllFeatures` is defined.",
keys:["_layer"],
includesDates: false,
},
(feature, freshness, layer) => {
if(feature.properties._layer === layer.id){
return false;
}
feature.properties._layer = layer.id
return true;
}
)
private static surfaceArea = new SimpleMetaTagger(
{
keys: ["_surface", "_surface:ha"],
@ -329,6 +344,7 @@ export default class SimpleMetaTagger {
)
public static metatags = [
SimpleMetaTagger.latlon,
SimpleMetaTagger.layerInfo,
SimpleMetaTagger.surfaceArea,
SimpleMetaTagger.lngth,
SimpleMetaTagger.canonicalize,
@ -346,7 +362,7 @@ export default class SimpleMetaTagger {
public readonly doc: string;
public readonly isLazy: boolean;
public readonly includesDates: boolean
public readonly applyMetaTagsOnFeature: (feature: any, freshness: Date) => boolean;
public readonly applyMetaTagsOnFeature: (feature: any, freshness: Date, layer: LayerConfig) => boolean;
/***
* A function that adds some extra data to a feature
@ -354,7 +370,7 @@ export default class SimpleMetaTagger {
* @param f: apply the changes. Returns true if something changed
*/
constructor(docs: { keys: string[], doc: string, includesDates?: boolean, isLazy?: boolean },
f: ((feature: any, freshness: Date) => boolean)) {
f: ((feature: any, freshness: Date, layer: LayerConfig) => boolean)) {
this.keys = docs.keys;
this.doc = docs.doc;
this.isLazy = docs.isLazy

31
Logic/UIEventSource.ts
View file

@ -75,6 +75,27 @@ export class UIEventSource<T> {
promise?.catch(err => console.warn("Promise failed:", err))
return src
}
public AsPromise(): Promise<T>{
const self = this;
return new Promise((resolve, reject) => {
if(self.data !== undefined){
resolve(self.data)
}else{
self.addCallbackD(data => {
resolve(data)
return true; // return true to unregister as we only need to be called once
})
}
})
}
public WaitForPromise(promise: Promise<T>, onFail: ((any) => void)): UIEventSource<T> {
const self = this;
promise?.then(d => self.setData(d))
promise?.catch(err =>onFail(err))
return this
}
/**
* Converts a promise into a UIVentsource, sets the UIEVentSource when the result is calculated.
@ -195,16 +216,20 @@ export class UIEventSource<T> {
const sink = new UIEventSource<X>(undefined)
const seenEventSources = new Set<UIEventSource<X>>();
mapped.addCallbackAndRun(newEventSource => {
if (newEventSource === undefined) {
if (newEventSource === null) {
sink.setData(null)
} else if (newEventSource === undefined) {
sink.setData(undefined)
} else if (!seenEventSources.has(newEventSource)) {
}else if (!seenEventSources.has(newEventSource)) {
seenEventSources.add(newEventSource)
newEventSource.addCallbackAndRun(resultData => {
if (mapped.data === newEventSource) {
sink.setData(resultData);
}
})
}else{
// Already seen, so we don't have to add a callback, just update the value
sink.setData(newEventSource.data)
}
})

286
Logic/Web/Wikidata.ts
View file

@ -2,22 +2,33 @@ import {Utils} from "../../Utils";
import {UIEventSource} from "../UIEventSource";
export interface WikidataResponse {
export class WikidataResponse {
public readonly id: string
public readonly labels: Map<string, string>
public readonly descriptions: Map<string, string>
public readonly claims: Map<string, Set<string>>
public readonly wikisites: Map<string, string>
public readonly commons: string
id: string,
labels: Map<string, string>,
descriptions: Map<string, string>,
claims: Map<string, Set<string>>,
wikisites: Map<string, string>
commons: string
}
constructor(
id: string,
labels: Map<string, string>,
descriptions: Map<string, string>,
claims: Map<string, Set<string>>,
wikisites: Map<string, string>,
commons: string
) {
/**
* Utility functions around wikidata
*/
export default class Wikidata {
this.id = id
this.labels = labels
this.descriptions = descriptions
this.claims = claims
this.wikisites = wikisites
this.commons = commons
private static ParseResponse(entity: any): WikidataResponse {
}
public static fromJson(entity: any): WikidataResponse {
const labels = new Map<string, string>()
for (const labelName in entity.labels) {
// The labelname is the language code
@ -37,64 +48,239 @@ export default class Wikidata {
const title = entity.sitelinks[labelName].title
sitelinks.set(language, title)
}
const commons = sitelinks.get("commons")
sitelinks.delete("commons")
const claims = WikidataResponse.extractClaims(entity.claims);
return new WikidataResponse(
entity.id,
labels,
descr,
claims,
sitelinks,
commons
)
}
static extractClaims(claimsJson: any): Map<string, Set<string>> {
const claims = new Map<string, Set<string>>();
for (const claimId of entity.claims) {
for (const claimId in claimsJson) {
const claimsList: any[] = entity.claims[claimId]
const claimsList: any[] = claimsJson[claimId]
const values = new Set<string>()
for (const claim of claimsList) {
const value = claim.mainsnak.datavalue.value;
let value = claim.mainsnak?.datavalue?.value;
if (value === undefined) {
continue;
}
if (value.id !== undefined) {
value = value.id
}
values.add(value)
}
claims.set(claimId, values);
}
return claims
}
}
return {
claims: claims,
descriptions: descr,
id: entity.id,
labels: labels,
wikisites: sitelinks,
commons: commons
export class WikidataLexeme {
id: string
lemma: Map<string, string>
senses: Map<string, string>
claims: Map<string, Set<string>>
constructor(json) {
this.id = json.id
this.claims = WikidataResponse.extractClaims(json.claims)
this.lemma = new Map<string, string>()
for (const language in json.lemmas) {
this.lemma.set(language, json.lemmas[language].value)
}
this.senses = new Map<string, string>()
for (const sense of json.senses) {
const glosses = sense.glosses
for (const language in glosses) {
let previousSenses = this.senses.get(language)
if(previousSenses === undefined){
previousSenses = ""
}else{
previousSenses = previousSenses+"; "
}
this.senses.set(language, previousSenses + glosses[language].value ?? "")
}
}
}
private static readonly _cache = new Map<number, UIEventSource<{success: WikidataResponse} | {error: any}>>()
public static LoadWikidataEntry(value: string | number): UIEventSource<{success: WikidataResponse} | {error: any}> {
asWikidataResponse() {
return new WikidataResponse(
this.id,
this.lemma,
this.senses,
this.claims,
new Map(),
undefined
);
}
}
export interface WikidataSearchoptions {
lang?: "en" | string,
maxCount?: 20 | number
}
/**
* Utility functions around wikidata
*/
export default class Wikidata {
private static readonly _identifierPrefixes = ["Q", "L"].map(str => str.toLowerCase())
private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:", "https://www.wikidata.org/wiki/", "Lexeme:"].map(str => str.toLowerCase())
private static readonly _cache = new Map<string, UIEventSource<{ success: WikidataResponse } | { error: any }>>()
public static LoadWikidataEntry(value: string | number): UIEventSource<{ success: WikidataResponse } | { error: any }> {
const key = this.ExtractKey(value)
const cached = Wikidata._cache.get(key)
if(cached !== undefined){
if (cached !== undefined) {
return cached
}
const src = UIEventSource.FromPromiseWithErr(Wikidata.LoadWikidataEntryAsync(key))
Wikidata._cache.set(key, src)
return src;
}
private static ExtractKey(value: string | number) : number{
public static async search(
search: string,
options?: WikidataSearchoptions,
page = 1
): Promise<{
id: string,
label: string,
description: string
}[]> {
const maxCount = options?.maxCount ?? 20
let pageCount = Math.min(maxCount, 50)
const start = page * pageCount - pageCount;
const lang = (options?.lang ?? "en")
const url =
"https://www.wikidata.org/w/api.php?action=wbsearchentities&search=" +
search +
"&language=" +
lang +
"&limit=" + pageCount + "&continue=" +
start +
"&format=json&uselang=" +
lang +
"&type=item&origin=*" +
"&props=";// props= removes some unused values in the result
const response = await Utils.downloadJson(url)
const result: any[] = response.search
if (result.length < pageCount) {
// No next page
return result;
}
if (result.length < maxCount) {
const newOptions = {...options}
newOptions.maxCount = maxCount - result.length
result.push(...await Wikidata.search(search,
newOptions,
page + 1
))
}
return result;
}
public static async searchAndFetch(
search: string,
options?: WikidataSearchoptions
): Promise<WikidataResponse[]> {
const maxCount = options.maxCount
// We provide some padding to filter away invalid values
options.maxCount = Math.ceil((options.maxCount ?? 20) * 1.5)
const searchResults = await Wikidata.search(search, options)
const maybeResponses = await Promise.all(searchResults.map(async r => {
try {
return await Wikidata.LoadWikidataEntry(r.id).AsPromise()
} catch (e) {
console.error(e)
return undefined;
}
}))
const responses = maybeResponses
.map(r => <WikidataResponse>r["success"])
.filter(wd => {
if (wd === undefined) {
return false;
}
if (wd.claims.get("P31" /*Instance of*/)?.has("Q4167410"/* Wikimedia Disambiguation page*/)) {
return false;
}
return true;
})
responses.splice(maxCount, responses.length - maxCount)
return responses
}
public static ExtractKey(value: string | number): string {
if (typeof value === "number") {
return value
return "Q" + value
}
const wikidataUrl = "https://www.wikidata.org/wiki/"
if (value.startsWith(wikidataUrl)) {
value = value.substring(wikidataUrl.length)
if (value === undefined) {
console.error("ExtractKey: value is undefined")
return undefined;
}
if (value.startsWith("http")) {
value = value.trim().toLowerCase()
for (const prefix of Wikidata._prefixesToRemove) {
if (value.startsWith(prefix)) {
value = value.substring(prefix.length)
}
}
if (value.startsWith("http") && value === "") {
// Probably some random link in the image field - we skip it
return undefined
}
if (value.startsWith("Q")) {
value = value.substring(1)
for (const identifierPrefix of Wikidata._identifierPrefixes) {
if (value.startsWith(identifierPrefix)) {
const trimmed = value.substring(identifierPrefix.length);
if(trimmed === ""){
return undefined
}
const n = Number(trimmed)
if (isNaN(n)) {
return undefined
}
return value.toUpperCase();
}
}
const n = Number(value)
if(isNaN(n)){
return undefined
if (value !== "" && !isNaN(Number(value))) {
return "Q" + value
}
return n;
return undefined;
}
public static IdToArticle(id: string){
if(id.startsWith("Q")){
return "https://wikidata.org/wiki/"+id
}
if(id.startsWith("L")){
return "https://wikidata.org/wiki/Lexeme:"+id
}
throw "Unknown id type: "+id
}
/**
@ -103,14 +289,22 @@ export default class Wikidata {
*/
public static async LoadWikidataEntryAsync(value: string | number): Promise<WikidataResponse> {
const id = Wikidata.ExtractKey(value)
if(id === undefined){
if (id === undefined) {
console.warn("Could not extract a wikidata entry from", value)
return undefined;
throw "Could not extract a wikidata entry from " + value
}
console.log("Requesting wikidata with id", id)
const url = "https://www.wikidata.org/wiki/Special:EntityData/Q" + id + ".json";
const response = await Utils.downloadJson(url)
return Wikidata.ParseResponse(response.entities["Q" + id])
const url = "https://www.wikidata.org/wiki/Special:EntityData/" + id + ".json";
const entities = (await Utils.downloadJson(url)).entities
const firstKey = <string> Array.from(Object.keys(entities))[0] // Roundabout way to fetch the entity; it might have been a redirect
const response = entities[firstKey]
if (id.startsWith("L")) {
// This is a lexeme:
return new WikidataLexeme(response).asWikidataResponse()
}
return WikidataResponse.fromJson(response)
}
}

47
Logic/Web/Wikimedia.ts Normal file
View file

@ -0,0 +1,47 @@
import {Utils} from "../../Utils";
export default class Wikimedia {
/**
* Recursively walks a wikimedia commons category in order to search for entries, which can be File: or Category: entries
* Returns (a promise of) a list of URLS
* @param categoryName The name of the wikimedia category
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
public static async GetCategoryContents(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
}
if (maxLoad - imageOverview.length <= 0) {
console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
}
// We do have a continue token - let's load the next page
const recursive = await Wikimedia.GetCategoryContents(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
imageOverview.push(...recursive)
return imageOverview
}
}

11
Logic/Web/Wikipedia.ts
View file

@ -22,6 +22,10 @@ export default class Wikipedia {
"mw-selflink",
"hatnote" // Often redirects
]
private static readonly idsToRemove = [
"sjabloon_zie"
]
private static readonly _cache = new Map<string, UIEventSource<{ success: string } | { error: any }>>()
@ -59,6 +63,13 @@ export default class Wikipedia {
}
}
for (const forbiddenId of Wikipedia.idsToRemove) {
const toRemove = content.querySelector("#"+forbiddenId)
toRemove?.parentElement?.removeChild(toRemove)
}
const links = Array.from(content.getElementsByTagName("a"))
// Rewrite relative links to absolute links + open them in a new tab