midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Reformat all files with prettier

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-09-08 21:40:48 +02:00
parent e22d189376
commit b541d3eab4
382 changed files with 50893 additions and 35566 deletions
Download patch file Download diff file Expand all files Collapse all files

41
Logic/Web/Hash.ts
View file

@ -1,12 +1,11 @@
import {UIEventSource} from "../UIEventSource";
import {Utils} from "../../Utils";
import { UIEventSource } from "../UIEventSource"
import { Utils } from "../../Utils"
/**
* Wrapper around the hash to create an UIEventSource from it
*/
export default class Hash {
public static hash: UIEventSource<string> = Hash.Get();
public static hash: UIEventSource<string> = Hash.Get()
/**
* Gets the current string, including the pound sign if there is any
@ -16,48 +15,46 @@ export default class Hash {
if (Hash.hash.data === undefined || Hash.hash.data === "") {
return ""
} else {
return "#" + Hash.hash.data;
return "#" + Hash.hash.data
}
}
private static Get(): UIEventSource<string> {
if (Utils.runningFromConsole) {
return new UIEventSource<string>(undefined);
return new UIEventSource<string>(undefined)
}
const hash = new UIEventSource<string>(window.location.hash.substr(1));
hash.addCallback(h => {
const hash = new UIEventSource<string>(window.location.hash.substr(1))
hash.addCallback((h) => {
if (h === "undefined") {
console.warn("Got a literal 'undefined' as hash, ignoring")
h = undefined;
h = undefined
}
if (h === undefined || h === "") {
window.location.hash = "";
return;
window.location.hash = ""
return
}
history.pushState({}, "")
window.location.hash = "#" + h;
});
window.location.hash = "#" + h
})
window.onhashchange = () => {
let newValue = window.location.hash.substr(1);
let newValue = window.location.hash.substr(1)
if (newValue === "") {
newValue = undefined;
newValue = undefined
}
hash.setData(newValue)
}
window.addEventListener('popstate', _ => {
let newValue = window.location.hash.substr(1);
window.addEventListener("popstate", (_) => {
let newValue = window.location.hash.substr(1)
if (newValue === "") {
newValue = undefined;
newValue = undefined
}
hash.setData(newValue)
})
return hash;
return hash
}
}
}

47
Logic/Web/IdbLocalStorage.ts
View file

@ -1,38 +1,41 @@
import {UIEventSource} from "../UIEventSource";
import { UIEventSource } from "../UIEventSource"
import * as idb from "idb-keyval"
import {Utils} from "../../Utils";
import { Utils } from "../../Utils"
/**
* UIEventsource-wrapper around indexedDB key-value
*/
export class IdbLocalStorage {
private static readonly _sourceCache: Record<string, UIEventSource<any>> = {}
public static Get<T>(key: string, options?: { defaultValue?: T, whenLoaded?: (t: T | null) => void }): UIEventSource<T> {
if(IdbLocalStorage._sourceCache[key] !== undefined){
public static Get<T>(
key: string,
options?: { defaultValue?: T; whenLoaded?: (t: T | null) => void }
): UIEventSource<T> {
if (IdbLocalStorage._sourceCache[key] !== undefined) {
return IdbLocalStorage._sourceCache[key]
}
const src = new UIEventSource<T>(options?.defaultValue, "idb-local-storage:" + key)
if (Utils.runningFromConsole) {
return src;
return src
}
src.addCallback(v => idb.set(key, v))
idb.get(key).then(v => {
src.setData(v ?? options?.defaultValue);
if (options?.whenLoaded !== undefined) {
options?.whenLoaded(v)
}
}).catch(err => {
console.warn("Loading from local storage failed due to", err)
if (options?.whenLoaded !== undefined) {
options?.whenLoaded(null)
}
})
IdbLocalStorage._sourceCache[key] = src;
return src;
src.addCallback((v) => idb.set(key, v))
idb.get(key)
.then((v) => {
src.setData(v ?? options?.defaultValue)
if (options?.whenLoaded !== undefined) {
options?.whenLoaded(v)
}
})
.catch((err) => {
console.warn("Loading from local storage failed due to", err)
if (options?.whenLoaded !== undefined) {
options?.whenLoaded(null)
}
})
IdbLocalStorage._sourceCache[key] = src
return src
}
public static SetDirectly(key: string, value) {

46
Logic/Web/LiveQueryHandler.ts
View file

@ -1,51 +1,47 @@
import {UIEventSource} from "../UIEventSource";
import {Utils} from "../../Utils";
import { UIEventSource } from "../UIEventSource"
import { Utils } from "../../Utils"
/**
* Fetches data from random data sources, used in the metatagging
*/
export default class LiveQueryHandler {
private static neededShorthands = {} // url -> (shorthand:paths)[]
public static FetchLiveData(url: string, shorthands: string[]): UIEventSource<any /* string -> string */> {
public static FetchLiveData(
url: string,
shorthands: string[]
): UIEventSource<any /* string -> string */> {
const shorthandsSet: string[] = LiveQueryHandler.neededShorthands[url] ?? []
for (const shorthand of shorthands) {
if (shorthandsSet.indexOf(shorthand) < 0) {
shorthandsSet.push(shorthand);
shorthandsSet.push(shorthand)
}
}
LiveQueryHandler.neededShorthands[url] = shorthandsSet;
LiveQueryHandler.neededShorthands[url] = shorthandsSet
if (LiveQueryHandler[url] === undefined) {
const source = new UIEventSource({});
LiveQueryHandler[url] = source;
const source = new UIEventSource({})
LiveQueryHandler[url] = source
console.log("Fetching live data from a third-party (unknown) API:", url)
Utils.downloadJson(url).then(data => {
Utils.downloadJson(url).then((data) => {
for (const shorthandDescription of shorthandsSet) {
const descr = shorthandDescription.trim().split(":");
const shorthand = descr[0];
const path = descr[1];
const parts = path.split(".");
let trail = data;
const descr = shorthandDescription.trim().split(":")
const shorthand = descr[0]
const path = descr[1]
const parts = path.split(".")
let trail = data
for (const part of parts) {
if (trail !== undefined) {
trail = trail[part];
trail = trail[part]
}
}
source.data[shorthand] = trail;
source.data[shorthand] = trail
}
source.ping();
source.ping()
})
}
return LiveQueryHandler[url];
return LiveQueryHandler[url]
}
}
}

23
Logic/Web/LocalStorageSource.ts
View file

@ -1,13 +1,12 @@
import {UIEventSource} from "../UIEventSource";
import { UIEventSource } from "../UIEventSource"
/**
* UIEventsource-wrapper around localStorage
*/
export class LocalStorageSource {
static GetParsed<T>(key: string, defaultValue: T): UIEventSource<T> {
return LocalStorageSource.Get(key).sync(
str => {
(str) => {
if (str === undefined) {
return defaultValue
}
@ -16,29 +15,29 @@ export class LocalStorageSource {
} catch {
return defaultValue
}
}, [],
value => JSON.stringify(value)
},
[],
(value) => JSON.stringify(value)
)
}
static Get(key: string, defaultValue: string = undefined): UIEventSource<string> {
try {
const saved = localStorage.getItem(key);
const source = new UIEventSource<string>(saved ?? defaultValue, "localstorage:" + key);
const saved = localStorage.getItem(key)
const source = new UIEventSource<string>(saved ?? defaultValue, "localstorage:" + key)
source.addCallback((data) => {
try {
localStorage.setItem(key, data);
localStorage.setItem(key, data)
} catch (e) {
// Probably exceeded the quota with this item!
// Lets nuke everything
localStorage.clear()
}
});
return source;
})
return source
} catch (e) {
return new UIEventSource<string>(defaultValue);
return new UIEventSource<string>(defaultValue)
}
}
}

219
Logic/Web/MangroveReviews.ts
View file

@ -1,31 +1,31 @@
import * as mangrove from 'mangrove-reviews'
import {UIEventSource} from "../UIEventSource";
import {Review} from "./Review";
import {Utils} from "../../Utils";
import * as mangrove from "mangrove-reviews"
import { UIEventSource } from "../UIEventSource"
import { Review } from "./Review"
import { Utils } from "../../Utils"
export class MangroveIdentity {
public keypair: any = undefined;
public readonly kid: UIEventSource<string> = new UIEventSource<string>(undefined);
private readonly _mangroveIdentity: UIEventSource<string>;
public keypair: any = undefined
public readonly kid: UIEventSource<string> = new UIEventSource<string>(undefined)
private readonly _mangroveIdentity: UIEventSource<string>
constructor(mangroveIdentity: UIEventSource<string>) {
const self = this;
this._mangroveIdentity = mangroveIdentity;
mangroveIdentity.addCallbackAndRunD(str => {
const self = this
this._mangroveIdentity = mangroveIdentity
mangroveIdentity.addCallbackAndRunD((str) => {
if (str === "") {
return;
return
}
mangrove.jwkToKeypair(JSON.parse(str)).then(keypair => {
self.keypair = keypair;
mangrove.publicToPem(keypair.publicKey).then(pem => {
mangrove.jwkToKeypair(JSON.parse(str)).then((keypair) => {
self.keypair = keypair
mangrove.publicToPem(keypair.publicKey).then((pem) => {
console.log("Identity loaded")
self.kid.setData(pem);
self.kid.setData(pem)
})
})
})
try {
if (!Utils.runningFromConsole && (mangroveIdentity.data ?? "") === "") {
this.CreateIdentity();
this.CreateIdentity()
}
} catch (e) {
console.error("Could not create identity: ", e)
@ -41,58 +41,62 @@ export class MangroveIdentity {
if ("" !== (this._mangroveIdentity.data ?? "")) {
throw "Identity already defined - not creating a new one"
}
const self = this;
mangrove.generateKeypair().then(
keypair => {
self.keypair = keypair;
mangrove.keypairToJwk(keypair).then(jwk => {
self._mangroveIdentity.setData(JSON.stringify(jwk));
})
});
const self = this
mangrove.generateKeypair().then((keypair) => {
self.keypair = keypair
mangrove.keypairToJwk(keypair).then((jwk) => {
self._mangroveIdentity.setData(JSON.stringify(jwk))
})
})
}
}
export default class MangroveReviews {
private static _reviewsCache = {};
private static didWarn = false;
private readonly _lon: number;
private readonly _lat: number;
private readonly _name: string;
private readonly _reviews: UIEventSource<Review[]> = new UIEventSource<Review[]>([]);
private _dryRun: boolean;
private _mangroveIdentity: MangroveIdentity;
private _lastUpdate: Date = undefined;
private static _reviewsCache = {}
private static didWarn = false
private readonly _lon: number
private readonly _lat: number
private readonly _name: string
private readonly _reviews: UIEventSource<Review[]> = new UIEventSource<Review[]>([])
private _dryRun: boolean
private _mangroveIdentity: MangroveIdentity
private _lastUpdate: Date = undefined
private constructor(lon: number, lat: number, name: string,
identity: MangroveIdentity,
dryRun?: boolean) {
this._lon = lon;
this._lat = lat;
this._name = name;
this._mangroveIdentity = identity;
this._dryRun = dryRun;
private constructor(
lon: number,
lat: number,
name: string,
identity: MangroveIdentity,
dryRun?: boolean
) {
this._lon = lon
this._lat = lat
this._name = name
this._mangroveIdentity = identity
this._dryRun = dryRun
if (dryRun && !MangroveReviews.didWarn) {
MangroveReviews.didWarn = true;
MangroveReviews.didWarn = true
console.warn("Mangrove reviews will _not_ be saved as dryrun is specified")
}
}
public static Get(lon: number, lat: number, name: string,
identity: MangroveIdentity,
dryRun?: boolean) {
const newReviews = new MangroveReviews(lon, lat, name, identity, dryRun);
public static Get(
lon: number,
lat: number,
name: string,
identity: MangroveIdentity,
dryRun?: boolean
) {
const newReviews = new MangroveReviews(lon, lat, name, identity, dryRun)
const uri = newReviews.GetSubjectUri();
const cached = MangroveReviews._reviewsCache[uri];
const uri = newReviews.GetSubjectUri()
const cached = MangroveReviews._reviewsCache[uri]
if (cached !== undefined) {
return cached;
return cached
}
MangroveReviews._reviewsCache[uri] = newReviews;
MangroveReviews._reviewsCache[uri] = newReviews
return newReviews;
return newReviews
}
/**
@ -100,63 +104,64 @@ export default class MangroveReviews {
* @constructor
*/
public GetSubjectUri() {
let uri = `geo:${this._lat},${this._lon}?u=50`;
let uri = `geo:${this._lat},${this._lon}?u=50`
if (this._name !== undefined && this._name !== null) {
uri += "&q=" + this._name;
uri += "&q=" + this._name
}
return uri;
return uri
}
/**
* Gives a UIEVentsource with all reviews.
* Note: rating is between 1 and 100
*/
public GetReviews(): UIEventSource<Review[]> {
if (this._lastUpdate !== undefined && this._reviews.data !== undefined &&
(new Date().getTime() - this._lastUpdate.getTime()) < 15000
if (
this._lastUpdate !== undefined &&
this._reviews.data !== undefined &&
new Date().getTime() - this._lastUpdate.getTime() < 15000
) {
// Last update was pretty recent
return this._reviews;
return this._reviews
}
this._lastUpdate = new Date();
this._lastUpdate = new Date()
const self = this;
mangrove.getReviews({sub: this.GetSubjectUri()}).then(
(data) => {
const reviews = [];
const reviewsByUser = [];
for (const review of data.reviews) {
const r = review.payload;
const self = this
mangrove.getReviews({ sub: this.GetSubjectUri() }).then((data) => {
const reviews = []
const reviewsByUser = []
for (const review of data.reviews) {
const r = review.payload
console.log("PublicKey is ", self._mangroveIdentity.kid.data, "reviews.kid is", review.kid);
const byUser = self._mangroveIdentity.kid.map(data => data === review.signature);
const rev: Review = {
made_by_user: byUser,
date: new Date(r.iat * 1000),
comment: r.opinion,
author: r.metadata.nickname,
affiliated: r.metadata.is_affiliated,
rating: r.rating // percentage points
};
(rev.made_by_user ? reviewsByUser : reviews).push(rev);
console.log(
"PublicKey is ",
self._mangroveIdentity.kid.data,
"reviews.kid is",
review.kid
)
const byUser = self._mangroveIdentity.kid.map((data) => data === review.signature)
const rev: Review = {
made_by_user: byUser,
date: new Date(r.iat * 1000),
comment: r.opinion,
author: r.metadata.nickname,
affiliated: r.metadata.is_affiliated,
rating: r.rating, // percentage points
}
self._reviews.setData(reviewsByUser.concat(reviews))
;(rev.made_by_user ? reviewsByUser : reviews).push(rev)
}
);
return this._reviews;
self._reviews.setData(reviewsByUser.concat(reviews))
})
return this._reviews
}
AddReview(r: Review, callback?: (() => void)) {
callback = callback ?? (() => {
return undefined;
});
AddReview(r: Review, callback?: () => void) {
callback =
callback ??
(() => {
return undefined
})
const payload = {
sub: this.GetSubjectUri(),
@ -164,35 +169,29 @@ export default class MangroveReviews {
opinion: r.comment,
metadata: {
nickname: r.author,
}
};
},
}
if (r.affiliated) {
// @ts-ignore
payload.metadata.is_affiliated = true;
payload.metadata.is_affiliated = true
}
if (this._dryRun) {
console.warn("DRYRUNNING mangrove reviews: ", payload);
console.warn("DRYRUNNING mangrove reviews: ", payload)
if (callback) {
if (callback) {
callback();
callback()
}
this._reviews.data.push(r);
this._reviews.ping();
this._reviews.data.push(r)
this._reviews.ping()
}
} else {
mangrove.signAndSubmitReview(this._mangroveIdentity.keypair, payload).then(() => {
if (callback) {
callback();
callback()
}
this._reviews.data.push(r);
this._reviews.ping();
this._reviews.data.push(r)
this._reviews.ping()
})
}
}
}
}

1266
Logic/Web/PlantNet.ts
View file

File diff suppressed because one or more lines are too long

87
Logic/Web/QueryParameters.ts
View file

@ -1,42 +1,52 @@
/**
* Wraps the query parameters into UIEventSources
*/
import {UIEventSource} from "../UIEventSource";
import Hash from "./Hash";
import {Utils} from "../../Utils";
import { UIEventSource } from "../UIEventSource"
import Hash from "./Hash"
import { Utils } from "../../Utils"
export class QueryParameters {
static defaults = {}
static documentation: Map<string, string> = new Map<string, string>()
private static order: string [] = ["layout", "test", "z", "lat", "lon"];
private static order: string[] = ["layout", "test", "z", "lat", "lon"]
private static _wasInitialized: Set<string> = new Set()
private static knownSources = {};
private static initialized = false;
private static knownSources = {}
private static initialized = false
public static GetQueryParameter(key: string, deflt: string, documentation?: string): UIEventSource<string> {
public static GetQueryParameter(
key: string,
deflt: string,
documentation?: string
): UIEventSource<string> {
if (!this.initialized) {
this.init();
this.init()
}
QueryParameters.documentation.set(key, documentation);
QueryParameters.documentation.set(key, documentation)
if (deflt !== undefined) {
QueryParameters.defaults[key] = deflt;
QueryParameters.defaults[key] = deflt
}
if (QueryParameters.knownSources[key] !== undefined) {
return QueryParameters.knownSources[key];
return QueryParameters.knownSources[key]
}
QueryParameters.addOrder(key);
const source = new UIEventSource<string>(deflt, "&" + key);
QueryParameters.knownSources[key] = source;
QueryParameters.addOrder(key)
const source = new UIEventSource<string>(deflt, "&" + key)
QueryParameters.knownSources[key] = source
source.addCallback(() => QueryParameters.Serialize())
return source;
return source
}
public static GetBooleanQueryParameter(key: string, deflt: boolean, documentation?: string): UIEventSource<boolean> {
return QueryParameters.GetQueryParameter(key, ""+ deflt, documentation).sync(str => str === "true", [], b => "" + b)
public static GetBooleanQueryParameter(
key: string,
deflt: boolean,
documentation?: string
): UIEventSource<boolean> {
return QueryParameters.GetQueryParameter(key, "" + deflt, documentation).sync(
(str) => str === "true",
[],
(b) => "" + b
)
}
public static wasInitialized(key: string): boolean {
return QueryParameters._wasInitialized.has(key)
}
@ -48,53 +58,54 @@ export class QueryParameters {
}
private static init() {
if (this.initialized) {
return;
return
}
this.initialized = true;
this.initialized = true
if (Utils.runningFromConsole) {
return;
return
}
if (window?.location?.search) {
const params = window.location.search.substr(1).split("&");
const params = window.location.search.substr(1).split("&")
for (const param of params) {
const kv = param.split("=");
const key = decodeURIComponent(kv[0]);
const kv = param.split("=")
const key = decodeURIComponent(kv[0])
QueryParameters.addOrder(key)
QueryParameters._wasInitialized.add(key)
const v = decodeURIComponent(kv[1]);
const source = new UIEventSource<string>(v);
const v = decodeURIComponent(kv[1])
const source = new UIEventSource<string>(v)
source.addCallback(() => QueryParameters.Serialize())
QueryParameters.knownSources[key] = source;
QueryParameters.knownSources[key] = source
}
}
}
private static Serialize() {
const parts = []
for (const key of QueryParameters.order) {
if (QueryParameters.knownSources[key]?.data === undefined) {
continue;
continue
}
if (QueryParameters.knownSources[key].data === "undefined") {
continue;
continue
}
if (QueryParameters.knownSources[key].data === QueryParameters.defaults[key]) {
continue;
continue
}
parts.push(encodeURIComponent(key) + "=" + encodeURIComponent(QueryParameters.knownSources[key].data))
parts.push(
encodeURIComponent(key) +
"=" +
encodeURIComponent(QueryParameters.knownSources[key].data)
)
}
if(!Utils.runningFromConsole){
if (!Utils.runningFromConsole) {
// Don't pollute the history every time a parameter changes
history.replaceState(null, "", "?" + parts.join("&") + Hash.Current());
history.replaceState(null, "", "?" + parts.join("&") + Hash.Current())
}
}
}
}

14
Logic/Web/Review.ts
View file

@ -1,13 +1,13 @@
import {Store} from "../UIEventSource";
import { Store } from "../UIEventSource"
export interface Review {
comment?: string,
author: string,
date: Date,
rating: number,
affiliated: boolean,
comment?: string
author: string
date: Date
rating: number
affiliated: boolean
/**
* True if the current logged in user is the creator of this comment
*/
made_by_user: Store<boolean>
}
}

166
Logic/Web/Wikidata.ts
View file

@ -1,5 +1,5 @@
import {Utils} from "../../Utils";
import {UIEventSource} from "../UIEventSource";
import { Utils } from "../../Utils"
import { UIEventSource } from "../UIEventSource"
import * as wds from "wikidata-sdk"
export class WikidataResponse {
@ -18,14 +18,12 @@ export class WikidataResponse {
wikisites: Map<string, string>,
commons: string
) {
this.id = id
this.labels = labels
this.descriptions = descriptions
this.claims = claims
this.wikisites = wikisites
this.commons = commons
}
public static fromJson(entity: any): WikidataResponse {
@ -41,7 +39,7 @@ export class WikidataResponse {
descr.set(labelName, entity.descriptions[labelName].value)
}
const sitelinks = new Map<string, string>();
const sitelinks = new Map<string, string>()
for (const labelName in entity.sitelinks) {
// labelName is `${language}wiki`
const language = labelName.substring(0, labelName.length - 4)
@ -51,28 +49,19 @@ export class WikidataResponse {
const commons = sitelinks.get("commons")
sitelinks.delete("commons")
const claims = WikidataResponse.extractClaims(entity.claims);
return new WikidataResponse(
entity.id,
labels,
descr,
claims,
sitelinks,
commons
)
const claims = WikidataResponse.extractClaims(entity.claims)
return new WikidataResponse(entity.id, labels, descr, claims, sitelinks, commons)
}
static extractClaims(claimsJson: any): Map<string, Set<string>> {
const simplified = wds.simplify.claims(claimsJson, {
timeConverter: 'simple-day'
timeConverter: "simple-day",
})
const claims = new Map<string, Set<string>>();
const claims = new Map<string, Set<string>>()
for (const claimId in simplified) {
const claimsList: any[] = simplified[claimId]
claims.set(claimId, new Set(claimsList));
claims.set(claimId, new Set(claimsList))
}
return claims
}
@ -84,7 +73,6 @@ export class WikidataLexeme {
senses: Map<string, string>
claims: Map<string, Set<string>>
constructor(json) {
this.id = json.id
this.claims = WikidataResponse.extractClaims(json.claims)
@ -117,36 +105,40 @@ export class WikidataLexeme {
this.claims,
new Map(),
undefined
);
)
}
}
export interface WikidataSearchoptions {
lang?: "en" | string,
lang?: "en" | string
maxCount?: 20 | number
}
export interface WikidataAdvancedSearchoptions extends WikidataSearchoptions {
instanceOf?: number[];
instanceOf?: number[]
notInstanceOf?: number[]
}
/**
* Utility functions around wikidata
*/
export default class Wikidata {
private static readonly _identifierPrefixes = ["Q", "L"].map(str => str.toLowerCase())
private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:",
private static readonly _identifierPrefixes = ["Q", "L"].map((str) => str.toLowerCase())
private static readonly _prefixesToRemove = [
"https://www.wikidata.org/wiki/Lexeme:",
"https://www.wikidata.org/wiki/",
"http://www.wikidata.org/entity/",
"Lexeme:"].map(str => str.toLowerCase())
"Lexeme:",
].map((str) => str.toLowerCase())
private static readonly _cache = new Map<
string,
UIEventSource<{ success: WikidataResponse } | { error: any }>
>()
private static readonly _cache = new Map<string, UIEventSource<{ success: WikidataResponse } | { error: any }>>()
public static LoadWikidataEntry(value: string | number): UIEventSource<{ success: WikidataResponse } | { error: any }> {
public static LoadWikidataEntry(
value: string | number
): UIEventSource<{ success: WikidataResponse } | { error: any }> {
const key = this.ExtractKey(value)
const cached = Wikidata._cache.get(key)
if (cached !== undefined) {
@ -154,27 +146,31 @@ export default class Wikidata {
}
const src = UIEventSource.FromPromiseWithErr(Wikidata.LoadWikidataEntryAsync(key))
Wikidata._cache.set(key, src)
return src;
return src
}
/**
* Given a search text, searches for the relevant wikidata entries, excluding pages "outside of the main tree", e.g. disambiguation pages.
* Optionally, an 'instance of' can be given to limit the scope, e.g. instanceOf:5 (humans) will only search for humans
*/
public static async searchAdvanced(text: string, options: WikidataAdvancedSearchoptions): Promise<{
id: string,
relevance?: number,
label: string,
description?: string
}[]> {
public static async searchAdvanced(
text: string,
options: WikidataAdvancedSearchoptions
): Promise<
{
id: string
relevance?: number
label: string
description?: string
}[]
> {
let instanceOf = ""
if (options?.instanceOf !== undefined && options.instanceOf.length > 0) {
const phrases = options.instanceOf.map(q => `{ ?item wdt:P31/wdt:P279* wd:Q${q}. }`)
instanceOf = "{"+ phrases.join(" UNION ") + "}"
const phrases = options.instanceOf.map((q) => `{ ?item wdt:P31/wdt:P279* wd:Q${q}. }`)
instanceOf = "{" + phrases.join(" UNION ") + "}"
}
const forbidden = (options?.notInstanceOf ?? [])
.concat([17379835]) // blacklist 'wikimedia pages outside of the main knowledge tree', e.g. disambiguation pages
const minusPhrases = forbidden.map(q => `MINUS {?item wdt:P31/wdt:P279* wd:Q${q} .}`)
const forbidden = (options?.notInstanceOf ?? []).concat([17379835]) // blacklist 'wikimedia pages outside of the main knowledge tree', e.g. disambiguation pages
const minusPhrases = forbidden.map((q) => `MINUS {?item wdt:P31/wdt:P279* wd:Q${q} .}`)
const sparql = `SELECT * WHERE {
SERVICE wikibase:mwapi {
bd:serviceParam wikibase:api "EntitySearch" .
@ -183,7 +179,11 @@ export default class Wikidata {
bd:serviceParam mwapi:language "${options.lang}" .
?item wikibase:apiOutputItem mwapi:item .
?num wikibase:apiOrdinal true .
bd:serviceParam wikibase:limit ${Math.round((options.maxCount ?? 20) * 1.5) /*Some padding for disambiguation pages */} .
bd:serviceParam wikibase:limit ${
Math.round(
(options.maxCount ?? 20) * 1.5
) /*Some padding for disambiguation pages */
} .
?label wikibase:apiOutput mwapi:label .
?description wikibase:apiOutput "@description" .
}
@ -195,11 +195,11 @@ export default class Wikidata {
const result = await Utils.downloadJson(url)
/*The full uri of the wikidata-item*/
return result.results.bindings.map(({item, label, description, num}) => ({
return result.results.bindings.map(({ item, label, description, num }) => ({
relevance: num?.value,
id: item?.value,
label: label?.value,
description: description?.value
description: description?.value,
}))
}
@ -207,47 +207,47 @@ export default class Wikidata {
search: string,
options?: WikidataSearchoptions,
page = 1
): Promise<{
id: string,
label: string,
description: string
}[]> {
): Promise<
{
id: string
label: string
description: string
}[]
> {
const maxCount = options?.maxCount ?? 20
let pageCount = Math.min(maxCount, 50)
const start = page * pageCount - pageCount;
const lang = (options?.lang ?? "en")
const start = page * pageCount - pageCount
const lang = options?.lang ?? "en"
const url =
"https://www.wikidata.org/w/api.php?action=wbsearchentities&search=" +
search +
"&language=" +
lang +
"&limit=" + pageCount + "&continue=" +
"&limit=" +
pageCount +
"&continue=" +
start +
"&format=json&uselang=" +
lang +
"&type=item&origin=*" +
"&props=";// props= removes some unused values in the result
"&props=" // props= removes some unused values in the result
const response = await Utils.downloadJsonCached(url, 10000)
const result: any[] = response.search
if (result.length < pageCount) {
// No next page
return result;
return result
}
if (result.length < maxCount) {
const newOptions = {...options}
const newOptions = { ...options }
newOptions.maxCount = maxCount - result.length
result.push(...await Wikidata.search(search,
newOptions,
page + 1
))
result.push(...(await Wikidata.search(search, newOptions, page + 1)))
}
return result;
return result
}
public static async searchAndFetch(
search: string,
options?: WikidataAdvancedSearchoptions
@ -255,16 +255,17 @@ export default class Wikidata {
// We provide some padding to filter away invalid values
const searchResults = await Wikidata.searchAdvanced(search, options)
const maybeResponses = await Promise.all(
searchResults.map(async r => {
searchResults.map(async (r) => {
try {
console.log("Loading ", r.id)
return await Wikidata.LoadWikidataEntry(r.id).AsPromise()
} catch (e) {
console.error(e)
return undefined;
return undefined
}
}))
return Utils.NoNull(maybeResponses.map(r => <WikidataResponse>r["success"]))
})
)
return Utils.NoNull(maybeResponses.map((r) => <WikidataResponse>r["success"]))
}
/**
@ -279,7 +280,7 @@ export default class Wikidata {
}
if (value === undefined) {
console.error("ExtractKey: value is undefined")
return undefined;
return undefined
}
value = value.trim().toLowerCase()
@ -296,7 +297,7 @@ export default class Wikidata {
for (const identifierPrefix of Wikidata._identifierPrefixes) {
if (value.startsWith(identifierPrefix)) {
const trimmed = value.substring(identifierPrefix.length);
const trimmed = value.substring(identifierPrefix.length)
if (trimmed === "") {
return undefined
}
@ -304,7 +305,7 @@ export default class Wikidata {
if (isNaN(n)) {
return undefined
}
return value.toUpperCase();
return value.toUpperCase()
}
}
@ -312,7 +313,7 @@ export default class Wikidata {
return "Q" + value
}
return undefined;
return undefined
}
/**
@ -326,10 +327,10 @@ export default class Wikidata {
* Wikidata.QIdToNumber(123) // => 123
*/
public static QIdToNumber(q: string | number): number | undefined {
if(q === undefined || q === null){
if (q === undefined || q === null) {
return
}
if(typeof q === "number"){
if (typeof q === "number") {
return q
}
q = q.trim()
@ -356,17 +357,23 @@ export default class Wikidata {
/**
* Build a SPARQL-query, return the result
*
*
* @param keys: how variables are named. Every key not ending with 'Label' should appear in at least one statement
* @param statements
* @constructor
*/
public static async Sparql<T>(keys: string[], statements: string[]):Promise< (T & Record<string, {type: string, value: string}>) []> {
const query = "SELECT "+keys.map(k => k.startsWith("?") ? k : "?"+k).join(" ")+"\n" +
public static async Sparql<T>(
keys: string[],
statements: string[]
): Promise<(T & Record<string, { type: string; value: string }>)[]> {
const query =
"SELECT " +
keys.map((k) => (k.startsWith("?") ? k : "?" + k)).join(" ") +
"\n" +
"WHERE\n" +
"{\n" +
statements.map(stmt => stmt.endsWith(".") ? stmt : stmt+".").join("\n") +
" SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE]\". }\n" +
statements.map((stmt) => (stmt.endsWith(".") ? stmt : stmt + ".")).join("\n") +
' SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE]". }\n' +
"}"
const url = wds.sparqlQuery(query)
const result = await Utils.downloadJsonCached(url, 24 * 60 * 60 * 1000)
@ -384,7 +391,7 @@ export default class Wikidata {
return undefined
}
const url = "https://www.wikidata.org/wiki/Special:EntityData/" + id + ".json";
const url = "https://www.wikidata.org/wiki/Special:EntityData/" + id + ".json"
const entities = (await Utils.downloadJsonCached(url, 10000)).entities
const firstKey = <string>Array.from(Object.keys(entities))[0] // Roundabout way to fetch the entity; it might have been a redirect
const response = entities[firstKey]
@ -396,5 +403,4 @@ export default class Wikidata {
return WikidataResponse.fromJson(response)
}
}

38
Logic/Web/Wikimedia.ts
View file

@ -1,4 +1,4 @@
import {Utils} from "../../Utils";
import { Utils } from "../../Utils"
export default class Wikimedia {
/**
@ -8,40 +8,48 @@ export default class Wikimedia {
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
public static async GetCategoryContents(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
public static async GetCategoryContents(
categoryName: string,
maxLoad = 10,
continueParameter: string = undefined
): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
return []
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
categoryName = "Category:" + categoryName
}
let url = "https://commons.wikimedia.org/w/api.php?" +
let url =
"https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
"&cmtitle=" +
encodeURIComponent(categoryName)
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
url = `${url}&cmcontinue=${continueParameter}`
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
const members = response.query?.categorymembers ?? []
const imageOverview: string[] = members.map((member) => member.title)
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
return imageOverview
}
if (maxLoad - imageOverview.length <= 0) {
console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
return imageOverview
}
// We do have a continue token - let's load the next page
const recursive = await Wikimedia.GetCategoryContents(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
const recursive = await Wikimedia.GetCategoryContents(
categoryName,
maxLoad - imageOverview.length,
response.continue.cmcontinue
)
imageOverview.push(...recursive)
return imageOverview
}
}
}

134
Logic/Web/Wikipedia.ts
View file

@ -1,12 +1,11 @@
/**
* Some usefull utility functions around the wikipedia API
*/
import {Utils} from "../../Utils";
import {UIEventSource} from "../UIEventSource";
import {WikipediaBoxOptions} from "../../UI/Wikipedia/WikipediaBox";
import { Utils } from "../../Utils"
import { UIEventSource } from "../UIEventSource"
import { WikipediaBoxOptions } from "../../UI/Wikipedia/WikipediaBox"
export default class Wikipedia {
/**
* When getting a wikipedia page data result, some elements (e.g. navigation, infoboxes, ...) should be removed if 'removeInfoBoxes' is set.
* We do this based on the classes. This set contains a blacklist of the classes to remove
@ -15,26 +14,27 @@ export default class Wikipedia {
private static readonly classesToRemove = [
"shortdescription",
"sidebar",
"infobox", "infobox_v2",
"infobox",
"infobox_v2",
"noprint",
"ambox",
"mw-editsection",
"mw-selflink",
"mw-empty-elt",
"hatnote" // Often redirects
"hatnote", // Often redirects
]
private static readonly idsToRemove = [
"sjabloon_zie"
]
private static readonly idsToRemove = ["sjabloon_zie"]
private static readonly _cache = new Map<string, UIEventSource<{ success: string } | { error: any }>>()
private static readonly _cache = new Map<
string,
UIEventSource<{ success: string } | { error: any }>
>()
public readonly backend: string
public readonly backend: string;
constructor(options?: ({ language?: "en" | string } | { backend?: string })) {
this.backend = Wikipedia.getBackendUrl(options ?? {});
constructor(options?: { language?: "en" | string } | { backend?: string }) {
this.backend = Wikipedia.getBackendUrl(options ?? {})
}
/**
@ -43,30 +43,31 @@ export default class Wikipedia {
* Wikipedia.extractLanguageAndName("qsdf") // => undefined
* Wikipedia.extractLanguageAndName("nl:Warandeputten") // => {language: "nl", pageName: "Warandeputten"}
*/
public static extractLanguageAndName(input: string): { language: string, pageName: string } {
public static extractLanguageAndName(input: string): { language: string; pageName: string } {
const matched = input.match("([^:]+):(.*)")
if (matched === undefined || matched === null) {
return undefined
}
const [_, language, pageName] = matched
return {
language, pageName
language,
pageName,
}
}
/**
* Extracts the actual pagename; returns undefined if this came from a different wikimedia entry
*
*
* new Wikipedia({backend: "https://wiki.openstreetmap.org"}).extractPageName("https://wiki.openstreetmap.org/wiki/NL:Speelbos") // => "NL:Speelbos"
* new Wikipedia().extractPageName("https://wiki.openstreetmap.org/wiki/NL:Speelbos") // => undefined
*/
public extractPageName(input: string):string | undefined{
if(!input.startsWith(this.backend)){
public extractPageName(input: string): string | undefined {
if (!input.startsWith(this.backend)) {
return undefined
}
input = input.substring(this.backend.length);
const matched = input.match("/?wiki/\(.+\)")
input = input.substring(this.backend.length)
const matched = input.match("/?wiki/(.+)")
if (matched === undefined || matched === null) {
return undefined
}
@ -74,7 +75,9 @@ export default class Wikipedia {
return pageName
}
private static getBackendUrl(options: { language?: "en" | string } | { backend?: "en.wikipedia.org" | string }): string {
private static getBackendUrl(
options: { language?: "en" | string } | { backend?: "en.wikipedia.org" | string }
): string {
let backend = "en.wikipedia.org"
if (options["backend"]) {
backend = options["backend"]
@ -87,7 +90,10 @@ export default class Wikipedia {
return backend
}
public GetArticle(pageName: string, options: WikipediaBoxOptions): UIEventSource<{ success: string } | { error: any }> {
public GetArticle(
pageName: string,
options: WikipediaBoxOptions
): UIEventSource<{ success: string } | { error: any }> {
const key = this.backend + ":" + pageName + ":" + (options.firstParagraphOnly ?? false)
const cached = Wikipedia._cache.get(key)
if (cached !== undefined) {
@ -95,11 +101,13 @@ export default class Wikipedia {
}
const v = UIEventSource.FromPromiseWithErr(this.GetArticleAsync(pageName, options))
Wikipedia._cache.set(key, v)
return v;
return v
}
public getDataUrl(pageName: string): string {
return `${this.backend}/w/api.php?action=parse&format=json&origin=*&prop=text&page=` + pageName
return (
`${this.backend}/w/api.php?action=parse&format=json&origin=*&prop=text&page=` + pageName
)
}
public getPageUrl(pageName: string): string {
@ -110,9 +118,12 @@ export default class Wikipedia {
* Textual search of the specified wiki-instance. If searching Wikipedia, we recommend using wikidata.search instead
* @param searchTerm
*/
public async search(searchTerm: string): Promise<{ title: string, snippet: string }[]> {
const url = this.backend + "/w/api.php?action=query&format=json&list=search&srsearch=" + encodeURIComponent(searchTerm);
return (await Utils.downloadJson(url))["query"]["search"];
public async search(searchTerm: string): Promise<{ title: string; snippet: string }[]> {
const url =
this.backend +
"/w/api.php?action=query&format=json&list=search&srsearch=" +
encodeURIComponent(searchTerm)
return (await Utils.downloadJson(url))["query"]["search"]
}
/**
@ -120,46 +131,55 @@ export default class Wikipedia {
* This gives better results then via the API
* @param searchTerm
*/
public async searchViaIndex(searchTerm: string): Promise<{ title: string, snippet: string, url: string } []> {
public async searchViaIndex(
searchTerm: string
): Promise<{ title: string; snippet: string; url: string }[]> {
const url = `${this.backend}/w/index.php?search=${encodeURIComponent(searchTerm)}&ns0=1`
const result = await Utils.downloadAdvanced(url);
if(result["redirect"] ){
const result = await Utils.downloadAdvanced(url)
if (result["redirect"]) {
const targetUrl = result["redirect"]
// This is an exact match
return [{
title: this.extractPageName(targetUrl)?.trim(),
url: targetUrl,
snippet: ""
}]
return [
{
title: this.extractPageName(targetUrl)?.trim(),
url: targetUrl,
snippet: "",
},
]
}
const el = document.createElement('html');
el.innerHTML = result["content"].replace(/href="\//g, "href=\""+this.backend+"/");
const el = document.createElement("html")
el.innerHTML = result["content"].replace(/href="\//g, 'href="' + this.backend + "/")
const searchResults = el.getElementsByClassName("mw-search-results")
const individualResults = Array.from(searchResults[0]?.getElementsByClassName("mw-search-result") ?? [])
return individualResults.map(result => {
const individualResults = Array.from(
searchResults[0]?.getElementsByClassName("mw-search-result") ?? []
)
return individualResults.map((result) => {
const toRemove = Array.from(result.getElementsByClassName("searchalttitle"))
for (const toRm of toRemove) {
toRm.parentElement.removeChild(toRm)
}
return {
title: result.getElementsByClassName("mw-search-result-heading")[0].textContent.trim(),
title: result
.getElementsByClassName("mw-search-result-heading")[0]
.textContent.trim(),
url: result.getElementsByTagName("a")[0].href,
snippet: result.getElementsByClassName("searchresult")[0].textContent
snippet: result.getElementsByClassName("searchresult")[0].textContent,
}
})
}
public async GetArticleAsync(pageName: string, options:
{
public async GetArticleAsync(
pageName: string,
options: {
firstParagraphOnly?: false | boolean
}): Promise<string | undefined> {
}
): Promise<string | undefined> {
const response = await Utils.downloadJson(this.getDataUrl(pageName))
if (response?.parse?.text === undefined) {
return undefined
}
const html = response["parse"]["text"]["*"];
const html = response["parse"]["text"]["*"]
if (html === undefined) {
return undefined
}
@ -179,15 +199,16 @@ export default class Wikipedia {
toRemove?.parentElement?.removeChild(toRemove)
}
const links = Array.from(content.getElementsByTagName("a"))
// Rewrite relative links to absolute links + open them in a new tab
links.filter(link => link.getAttribute("href")?.startsWith("/") ?? false).forEach(link => {
link.target = '_blank'
// note: link.getAttribute("href") gets the textual value, link.href is the rewritten version which'll contain the host for relative paths
link.href = `${this.backend}${link.getAttribute("href")}`;
})
links
.filter((link) => link.getAttribute("href")?.startsWith("/") ?? false)
.forEach((link) => {
link.target = "_blank"
// note: link.getAttribute("href") gets the textual value, link.href is the rewritten version which'll contain the host for relative paths
link.href = `${this.backend}${link.getAttribute("href")}`
})
if (options?.firstParagraphOnly) {
return content.getElementsByTagName("p").item(0).innerHTML
@ -195,5 +216,4 @@ export default class Wikipedia {
return content.innerHTML
}
}
}