midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add a wikidata search box

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-10-08 04:33:39 +02:00
parent 54bc4f24da
commit b5a2ee1757
21 changed files with 4141 additions and 3590 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Logic/ImageProviders/AllImageProviders.ts
View file

@ -26,19 +26,19 @@ export default class AllImageProviders {
private static _cache: Map<string, UIEventSource<ProvidedImage[]>> = new Map<string, UIEventSource<ProvidedImage[]>>()
public static LoadImagesFor(tags: UIEventSource<any>, tagKey?: string): UIEventSource<ProvidedImage[]> {
const id = tags.data.id
if (id === undefined) {
if (tags.data.id === undefined) {
return undefined;
}
const cached = this._cache.get(tags.data.id)
const cacheKey = tags.data.id+tagKey
const cached = this._cache.get(cacheKey)
if (cached !== undefined) {
return cached
}
const source = new UIEventSource([])
this._cache.set(id, source)
this._cache.set(cacheKey, source)
const allSources = []
for (const imageProvider of AllImageProviders.ImageAttributionSource) {

32
Logic/ImageProviders/WikimediaImageProvider.ts
View file

@ -44,7 +44,7 @@ export class WikimediaImageProvider extends ImageProvider {
}
private PrepareUrl(value: string): string {
private static PrepareUrl(value: string): string {
if (value.toLowerCase().startsWith("https://commons.wikimedia.org/wiki/")) {
return value;
@ -97,18 +97,18 @@ export class WikimediaImageProvider extends ImageProvider {
}
private async UrlForImage(image: string): Promise<ProvidedImage> {
private UrlForImage(image: string): ProvidedImage {
if (!image.startsWith("File:")) {
image = "File:" + image
}
return {url: this.PrepareUrl(image), key: undefined, provider: this}
return {url: WikimediaImageProvider.PrepareUrl(image), key: undefined, provider: this}
}
private startsWithCommonsPrefix(value: string){
private static startsWithCommonsPrefix(value: string): boolean{
return WikimediaImageProvider.commonsPrefixes.some(prefix => value.startsWith(prefix))
}
private removeCommonsPrefix(value: string){
private static removeCommonsPrefix(value: string): string{
if(value.startsWith("https://upload.wikimedia.org/")){
value = value.substring(value.lastIndexOf("/") + 1)
value = decodeURIComponent(value)
@ -130,26 +130,38 @@ export class WikimediaImageProvider extends ImageProvider {
return value;
}
public PrepUrl(value: string): ProvidedImage {
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("File:")) {
return this.UrlForImage(value)
}
// We do a last effort and assume this is a file
return this.UrlForImage("File:" + value)
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
const hasCommonsPrefix = this.startsWithCommonsPrefix(value)
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
if(key !== undefined && key !== this.commons_key && !hasCommonsPrefix){
return []
}
value = this.removeCommonsPrefix(value)
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("Category:")) {
const urls = await Wikimedia.GetCategoryContents(value)
return urls.filter(url => url.startsWith("File:")).map(image => this.UrlForImage(image))
return urls.filter(url => url.startsWith("File:")).map(image => Promise.resolve(this.UrlForImage(image)))
}
if (value.startsWith("File:")) {
return [this.UrlForImage(value)]
return [Promise.resolve(this.UrlForImage(value))]
}
if (value.startsWith("http")) {
// PRobably an error
return []
}
// We do a last effort and assume this is a file
return [this.UrlForImage("File:" + value)]
return [Promise.resolve(this.UrlForImage("File:" + value))]
}

31
Logic/UIEventSource.ts
View file

@ -75,6 +75,27 @@ export class UIEventSource<T> {
promise?.catch(err => console.warn("Promise failed:", err))
return src
}
public AsPromise(): Promise<T>{
const self = this;
return new Promise((resolve, reject) => {
if(self.data !== undefined){
resolve(self.data)
}else{
self.addCallbackD(data => {
resolve(data)
return true; // return true to unregister as we only need to be called once
})
}
})
}
public WaitForPromise(promise: Promise<T>, onFail: ((any) => void)): UIEventSource<T> {
const self = this;
promise?.then(d => self.setData(d))
promise?.catch(err =>onFail(err))
return this
}
/**
* Converts a promise into a UIVentsource, sets the UIEVentSource when the result is calculated.
@ -195,16 +216,20 @@ export class UIEventSource<T> {
const sink = new UIEventSource<X>(undefined)
const seenEventSources = new Set<UIEventSource<X>>();
mapped.addCallbackAndRun(newEventSource => {
if (newEventSource === undefined) {
if (newEventSource === null) {
sink.setData(null)
} else if (newEventSource === undefined) {
sink.setData(undefined)
} else if (!seenEventSources.has(newEventSource)) {
}else if (!seenEventSources.has(newEventSource)) {
seenEventSources.add(newEventSource)
newEventSource.addCallbackAndRun(resultData => {
if (mapped.data === newEventSource) {
sink.setData(resultData);
}
})
}else{
// Already seen, so we don't have to add a callback, just update the value
sink.setData(newEventSource.data)
}
})

94
Logic/Web/Wikidata.ts
View file

@ -12,6 +12,11 @@ export interface WikidataResponse {
commons: string
}
export interface WikidataSearchoptions {
lang?: "en" | string,
maxCount?: 20 | number
}
/**
* Utility functions around wikidata
*/
@ -47,10 +52,14 @@ export default class Wikidata {
const claimsList: any[] = entity.claims[claimId]
const values = new Set<string>()
for (const claim of claimsList) {
const value = claim.mainsnak?.datavalue?.value;
if(value !== undefined){
values.add(value)
let value = claim.mainsnak?.datavalue?.value;
if (value === undefined) {
continue;
}
if(value.id !== undefined){
value = value.id
}
values.add(value)
}
claims.set(claimId, values);
}
@ -77,6 +86,82 @@ export default class Wikidata {
return src;
}
public static async search(
search: string,
options?:WikidataSearchoptions,
page = 1
): Promise<{
id: string,
label: string,
description: string
}[]> {
const maxCount = options?.maxCount ?? 20
let pageCount = Math.min(maxCount,50)
const start = page * pageCount - pageCount;
const lang = (options?.lang ?? "en")
const url =
"https://www.wikidata.org/w/api.php?action=wbsearchentities&search=" +
search +
"&language=" +
lang +
"&limit="+pageCount+"&continue=" +
start +
"&format=json&uselang=" +
lang +
"&type=item&origin=*"+
"&props=" ;// props= removes some unused values in the result
const response = await Utils.downloadJson(url)
const result : any[] = response.search
if(result.length < pageCount){
// No next page
return result;
}
if(result.length < maxCount){
const newOptions = {...options}
newOptions.maxCount = maxCount - result.length
result.push(...await Wikidata.search(search,
newOptions,
page + 1
))
}
return result;
}
public static async searchAndFetch(
search: string,
options?:WikidataSearchoptions
) : Promise<WikidataResponse[]>
{
const maxCount = options.maxCount
// We provide some padding to filter away invalid values
options.maxCount = Math.ceil((options.maxCount ?? 20) * 1.5)
const searchResults = await Wikidata.search(search, options)
const maybeResponses = await Promise.all(searchResults.map(async r => {
try{
return await Wikidata.LoadWikidataEntry(r.id).AsPromise()
}catch(e){
console.error(e)
return undefined;
}
}))
const responses = maybeResponses
.map(r => <WikidataResponse> r["success"])
.filter(wd => {
if(wd === undefined){
return false;
}
if(wd.claims.get("P31" /*Instance of*/)?.has("Q4167410"/* Wikimedia Disambiguation page*/)){
return false;
}
return true;
})
responses.splice(maxCount, responses.length - maxCount)
return responses
}
private static ExtractKey(value: string | number) : number{
if (typeof value === "number") {
return value
@ -99,6 +184,7 @@ export default class Wikidata {
return n;
}
/**
* Loads a wikidata page
* @returns the entity of the given value
@ -109,7 +195,7 @@ export default class Wikidata {
console.warn("Could not extract a wikidata entry from", value)
return undefined;
}
console.log("Requesting wikidata with id", id)
const url = "https://www.wikidata.org/wiki/Special:EntityData/Q" + id + ".json";
const response = await Utils.downloadJson(url)
return Wikidata.ParseResponse(response.entities["Q" + id])