midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add wikidata-images to etymology theme, various fixes for custom image carousels and gracious handling of wikidata/wikimedia

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-10-06 02:30:23 +02:00
parent 54abe7d057
commit ff11f96e91
10 changed files with 155 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Logic/ImageProviders/AllImageProviders.ts
View file

@ -21,7 +21,7 @@ export default class AllImageProviders {
private static _cache: Map<string, UIEventSource<ProvidedImage[]>> = new Map<string, UIEventSource<ProvidedImage[]>>()
public static LoadImagesFor(tags: UIEventSource<any>, imagePrefix?: string): UIEventSource<ProvidedImage[]> {
public static LoadImagesFor(tags: UIEventSource<any>, tagKey?: string): UIEventSource<ProvidedImage[]> {
const id = tags.data.id
if (id === undefined) {
return undefined;
@ -39,12 +39,8 @@ export default class AllImageProviders {
for (const imageProvider of AllImageProviders.ImageAttributionSource) {
let prefixes = imageProvider.defaultKeyPrefixes
if(imagePrefix !== undefined){
prefixes = [...prefixes]
if(prefixes.indexOf("image") >= 0){
prefixes.splice(prefixes.indexOf("image"), 1)
}
prefixes.push(imagePrefix)
if(tagKey !== undefined){
prefixes = [tagKey]
}
const singleSource = imageProvider.GetRelevantUrls(tags, {

9
Logic/ImageProviders/GenericImageProvider.ts
View file

@ -20,7 +20,14 @@ export default class GenericImageProvider extends ImageProvider {
if (this._valuePrefixBlacklist.some(prefix => value.startsWith(prefix))) {
return []
}
try{
new URL(value)
}catch (_){
// Not a valid URL
return []
}
return [Promise.resolve({
key: key,
url: value,

3
Logic/ImageProviders/ImageProvider.ts
View file

@ -17,7 +17,7 @@ export default abstract class ImageProvider {
if (cached !== undefined) {
return cached;
}
const src =UIEventSource.FromPromise(this.DownloadAttribution(url))
const src = UIEventSource.FromPromise(this.DownloadAttribution(url))
this._cache.set(url, src)
return src;
}
@ -38,6 +38,7 @@ export default abstract class ImageProvider {
}
const relevantUrls = new UIEventSource<{ url: string; key: string; provider: ImageProvider }[]>([])
const seenValues = new Set<string>()
const self = this
allTags.addCallbackAndRunD(tags => {
for (const key in tags) {
if(!prefixes.some(prefix => key.startsWith(prefix))){

11
Logic/ImageProviders/WikidataImageProvider.ts
View file

@ -27,6 +27,7 @@ export class WikidataImageProvider extends ImageProvider {
if(entity === undefined){
return []
}
console.log("Entity:", entity)
const allImages : Promise<ProvidedImage>[] = []
// P18 is the claim 'depicted in this image'
@ -34,9 +35,17 @@ export class WikidataImageProvider extends ImageProvider {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, img)
allImages.push(...promises)
}
// P373 is 'commons category'
for (let cat of Array.from(entity.claims.get("P373") ?? [])) {
if(!cat.startsWith("Category:")){
cat = "Category:"+cat
}
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, cat)
allImages.push(...promises)
}
const commons = entity.commons
if (commons !== undefined) {
if (commons !== undefined && (commons.startsWith("Category:") || commons.startsWith("File:"))) {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
allImages.push(...promises)
}

51
Logic/ImageProviders/WikimediaImageProvider.ts
View file

@ -4,6 +4,7 @@ import Svg from "../../Svg";
import Link from "../../UI/Base/Link";
import {Utils} from "../../Utils";
import {LicenseInfo} from "./LicenseInfo";
import Wikimedia from "../Web/Wikimedia";
/**
* This module provides endpoints for wikimedia and others
@ -20,50 +21,6 @@ export class WikimediaImageProvider extends ImageProvider {
super();
}
/**
* Recursively walks a wikimedia commons category in order to search for (image) files
* Returns (a promise of) a list of URLS
* @param categoryName The name of the wikimedia category
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
private static async GetImagesInCategory(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
}
if (maxLoad - imageOverview.length <= 0) {
console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
}
// We do have a continue token - let's load the next page
const recursive = await this.GetImagesInCategory(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
imageOverview.push(...recursive)
return imageOverview
}
private static ExtractFileName(url: string) {
if (!url.startsWith("http")) {
return url;
@ -110,7 +67,7 @@ export class WikimediaImageProvider extends ImageProvider {
const licenseInfo = new LicenseInfo();
const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata;
if (license === undefined) {
console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!")
console.warn("The file", filename ,"has no usable metedata or license attached... Please fix the license info file yourself!")
return undefined;
}
@ -149,8 +106,8 @@ export class WikimediaImageProvider extends ImageProvider {
return [Promise.resolve(result)]
}
if (value.startsWith("Category:")) {
const urls = await WikimediaImageProvider.GetImagesInCategory(value)
return urls.map(image => this.UrlForImage(image))
const urls = await Wikimedia.GetCategoryContents(value)
return urls.filter(url => url.startsWith("File:")).map(image => this.UrlForImage(image))
}
if (value.startsWith("File:")) {
return [this.UrlForImage(value)]

2
Logic/Web/Wikidata.ts
View file

@ -42,7 +42,7 @@ export default class Wikidata {
sitelinks.delete("commons")
const claims = new Map<string, Set<string>>();
for (const claimId of entity.claims) {
for (const claimId in entity.claims) {
const claimsList: any[] = entity.claims[claimId]
const values = new Set<string>()

47
Logic/Web/Wikimedia.ts Normal file
View file

@ -0,0 +1,47 @@
import {Utils} from "../../Utils";
export default class Wikimedia {
/**
* Recursively walks a wikimedia commons category in order to search for entries, which can be File: or Category: entries
* Returns (a promise of) a list of URLS
* @param categoryName The name of the wikimedia category
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
public static async GetCategoryContents(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
}
if (maxLoad - imageOverview.length <= 0) {
console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
}
// We do have a continue token - let's load the next page
const recursive = await Wikimedia.GetCategoryContents(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
imageOverview.push(...recursive)
return imageOverview
}
}