midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add wikidata-images to etymology theme, various fixes for custom image carousels and gracious handling of wikidata/wikimedia

Browse source
This commit is contained in:
pietervdvn 2021-10-06 02:30:23 +02:00
parent 54abe7d057
commit ff11f96e91
10 changed files with 155 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Logic/ImageProviders/AllImageProviders.ts
View file

@ -21,7 +21,7 @@ export default class AllImageProviders {
private static _cache: Map<string, UIEventSource<ProvidedImage[]>> = new Map<string, UIEventSource<ProvidedImage[]>>()
public static LoadImagesFor(tags: UIEventSource<any>, imagePrefix?: string): UIEventSource<ProvidedImage[]> {
public static LoadImagesFor(tags: UIEventSource<any>, tagKey?: string): UIEventSource<ProvidedImage[]> {
const id = tags.data.id
if (id === undefined) {
return undefined;
@ -39,12 +39,8 @@ export default class AllImageProviders {
for (const imageProvider of AllImageProviders.ImageAttributionSource) {
let prefixes = imageProvider.defaultKeyPrefixes
if(imagePrefix !== undefined){
prefixes = [...prefixes]
if(prefixes.indexOf("image") >= 0){
prefixes.splice(prefixes.indexOf("image"), 1)
}
prefixes.push(imagePrefix)
if(tagKey !== undefined){
prefixes = [tagKey]
}
const singleSource = imageProvider.GetRelevantUrls(tags, {

9
Logic/ImageProviders/GenericImageProvider.ts
View file

@ -20,7 +20,14 @@ export default class GenericImageProvider extends ImageProvider {
if (this._valuePrefixBlacklist.some(prefix => value.startsWith(prefix))) {
return []
}
try{
new URL(value)
}catch (_){
// Not a valid URL
return []
}
return [Promise.resolve({
key: key,
url: value,

3
Logic/ImageProviders/ImageProvider.ts
View file

@ -17,7 +17,7 @@ export default abstract class ImageProvider {
if (cached !== undefined) {
return cached;
}
const src =UIEventSource.FromPromise(this.DownloadAttribution(url))
const src = UIEventSource.FromPromise(this.DownloadAttribution(url))
this._cache.set(url, src)
return src;
}
@ -38,6 +38,7 @@ export default abstract class ImageProvider {
}
const relevantUrls = new UIEventSource<{ url: string; key: string; provider: ImageProvider }[]>([])
const seenValues = new Set<string>()
const self = this
allTags.addCallbackAndRunD(tags => {
for (const key in tags) {
if(!prefixes.some(prefix => key.startsWith(prefix))){

11
Logic/ImageProviders/WikidataImageProvider.ts
View file

@ -27,6 +27,7 @@ export class WikidataImageProvider extends ImageProvider {
if(entity === undefined){
return []
}
console.log("Entity:", entity)
const allImages : Promise<ProvidedImage>[] = []
// P18 is the claim 'depicted in this image'
@ -34,9 +35,17 @@ export class WikidataImageProvider extends ImageProvider {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, img)
allImages.push(...promises)
}
// P373 is 'commons category'
for (let cat of Array.from(entity.claims.get("P373") ?? [])) {
if(!cat.startsWith("Category:")){
cat = "Category:"+cat
}
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, cat)
allImages.push(...promises)
}
const commons = entity.commons
if (commons !== undefined) {
if (commons !== undefined && (commons.startsWith("Category:") || commons.startsWith("File:"))) {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
allImages.push(...promises)
}

51
Logic/ImageProviders/WikimediaImageProvider.ts
View file

@ -4,6 +4,7 @@ import Svg from "../../Svg";
import Link from "../../UI/Base/Link";
import {Utils} from "../../Utils";
import {LicenseInfo} from "./LicenseInfo";
import Wikimedia from "../Web/Wikimedia";
/**
* This module provides endpoints for wikimedia and others
@ -20,50 +21,6 @@ export class WikimediaImageProvider extends ImageProvider {
super();
}
/**
* Recursively walks a wikimedia commons category in order to search for (image) files
* Returns (a promise of) a list of URLS
* @param categoryName The name of the wikimedia category
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
private static async GetImagesInCategory(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
}
if (maxLoad - imageOverview.length <= 0) {
console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
}
// We do have a continue token - let's load the next page
const recursive = await this.GetImagesInCategory(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
imageOverview.push(...recursive)
return imageOverview
}
private static ExtractFileName(url: string) {
if (!url.startsWith("http")) {
return url;
@ -110,7 +67,7 @@ export class WikimediaImageProvider extends ImageProvider {
const licenseInfo = new LicenseInfo();
const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata;
if (license === undefined) {
console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!")
console.warn("The file", filename ,"has no usable metedata or license attached... Please fix the license info file yourself!")
return undefined;
}
@ -149,8 +106,8 @@ export class WikimediaImageProvider extends ImageProvider {
return [Promise.resolve(result)]
}
if (value.startsWith("Category:")) {
const urls = await WikimediaImageProvider.GetImagesInCategory(value)
return urls.map(image => this.UrlForImage(image))
const urls = await Wikimedia.GetCategoryContents(value)
return urls.filter(url => url.startsWith("File:")).map(image => this.UrlForImage(image))
}
if (value.startsWith("File:")) {
return [this.UrlForImage(value)]

2
Logic/Web/Wikidata.ts
View file

@ -42,7 +42,7 @@ export default class Wikidata {
sitelinks.delete("commons")
const claims = new Map<string, Set<string>>();
for (const claimId of entity.claims) {
for (const claimId in entity.claims) {
const claimsList: any[] = entity.claims[claimId]
const values = new Set<string>()

47
Logic/Web/Wikimedia.ts Normal file
View file

@ -0,0 +1,47 @@
import {Utils} from "../../Utils";
export default class Wikimedia {
/**
* Recursively walks a wikimedia commons category in order to search for entries, which can be File: or Category: entries
* Returns (a promise of) a list of URLS
* @param categoryName The name of the wikimedia category
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
public static async GetCategoryContents(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
}
if (maxLoad - imageOverview.length <= 0) {
console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
}
// We do have a continue token - let's load the next page
const recursive = await Wikimedia.GetCategoryContents(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
imageOverview.push(...recursive)
return imageOverview
}
}

9
UI/SpecialVisualizations.ts
View file

@ -64,13 +64,16 @@ export default class SpecialVisualizations {
funcName: "image_carousel",
docs: "Creates an image carousel for the given sources. An attempt will be made to guess what source is used. Supported: Wikidata identifiers, Wikipedia pages, Wikimedia categories, IMGUR (with attribution, direct links)",
args: [{
name: "image key/prefix",
name: "image key/prefix (multiple values allowed if comma-seperated)",
defaultValue: "image",
doc: "The keys given to the images, e.g. if <span class='literal-code'>image</span> is given, the first picture URL will be added as <span class='literal-code'>image</span>, the second as <span class='literal-code'>image:0</span>, the third as <span class='literal-code'>image:1</span>, etc... "
}],
constr: (state: State, tags, args) => {
const imagePrefix = args[0];
return new ImageCarousel(AllImageProviders.LoadImagesFor(tags, imagePrefix), tags);
let imagePrefixes = undefined;
if(args.length > 0){
imagePrefixes = args;
}
return new ImageCarousel(AllImageProviders.LoadImagesFor(tags, imagePrefixes), tags);
}
},
{

104
UI/WikipediaBox.ts
View file

@ -11,6 +11,7 @@ import Svg from "../Svg";
import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata";
import Locale from "./i18n/Locale";
import Toggle from "./Input/Toggle";
import Link from "./Base/Link";
export default class WikipediaBox extends Toggle {
@ -22,49 +23,78 @@ export default class WikipediaBox extends Toggle {
}
const wikibox = wikidataId
.bind(id => {
console.log("Wikidata is", id)
if(id === undefined){
return undefined
}
console.log("Initing load WIkidataentry with id", id)
return Wikidata.LoadWikidataEntry(id);
})
.map(maybewikidata => {
if (maybewikidata === undefined) {
return new Loading(wp.loading.Clone())
}
if (maybewikidata["error"] !== undefined) {
return wp.failed.Clone().SetClass("alert p-4")
}
const wikidata = <WikidataResponse>maybewikidata["success"]
console.log("Got wikidata response", wikidata)
if (wikidata.wikisites.size === 0) {
return wp.noWikipediaPage.Clone()
}
const wikiLink: UIEventSource<[string, string] | "loading" | "failed" | "no page"> =
wikidataId
.bind(id => {
if (id === undefined) {
return undefined
}
return Wikidata.LoadWikidataEntry(id);
})
.map(maybewikidata => {
if (maybewikidata === undefined) {
return "loading"
}
if (maybewikidata["error"] !== undefined) {
return "failed"
const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]]
let language
let pagetitle;
let i = 0
do {
language = preferredLanguage[i]
pagetitle = wikidata.wikisites.get(language)
i++;
} while (pagetitle === undefined)
return WikipediaBox.createContents(pagetitle, language)
}, [Locale.language])
}
const wikidata = <WikidataResponse>maybewikidata["success"]
if (wikidata.wikisites.size === 0) {
return "no page"
}
const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]]
let language
let pagetitle;
let i = 0
do {
language = preferredLanguage[i]
pagetitle = wikidata.wikisites.get(language)
i++;
} while (pagetitle === undefined)
return [pagetitle, language]
}, [Locale.language])
const contents = new VariableUiElement(
wikibox
wikiLink.map(status => {
if (status === "loading") {
return new Loading(wp.loading.Clone()).SetClass("pl-6 pt-2")
}
if (status === "failed") {
return wp.failed.Clone().SetClass("alert p-4")
}
if (status == "no page") {
return wp.noWikipediaPage.Clone()
}
const [pagetitle, language] = status
return WikipediaBox.createContents(pagetitle, language)
})
).SetClass("overflow-auto normal-background rounded-lg")
const linkElement = new VariableUiElement(wikiLink.map(state => {
if (typeof state !== "string") {
const [pagetitle, language] = state
const url= `https://${language}.wikipedia.org/wiki/${pagetitle}`
return new Link(Svg.pop_out_ui().SetStyle("width: 1.2rem").SetClass("block "), url, true)
}
return undefined}))
.SetClass("flex items-center")
const mainContent = new Combine([
new Combine([Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"),
new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2)]).SetClass("flex"),
new Combine([
new Combine([
Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"),
new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2),
]).SetClass("flex"),
linkElement
]).SetClass("flex justify-between"),
contents]).SetClass("block rounded-xl subtle-background m-1 p-2 flex flex-col")
.SetStyle("max-height: inherit")
super(
@ -102,8 +132,8 @@ export default class WikipediaBox extends Toggle {
return undefined
})
return new Combine([new VariableUiElement(contents).SetClass("block pl-6 pt-2")])
.SetClass("block")
return new Combine([new VariableUiElement(contents)
.SetClass("block pl-6 pt-2")])
}
}

8
assets/themes/etymology.json
View file

@ -50,6 +50,12 @@
"nl": "Alle lagen met een gelinkt etymology"
},
"tagRenderings": [
{
"id": "etymology_wikidata_image",
"render": {
"*": "{image_carousel(name:etymology:wikidata)}"
}
},
{
"id": "simple etymology",
"render": {
@ -63,7 +69,7 @@
{
"id": "wikipedia-etymology",
"render": {
"*": "{wikipedia(name:etymology:wikidata):max-height:20rem}"
"*": "{wikipedia(name:etymology:wikidata):max-height:30rem}"
}
}
],