Add wikidata-images to etymology theme, various fixes for custom image carousels and gracious handling of wikidata/wikimedia

Logic/ImageProviders/AllImageProviders.ts

@@ -21,7 +21,7 @@ export default class AllImageProviders {
 
     private static _cache: Map<string, UIEventSource<ProvidedImage[]>> = new Map<string, UIEventSource<ProvidedImage[]>>()
 
-    public static LoadImagesFor(tags: UIEventSource<any>, imagePrefix?: string): UIEventSource<ProvidedImage[]> {
+    public static LoadImagesFor(tags: UIEventSource<any>, tagKey?: string): UIEventSource<ProvidedImage[]> {
         const id = tags.data.id
         if (id === undefined) {
             return undefined;
@@ -39,12 +39,8 @@ export default class AllImageProviders {
         for (const imageProvider of AllImageProviders.ImageAttributionSource) {
             
             let prefixes = imageProvider.defaultKeyPrefixes
-            if(imagePrefix !== undefined){
+            if(tagKey !== undefined){
-                prefixes = [...prefixes]
+                prefixes = [tagKey]
-                if(prefixes.indexOf("image") >= 0){
-                    prefixes.splice(prefixes.indexOf("image"), 1)
-                }
-                prefixes.push(imagePrefix)
             }
             
             const singleSource = imageProvider.GetRelevantUrls(tags, {

Logic/ImageProviders/GenericImageProvider.ts

@@ -20,7 +20,14 @@ export default class GenericImageProvider extends ImageProvider {
         if (this._valuePrefixBlacklist.some(prefix => value.startsWith(prefix))) {
             return []
         }
-
+        
+        try{
+            new URL(value)
+        }catch (_){
+            // Not a valid URL
+            return []
+        }
+        
         return [Promise.resolve({
             key: key,
             url: value,

Logic/ImageProviders/ImageProvider.ts

@@ -17,7 +17,7 @@ export default abstract class ImageProvider {
         if (cached !== undefined) {
             return cached;
         }
-        const src =UIEventSource.FromPromise(this.DownloadAttribution(url))
+        const src = UIEventSource.FromPromise(this.DownloadAttribution(url))
         this._cache.set(url, src)
         return src;
     }
@@ -38,6 +38,7 @@ export default abstract class ImageProvider {
         }
         const relevantUrls = new UIEventSource<{ url: string; key: string; provider: ImageProvider }[]>([])
         const seenValues = new Set<string>()
+        const self = this
         allTags.addCallbackAndRunD(tags => {
             for (const key in tags) {
                 if(!prefixes.some(prefix => key.startsWith(prefix))){

Logic/ImageProviders/WikidataImageProvider.ts

@@ -27,6 +27,7 @@ export class WikidataImageProvider extends ImageProvider {
         if(entity === undefined){
             return []
         }
+        console.log("Entity:", entity)
        
         const allImages : Promise<ProvidedImage>[] = []
         // P18 is the claim 'depicted in this image'
@@ -34,9 +35,17 @@ export class WikidataImageProvider extends ImageProvider {
             const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, img)
             allImages.push(...promises)
         }
+        // P373 is 'commons category'
+        for (let cat of Array.from(entity.claims.get("P373") ?? [])) {
+            if(!cat.startsWith("Category:")){
+                cat = "Category:"+cat
+            }
+            const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, cat)
+            allImages.push(...promises)
+        }
         
         const commons = entity.commons
-        if (commons !== undefined) {
+        if (commons !== undefined && (commons.startsWith("Category:") || commons.startsWith("File:"))) {
             const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
             allImages.push(...promises)
         }

Logic/ImageProviders/WikimediaImageProvider.ts

@@ -4,6 +4,7 @@ import Svg from "../../Svg";
 import Link from "../../UI/Base/Link";
 import {Utils} from "../../Utils";
 import {LicenseInfo} from "./LicenseInfo";
+import Wikimedia from "../Web/Wikimedia";
 
 /**
  * This module provides endpoints for wikimedia and others
@@ -20,50 +21,6 @@ export class WikimediaImageProvider extends ImageProvider {
         super();
     }
 
-    /**
-     * Recursively walks a wikimedia commons category in order to search for (image) files
-     * Returns (a promise of) a list of URLS
-     * @param categoryName The name of the wikimedia category
-     * @param maxLoad: the maximum amount of images to return
-     * @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
-     */
-    private static async GetImagesInCategory(categoryName: string,
-                                             maxLoad = 10,
-                                             continueParameter: string = undefined): Promise<string[]> {
-        if (categoryName === undefined || categoryName === null || categoryName === "") {
-            return [];
-        }
-        if (!categoryName.startsWith("Category:")) {
-            categoryName = "Category:" + categoryName;
-        }
-
-        let url = "https://commons.wikimedia.org/w/api.php?" +
-            "action=query&list=categorymembers&format=json&" +
-            "&origin=*" +
-            "&cmtitle=" + encodeURIComponent(categoryName);
-        if (continueParameter !== undefined) {
-            url = `${url}&cmcontinue=${continueParameter}`;
-        }
-        const response = await Utils.downloadJson(url)
-        const members = response.query?.categorymembers ?? [];
-        const imageOverview: string[] = members.map(member => member.title);
-
-        if (response.continue === undefined) {
-            // We are done crawling through the category - no continuation in sight
-            return imageOverview;
-        }
-
-        if (maxLoad - imageOverview.length <= 0) {
-            console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
-            return imageOverview;
-        }
-
-        // We do have a continue token - let's load the next page
-        const recursive = await this.GetImagesInCategory(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
-        imageOverview.push(...recursive)
-        return imageOverview
-    }
-
     private static ExtractFileName(url: string) {
         if (!url.startsWith("http")) {
             return url;
@@ -110,7 +67,7 @@ export class WikimediaImageProvider extends ImageProvider {
         const licenseInfo = new LicenseInfo();
         const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata;
         if (license === undefined) {
-            console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!")
+            console.warn("The file", filename ,"has no usable metedata or license attached... Please fix the license info file yourself!")
             return undefined;
         }
 
@@ -149,8 +106,8 @@ export class WikimediaImageProvider extends ImageProvider {
             return [Promise.resolve(result)]
         }
         if (value.startsWith("Category:")) {
-            const urls = await WikimediaImageProvider.GetImagesInCategory(value)
+            const urls = await Wikimedia.GetCategoryContents(value)
-            return urls.map(image => this.UrlForImage(image))
+            return urls.filter(url => url.startsWith("File:")).map(image => this.UrlForImage(image))
         }
         if (value.startsWith("File:")) {
             return [this.UrlForImage(value)]

Logic/Web/Wikidata.ts

@@ -42,7 +42,7 @@ export default class Wikidata {
         sitelinks.delete("commons")
 
         const claims = new Map<string, Set<string>>();
-        for (const claimId of entity.claims) {
+        for (const claimId in entity.claims) {
 
             const claimsList: any[] = entity.claims[claimId]
             const values = new Set<string>()

Logic/Web/Wikimedia.ts

@@ -0,0 +1,47 @@
+import {Utils} from "../../Utils";
+
+export default class Wikimedia {
+    /**
+     * Recursively walks a wikimedia commons category in order to search for entries, which can be File: or Category: entries
+     * Returns (a promise of) a list of URLS
+     * @param categoryName The name of the wikimedia category
+     * @param maxLoad: the maximum amount of images to return
+     * @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
+     */
+    public static async GetCategoryContents(categoryName: string,
+                                             maxLoad = 10,
+                                             continueParameter: string = undefined): Promise<string[]> {
+        if (categoryName === undefined || categoryName === null || categoryName === "") {
+            return [];
+        }
+        if (!categoryName.startsWith("Category:")) {
+            categoryName = "Category:" + categoryName;
+        }
+
+        let url = "https://commons.wikimedia.org/w/api.php?" +
+            "action=query&list=categorymembers&format=json&" +
+            "&origin=*" +
+            "&cmtitle=" + encodeURIComponent(categoryName);
+        if (continueParameter !== undefined) {
+            url = `${url}&cmcontinue=${continueParameter}`;
+        }
+        const response = await Utils.downloadJson(url)
+        const members = response.query?.categorymembers ?? [];
+        const imageOverview: string[] = members.map(member => member.title);
+
+        if (response.continue === undefined) {
+            // We are done crawling through the category - no continuation in sight
+            return imageOverview;
+        }
+
+        if (maxLoad - imageOverview.length <= 0) {
+            console.debug(`Recursive wikimedia category load stopped for ${categoryName}`)
+            return imageOverview;
+        }
+
+        // We do have a continue token - let's load the next page
+        const recursive = await Wikimedia.GetCategoryContents(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
+        imageOverview.push(...recursive)
+        return imageOverview
+    }
+}

UI/SpecialVisualizations.ts

@@ -64,13 +64,16 @@ export default class SpecialVisualizations {
                 funcName: "image_carousel",
                 docs: "Creates an image carousel for the given sources. An attempt will be made to guess what source is used. Supported: Wikidata identifiers, Wikipedia pages, Wikimedia categories, IMGUR (with attribution, direct links)",
                 args: [{
-                    name: "image key/prefix",
+                    name: "image key/prefix (multiple values allowed if comma-seperated)",
                     defaultValue: "image",
                     doc: "The keys given to the images, e.g. if <span class='literal-code'>image</span> is given, the first picture URL will be added as <span class='literal-code'>image</span>, the second as <span class='literal-code'>image:0</span>, the third as <span class='literal-code'>image:1</span>, etc... "
                 }],
                 constr: (state: State, tags, args) => {
-                    const imagePrefix = args[0];
+                    let imagePrefixes = undefined;
-                    return new ImageCarousel(AllImageProviders.LoadImagesFor(tags, imagePrefix), tags);
+                    if(args.length > 0){
+                        imagePrefixes = args;
+                    }
+                    return new ImageCarousel(AllImageProviders.LoadImagesFor(tags, imagePrefixes), tags);
                 }
             },
             {

UI/WikipediaBox.ts

@@ -11,6 +11,7 @@ import Svg from "../Svg";
 import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata";
 import Locale from "./i18n/Locale";
 import Toggle from "./Input/Toggle";
+import Link from "./Base/Link";
 
 export default class WikipediaBox extends Toggle {
 
@@ -22,49 +23,78 @@ export default class WikipediaBox extends Toggle {
         }
 
 
-        const wikibox = wikidataId
+        const wikiLink: UIEventSource<[string, string] | "loading" | "failed" | "no page"> =
-            .bind(id => {
+            wikidataId
-                console.log("Wikidata is", id)
+                .bind(id => {
-                if(id === undefined){
+                    if (id === undefined) {
-                    return undefined
+                        return undefined
-                }
+                    }
-                console.log("Initing load WIkidataentry with id", id)
+                    return Wikidata.LoadWikidataEntry(id);
-                return Wikidata.LoadWikidataEntry(id);
+                })
-            })
+                .map(maybewikidata => {
-            .map(maybewikidata => {
+                    if (maybewikidata === undefined) {
-                if (maybewikidata === undefined) {
+                        return "loading"
-                    return new Loading(wp.loading.Clone())
+                    }
-                }
+                    if (maybewikidata["error"] !== undefined) {
-                if (maybewikidata["error"] !== undefined) {
+                        return "failed"
-                    return wp.failed.Clone().SetClass("alert p-4")
-                }
-                const wikidata = <WikidataResponse>maybewikidata["success"]
-                console.log("Got wikidata response", wikidata)
-                if (wikidata.wikisites.size === 0) {
-                    return wp.noWikipediaPage.Clone()
-                }
 
-                const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]]
+                    }
-                let language
+                    const wikidata = <WikidataResponse>maybewikidata["success"]
-                let pagetitle;
+                    if (wikidata.wikisites.size === 0) {
-                let i = 0
+                        return "no page"
-                do {
+                    }
-                    language = preferredLanguage[i]
+
-                    pagetitle = wikidata.wikisites.get(language)
+                    const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]]
-                    i++;
+                    let language
-                } while (pagetitle === undefined)
+                    let pagetitle;
-                return WikipediaBox.createContents(pagetitle, language)
+                    let i = 0
-            }, [Locale.language])
+                    do {
+                        language = preferredLanguage[i]
+                        pagetitle = wikidata.wikisites.get(language)
+                        i++;
+                    } while (pagetitle === undefined)
+                    return [pagetitle, language]
+                }, [Locale.language])
 
 
         const contents = new VariableUiElement(
-            wikibox
+            wikiLink.map(status => {
+                if (status === "loading") {
+                    return new Loading(wp.loading.Clone()).SetClass("pl-6 pt-2")
+                }
+
+                if (status === "failed") {
+                    return wp.failed.Clone().SetClass("alert p-4")
+                }
+                if (status == "no page") {
+                    return wp.noWikipediaPage.Clone()
+                }
+
+                const [pagetitle, language] = status
+                return WikipediaBox.createContents(pagetitle, language)
+               
+            })
         ).SetClass("overflow-auto normal-background rounded-lg")
 
 
+      const linkElement = new VariableUiElement(wikiLink.map(state => {
+            if (typeof state !== "string") {
+                const [pagetitle, language] = state
+                const url=  `https://${language}.wikipedia.org/wiki/${pagetitle}`
+                return new Link(Svg.pop_out_ui().SetStyle("width: 1.2rem").SetClass("block  "), url, true)
+            }
+            return undefined}))
+                    .SetClass("flex items-center")
+
         const mainContent = new Combine([
-            new Combine([Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"),
+           new Combine([
-                new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2)]).SetClass("flex"),
+               new Combine([
+                Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"),
+                new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2),
+               ]).SetClass("flex"),
+                
+               linkElement
+            ]).SetClass("flex justify-between"),
             contents]).SetClass("block rounded-xl subtle-background m-1 p-2 flex flex-col")
             .SetStyle("max-height: inherit")
         super(
@@ -102,8 +132,8 @@ export default class WikipediaBox extends Toggle {
             return undefined
         })
 
-        return new Combine([new VariableUiElement(contents).SetClass("block pl-6 pt-2")])
+        return new Combine([new VariableUiElement(contents)
-            .SetClass("block")
+            .SetClass("block pl-6 pt-2")])
     }
 
 }

assets/themes/etymology.json

@@ -50,6 +50,12 @@
         "nl": "Alle lagen met een gelinkt etymology"
       },
       "tagRenderings": [
+        {
+          "id": "etymology_wikidata_image",
+          "render": {
+            "*": "{image_carousel(name:etymology:wikidata)}"
+          }
+        },
         {
           "id": "simple etymology",
           "render": {
@@ -63,7 +69,7 @@
         {
           "id": "wikipedia-etymology",
           "render": {
-            "*": "{wikipedia(name:etymology:wikidata):max-height:20rem}"
+            "*": "{wikipedia(name:etymology:wikidata):max-height:30rem}"
           }
         }
       ],