diff --git a/Logic/ImageProviders/AllImageProviders.ts b/Logic/ImageProviders/AllImageProviders.ts index 92818379f..2f27539a1 100644 --- a/Logic/ImageProviders/AllImageProviders.ts +++ b/Logic/ImageProviders/AllImageProviders.ts @@ -21,7 +21,7 @@ export default class AllImageProviders { private static _cache: Map> = new Map>() - public static LoadImagesFor(tags: UIEventSource, imagePrefix?: string): UIEventSource { + public static LoadImagesFor(tags: UIEventSource, tagKey?: string): UIEventSource { const id = tags.data.id if (id === undefined) { return undefined; @@ -39,12 +39,8 @@ export default class AllImageProviders { for (const imageProvider of AllImageProviders.ImageAttributionSource) { let prefixes = imageProvider.defaultKeyPrefixes - if(imagePrefix !== undefined){ - prefixes = [...prefixes] - if(prefixes.indexOf("image") >= 0){ - prefixes.splice(prefixes.indexOf("image"), 1) - } - prefixes.push(imagePrefix) + if(tagKey !== undefined){ + prefixes = [tagKey] } const singleSource = imageProvider.GetRelevantUrls(tags, { diff --git a/Logic/ImageProviders/GenericImageProvider.ts b/Logic/ImageProviders/GenericImageProvider.ts index 170829fe0..2f2dab322 100644 --- a/Logic/ImageProviders/GenericImageProvider.ts +++ b/Logic/ImageProviders/GenericImageProvider.ts @@ -20,7 +20,14 @@ export default class GenericImageProvider extends ImageProvider { if (this._valuePrefixBlacklist.some(prefix => value.startsWith(prefix))) { return [] } - + + try{ + new URL(value) + }catch (_){ + // Not a valid URL + return [] + } + return [Promise.resolve({ key: key, url: value, diff --git a/Logic/ImageProviders/ImageProvider.ts b/Logic/ImageProviders/ImageProvider.ts index efce6aa92..be3d58ce5 100644 --- a/Logic/ImageProviders/ImageProvider.ts +++ b/Logic/ImageProviders/ImageProvider.ts @@ -17,7 +17,7 @@ export default abstract class ImageProvider { if (cached !== undefined) { return cached; } - const src =UIEventSource.FromPromise(this.DownloadAttribution(url)) + const src = UIEventSource.FromPromise(this.DownloadAttribution(url)) this._cache.set(url, src) return src; } @@ -38,6 +38,7 @@ export default abstract class ImageProvider { } const relevantUrls = new UIEventSource<{ url: string; key: string; provider: ImageProvider }[]>([]) const seenValues = new Set() + const self = this allTags.addCallbackAndRunD(tags => { for (const key in tags) { if(!prefixes.some(prefix => key.startsWith(prefix))){ diff --git a/Logic/ImageProviders/WikidataImageProvider.ts b/Logic/ImageProviders/WikidataImageProvider.ts index f24f58aca..9c3d6af3e 100644 --- a/Logic/ImageProviders/WikidataImageProvider.ts +++ b/Logic/ImageProviders/WikidataImageProvider.ts @@ -27,6 +27,7 @@ export class WikidataImageProvider extends ImageProvider { if(entity === undefined){ return [] } + console.log("Entity:", entity) const allImages : Promise[] = [] // P18 is the claim 'depicted in this image' @@ -34,9 +35,17 @@ export class WikidataImageProvider extends ImageProvider { const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, img) allImages.push(...promises) } + // P373 is 'commons category' + for (let cat of Array.from(entity.claims.get("P373") ?? [])) { + if(!cat.startsWith("Category:")){ + cat = "Category:"+cat + } + const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, cat) + allImages.push(...promises) + } const commons = entity.commons - if (commons !== undefined) { + if (commons !== undefined && (commons.startsWith("Category:") || commons.startsWith("File:"))) { const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons) allImages.push(...promises) } diff --git a/Logic/ImageProviders/WikimediaImageProvider.ts b/Logic/ImageProviders/WikimediaImageProvider.ts index 0d2f6c1d0..ae922c99a 100644 --- a/Logic/ImageProviders/WikimediaImageProvider.ts +++ b/Logic/ImageProviders/WikimediaImageProvider.ts @@ -4,6 +4,7 @@ import Svg from "../../Svg"; import Link from "../../UI/Base/Link"; import {Utils} from "../../Utils"; import {LicenseInfo} from "./LicenseInfo"; +import Wikimedia from "../Web/Wikimedia"; /** * This module provides endpoints for wikimedia and others @@ -20,50 +21,6 @@ export class WikimediaImageProvider extends ImageProvider { super(); } - /** - * Recursively walks a wikimedia commons category in order to search for (image) files - * Returns (a promise of) a list of URLS - * @param categoryName The name of the wikimedia category - * @param maxLoad: the maximum amount of images to return - * @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia - */ - private static async GetImagesInCategory(categoryName: string, - maxLoad = 10, - continueParameter: string = undefined): Promise { - if (categoryName === undefined || categoryName === null || categoryName === "") { - return []; - } - if (!categoryName.startsWith("Category:")) { - categoryName = "Category:" + categoryName; - } - - let url = "https://commons.wikimedia.org/w/api.php?" + - "action=query&list=categorymembers&format=json&" + - "&origin=*" + - "&cmtitle=" + encodeURIComponent(categoryName); - if (continueParameter !== undefined) { - url = `${url}&cmcontinue=${continueParameter}`; - } - const response = await Utils.downloadJson(url) - const members = response.query?.categorymembers ?? []; - const imageOverview: string[] = members.map(member => member.title); - - if (response.continue === undefined) { - // We are done crawling through the category - no continuation in sight - return imageOverview; - } - - if (maxLoad - imageOverview.length <= 0) { - console.debug(`Recursive wikimedia category load stopped for ${categoryName}`) - return imageOverview; - } - - // We do have a continue token - let's load the next page - const recursive = await this.GetImagesInCategory(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue) - imageOverview.push(...recursive) - return imageOverview - } - private static ExtractFileName(url: string) { if (!url.startsWith("http")) { return url; @@ -110,7 +67,7 @@ export class WikimediaImageProvider extends ImageProvider { const licenseInfo = new LicenseInfo(); const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata; if (license === undefined) { - console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!") + console.warn("The file", filename ,"has no usable metedata or license attached... Please fix the license info file yourself!") return undefined; } @@ -149,8 +106,8 @@ export class WikimediaImageProvider extends ImageProvider { return [Promise.resolve(result)] } if (value.startsWith("Category:")) { - const urls = await WikimediaImageProvider.GetImagesInCategory(value) - return urls.map(image => this.UrlForImage(image)) + const urls = await Wikimedia.GetCategoryContents(value) + return urls.filter(url => url.startsWith("File:")).map(image => this.UrlForImage(image)) } if (value.startsWith("File:")) { return [this.UrlForImage(value)] diff --git a/Logic/Web/Wikidata.ts b/Logic/Web/Wikidata.ts index ae981ee82..f92b419bf 100644 --- a/Logic/Web/Wikidata.ts +++ b/Logic/Web/Wikidata.ts @@ -42,7 +42,7 @@ export default class Wikidata { sitelinks.delete("commons") const claims = new Map>(); - for (const claimId of entity.claims) { + for (const claimId in entity.claims) { const claimsList: any[] = entity.claims[claimId] const values = new Set() diff --git a/Logic/Web/Wikimedia.ts b/Logic/Web/Wikimedia.ts new file mode 100644 index 000000000..8aa34068e --- /dev/null +++ b/Logic/Web/Wikimedia.ts @@ -0,0 +1,47 @@ +import {Utils} from "../../Utils"; + +export default class Wikimedia { + /** + * Recursively walks a wikimedia commons category in order to search for entries, which can be File: or Category: entries + * Returns (a promise of) a list of URLS + * @param categoryName The name of the wikimedia category + * @param maxLoad: the maximum amount of images to return + * @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia + */ + public static async GetCategoryContents(categoryName: string, + maxLoad = 10, + continueParameter: string = undefined): Promise { + if (categoryName === undefined || categoryName === null || categoryName === "") { + return []; + } + if (!categoryName.startsWith("Category:")) { + categoryName = "Category:" + categoryName; + } + + let url = "https://commons.wikimedia.org/w/api.php?" + + "action=query&list=categorymembers&format=json&" + + "&origin=*" + + "&cmtitle=" + encodeURIComponent(categoryName); + if (continueParameter !== undefined) { + url = `${url}&cmcontinue=${continueParameter}`; + } + const response = await Utils.downloadJson(url) + const members = response.query?.categorymembers ?? []; + const imageOverview: string[] = members.map(member => member.title); + + if (response.continue === undefined) { + // We are done crawling through the category - no continuation in sight + return imageOverview; + } + + if (maxLoad - imageOverview.length <= 0) { + console.debug(`Recursive wikimedia category load stopped for ${categoryName}`) + return imageOverview; + } + + // We do have a continue token - let's load the next page + const recursive = await Wikimedia.GetCategoryContents(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue) + imageOverview.push(...recursive) + return imageOverview + } +} \ No newline at end of file diff --git a/UI/SpecialVisualizations.ts b/UI/SpecialVisualizations.ts index de45911db..d4c4ea2cc 100644 --- a/UI/SpecialVisualizations.ts +++ b/UI/SpecialVisualizations.ts @@ -64,13 +64,16 @@ export default class SpecialVisualizations { funcName: "image_carousel", docs: "Creates an image carousel for the given sources. An attempt will be made to guess what source is used. Supported: Wikidata identifiers, Wikipedia pages, Wikimedia categories, IMGUR (with attribution, direct links)", args: [{ - name: "image key/prefix", + name: "image key/prefix (multiple values allowed if comma-seperated)", defaultValue: "image", doc: "The keys given to the images, e.g. if image is given, the first picture URL will be added as image, the second as image:0, the third as image:1, etc... " }], constr: (state: State, tags, args) => { - const imagePrefix = args[0]; - return new ImageCarousel(AllImageProviders.LoadImagesFor(tags, imagePrefix), tags); + let imagePrefixes = undefined; + if(args.length > 0){ + imagePrefixes = args; + } + return new ImageCarousel(AllImageProviders.LoadImagesFor(tags, imagePrefixes), tags); } }, { diff --git a/UI/WikipediaBox.ts b/UI/WikipediaBox.ts index 7f0d392a6..4b0f08d4f 100644 --- a/UI/WikipediaBox.ts +++ b/UI/WikipediaBox.ts @@ -11,6 +11,7 @@ import Svg from "../Svg"; import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata"; import Locale from "./i18n/Locale"; import Toggle from "./Input/Toggle"; +import Link from "./Base/Link"; export default class WikipediaBox extends Toggle { @@ -22,49 +23,78 @@ export default class WikipediaBox extends Toggle { } - const wikibox = wikidataId - .bind(id => { - console.log("Wikidata is", id) - if(id === undefined){ - return undefined - } - console.log("Initing load WIkidataentry with id", id) - return Wikidata.LoadWikidataEntry(id); - }) - .map(maybewikidata => { - if (maybewikidata === undefined) { - return new Loading(wp.loading.Clone()) - } - if (maybewikidata["error"] !== undefined) { - return wp.failed.Clone().SetClass("alert p-4") - } - const wikidata = maybewikidata["success"] - console.log("Got wikidata response", wikidata) - if (wikidata.wikisites.size === 0) { - return wp.noWikipediaPage.Clone() - } + const wikiLink: UIEventSource<[string, string] | "loading" | "failed" | "no page"> = + wikidataId + .bind(id => { + if (id === undefined) { + return undefined + } + return Wikidata.LoadWikidataEntry(id); + }) + .map(maybewikidata => { + if (maybewikidata === undefined) { + return "loading" + } + if (maybewikidata["error"] !== undefined) { + return "failed" - const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]] - let language - let pagetitle; - let i = 0 - do { - language = preferredLanguage[i] - pagetitle = wikidata.wikisites.get(language) - i++; - } while (pagetitle === undefined) - return WikipediaBox.createContents(pagetitle, language) - }, [Locale.language]) + } + const wikidata = maybewikidata["success"] + if (wikidata.wikisites.size === 0) { + return "no page" + } + + const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]] + let language + let pagetitle; + let i = 0 + do { + language = preferredLanguage[i] + pagetitle = wikidata.wikisites.get(language) + i++; + } while (pagetitle === undefined) + return [pagetitle, language] + }, [Locale.language]) const contents = new VariableUiElement( - wikibox + wikiLink.map(status => { + if (status === "loading") { + return new Loading(wp.loading.Clone()).SetClass("pl-6 pt-2") + } + + if (status === "failed") { + return wp.failed.Clone().SetClass("alert p-4") + } + if (status == "no page") { + return wp.noWikipediaPage.Clone() + } + + const [pagetitle, language] = status + return WikipediaBox.createContents(pagetitle, language) + + }) ).SetClass("overflow-auto normal-background rounded-lg") + const linkElement = new VariableUiElement(wikiLink.map(state => { + if (typeof state !== "string") { + const [pagetitle, language] = state + const url= `https://${language}.wikipedia.org/wiki/${pagetitle}` + return new Link(Svg.pop_out_ui().SetStyle("width: 1.2rem").SetClass("block "), url, true) + } + return undefined})) + .SetClass("flex items-center") + const mainContent = new Combine([ - new Combine([Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"), - new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2)]).SetClass("flex"), + new Combine([ + new Combine([ + Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"), + new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2), + ]).SetClass("flex"), + + linkElement + ]).SetClass("flex justify-between"), contents]).SetClass("block rounded-xl subtle-background m-1 p-2 flex flex-col") .SetStyle("max-height: inherit") super( @@ -102,8 +132,8 @@ export default class WikipediaBox extends Toggle { return undefined }) - return new Combine([new VariableUiElement(contents).SetClass("block pl-6 pt-2")]) - .SetClass("block") + return new Combine([new VariableUiElement(contents) + .SetClass("block pl-6 pt-2")]) } } \ No newline at end of file diff --git a/assets/themes/etymology.json b/assets/themes/etymology.json index a8ca9dfc9..ec170496a 100644 --- a/assets/themes/etymology.json +++ b/assets/themes/etymology.json @@ -50,6 +50,12 @@ "nl": "Alle lagen met een gelinkt etymology" }, "tagRenderings": [ + { + "id": "etymology_wikidata_image", + "render": { + "*": "{image_carousel(name:etymology:wikidata)}" + } + }, { "id": "simple etymology", "render": { @@ -63,7 +69,7 @@ { "id": "wikipedia-etymology", "render": { - "*": "{wikipedia(name:etymology:wikidata):max-height:20rem}" + "*": "{wikipedia(name:etymology:wikidata):max-height:30rem}" } } ],