Fix: fix wikimedia-commons image attribution; move code for this out of 'DownloadCommons'

scripts/downloadCommons.ts

@@ -5,56 +5,9 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
 import { unescape } from "querystring"
 import SmallLicense from "../src/Models/smallLicense"
+import Wikimedia, { ImageQueryAPIResponse } from "../src/Logic/Web/Wikimedia"
 
-interface ExtMetadataProp {
-    value: string
-    source: string
-    hidden: string
-}
 
-interface ImageQueryAPIResponse {
-    continue: {
-        iistart: string
-        continue: string
-    }
-    query: {
-        normalized?: {
-            from: string
-            to: string
-        }[]
-        pages: {
-            [key: string]: {
-                pageid: number
-                ns: number
-                title: string
-                imagerepository: string
-                imageinfo?: {
-                    user: string
-                    url: string
-                    descriptionurl: string
-                    descriptionshorturl: string
-                    extmetadata?: {
-                        DateTime: ExtMetadataProp
-                        ObjectName: ExtMetadataProp
-                        CommonsMetadataExtension?: ExtMetadataProp
-                        Categories?: ExtMetadataProp
-                        Assessments?: ExtMetadataProp
-                        ImageDescription?: ExtMetadataProp
-                        DateTimeOriginal?: ExtMetadataProp
-                        Credit?: ExtMetadataProp
-                        Artist?: ExtMetadataProp
-                        LicenseShortName?: ExtMetadataProp
-                        UsageTerms?: ExtMetadataProp
-                        AttributionRequired?: ExtMetadataProp
-                        Copyrighted?: ExtMetadataProp
-                        Restrictions?: ExtMetadataProp
-                        License?: ExtMetadataProp
-                    }
-                }[]
-            }
-        }
-    }
-}
 
 interface CategoryMember {
     pageid: number
@@ -93,36 +46,6 @@ interface ImagesQueryAPIResponse {
     }
 }
 
-interface TemplateQueryAPIResponse {
-    batchcomplete: string
-    query: {
-        normalized?: {
-            from: string
-            to: string
-        }[]
-        pages: {
-            [key: string]: {
-                pageid: number
-                ns: number
-                title: string
-                templates?: {
-                    ns: number
-                    title: string
-                }[]
-            }
-        }
-    }
-}
-
-// Map license names of Wikimedia Commons to different names
-const licenseMapping = {}
-
-// Map template names to license names
-const templateMapping = {
-    "Template:PD": "Public Domain",
-    "Template:CC0": "CC0 1.0",
-}
-
 export async function main(args: string[]) {
     if (args.length < 2) {
         console.log("Usage: downloadCommons.ts <output folder> <url> <?url> <?url> .. ")
@@ -136,59 +59,62 @@ export async function main(args: string[]) {
     for (const url of urls) {
         // Download details from the API
         const commonsFileNamePath = url.split("/").pop()
-        if (commonsFileNamePath !== undefined) {
+        if (commonsFileNamePath === undefined) {
-            const commonsFileName = commonsFileNamePath.split("?").shift()
+            console.log(
+                "\x1b[31m%s\x1b[0m",
+                `URL ${url} doesn't seem to be a valid URL! Skipping...`,
+            )
+            continue
+        }
 
-            if (commonsFileName !== undefined) {
-                console.log(`Processing ${commonsFileName}...`)
 
-                const baseUrl = url.split("/").slice(0, 3).join("/")
+        const commonsFileName = commonsFileNamePath.split("?").shift()
 
-                // Check if it is a file or a category
+        if (commonsFileName === undefined) {
-                if (url.includes("Category:")) {
+            console.log(
-                    // Download all files in the category
+                "\x1b[31m%s\x1b[0m",
-                    const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&list=categorymembers&cmtitle=${commonsFileName}&cmlimit=250&cmtype=file`
+                `URL ${url} doesn't seem to contain a filename or category! Skipping...`,
-                    const response = await fetch(apiUrl)
+            )
-                    const apiDetails: CategoryQueryAPIResponse = await response.json()
+            continue
-                    for (const member of apiDetails.query.categorymembers) {
+        }
-                        await downloadImage(member.title, outputFolder, baseUrl)
+
-                    }
+        console.log(`Processing ${commonsFileName}...`)
-                } else if (url.includes("File:")) {
+
-                    await downloadImage(commonsFileName, outputFolder, baseUrl)
+        const baseUrl = url.split("/").slice(0, 3).join("/")
-                } else {
+
-                    // Probably a page url, try to get all images from the page
+        // Check if it is a file or a category
-                    const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=images&titles=${commonsFileName}&imlimit=250`
+        if (url.includes("Category:")) {
-                    const response = await fetch(apiUrl)
+            // Download all files in the category
-                    const apiDetails: ImagesQueryAPIResponse = await response.json()
+            const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&list=categorymembers&cmtitle=${commonsFileName}&cmlimit=250&cmtype=file`
-                    const page = apiDetails.query.pages[Object.keys(apiDetails.query.pages)[0]]
+            const response = await fetch(apiUrl)
-                    if (page.images) {
+            const apiDetails: CategoryQueryAPIResponse = await response.json()
-                        for (const image of page.images) {
+            for (const member of apiDetails.query.categorymembers) {
-                            await downloadImage(image.title, outputFolder, baseUrl)
+                await downloadImage(member.title, outputFolder, baseUrl)
-                        }
+            }
-                    } else {
+        } else if (url.includes("File:")) {
-                        console.log(
+            await downloadImage(commonsFileName, outputFolder, baseUrl)
-                            "\x1b[31m%s\x1b[0m",
+        } else {
-                            `URL ${url} doesn't seem to contain any images! Skipping...`
+            // Probably a page url, try to get all images from the page
-                        )
+            const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=images&titles=${commonsFileName}&imlimit=250`
-                    }
+            const response = await fetch(apiUrl)
+            const apiDetails: ImagesQueryAPIResponse = await response.json()
+            const page = apiDetails.query.pages[Object.keys(apiDetails.query.pages)[0]]
+            if (page.images) {
+                for (const image of page.images) {
+                    await downloadImage(image.title, outputFolder, baseUrl)
                 }
             } else {
                 console.log(
                     "\x1b[31m%s\x1b[0m",
-                    `URL ${url} doesn't seem to contain a filename or category! Skipping...`
+                    `URL ${url} doesn't seem to contain any images! Skipping...`,
                 )
-                continue
             }
-        } else {
-            console.log(
-                "\x1b[31m%s\x1b[0m",
-                `URL ${url} doesn't seem to be a valid URL! Skipping...`
-            )
-            continue
         }
     }
 }
 
+
+
 async function downloadImage(filename: string, outputFolder: string, baseUrl: string) {
     const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=imageinfo&iiprop=url|extmetadata|user&iimetadataversion=latest&titles=${filename}`
     const response = await fetch(apiUrl)
@@ -252,11 +178,6 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
 
         // Check if we actually have image info
         if (wikiPage.imageinfo?.length !== undefined && wikiPage.imageinfo.length > 0) {
-            const wikiUrl = wikiPage.imageinfo[0].descriptionurl
-            const fileUrl = wikiPage.imageinfo[0].url
-            const author =
-                wikiPage.imageinfo[0].extmetadata?.Artist?.value || wikiPage.imageinfo[0].user
-            let license = wikiPage.imageinfo[0].extmetadata?.LicenseShortName?.value || null
 
             // Check if the output folder exists
             if (!existsSync(outputFolder)) {
@@ -270,42 +191,11 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
                 }
             }
 
-            // Check if the license is present
-            if (!license) {
-                console.log(
-                    `${filename} does not have a license, falling back to checking template...`
-                )
-                const templateUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=templates&titles=${filename}&tllimit=500`
-                const templateResponse = await fetch(templateUrl)
-                const templateDetails: TemplateQueryAPIResponse = await templateResponse.json()
 
-                // Loop through all templates and check if one of them is a license
-                const wikiPage =
-                    templateDetails.query.pages[Object.keys(templateDetails.query.pages)[0]]
-                if (wikiPage.templates) {
-                    for (const template of wikiPage.templates) {
-                        if (templateMapping[template.title]) {
-                            console.log(
-                                `Found license ${templateMapping[template.title]} for ${filename}`
-                            )
-                            license = templateMapping[template.title]
-                        }
-                    }
-                }
-
-                // If no license was found, skip the file
-                if (!license) {
-                    // Log in yellow
-                    console.log(
-                        `\x1b[33m%s\x1b[0m`,
-                        `No license found for ${filename}, skipping...`
-                    )
-                    return
-                }
-            }
 
             // Download the file and save it
             const cleanFileName = unescape(filename).replace("File:", "")
+            const fileUrl = wikiPage.imageinfo[0].url
             console.log(
                 `Downloading ${cleanFileName} from ${fileUrl} and saving it to ${outputFolder}/${cleanFileName}...`
             )
@@ -316,12 +206,7 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
             writeFileSync(filePath, file)
 
             // Save the license information
-            const licenseInfo: SmallLicense = {
+            const licenseInfo: SmallLicense = await Wikimedia.getLicenseFor(baseUrl, filename)
-                path: cleanFileName,
-                license: licenseMapping[license] || license.replace("CC BY", "CC-BY"),
-                authors: [removeLinks(author)],
-                sources: [wikiUrl],
-            }
 
             const licensePath = `${outputFolder}/license_info.json`
             if (!existsSync(licensePath)) {
@@ -340,9 +225,5 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
     }
 }
 
-function removeLinks(text: string): string {
-    // Remove <a> tags
-    return text.replace(/<a.*?>(.*?)<\/a>/g, "$1")
-}
 
 // main(process.argv.slice(2))

src/Logic/ImageProviders/WikimediaImageProvider.ts

@@ -156,7 +156,7 @@ export class WikimediaImageProvider extends ImageProvider {
 
         value = WikimediaImageProvider.removeCommonsPrefix(value)
         if (value.startsWith("Category:")) {
-            const urls = await Wikimedia.GetCategoryContents(value)
+            const urls = await Wikimedia.getCategoryContents(value)
             return urls
                 .filter((url) => url.startsWith("File:"))
                 .map((image) => this.UrlForImage(image))
@@ -173,59 +173,15 @@ export class WikimediaImageProvider extends ImageProvider {
     }
 
     public async DownloadAttribution(img: { id: string }): Promise<LicenseInfo> {
-        const filename = "File:" + WikimediaImageProvider.extractFileName(img.id)
+        const l = await Wikimedia.getLicenseFor("https://commons.wikimedia.org", img.id)
-        console.log("Downloading attribution for", filename, img.id)
+        return <LicenseInfo>{
-        if (filename === "") {
+            title: img.id,
-            return undefined
+            artist: l.authors.join(", "),
+            licenseShortName : l.license,
+            informationLocation: {href:
+                this.visitUrl(img)
+            }
         }
-
-        const url =
-            "https://en.wikipedia.org/w/" +
-            "api.php?action=query&prop=imageinfo&iiprop=extmetadata&" +
-            "titles=" +
-            filename +
-            "&format=json&origin=*"
-        const data = await Utils.downloadJsonCached<{
-            query: { pages: { title: string; imageinfo: { extmetadata }[] }[] }
-        }>(url, 365 * 24 * 60 * 60)
-        const licenseInfo = new LicenseInfo()
-        const pages = data.query.pages
-        /*jup, a literal "-1" in an object, not a list!*/
-        let pageInfo = pages["-1"]
-        if (Array.isArray(pages)) {
-            pageInfo = pages.at(-1)
-        }
-        if (pageInfo === undefined) {
-            console.warn("No attribution found for wikimedia image:", filename)
-            return undefined
-        }
-
-        const license = (pageInfo.imageinfo ?? [])[0]?.extmetadata
-        if (license === undefined) {
-            console.warn(
-                "The file",
-                filename,
-                "has no usable metedata or license attached... Please fix the license info file yourself!"
-            )
-            return undefined
-        }
-
-        let title = WikimediaImageProvider.makeCanonical(pageInfo.title)
-        if (title.endsWith(".jpg") || title.endsWith(".png")) {
-            title = title.substring(0, title.length - 4)
-        }
-
-        licenseInfo.title = title
-        licenseInfo.artist = license.Artist?.value
-        licenseInfo.license = license.License?.value
-        licenseInfo.copyrighted = license.Copyrighted?.value
-        licenseInfo.attributionRequired = license.AttributionRequired?.value
-        licenseInfo.usageTerms = license.UsageTerms?.value
-        licenseInfo.licenseShortName = license.LicenseShortName?.value
-        licenseInfo.credit = license.Credit?.value
-        licenseInfo.description = license.ImageDescription?.value
-        licenseInfo.informationLocation = new URL("https://en.wikipedia.org/wiki/" + pageInfo.title)
-        return licenseInfo
     }
 
     private UrlForImage(image: string, key?: string, value?: string): ProvidedImage {
@@ -248,7 +204,7 @@ export class WikimediaImageProvider extends ImageProvider {
         return undefined
     }
 
-    visitUrl(): string | undefined {
+    visitUrl(img: ProvidedImage): string | undefined {
-        return undefined
+        return "https://commons.wikimedia.org/wiki/"+img.id
     }
 }

src/Logic/Web/Wikimedia.ts

@@ -1,4 +1,84 @@
 import { Utils } from "../../Utils"
+import SmallLicense from "../../Models/smallLicense"
+
+
+interface ExtMetadataProp {
+    value: string
+    source: string
+    hidden: string
+}
+
+interface TemplateQueryAPIResponse {
+    batchcomplete: string
+    query: {
+        normalized?: {
+            from: string
+            to: string
+        }[]
+        pages: {
+            [key: string]: {
+                pageid: number
+                ns: number
+                title: string
+                templates?: {
+                    ns: number
+                    title: string
+                }[]
+            }
+        }
+    }
+}
+// Map license names of Wikimedia Commons to different names
+const licenseMapping = {}
+
+// Map template names to license names
+const templateMapping = {
+    "Template:PD": "Public Domain",
+    "Template:CC0": "CC0 1.0",
+}
+export interface ImageQueryAPIResponse {
+    continue: {
+        iistart: string
+        continue: string
+    }
+    query: {
+        normalized?: {
+            from: string
+            to: string
+        }[]
+        pages: {
+            [key: string]: {
+                pageid: number
+                ns: number
+                title: string
+                imagerepository: string
+                imageinfo?: {
+                    user: string
+                    url: string
+                    descriptionurl: string
+                    descriptionshorturl: string
+                    extmetadata?: {
+                        DateTime: ExtMetadataProp
+                        ObjectName: ExtMetadataProp
+                        CommonsMetadataExtension?: ExtMetadataProp
+                        Categories?: ExtMetadataProp
+                        Assessments?: ExtMetadataProp
+                        ImageDescription?: ExtMetadataProp
+                        DateTimeOriginal?: ExtMetadataProp
+                        Credit?: ExtMetadataProp
+                        Artist?: ExtMetadataProp
+                        LicenseShortName?: ExtMetadataProp
+                        UsageTerms?: ExtMetadataProp
+                        AttributionRequired?: ExtMetadataProp
+                        Copyrighted?: ExtMetadataProp
+                        Restrictions?: ExtMetadataProp
+                        License?: ExtMetadataProp
+                    }
+                }[]
+            }
+        }
+    }
+}
 
 export default class Wikimedia {
     /**
@@ -8,7 +88,7 @@ export default class Wikimedia {
      * @param maxLoad: the maximum amount of images to return
      * @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
      */
-    public static async GetCategoryContents(
+    public static async getCategoryContents(
         categoryName: string,
         maxLoad = 10,
         continueParameter: string = undefined
@@ -30,10 +110,10 @@ export default class Wikimedia {
             url = `${url}&cmcontinue=${continueParameter}`
         }
         const response = await Utils.downloadJson(url)
-        const members = response.query?.categorymembers ?? []
+        const members = response["query"]?.categorymembers ?? []
         const imageOverview: string[] = members.map((member) => member.title)
 
-        if (response.continue === undefined) {
+        if (response["continue"] === undefined) {
             // We are done crawling through the category - no continuation in sight
             return imageOverview
         }
@@ -44,12 +124,82 @@ export default class Wikimedia {
         }
 
         // We do have a continue token - let's load the next page
-        const recursive = await Wikimedia.GetCategoryContents(
+        const recursive = await Wikimedia.getCategoryContents(
             categoryName,
             maxLoad - imageOverview.length,
-            response.continue.cmcontinue
+            response["continue"].cmcontinue
         )
         imageOverview.push(...recursive)
         return imageOverview
     }
+    private static removeLinks(text: string): string {
+        // Remove <a> tags
+        return text.replace(/<a.*?>(.*?)<\/a>/g, "$1")
+    }
+
+    /**
+     * Queries the wikimedia api and tries to construct a license based on the response
+     * @param baseUrl
+     * @param filename
+     */
+    static async getLicenseFor(baseUrl: string, filename: string): Promise<SmallLicense> {
+        let cors = ""
+        if(!Utils.runningFromConsole){
+            // https://commons.wikimedia.org/w/api.php?action=help&modules=main
+            cors = "&origin=*"
+        }
+        const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=imageinfo&iiprop=url|extmetadata|user&iimetadataversion=latest&titles=${filename}${cors}`
+        const response = await fetch(apiUrl)
+        const apiDetails: ImageQueryAPIResponse = await response.json()
+        const wikiPage = apiDetails.query.pages[Object.keys(apiDetails.query.pages)[0]]
+
+        const fileUrl = wikiPage.imageinfo[0].url
+        const wikiUrl = wikiPage.imageinfo[0].descriptionurl
+        const author =
+            wikiPage.imageinfo[0].extmetadata?.Artist?.value || wikiPage.imageinfo[0].user
+        let license = wikiPage.imageinfo[0].extmetadata?.LicenseShortName?.value || null
+        // Check if the license is present in the response
+        if (!license) {
+            console.log(
+                `${filename} does not have a license, falling back to checking template...`,
+            )
+            const templateUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=templates&titles=${filename}&tllimit=500`
+            const templateResponse = await fetch(templateUrl)
+            const templateDetails: TemplateQueryAPIResponse = await templateResponse.json()
+
+            // Loop through all templates and check if one of them is a license
+            const wikiPage =
+                templateDetails.query.pages[Object.keys(templateDetails.query.pages)[0]]
+            if (!wikiPage.templates) {
+                return undefined
+            }
+            for (const template of wikiPage.templates) {
+                if (templateMapping[template.title]) {
+                    console.log(
+                        `Found license ${templateMapping[template.title]} for ${filename}`,
+                    )
+                    license = templateMapping[template.title]
+                }
+            }
+
+        }
+
+        // If no license was found, skip the file
+        if (!license) {
+            // Log in yellow
+            console.log(
+                `\x1b[33m%s\x1b[0m`,
+                `No license found for ${filename}...`,
+            )
+            return undefined
+        }
+
+        const cleanFileName = unescape(filename).replace("File:", "")
+        return <SmallLicense>{
+            path: cleanFileName,
+            license: licenseMapping[license] || license.replace("CC BY", "CC-BY"),
+            authors: [Wikimedia.removeLinks(author)],
+            sources: [wikiUrl],
+        }
+    }
 }

src/UI/Image/AttributedImage.svelte

@@ -3,7 +3,7 @@
    * Shows an image with attribution
    */
   import ImageAttribution from "./ImageAttribution.svelte"
-  import { Store } from "../../Logic/UIEventSource"
+  import { ImmutableStore, Store } from "../../Logic/UIEventSource"
 
   import type { HotspotProperties, ProvidedImage } from "../../Logic/ImageProviders/ImageProvider"
   import { Mapillary } from "../../Logic/ImageProviders/Mapillary"
@@ -111,6 +111,8 @@
     }
     state?.geocodedImages.set([f])
   }
+
+  let debug = state?.featureSwitches?.featureSwitchIsDebugging ?? new ImmutableStore(false)
 </script>
 
 <Popup shown={showBigPreview} bodyPadding="p-0" dismissable={true}>
@@ -218,6 +220,11 @@
     <div class="absolute bottom-0 left-0">
       <ImageAttribution {image} {attributionFormat} />
     </div>
+    {#if $debug}
+      <div class="subtle as-link" on:click={() => console.log("Image is", image)}>
+        {image.id}
+      </div>
+    {/if}
   </div>
 {:else if image.status === "hidden"}
   <div class="flex h-80 w-60 flex-col items-center justify-center break-words p-4 text-center">

src/UI/Image/ImageAttribution.svelte

@@ -13,6 +13,7 @@
   export let image: Partial<ProvidedImage> & { id: string; url: string }
   export let attributionFormat: "minimal" | "medium" | "large" = "medium"
 
+  console.log("Initial license:", image.license, image.provider)
   let license: Store<LicenseInfo> = image.license
     ? new ImmutableStore(image.license)
     : UIEventSource.FromPromise(image.provider?.DownloadAttribution(image))
@@ -37,7 +38,7 @@
               {$license.title}
             </a>
           {:else}
-            $license.title
+            {$license.title}
           {/if}
         {/if}
       {/if}