Fix: fix wikimedia-commons image attribution; move code for this out of 'DownloadCommons'

This commit is contained in:
Pieter Vander Vennet 2025-07-05 20:54:17 +02:00
parent d8b86703c6
commit eab010a13e
5 changed files with 222 additions and 227 deletions

View file

@ -5,56 +5,9 @@
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { unescape } from "querystring"
import SmallLicense from "../src/Models/smallLicense"
import Wikimedia, { ImageQueryAPIResponse } from "../src/Logic/Web/Wikimedia"
interface ExtMetadataProp {
value: string
source: string
hidden: string
}
interface ImageQueryAPIResponse {
continue: {
iistart: string
continue: string
}
query: {
normalized?: {
from: string
to: string
}[]
pages: {
[key: string]: {
pageid: number
ns: number
title: string
imagerepository: string
imageinfo?: {
user: string
url: string
descriptionurl: string
descriptionshorturl: string
extmetadata?: {
DateTime: ExtMetadataProp
ObjectName: ExtMetadataProp
CommonsMetadataExtension?: ExtMetadataProp
Categories?: ExtMetadataProp
Assessments?: ExtMetadataProp
ImageDescription?: ExtMetadataProp
DateTimeOriginal?: ExtMetadataProp
Credit?: ExtMetadataProp
Artist?: ExtMetadataProp
LicenseShortName?: ExtMetadataProp
UsageTerms?: ExtMetadataProp
AttributionRequired?: ExtMetadataProp
Copyrighted?: ExtMetadataProp
Restrictions?: ExtMetadataProp
License?: ExtMetadataProp
}
}[]
}
}
}
}
interface CategoryMember {
pageid: number
@ -93,36 +46,6 @@ interface ImagesQueryAPIResponse {
}
}
interface TemplateQueryAPIResponse {
batchcomplete: string
query: {
normalized?: {
from: string
to: string
}[]
pages: {
[key: string]: {
pageid: number
ns: number
title: string
templates?: {
ns: number
title: string
}[]
}
}
}
}
// Map license names of Wikimedia Commons to different names
const licenseMapping = {}
// Map template names to license names
const templateMapping = {
"Template:PD": "Public Domain",
"Template:CC0": "CC0 1.0",
}
export async function main(args: string[]) {
if (args.length < 2) {
console.log("Usage: downloadCommons.ts <output folder> <url> <?url> <?url> .. ")
@ -136,59 +59,62 @@ export async function main(args: string[]) {
for (const url of urls) {
// Download details from the API
const commonsFileNamePath = url.split("/").pop()
if (commonsFileNamePath !== undefined) {
const commonsFileName = commonsFileNamePath.split("?").shift()
if (commonsFileNamePath === undefined) {
console.log(
"\x1b[31m%s\x1b[0m",
`URL ${url} doesn't seem to be a valid URL! Skipping...`,
)
continue
}
if (commonsFileName !== undefined) {
console.log(`Processing ${commonsFileName}...`)
const baseUrl = url.split("/").slice(0, 3).join("/")
const commonsFileName = commonsFileNamePath.split("?").shift()
// Check if it is a file or a category
if (url.includes("Category:")) {
// Download all files in the category
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&list=categorymembers&cmtitle=${commonsFileName}&cmlimit=250&cmtype=file`
const response = await fetch(apiUrl)
const apiDetails: CategoryQueryAPIResponse = await response.json()
for (const member of apiDetails.query.categorymembers) {
await downloadImage(member.title, outputFolder, baseUrl)
}
} else if (url.includes("File:")) {
await downloadImage(commonsFileName, outputFolder, baseUrl)
} else {
// Probably a page url, try to get all images from the page
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=images&titles=${commonsFileName}&imlimit=250`
const response = await fetch(apiUrl)
const apiDetails: ImagesQueryAPIResponse = await response.json()
const page = apiDetails.query.pages[Object.keys(apiDetails.query.pages)[0]]
if (page.images) {
for (const image of page.images) {
await downloadImage(image.title, outputFolder, baseUrl)
}
} else {
console.log(
"\x1b[31m%s\x1b[0m",
`URL ${url} doesn't seem to contain any images! Skipping...`
)
}
if (commonsFileName === undefined) {
console.log(
"\x1b[31m%s\x1b[0m",
`URL ${url} doesn't seem to contain a filename or category! Skipping...`,
)
continue
}
console.log(`Processing ${commonsFileName}...`)
const baseUrl = url.split("/").slice(0, 3).join("/")
// Check if it is a file or a category
if (url.includes("Category:")) {
// Download all files in the category
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&list=categorymembers&cmtitle=${commonsFileName}&cmlimit=250&cmtype=file`
const response = await fetch(apiUrl)
const apiDetails: CategoryQueryAPIResponse = await response.json()
for (const member of apiDetails.query.categorymembers) {
await downloadImage(member.title, outputFolder, baseUrl)
}
} else if (url.includes("File:")) {
await downloadImage(commonsFileName, outputFolder, baseUrl)
} else {
// Probably a page url, try to get all images from the page
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=images&titles=${commonsFileName}&imlimit=250`
const response = await fetch(apiUrl)
const apiDetails: ImagesQueryAPIResponse = await response.json()
const page = apiDetails.query.pages[Object.keys(apiDetails.query.pages)[0]]
if (page.images) {
for (const image of page.images) {
await downloadImage(image.title, outputFolder, baseUrl)
}
} else {
console.log(
"\x1b[31m%s\x1b[0m",
`URL ${url} doesn't seem to contain a filename or category! Skipping...`
`URL ${url} doesn't seem to contain any images! Skipping...`,
)
continue
}
} else {
console.log(
"\x1b[31m%s\x1b[0m",
`URL ${url} doesn't seem to be a valid URL! Skipping...`
)
continue
}
}
}
async function downloadImage(filename: string, outputFolder: string, baseUrl: string) {
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=imageinfo&iiprop=url|extmetadata|user&iimetadataversion=latest&titles=${filename}`
const response = await fetch(apiUrl)
@ -252,11 +178,6 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
// Check if we actually have image info
if (wikiPage.imageinfo?.length !== undefined && wikiPage.imageinfo.length > 0) {
const wikiUrl = wikiPage.imageinfo[0].descriptionurl
const fileUrl = wikiPage.imageinfo[0].url
const author =
wikiPage.imageinfo[0].extmetadata?.Artist?.value || wikiPage.imageinfo[0].user
let license = wikiPage.imageinfo[0].extmetadata?.LicenseShortName?.value || null
// Check if the output folder exists
if (!existsSync(outputFolder)) {
@ -270,42 +191,11 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
}
}
// Check if the license is present
if (!license) {
console.log(
`${filename} does not have a license, falling back to checking template...`
)
const templateUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=templates&titles=${filename}&tllimit=500`
const templateResponse = await fetch(templateUrl)
const templateDetails: TemplateQueryAPIResponse = await templateResponse.json()
// Loop through all templates and check if one of them is a license
const wikiPage =
templateDetails.query.pages[Object.keys(templateDetails.query.pages)[0]]
if (wikiPage.templates) {
for (const template of wikiPage.templates) {
if (templateMapping[template.title]) {
console.log(
`Found license ${templateMapping[template.title]} for ${filename}`
)
license = templateMapping[template.title]
}
}
}
// If no license was found, skip the file
if (!license) {
// Log in yellow
console.log(
`\x1b[33m%s\x1b[0m`,
`No license found for ${filename}, skipping...`
)
return
}
}
// Download the file and save it
const cleanFileName = unescape(filename).replace("File:", "")
const fileUrl = wikiPage.imageinfo[0].url
console.log(
`Downloading ${cleanFileName} from ${fileUrl} and saving it to ${outputFolder}/${cleanFileName}...`
)
@ -316,12 +206,7 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
writeFileSync(filePath, file)
// Save the license information
const licenseInfo: SmallLicense = {
path: cleanFileName,
license: licenseMapping[license] || license.replace("CC BY", "CC-BY"),
authors: [removeLinks(author)],
sources: [wikiUrl],
}
const licenseInfo: SmallLicense = await Wikimedia.getLicenseFor(baseUrl, filename)
const licensePath = `${outputFolder}/license_info.json`
if (!existsSync(licensePath)) {
@ -340,9 +225,5 @@ async function downloadImage(filename: string, outputFolder: string, baseUrl: st
}
}
function removeLinks(text: string): string {
// Remove <a> tags
return text.replace(/<a.*?>(.*?)<\/a>/g, "$1")
}
// main(process.argv.slice(2))

View file

@ -156,7 +156,7 @@ export class WikimediaImageProvider extends ImageProvider {
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("Category:")) {
const urls = await Wikimedia.GetCategoryContents(value)
const urls = await Wikimedia.getCategoryContents(value)
return urls
.filter((url) => url.startsWith("File:"))
.map((image) => this.UrlForImage(image))
@ -173,59 +173,15 @@ export class WikimediaImageProvider extends ImageProvider {
}
public async DownloadAttribution(img: { id: string }): Promise<LicenseInfo> {
const filename = "File:" + WikimediaImageProvider.extractFileName(img.id)
console.log("Downloading attribution for", filename, img.id)
if (filename === "") {
return undefined
const l = await Wikimedia.getLicenseFor("https://commons.wikimedia.org", img.id)
return <LicenseInfo>{
title: img.id,
artist: l.authors.join(", "),
licenseShortName : l.license,
informationLocation: {href:
this.visitUrl(img)
}
}
const url =
"https://en.wikipedia.org/w/" +
"api.php?action=query&prop=imageinfo&iiprop=extmetadata&" +
"titles=" +
filename +
"&format=json&origin=*"
const data = await Utils.downloadJsonCached<{
query: { pages: { title: string; imageinfo: { extmetadata }[] }[] }
}>(url, 365 * 24 * 60 * 60)
const licenseInfo = new LicenseInfo()
const pages = data.query.pages
/*jup, a literal "-1" in an object, not a list!*/
let pageInfo = pages["-1"]
if (Array.isArray(pages)) {
pageInfo = pages.at(-1)
}
if (pageInfo === undefined) {
console.warn("No attribution found for wikimedia image:", filename)
return undefined
}
const license = (pageInfo.imageinfo ?? [])[0]?.extmetadata
if (license === undefined) {
console.warn(
"The file",
filename,
"has no usable metedata or license attached... Please fix the license info file yourself!"
)
return undefined
}
let title = WikimediaImageProvider.makeCanonical(pageInfo.title)
if (title.endsWith(".jpg") || title.endsWith(".png")) {
title = title.substring(0, title.length - 4)
}
licenseInfo.title = title
licenseInfo.artist = license.Artist?.value
licenseInfo.license = license.License?.value
licenseInfo.copyrighted = license.Copyrighted?.value
licenseInfo.attributionRequired = license.AttributionRequired?.value
licenseInfo.usageTerms = license.UsageTerms?.value
licenseInfo.licenseShortName = license.LicenseShortName?.value
licenseInfo.credit = license.Credit?.value
licenseInfo.description = license.ImageDescription?.value
licenseInfo.informationLocation = new URL("https://en.wikipedia.org/wiki/" + pageInfo.title)
return licenseInfo
}
private UrlForImage(image: string, key?: string, value?: string): ProvidedImage {
@ -248,7 +204,7 @@ export class WikimediaImageProvider extends ImageProvider {
return undefined
}
visitUrl(): string | undefined {
return undefined
visitUrl(img: ProvidedImage): string | undefined {
return "https://commons.wikimedia.org/wiki/"+img.id
}
}

View file

@ -1,4 +1,84 @@
import { Utils } from "../../Utils"
import SmallLicense from "../../Models/smallLicense"
interface ExtMetadataProp {
value: string
source: string
hidden: string
}
interface TemplateQueryAPIResponse {
batchcomplete: string
query: {
normalized?: {
from: string
to: string
}[]
pages: {
[key: string]: {
pageid: number
ns: number
title: string
templates?: {
ns: number
title: string
}[]
}
}
}
}
// Map license names of Wikimedia Commons to different names
const licenseMapping = {}
// Map template names to license names
const templateMapping = {
"Template:PD": "Public Domain",
"Template:CC0": "CC0 1.0",
}
export interface ImageQueryAPIResponse {
continue: {
iistart: string
continue: string
}
query: {
normalized?: {
from: string
to: string
}[]
pages: {
[key: string]: {
pageid: number
ns: number
title: string
imagerepository: string
imageinfo?: {
user: string
url: string
descriptionurl: string
descriptionshorturl: string
extmetadata?: {
DateTime: ExtMetadataProp
ObjectName: ExtMetadataProp
CommonsMetadataExtension?: ExtMetadataProp
Categories?: ExtMetadataProp
Assessments?: ExtMetadataProp
ImageDescription?: ExtMetadataProp
DateTimeOriginal?: ExtMetadataProp
Credit?: ExtMetadataProp
Artist?: ExtMetadataProp
LicenseShortName?: ExtMetadataProp
UsageTerms?: ExtMetadataProp
AttributionRequired?: ExtMetadataProp
Copyrighted?: ExtMetadataProp
Restrictions?: ExtMetadataProp
License?: ExtMetadataProp
}
}[]
}
}
}
}
export default class Wikimedia {
/**
@ -8,7 +88,7 @@ export default class Wikimedia {
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
public static async GetCategoryContents(
public static async getCategoryContents(
categoryName: string,
maxLoad = 10,
continueParameter: string = undefined
@ -30,10 +110,10 @@ export default class Wikimedia {
url = `${url}&cmcontinue=${continueParameter}`
}
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? []
const members = response["query"]?.categorymembers ?? []
const imageOverview: string[] = members.map((member) => member.title)
if (response.continue === undefined) {
if (response["continue"] === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview
}
@ -44,12 +124,82 @@ export default class Wikimedia {
}
// We do have a continue token - let's load the next page
const recursive = await Wikimedia.GetCategoryContents(
const recursive = await Wikimedia.getCategoryContents(
categoryName,
maxLoad - imageOverview.length,
response.continue.cmcontinue
response["continue"].cmcontinue
)
imageOverview.push(...recursive)
return imageOverview
}
private static removeLinks(text: string): string {
// Remove <a> tags
return text.replace(/<a.*?>(.*?)<\/a>/g, "$1")
}
/**
* Queries the wikimedia api and tries to construct a license based on the response
* @param baseUrl
* @param filename
*/
static async getLicenseFor(baseUrl: string, filename: string): Promise<SmallLicense> {
let cors = ""
if(!Utils.runningFromConsole){
// https://commons.wikimedia.org/w/api.php?action=help&modules=main
cors = "&origin=*"
}
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=imageinfo&iiprop=url|extmetadata|user&iimetadataversion=latest&titles=${filename}${cors}`
const response = await fetch(apiUrl)
const apiDetails: ImageQueryAPIResponse = await response.json()
const wikiPage = apiDetails.query.pages[Object.keys(apiDetails.query.pages)[0]]
const fileUrl = wikiPage.imageinfo[0].url
const wikiUrl = wikiPage.imageinfo[0].descriptionurl
const author =
wikiPage.imageinfo[0].extmetadata?.Artist?.value || wikiPage.imageinfo[0].user
let license = wikiPage.imageinfo[0].extmetadata?.LicenseShortName?.value || null
// Check if the license is present in the response
if (!license) {
console.log(
`${filename} does not have a license, falling back to checking template...`,
)
const templateUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=templates&titles=${filename}&tllimit=500`
const templateResponse = await fetch(templateUrl)
const templateDetails: TemplateQueryAPIResponse = await templateResponse.json()
// Loop through all templates and check if one of them is a license
const wikiPage =
templateDetails.query.pages[Object.keys(templateDetails.query.pages)[0]]
if (!wikiPage.templates) {
return undefined
}
for (const template of wikiPage.templates) {
if (templateMapping[template.title]) {
console.log(
`Found license ${templateMapping[template.title]} for ${filename}`,
)
license = templateMapping[template.title]
}
}
}
// If no license was found, skip the file
if (!license) {
// Log in yellow
console.log(
`\x1b[33m%s\x1b[0m`,
`No license found for ${filename}...`,
)
return undefined
}
const cleanFileName = unescape(filename).replace("File:", "")
return <SmallLicense>{
path: cleanFileName,
license: licenseMapping[license] || license.replace("CC BY", "CC-BY"),
authors: [Wikimedia.removeLinks(author)],
sources: [wikiUrl],
}
}
}

View file

@ -3,7 +3,7 @@
* Shows an image with attribution
*/
import ImageAttribution from "./ImageAttribution.svelte"
import { Store } from "../../Logic/UIEventSource"
import { ImmutableStore, Store } from "../../Logic/UIEventSource"
import type { HotspotProperties, ProvidedImage } from "../../Logic/ImageProviders/ImageProvider"
import { Mapillary } from "../../Logic/ImageProviders/Mapillary"
@ -111,6 +111,8 @@
}
state?.geocodedImages.set([f])
}
let debug = state?.featureSwitches?.featureSwitchIsDebugging ?? new ImmutableStore(false)
</script>
<Popup shown={showBigPreview} bodyPadding="p-0" dismissable={true}>
@ -218,6 +220,11 @@
<div class="absolute bottom-0 left-0">
<ImageAttribution {image} {attributionFormat} />
</div>
{#if $debug}
<div class="subtle as-link" on:click={() => console.log("Image is", image)}>
{image.id}
</div>
{/if}
</div>
{:else if image.status === "hidden"}
<div class="flex h-80 w-60 flex-col items-center justify-center break-words p-4 text-center">

View file

@ -13,6 +13,7 @@
export let image: Partial<ProvidedImage> & { id: string; url: string }
export let attributionFormat: "minimal" | "medium" | "large" = "medium"
console.log("Initial license:", image.license, image.provider)
let license: Store<LicenseInfo> = image.license
? new ImmutableStore(image.license)
: UIEventSource.FromPromise(image.provider?.DownloadAttribution(image))
@ -37,7 +38,7 @@
{$license.title}
</a>
{:else}
$license.title
{$license.title}
{/if}
{/if}
{/if}