contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Improve wikipedia fetcher

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-04-30 00:30:15 +02:00
parent f5ae73aac1
commit 7a76109d61
1 changed files with 19 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

25
Logic/Web/Wikipedia.ts
View file

@ -31,9 +31,10 @@ export default class Wikipedia {
public static GetArticle(options: { public static GetArticle(options: {
pageName: string, pageName: string,
language?: "en" | string language?: "en" | string,
firstParagraphOnly?: false | boolean
}): UIEventSource<{ success: string } | { error: any }> { }): UIEventSource<{ success: string } | { error: any }> {
const key = (options.language ?? "en") + ":" + options.pageName const key = (options.language ?? "en") + ":" + options.pageName + ":" + (options.firstParagraphOnly ?? false)
const cached = Wikipedia._cache.get(key) const cached = Wikipedia._cache.get(key)
if (cached !== undefined) { if (cached !== undefined) {
return cached return cached
@ -43,14 +44,21 @@ export default class Wikipedia {
return v; return v;
} }
public static getDataUrl(options: {language?: "en" | string, pageName: string}): string{
return `https://${options.language ?? "en"}.wikipedia.org/w/api.php?action=parse&format=json&origin=*&prop=text&page=` + options.pageName
}
public static getPageUrl(options: {language?: "en" | string, pageName: string}): string{
return `https://${options.language ?? "en"}.wikipedia.org/wiki/` + options.pageName
}
public static async GetArticleAsync(options: { public static async GetArticleAsync(options: {
pageName: string, pageName: string,
language?: "en" | string language?: "en" | string,
firstParagraphOnly?: false | boolean
}): Promise<string> { }): Promise<string> {
const language = options.language ?? "en" const response = await Utils.downloadJson(Wikipedia.getDataUrl(options))
const url = `https://${language}.wikipedia.org/w/api.php?action=parse&format=json&origin=*&prop=text&page=` + options.pageName
const response = await Utils.downloadJson(url)
const html = response["parse"]["text"]["*"]; const html = response["parse"]["text"]["*"];
const div = document.createElement("div") const div = document.createElement("div")
@ -73,12 +81,17 @@ export default class Wikipedia {
const links = Array.from(content.getElementsByTagName("a")) const links = Array.from(content.getElementsByTagName("a"))
// Rewrite relative links to absolute links + open them in a new tab // Rewrite relative links to absolute links + open them in a new tab
const language = options.language ?? "en"
links.filter(link => link.getAttribute("href")?.startsWith("/") ?? false).forEach(link => { links.filter(link => link.getAttribute("href")?.startsWith("/") ?? false).forEach(link => {
link.target = '_blank' link.target = '_blank'
// note: link.getAttribute("href") gets the textual value, link.href is the rewritten version which'll contain the host for relative paths // note: link.getAttribute("href") gets the textual value, link.href is the rewritten version which'll contain the host for relative paths
link.href = `https://${language}.wikipedia.org${link.getAttribute("href")}`; link.href = `https://${language}.wikipedia.org${link.getAttribute("href")}`;
}) })
if (options?.firstParagraphOnly) {
return content.getElementsByTagName("p").item(0).innerHTML
}
return content.innerHTML return content.innerHTML
} }