ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add caching

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-02-22 15:21:04 +01:00
parent f971d7fd41
commit a124b7f234
2 changed files with 14 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

15
scripts/serverLdScrape.ts
View file

@ -8,15 +8,26 @@ class ServerLdScrape extends Script {
}
async main(args: string[]): Promise<void> {
const port = Number(args[0] ?? 2346)
const cache: Record<string, { date: Date; contents: any }> = {}
new Server(port, {}, [
{
mustMatch: "extractgraph",
mimetype: "application/ld+json",
async handle(content, searchParams: URLSearchParams) {
const url = searchParams.get("url")
if (cache[url] !== undefined) {
const { date, contents } = cache[url]
// In seconds
const tdiff = (new Date().getTime() - date.getTime()) / 1000
if (tdiff < 24 * 60 * 60) {
return contents
}
}
const dloaded = await Utils.download(url, {
"User-Agent":
"MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
})
const parsed = parse(dloaded)
const scripts = Array.from(parsed.getElementsByTagName("script"))
@ -32,7 +43,7 @@ class ServerLdScrape extends Script {
console.error(e)
}
}
cache[url] = { contents: snippets, date: new Date() }
return JSON.stringify(snippets)
},
},

2
src/Utils.ts
View file

@ -980,7 +980,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
resolve({ error: "rate limited", url, statuscode: xhr.status })
} else {
resolve({
error: "other error: " + xhr.statusText,
error: "other error: " + xhr.statusText + ", " + xhr.responseText,
url,
statuscode: xhr.status,
})