From a124b7f2345f9059a18118091bf8baa6a8e10d6c Mon Sep 17 00:00:00 2001
From: Pieter Vander Vennet <pietervdvn@posteo.net>
Date: Thu, 22 Feb 2024 15:21:04 +0100
Subject: [PATCH] Add caching

---
 scripts/serverLdScrape.ts | 15 +++++++++++++--
 src/Utils.ts              |  2 +-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/scripts/serverLdScrape.ts b/scripts/serverLdScrape.ts
index 31d66b0e80..1157ef1c29 100644
--- a/scripts/serverLdScrape.ts
+++ b/scripts/serverLdScrape.ts
@@ -8,15 +8,26 @@ class ServerLdScrape extends Script {
     }
     async main(args: string[]): Promise<void> {
         const port = Number(args[0] ?? 2346)
+
+        const cache: Record<string, { date: Date; contents: any }> = {}
+
         new Server(port, {}, [
             {
                 mustMatch: "extractgraph",
                 mimetype: "application/ld+json",
                 async handle(content, searchParams: URLSearchParams) {
                     const url = searchParams.get("url")
+                    if (cache[url] !== undefined) {
+                        const { date, contents } = cache[url]
+                        // In seconds
+                        const tdiff = (new Date().getTime() - date.getTime()) / 1000
+                        if (tdiff < 24 * 60 * 60) {
+                            return contents
+                        }
+                    }
                     const dloaded = await Utils.download(url, {
                         "User-Agent":
-                            "MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
+                            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
                     })
                     const parsed = parse(dloaded)
                     const scripts = Array.from(parsed.getElementsByTagName("script"))
@@ -32,7 +43,7 @@ class ServerLdScrape extends Script {
                             console.error(e)
                         }
                     }
-
+                    cache[url] = { contents: snippets, date: new Date() }
                     return JSON.stringify(snippets)
                 },
             },
diff --git a/src/Utils.ts b/src/Utils.ts
index 3c673f2d93..4f099ed2e7 100644
--- a/src/Utils.ts
+++ b/src/Utils.ts
@@ -980,7 +980,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
                     resolve({ error: "rate limited", url, statuscode: xhr.status })
                 } else {
                     resolve({
-                        error: "other error: " + xhr.statusText,
+                        error: "other error: " + xhr.statusText + ", " + xhr.responseText,
                         url,
                         statuscode: xhr.status,
                     })