Add caching

scripts/serverLdScrape.ts

@@ -8,15 +8,26 @@ class ServerLdScrape extends Script {
     }
     async main(args: string[]): Promise<void> {
         const port = Number(args[0] ?? 2346)
+
+        const cache: Record<string, { date: Date; contents: any }> = {}
+
         new Server(port, {}, [
             {
                 mustMatch: "extractgraph",
                 mimetype: "application/ld+json",
                 async handle(content, searchParams: URLSearchParams) {
                     const url = searchParams.get("url")
+                    if (cache[url] !== undefined) {
+                        const { date, contents } = cache[url]
+                        // In seconds
+                        const tdiff = (new Date().getTime() - date.getTime()) / 1000
+                        if (tdiff < 24 * 60 * 60) {
+                            return contents
+                        }
+                    }
                     const dloaded = await Utils.download(url, {
                         "User-Agent":
-                            "MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
+                            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
                     })
                     const parsed = parse(dloaded)
                     const scripts = Array.from(parsed.getElementsByTagName("script"))
@@ -32,7 +43,7 @@ class ServerLdScrape extends Script {
                             console.error(e)
                         }
                     }
-
+                    cache[url] = { contents: snippets, date: new Date() }
                     return JSON.stringify(snippets)
                 },
             },

src/Utils.ts

@@ -980,7 +980,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
                     resolve({ error: "rate limited", url, statuscode: xhr.status })
                 } else {
                     resolve({
-                        error: "other error: " + xhr.statusText,
+                        error: "other error: " + xhr.statusText + ", " + xhr.responseText,
                         url,
                         statuscode: xhr.status,
                     })