Linked data loader: prettier UI, some bugfixes

langs/en.json

@@ -46,6 +46,21 @@
         "useSomethingElse": "Use another OpenStreetMap-editor to delete it instead",
         "whyDelete": "Why should this feature be deleted?"
     },
+    "external": {
+        "allAreApplied": "All missing, external values have been copied into OpenStreetMap",
+        "allIncluded": "The external data and OpenStreetMap-data are the same",
+        "apply": "Apply",
+        "applyAll": "Apply all missing values",
+        "conflicting": {
+            "intro": "OpenStreetMap has a different value then the source website for the following values.",
+            "title": "Conflicting items"
+        },
+        "currentInOsmIs": "At the moment, OpenStreetMap has the following value recorded:",
+        "done": "Done",
+        "error": "Error",
+        "loadedFrom": "The following data is loaded from <a href={url}>{source}</a> using the embedded JSON-LD",
+        "overwrite": "Overwrite"
+    },
     "favourite": {
         "loginNeeded": "<h3>Log in</h3>A personal layout is only available for OpenStreetMap users",
         "panelIntro": "<h3>Your personal theme</h3>Activate your favourite layers from all the official themes",

public/css/index-tailwind-output.css

@@ -1080,14 +1080,14 @@ video {
   height: 6rem;
 }
 
-.h-full {
-  height: 100%;
-}
-
 .h-screen {
   height: 100vh;
 }
 
+.h-full {
+  height: 100%;
+}
+
 .h-32 {
   height: 8rem;
 }
@@ -1163,10 +1163,6 @@ video {
   height: 20rem;
 }
 
-.h-5\/6 {
-  height: 83.333333%;
-}
-
 .h-56 {
   height: 14rem;
 }
@@ -1484,6 +1480,10 @@ video {
           column-gap: 0.5rem;
 }
 
+.gap-y-8 {
+  row-gap: 2rem;
+}
+
 .space-x-0\.5 > :not([hidden]) ~ :not([hidden]) {
   --tw-space-x-reverse: 0;
   margin-right: calc(0.125rem * var(--tw-space-x-reverse));
@@ -1610,14 +1610,14 @@ video {
   word-break: break-all;
 }
 
-.rounded-full {
-  border-radius: 9999px;
-}
-
 .rounded {
   border-radius: 0.25rem;
 }
 
+.rounded-full {
+  border-radius: 9999px;
+}
+
 .rounded-xl {
   border-radius: 0.75rem;
 }

scripts/ScriptUtils.ts

@@ -5,6 +5,7 @@ import * as https from "https"
 import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"
 import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
 import xml2js from "xml2js"
+import { resolve } from "node:dns"
 
 export default class ScriptUtils {
     public static fixUtils() {
@@ -161,8 +162,18 @@ export default class ScriptUtils {
     public static Download(
         url: string,
         headers?: any
-    ): Promise<{ content: string } | { redirect: string }> {
-        return new Promise((resolve, reject) => {
+    ): Promise<{ content: string } | { redirect: string }>
+    public static Download(
+        url: string,
+        headers?: any,
+        timeoutSecs?: number
+    ): Promise<{ content: string } | { redirect: string } | "timeout">
+    public static Download(
+        url: string,
+        headers?: any,
+        timeoutSecs?: number
+    ): Promise<{ content: string } | { redirect: string } | "timeout"> {
+        const requestPromise = new Promise((resolve, reject) => {
             try {
                 headers = headers ?? {}
                 headers.accept = "application/json"
@@ -209,5 +220,9 @@ export default class ScriptUtils {
                 reject(e)
             }
         })
+        const timeoutPromise = new Promise<any>((resolve, reject) => {
+            setTimeout(() => timeoutSecs === undefined ? reject(new Error("Timout reached")) : resolve("timeout"), (timeoutSecs ?? 10) * 1000)
+        })
+        return Promise.race([requestPromise, timeoutPromise])
     }
 }

scripts/serverLdScrape.ts

@@ -1,11 +1,13 @@
 import Script from "../scripts/Script"
 import { Server } from "./server"
-import { Utils } from "../src/Utils"
 import parse from "node-html-parser"
+import ScriptUtils from "./ScriptUtils"
+
 class ServerLdScrape extends Script {
     constructor() {
         super("Starts a server which fetches a webpage and returns embedded LD+JSON")
     }
+
     async main(args: string[]): Promise<void> {
         const port = Number(args[0] ?? 2346)
         const cache: Record<string, { date: Date; contents: any }> = {}
@@ -24,12 +26,18 @@ class ServerLdScrape extends Script {
                             return JSON.stringify(contents)
                         }
                     }
-                    const dloaded = await Utils.download(url, {
-                        "User-Agent":
-                            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
-                    })
-                    // return dloaded
-                    const parsed = parse(dloaded)
+                    let dloaded: { content: string } | { redirect: string } | "timeout" = { redirect: url }
+                    do {
+
+                        dloaded = await ScriptUtils.Download(dloaded["redirect"], {
+                            "User-Agent":
+                                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
+                        }, 10)
+                        if (dloaded === "timeout") {
+                            return "{\"#\":\"timout reached\"}"
+                        }
+                    } while (dloaded["redirect"])
+                    const parsed = parse(dloaded["content"])
                     const scripts = Array.from(parsed.getElementsByTagName("script"))
                     for (const script of scripts) {
                         const tp = script.attributes["type"]