Better documentation of server configuration, add error reporting server

Docs/ServerConfig/hetzner/Caddyfile

@@ -17,7 +17,7 @@ countrycoder.mapcomplete.org {
 
 report.mapcomplete.org {
 	reverse_proxy {
-		to http://127.0.0.1:2600
+		to http://127.0.0.1:2348
 	}
 }
 

Docs/ServerConfig/hetzner/hetzner.txt

@@ -2,11 +2,49 @@
 
 This server hosts the studio files and is used for expermintal builds.
 
-For used hosts, see the Caddyfile
+For used ports, see the Caddyfile
+
+To update caddy
+
+```
+    cp Caddyfile /etc/caddy/
+    # If caddy was running via a console instead of as a service, do `caddy stop` now
+    systemctl reload caddy
+```
+
+Debug logs with: `journalctl -u caddy --no-pager | less +G`
+
+Services:
 
 
-## Cache forwarding
+### studio + theme sync
 
-As the ISP of Nerdlab is a bit picky, we use SSH-port-forwarding on the cache server:
+The studio server, handling those requests.
 
-`ssh -R 5445:127.0.0.1:5445 hetzner`
+`npm run server:studio`
+
+Additionally, this runs syncthing to make a backup of all theme files.
+
+### LOD-server
+
+A server scraping other websites.
+
+`npm run server:ldjson`
+
+### Error report server
+
+A simple server logging everything it receives
+
+`npm run server:errorreport`
+
+### geo-ip
+
+Provides geolocation based on
+
+    ```
+    git clone https://github.com/pietervdvn/geoip-server
+    cd geoip-server
+    mkdir data
+    # Drop the databases from https://lite.ip2location.com/ in the data dir
+    npm run start
+    ```

package.json

@@ -114,8 +114,10 @@
     "dloadVelopark": "vite-node scripts/velopark/veloParkToGeojson.ts ",
     "compareVelopark": "vite-node scripts/velopark/compare.ts -- velopark_nonsynced_.geojson ~/Projecten/OSM/Fietsberaad/2024-02-02\\ Fietsenstallingen_OSM_met_velopark_ref.geojson\n",
     "scrapeWebsites": "vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/",
-    "summary-server": "vite-node scripts/osm2pgsql/tilecountServer.ts",
-    "ldjson-server": "vite-node scripts/serverLdScrape.ts",
+    "server:summary": "vite-node scripts/osm2pgsql/tilecountServer.ts",
+    "server:ldjson": "vite-node scripts/serverLdScrape.ts",
+    "sever:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets",
+    "server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/",
     "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts"
   },
   "keywords": [

scripts/server.ts

@@ -1,17 +1,36 @@
 import http from "node:http"
 
+export interface Handler {
+    mustMatch: string | RegExp
+    mimetype: string
+    addHeaders?: Record<string, string>
+    handle: (path: string, queryParams: URLSearchParams, req: http.IncomingMessage) => Promise<string>
+
+}
+
+class ServerUtils {
+
+    public static getBody(req: http.IncomingMessage): Promise<string> {
+        return new Promise<string>((resolve) => {
+            let body = '';
+            req.on('data', (chunk) => {
+                body += chunk;
+            });
+            req.on('end', () => {
+                resolve(body)
+            });
+        })
+    }
+
+}
+
 export class Server {
     constructor(
         port: number,
         options: {
             ignorePathPrefix?: string[]
         },
-        handle: {
-            mustMatch: string | RegExp
-            mimetype: string
-            addHeaders?: Record<string, string>
-            handle: (path: string, queryParams: URLSearchParams) => Promise<string>
-        }[]
+        handle: Handler[]
     ) {
         handle.push({
             mustMatch: "",
@@ -81,8 +100,9 @@ export class Server {
                     res.end()
                     return
                 }
+                let body = undefined
                 if (req.method === "POST" || req.method === "UPDATE") {
-                    return
+                    body = await ServerUtils.getBody(req)
                 }
 
                 if (req.method === "DELETE") {
@@ -90,7 +110,7 @@ export class Server {
                 }
 
                 try {
-                    const result = await handler.handle(path, url.searchParams)
+                    const result = await handler.handle(path, url.searchParams, req, body)
                     if (result === undefined) {
                         res.writeHead(500)
                         res.write("Could not fetch this website, probably blocked by them")

scripts/serverErrorReport.ts

@@ -0,0 +1,52 @@
+import { Handler, Server } from "./server"
+import Script from "./Script"
+import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "fs"
+import { mkdir } from "node:fs"
+import ScriptUtils from "./ScriptUtils"
+
+class ServerErrorReport extends Script {
+    constructor() {
+        super("A server which receives and logs error reports")
+    }
+
+    async main(args: string[]): Promise<void> {
+        const logDirectory = args[0] ?? "./error_logs"
+        console.log("Logging to directory", logDirectory)
+        if (!existsSync(logDirectory)) {
+            mkdirSync(logDirectory)
+            console.log("Created this directory")
+        }
+        let errorReport = 0
+        new Server(2348, {},
+            [<Handler>{
+                mustMatch: "report",
+                mimetype: "application/json",
+                handle: async (_, queryParams, req, body) => {
+                    if (!body) {
+                        throw "{\"error\": \"No body; use a post request\"}"
+                    }
+                    console.log(body)
+                    const ip = <string>req.headers["x-forwarded-for"]
+                    const d = new Date()
+                    const date = d.toISOString()
+                    const file = logDirectory + "/" + d.getUTCFullYear() + "_" + d.getUTCMonth() + "_" + d.getUTCDay() + ".lines.json"
+                    try{
+                        body = JSON.parse(body)
+                    }catch (e) {
+                        // could not parse, we'll save it as is
+                    }
+                    const contents = "\n"+JSON.stringify({ ip, index: errorReport, date, message: body })
+                    if (!existsSync(file)) {
+                        writeFileSync(file, contents)
+                    } else {
+                        appendFileSync(file, contents)
+                    }
+                    errorReport++
+                    return `{"status":"ok", "nr": ${errorReport}}`
+                },
+
+            }])
+    }
+}
+
+new ServerErrorReport().run()