From 13ea16317f818b0ae42d58b8393ec834ba9c7815 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Thu, 20 Jun 2024 14:22:33 +0200 Subject: [PATCH] Better documentation of server configuration, add error reporting server --- Docs/ServerConfig/hetzner/Caddyfile | 2 +- Docs/ServerConfig/hetzner/hetzner.txt | 46 +++++++++++++++++++++--- package.json | 6 ++-- scripts/server.ts | 36 ++++++++++++++----- scripts/serverErrorReport.ts | 52 +++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 15 deletions(-) create mode 100644 scripts/serverErrorReport.ts diff --git a/Docs/ServerConfig/hetzner/Caddyfile b/Docs/ServerConfig/hetzner/Caddyfile index 9d7516b4b..5164ad482 100644 --- a/Docs/ServerConfig/hetzner/Caddyfile +++ b/Docs/ServerConfig/hetzner/Caddyfile @@ -17,7 +17,7 @@ countrycoder.mapcomplete.org { report.mapcomplete.org { reverse_proxy { - to http://127.0.0.1:2600 + to http://127.0.0.1:2348 } } diff --git a/Docs/ServerConfig/hetzner/hetzner.txt b/Docs/ServerConfig/hetzner/hetzner.txt index 4009a6f92..fd39098f4 100644 --- a/Docs/ServerConfig/hetzner/hetzner.txt +++ b/Docs/ServerConfig/hetzner/hetzner.txt @@ -2,11 +2,49 @@ This server hosts the studio files and is used for expermintal builds. -For used hosts, see the Caddyfile +For used ports, see the Caddyfile + +To update caddy + +``` + cp Caddyfile /etc/caddy/ + # If caddy was running via a console instead of as a service, do `caddy stop` now + systemctl reload caddy +``` + +Debug logs with: `journalctl -u caddy --no-pager | less +G` + +Services: -## Cache forwarding +### studio + theme sync -As the ISP of Nerdlab is a bit picky, we use SSH-port-forwarding on the cache server: +The studio server, handling those requests. -`ssh -R 5445:127.0.0.1:5445 hetzner` +`npm run server:studio` + +Additionally, this runs syncthing to make a backup of all theme files. + +### LOD-server + +A server scraping other websites. + +`npm run server:ldjson` + +### Error report server + +A simple server logging everything it receives + +`npm run server:errorreport` + +### geo-ip + +Provides geolocation based on + + ``` + git clone https://github.com/pietervdvn/geoip-server + cd geoip-server + mkdir data + # Drop the databases from https://lite.ip2location.com/ in the data dir + npm run start + ``` diff --git a/package.json b/package.json index cd6be5755..87507f17c 100644 --- a/package.json +++ b/package.json @@ -114,8 +114,10 @@ "dloadVelopark": "vite-node scripts/velopark/veloParkToGeojson.ts ", "compareVelopark": "vite-node scripts/velopark/compare.ts -- velopark_nonsynced_.geojson ~/Projecten/OSM/Fietsberaad/2024-02-02\\ Fietsenstallingen_OSM_met_velopark_ref.geojson\n", "scrapeWebsites": "vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/", - "summary-server": "vite-node scripts/osm2pgsql/tilecountServer.ts", - "ldjson-server": "vite-node scripts/serverLdScrape.ts", + "server:summary": "vite-node scripts/osm2pgsql/tilecountServer.ts", + "server:ldjson": "vite-node scripts/serverLdScrape.ts", + "sever:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets", + "server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/", "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts" }, "keywords": [ diff --git a/scripts/server.ts b/scripts/server.ts index 9670433a2..df7205c2f 100644 --- a/scripts/server.ts +++ b/scripts/server.ts @@ -1,17 +1,36 @@ import http from "node:http" +export interface Handler { + mustMatch: string | RegExp + mimetype: string + addHeaders?: Record + handle: (path: string, queryParams: URLSearchParams, req: http.IncomingMessage) => Promise + +} + +class ServerUtils { + + public static getBody(req: http.IncomingMessage): Promise { + return new Promise((resolve) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + resolve(body) + }); + }) + } + +} + export class Server { constructor( port: number, options: { ignorePathPrefix?: string[] }, - handle: { - mustMatch: string | RegExp - mimetype: string - addHeaders?: Record - handle: (path: string, queryParams: URLSearchParams) => Promise - }[] + handle: Handler[] ) { handle.push({ mustMatch: "", @@ -81,8 +100,9 @@ export class Server { res.end() return } + let body = undefined if (req.method === "POST" || req.method === "UPDATE") { - return + body = await ServerUtils.getBody(req) } if (req.method === "DELETE") { @@ -90,7 +110,7 @@ export class Server { } try { - const result = await handler.handle(path, url.searchParams) + const result = await handler.handle(path, url.searchParams, req, body) if (result === undefined) { res.writeHead(500) res.write("Could not fetch this website, probably blocked by them") diff --git a/scripts/serverErrorReport.ts b/scripts/serverErrorReport.ts new file mode 100644 index 000000000..cbe6055c9 --- /dev/null +++ b/scripts/serverErrorReport.ts @@ -0,0 +1,52 @@ +import { Handler, Server } from "./server" +import Script from "./Script" +import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "fs" +import { mkdir } from "node:fs" +import ScriptUtils from "./ScriptUtils" + +class ServerErrorReport extends Script { + constructor() { + super("A server which receives and logs error reports") + } + + async main(args: string[]): Promise { + const logDirectory = args[0] ?? "./error_logs" + console.log("Logging to directory", logDirectory) + if (!existsSync(logDirectory)) { + mkdirSync(logDirectory) + console.log("Created this directory") + } + let errorReport = 0 + new Server(2348, {}, + [{ + mustMatch: "report", + mimetype: "application/json", + handle: async (_, queryParams, req, body) => { + if (!body) { + throw "{\"error\": \"No body; use a post request\"}" + } + console.log(body) + const ip = req.headers["x-forwarded-for"] + const d = new Date() + const date = d.toISOString() + const file = logDirectory + "/" + d.getUTCFullYear() + "_" + d.getUTCMonth() + "_" + d.getUTCDay() + ".lines.json" + try{ + body = JSON.parse(body) + }catch (e) { + // could not parse, we'll save it as is + } + const contents = "\n"+JSON.stringify({ ip, index: errorReport, date, message: body }) + if (!existsSync(file)) { + writeFileSync(file, contents) + } else { + appendFileSync(file, contents) + } + errorReport++ + return `{"status":"ok", "nr": ${errorReport}}` + }, + + }]) + } +} + +new ServerErrorReport().run()