forked from MapComplete/MapComplete
Better documentation of server configuration, add error reporting server
This commit is contained in:
parent
cd0d275965
commit
13ea16317f
5 changed files with 127 additions and 15 deletions
|
@ -17,7 +17,7 @@ countrycoder.mapcomplete.org {
|
|||
|
||||
report.mapcomplete.org {
|
||||
reverse_proxy {
|
||||
to http://127.0.0.1:2600
|
||||
to http://127.0.0.1:2348
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -2,11 +2,49 @@
|
|||
|
||||
This server hosts the studio files and is used for expermintal builds.
|
||||
|
||||
For used hosts, see the Caddyfile
|
||||
For used ports, see the Caddyfile
|
||||
|
||||
To update caddy
|
||||
|
||||
```
|
||||
cp Caddyfile /etc/caddy/
|
||||
# If caddy was running via a console instead of as a service, do `caddy stop` now
|
||||
systemctl reload caddy
|
||||
```
|
||||
|
||||
Debug logs with: `journalctl -u caddy --no-pager | less +G`
|
||||
|
||||
Services:
|
||||
|
||||
|
||||
## Cache forwarding
|
||||
### studio + theme sync
|
||||
|
||||
As the ISP of Nerdlab is a bit picky, we use SSH-port-forwarding on the cache server:
|
||||
The studio server, handling those requests.
|
||||
|
||||
`ssh -R 5445:127.0.0.1:5445 hetzner`
|
||||
`npm run server:studio`
|
||||
|
||||
Additionally, this runs syncthing to make a backup of all theme files.
|
||||
|
||||
### LOD-server
|
||||
|
||||
A server scraping other websites.
|
||||
|
||||
`npm run server:ldjson`
|
||||
|
||||
### Error report server
|
||||
|
||||
A simple server logging everything it receives
|
||||
|
||||
`npm run server:errorreport`
|
||||
|
||||
### geo-ip
|
||||
|
||||
Provides geolocation based on
|
||||
|
||||
```
|
||||
git clone https://github.com/pietervdvn/geoip-server
|
||||
cd geoip-server
|
||||
mkdir data
|
||||
# Drop the databases from https://lite.ip2location.com/ in the data dir
|
||||
npm run start
|
||||
```
|
||||
|
|
|
@ -114,8 +114,10 @@
|
|||
"dloadVelopark": "vite-node scripts/velopark/veloParkToGeojson.ts ",
|
||||
"compareVelopark": "vite-node scripts/velopark/compare.ts -- velopark_nonsynced_.geojson ~/Projecten/OSM/Fietsberaad/2024-02-02\\ Fietsenstallingen_OSM_met_velopark_ref.geojson\n",
|
||||
"scrapeWebsites": "vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/",
|
||||
"summary-server": "vite-node scripts/osm2pgsql/tilecountServer.ts",
|
||||
"ldjson-server": "vite-node scripts/serverLdScrape.ts",
|
||||
"server:summary": "vite-node scripts/osm2pgsql/tilecountServer.ts",
|
||||
"server:ldjson": "vite-node scripts/serverLdScrape.ts",
|
||||
"sever:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets",
|
||||
"server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/",
|
||||
"generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts"
|
||||
},
|
||||
"keywords": [
|
||||
|
|
|
@ -1,17 +1,36 @@
|
|||
import http from "node:http"
|
||||
|
||||
export interface Handler {
|
||||
mustMatch: string | RegExp
|
||||
mimetype: string
|
||||
addHeaders?: Record<string, string>
|
||||
handle: (path: string, queryParams: URLSearchParams, req: http.IncomingMessage) => Promise<string>
|
||||
|
||||
}
|
||||
|
||||
class ServerUtils {
|
||||
|
||||
public static getBody(req: http.IncomingMessage): Promise<string> {
|
||||
return new Promise<string>((resolve) => {
|
||||
let body = '';
|
||||
req.on('data', (chunk) => {
|
||||
body += chunk;
|
||||
});
|
||||
req.on('end', () => {
|
||||
resolve(body)
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class Server {
|
||||
constructor(
|
||||
port: number,
|
||||
options: {
|
||||
ignorePathPrefix?: string[]
|
||||
},
|
||||
handle: {
|
||||
mustMatch: string | RegExp
|
||||
mimetype: string
|
||||
addHeaders?: Record<string, string>
|
||||
handle: (path: string, queryParams: URLSearchParams) => Promise<string>
|
||||
}[]
|
||||
handle: Handler[]
|
||||
) {
|
||||
handle.push({
|
||||
mustMatch: "",
|
||||
|
@ -81,8 +100,9 @@ export class Server {
|
|||
res.end()
|
||||
return
|
||||
}
|
||||
let body = undefined
|
||||
if (req.method === "POST" || req.method === "UPDATE") {
|
||||
return
|
||||
body = await ServerUtils.getBody(req)
|
||||
}
|
||||
|
||||
if (req.method === "DELETE") {
|
||||
|
@ -90,7 +110,7 @@ export class Server {
|
|||
}
|
||||
|
||||
try {
|
||||
const result = await handler.handle(path, url.searchParams)
|
||||
const result = await handler.handle(path, url.searchParams, req, body)
|
||||
if (result === undefined) {
|
||||
res.writeHead(500)
|
||||
res.write("Could not fetch this website, probably blocked by them")
|
||||
|
|
52
scripts/serverErrorReport.ts
Normal file
52
scripts/serverErrorReport.ts
Normal file
|
@ -0,0 +1,52 @@
|
|||
import { Handler, Server } from "./server"
|
||||
import Script from "./Script"
|
||||
import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "fs"
|
||||
import { mkdir } from "node:fs"
|
||||
import ScriptUtils from "./ScriptUtils"
|
||||
|
||||
class ServerErrorReport extends Script {
|
||||
constructor() {
|
||||
super("A server which receives and logs error reports")
|
||||
}
|
||||
|
||||
async main(args: string[]): Promise<void> {
|
||||
const logDirectory = args[0] ?? "./error_logs"
|
||||
console.log("Logging to directory", logDirectory)
|
||||
if (!existsSync(logDirectory)) {
|
||||
mkdirSync(logDirectory)
|
||||
console.log("Created this directory")
|
||||
}
|
||||
let errorReport = 0
|
||||
new Server(2348, {},
|
||||
[<Handler>{
|
||||
mustMatch: "report",
|
||||
mimetype: "application/json",
|
||||
handle: async (_, queryParams, req, body) => {
|
||||
if (!body) {
|
||||
throw "{\"error\": \"No body; use a post request\"}"
|
||||
}
|
||||
console.log(body)
|
||||
const ip = <string>req.headers["x-forwarded-for"]
|
||||
const d = new Date()
|
||||
const date = d.toISOString()
|
||||
const file = logDirectory + "/" + d.getUTCFullYear() + "_" + d.getUTCMonth() + "_" + d.getUTCDay() + ".lines.json"
|
||||
try{
|
||||
body = JSON.parse(body)
|
||||
}catch (e) {
|
||||
// could not parse, we'll save it as is
|
||||
}
|
||||
const contents = "\n"+JSON.stringify({ ip, index: errorReport, date, message: body })
|
||||
if (!existsSync(file)) {
|
||||
writeFileSync(file, contents)
|
||||
} else {
|
||||
appendFileSync(file, contents)
|
||||
}
|
||||
errorReport++
|
||||
return `{"status":"ok", "nr": ${errorReport}}`
|
||||
},
|
||||
|
||||
}])
|
||||
}
|
||||
}
|
||||
|
||||
new ServerErrorReport().run()
|
Loading…
Reference in a new issue