Chore: linting

This commit is contained in:
Pieter Vander Vennet 2024-04-13 02:40:21 +02:00
parent 4625ad9a5c
commit 097141f944
307 changed files with 5346 additions and 2147 deletions

View file

@ -15,7 +15,7 @@ export default abstract class Script {
args.splice(0, 2)
const start = new Date()
this.main(args)
.then((_) =>{
.then((_) => {
const end = new Date()
const millisNeeded = end.getTime() - start.getTime()

View file

@ -154,10 +154,10 @@ export default class ScriptUtils {
headers?: any
): Promise<{ content: string } | { redirect: string }> {
console.log("Fetching", url)
const req = await fetch(url, {headers})
const data= await req.text()
console.log("Fetched", url,data)
return {content: data}
const req = await fetch(url, { headers })
const data = await req.text()
console.log("Fetched", url, data)
return { content: data }
}
public static Download(
url: string,
@ -221,7 +221,13 @@ export default class ScriptUtils {
}
})
const timeoutPromise = new Promise<any>((resolve, reject) => {
setTimeout(() => timeoutSecs === undefined ? reject(new Error("Timout reached")) : resolve("timeout"), (timeoutSecs ?? 10) * 1000)
setTimeout(
() =>
timeoutSecs === undefined
? reject(new Error("Timout reached"))
: resolve("timeout"),
(timeoutSecs ?? 10) * 1000
)
})
return Promise.race([requestPromise, timeoutPromise])
}

View file

@ -10,12 +10,9 @@ export default class DownloadLinkedDataList extends Script {
async main([url, noProxy]: string[]): Promise<void> {
const useProxy = noProxy !== "--no-proxy"
const data = await LinkedDataLoader.fetchJsonLd(url, {}, useProxy)
const path = "linked_data_"+url.replace(/[^a-zA-Z0-9_]/g, "_")+".jsonld"
writeFileSync(path,
JSON.stringify(data),
"utf8"
)
console.log("Written",path)
const path = "linked_data_" + url.replace(/[^a-zA-Z0-9_]/g, "_") + ".jsonld"
writeFileSync(path, JSON.stringify(data), "utf8")
console.log("Written", path)
}
}

View file

@ -95,7 +95,7 @@ export default class GenerateImageAnalysis extends Script {
if (fs.existsSync(targetPath)) {
return false
}
const attribution = await Imgur.singleton.DownloadAttribution({ url: image, })
const attribution = await Imgur.singleton.DownloadAttribution({ url: image })
if ((attribution.artist ?? "") === "") {
// This is an invalid attribution. We save the raw response as well
@ -451,7 +451,7 @@ export default class GenerateImageAnalysis extends Script {
args = args.filter((a) => a !== "--cached")
const datapath = args[1] ?? "../../git/MapComplete-data/ImageLicenseInfo"
const imageBackupPath = args[0]
if(imageBackupPath === "" || imageBackupPath === undefined){
if (imageBackupPath === "" || imageBackupPath === undefined) {
throw "No imageBackup path specified"
}
await this.downloadData(datapath, cached)

View file

@ -104,9 +104,9 @@ class GenerateLayouts extends Script {
if (!layout.icon.endsWith(".svg")) {
console.warn(
"Not creating a social image for " +
layout.id +
" as it is _not_ a .svg: " +
layout.icon
layout.id +
" as it is _not_ a .svg: " +
layout.icon
)
return undefined
}
@ -139,9 +139,9 @@ class GenerateLayouts extends Script {
id: "icon",
transform: `translate(${cx - r},${cy - r}) scale(${
(r * 2) / Number(width)
}) `
}) `,
},
g: [svg]
g: [svg],
}
},
(mightBeTokenToReplace) => {
@ -202,19 +202,19 @@ class GenerateLayouts extends Script {
icons.push({
src: name,
sizes: size + "x" + size,
type: "image/png"
type: "image/png",
})
}
icons.push({
src: path,
sizes: "513x513",
type: "image/svg"
type: "image/svg",
})
} else if (icon.endsWith(".png")) {
icons.push({
src: icon,
sizes: "513x513",
type: "image/png"
type: "image/png",
})
} else {
console.log(icon)
@ -233,11 +233,11 @@ class GenerateLayouts extends Script {
description: ogDescr,
orientation: "portrait-primary, landscape-primary",
icons: icons,
categories: ["map", "navigation"]
categories: ["map", "navigation"],
}
return {
manifest,
whiteIcons
whiteIcons,
}
}
@ -261,7 +261,7 @@ class GenerateLayouts extends Script {
const rasterLayers = [
AvailableRasterLayers.defaultBackgroundLayer,
...eli.features,
...eli_global.layers.map((properties) => ({ properties }))
...eli_global.layers.map((properties) => ({ properties })),
]
for (const feature of rasterLayers) {
const f = <RasterLayerPolygon>feature
@ -282,9 +282,9 @@ class GenerateLayouts extends Script {
url = url.substring("pmtiles://".length)
}
const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
Origin: "https://mapcomplete.org"
Origin: "https://mapcomplete.org",
})
urls.push(...(f.properties["connect-src"]??[]))
urls.push(...(f.properties["connect-src"] ?? []))
for (const key of Object.keys(styleSpec?.sources ?? {})) {
const url = styleSpec.sources[key].url
if (!url) {
@ -298,7 +298,7 @@ class GenerateLayouts extends Script {
urls.push(url)
if (urlClipped.endsWith(".json")) {
const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
Origin: "https://mapcomplete.org"
Origin: "https://mapcomplete.org",
})
urls.push(tileInfo["tiles"] ?? [])
}
@ -326,7 +326,7 @@ class GenerateLayouts extends Script {
"https://www.openstreetmap.org",
"https://api.openstreetmap.org",
"https://pietervdvn.goatcounter.com",
"https://cache.mapcomplete.org"
"https://cache.mapcomplete.org",
].concat(...(await this.eliUrls()))
SpecialVisualizations.specialVisualizations.forEach((sv) => {
@ -368,12 +368,14 @@ class GenerateLayouts extends Script {
const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
const vectorSources = vectorLayers.map((l) => l.properties.url)
vectorSources.push(...vectorLayers.map((l) => l.properties.style))
apiUrls.push(...vectorSources.map(url => {
if (url?.startsWith("pmtiles://")) {
return url.substring("pmtiles://".length)
}
return url
}))
apiUrls.push(
...vectorSources.map((url) => {
if (url?.startsWith("pmtiles://")) {
return url.substring("pmtiles://".length)
}
return url
})
)
}
for (let connectSource of apiUrls.concat(geojsonSources)) {
if (!connectSource) {
@ -418,8 +420,8 @@ class GenerateLayouts extends Script {
"script-src": [
"'self'",
"https://gc.zgo.at/count.js",
...(options?.scriptSrcs?.map((s) => "'" + s + "'") ?? [])
].join(" ")
...(options?.scriptSrcs?.map((s) => "'" + s + "'") ?? []),
].join(" "),
}
const content = Object.keys(csp)
.map((k) => k + " " + csp[k])
@ -427,7 +429,7 @@ class GenerateLayouts extends Script {
return [
`<meta http-equiv ="Report-To" content='{"group":"csp-endpoint", "max_age": 86400,"endpoints": [\{"url": "https://report.mapcomplete.org/csp"}], "include_subdomains": true}'>`,
`<meta http-equiv="Content-Security-Policy" content="${content}">`
`<meta http-equiv="Content-Security-Policy" content="${content}">`,
].join("\n")
}
@ -439,12 +441,12 @@ class GenerateLayouts extends Script {
) {
Locale.language.setData(layout.language[0])
const targetLanguage = layout.language[0]
const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, "\\\"")
const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"')
const ogDescr = Translations.T(
layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap"
)
.textFor(targetLanguage)
.replace(/"/g, "\\\"")
.replace(/"/g, '\\"')
let ogImage = layout.socialImage
let twitterImage = ogImage
if (ogImage === LayoutConfig.defaultSocialImage && layout.official) {
@ -503,7 +505,7 @@ class GenerateLayouts extends Script {
og,
customCss,
`<link rel="icon" href="${icon}" sizes="any" type="image/svg+xml">`,
...apple_icons
...apple_icons,
].join("\n")
let branchname = await this.getBranchName()
@ -526,7 +528,7 @@ class GenerateLayouts extends Script {
.replace(
/<!-- CSP -->/,
await this.generateCsp(layout, layoutJson, {
scriptSrcs: [this.removeOtherLanguagesHash]
scriptSrcs: [this.removeOtherLanguagesHash],
})
)
.replace(
@ -557,7 +559,7 @@ class GenerateLayouts extends Script {
const imports = [
`import layout from "./src/assets/generated/themes/${theme.id}.json"`,
`import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`
`import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`,
]
for (const layerName of Constants.added_by_default) {
imports.push(
@ -604,7 +606,7 @@ class GenerateLayouts extends Script {
"account",
"openstreetmap",
"custom",
"theme"
"theme",
]
// @ts-ignore
const all: LayoutConfigJson[] = all_known_layouts.themes
@ -656,7 +658,7 @@ class GenerateLayouts extends Script {
startLon: 0,
startZoom: 0,
title: { en: "MapComplete" },
description: { en: "A thematic map viewer and editor based on OpenStreetMap" }
description: { en: "A thematic map viewer and editor based on OpenStreetMap" },
}),
alreadyWritten
)

View file

@ -8,9 +8,11 @@ export class GenerateSunnyUnlabeled extends Script {
}
async main(args: string[]): Promise<void> {
const unlabeled = { "#":"AUTOMATICALLY GENERATED! Do not edit.", ...sunny }
unlabeled.name = unlabeled.name+"-unlabeled"
unlabeled.layers = sunny.layers.filter(l => l.type !== "symbol" || !l.layout["text-field"])
const unlabeled = { "#": "AUTOMATICALLY GENERATED! Do not edit.", ...sunny }
unlabeled.name = unlabeled.name + "-unlabeled"
unlabeled.layers = sunny.layers.filter(
(l) => l.type !== "symbol" || !l.layout["text-field"]
)
writeFileSync("public/assets/sunny-unlabeled.json", JSON.stringify(unlabeled, null, " "))
}
}

View file

@ -6,13 +6,15 @@ import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator"
// vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/
class CompareWebsiteData extends Script {
constructor() {
super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir")
super(
"Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir"
)
}
private readonly urlFormatter = new UrlValidator()
async getWithCache(cachedir : string, url: string): Promise<any>{
const filename= cachedir+"/"+encodeURIComponent(url)
if(fs.existsSync(filename)){
async getWithCache(cachedir: string, url: string): Promise<any> {
const filename = cachedir + "/" + encodeURIComponent(url)
if (fs.existsSync(filename)) {
return JSON.parse(fs.readFileSync(filename, "utf-8"))
}
const jsonLd = await LinkedDataLoader.fetchJsonLd(url, undefined, true)
@ -20,25 +22,24 @@ class CompareWebsiteData extends Script {
fs.writeFileSync(filename, JSON.stringify(jsonLd))
return jsonLd
}
async handleEntry(line: string, cachedir: string, targetfile: string) : Promise<boolean>{
async handleEntry(line: string, cachedir: string, targetfile: string): Promise<boolean> {
const id = JSON.parse(line.split(",")[0])
let tags = line.substring(line.indexOf("{") - 1)
tags = tags.substring(1, tags.length - 1)
tags = tags.replace(/""/g, "\"")
tags = tags.replace(/""/g, '"')
const data = JSON.parse(tags)
try{
const website = this.urlFormatter.reformat(data.website)
console.log(website)
const jsonld = await this.getWithCache(cachedir, website)
if(Object.keys(jsonld).length === 0){
return false
}
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n\n")
return true
}catch (e) {
try {
const website = this.urlFormatter.reformat(data.website)
console.log(website)
const jsonld = await this.getWithCache(cachedir, website)
if (Object.keys(jsonld).length === 0) {
return false
}
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
fs.appendFileSync(targetfile, id + ", " + JSON.stringify(diff) + "\n\n")
return true
} catch (e) {
console.error("Could not download ", data.website)
}
}
@ -48,7 +49,6 @@ class CompareWebsiteData extends Script {
throw "Not enough arguments"
}
const readInterface = readline.createInterface({
input: fs.createReadStream(args[0]),
})
@ -59,20 +59,19 @@ class CompareWebsiteData extends Script {
fs.writeFileSync(targetfile, "id, diff-json\n")
for await (const line of readInterface) {
try {
if(line.startsWith("\"id\"")){
if (line.startsWith('"id"')) {
continue
}
const madeComparison = await this.handleEntry(line, args[1], targetfile)
handled ++
handled++
diffed = diffed + (madeComparison ? 1 : 0)
if(handled % 1000 == 0){
console.log("Handled ",handled," got ",diffed,"diff results")
if (handled % 1000 == 0) {
console.log("Handled ", handled, " got ", diffed, "diff results")
}
} catch (e) {
// console.error(e)
// console.error(e)
}
}
}
}

View file

@ -115,7 +115,7 @@ class OsmPoiDatabase {
meta[property] = value
}
this.metaCacheDate = now
this.metaCache = <PoiDatabaseMeta> meta
this.metaCache = <PoiDatabaseMeta>meta
return this.metaCache
}
}
@ -161,10 +161,11 @@ class CachedSqlCount {
}
}
class TileCountServer extends Script {
constructor() {
super("Starts the tilecount server which calculates summary for a given tile, based on the database. Usage: [db-connection-string] [port=2345]")
super(
"Starts the tilecount server which calculates summary for a given tile, based on the database. Usage: [db-connection-string] [port=2345]"
)
}
async main(args: string[]): Promise<void> {
@ -226,7 +227,6 @@ class TileCountServer extends Script {
[3.2839964396059145, 51.172701162680994],
])
)
}
}

View file

@ -8,8 +8,8 @@ export class Server {
},
handle: {
mustMatch: string | RegExp
mimetype: string,
addHeaders?: Record<string, string>,
mimetype: string
addHeaders?: Record<string, string>
handle: (path: string, queryParams: URLSearchParams) => Promise<string>
}[]
) {
@ -91,11 +91,18 @@ export class Server {
try {
const result = await handler.handle(path, url.searchParams)
if(typeof result !== "string"){
console.error("Internal server error: handling", url,"resulted in a ",typeof result," instead of a string:", result)
if (typeof result !== "string") {
console.error(
"Internal server error: handling",
url,
"resulted in a ",
typeof result,
" instead of a string:",
result
)
}
const extraHeaders = handler.addHeaders ?? {}
res.writeHead(200, { "Content-Type": handler.mimetype , ...extraHeaders})
res.writeHead(200, { "Content-Type": handler.mimetype, ...extraHeaders })
res.write(result)
res.end()
} catch (e) {

View file

@ -16,7 +16,7 @@ class ServerLdScrape extends Script {
mustMatch: "extractgraph",
mimetype: "application/ld+json",
addHeaders: {
"Cache-control":"max-age=3600, public"
"Cache-control": "max-age=3600, public",
},
async handle(content, searchParams: URLSearchParams) {
const url = searchParams.get("url")
@ -30,19 +30,24 @@ class ServerLdScrape extends Script {
return JSON.stringify(contents)
}
}
let dloaded: { content: string } | { redirect: string } | "timeout" = { redirect: url }
let dloaded: { content: string } | { redirect: string } | "timeout" = {
redirect: url,
}
do {
dloaded = await ScriptUtils.Download(dloaded["redirect"], {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
}, 10)
dloaded = await ScriptUtils.Download(
dloaded["redirect"],
{
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
},
10
)
if (dloaded === "timeout") {
return "{\"#\":\"timout reached\"}"
return '{"#":"timout reached"}'
}
} while (dloaded["redirect"])
if(dloaded["content"].startsWith("{")){
if (dloaded["content"].startsWith("{")) {
// This is probably a json
const snippet = JSON.parse(dloaded["content"])
console.log("Snippet is", snippet)

View file

@ -30,7 +30,7 @@ class Compare extends Script {
Object.keys(osmParking.properties).concat(Object.keys(veloParking.properties))
)
for (const key of allKeys) {
if(["name","numberOfLevels"].indexOf(key) >= 0){
if (["name", "numberOfLevels"].indexOf(key) >= 0) {
continue // We don't care about these tags
}
if (osmParking.properties[key] === veloParking.properties[key]) {
@ -45,10 +45,12 @@ class Compare extends Script {
diffs.push({
key,
osm: osmParking.properties[key],
velopark: veloParking.properties[key]
velopark: veloParking.properties[key],
})
}
let osmid = osmParking.properties["@id"] ?? osmParking["id"] /*Not in the properties, that is how overpass returns it*/
let osmid =
osmParking.properties["@id"] ??
osmParking["id"] /*Not in the properties, that is how overpass returns it*/
if (!osmid.startsWith("http")) {
osmid = "https://openstreetmap.org/" + osmid
}
@ -57,7 +59,7 @@ class Compare extends Script {
ref: veloId,
osmid,
distance,
diffs
diffs,
}
}
@ -82,13 +84,16 @@ class Compare extends Script {
}
diffs.push(this.compare(veloId, parking, veloparking))
}
console.log("Found ", diffs.length, " items with differences between OSM and the provided data")
console.log(
"Found ",
diffs.length,
" items with differences between OSM and the provided data"
)
const maxDistance = Math.max(...diffs.map(d => d.distance))
const maxDistance = Math.max(...diffs.map((d) => d.distance))
const distanceBins = []
const binSize = 5
for (let i = 0; i < Math.ceil(maxDistance / binSize) ; i++) {
for (let i = 0; i < Math.ceil(maxDistance / binSize); i++) {
distanceBins.push(0)
}
for (const diff of diffs) {

View file

@ -17,13 +17,16 @@ class VeloParkToGeojson extends Script {
}
private static exportGeojsonTo(filename: string, features: Feature[], extension = ".geojson") {
const file = filename + "_" + /*new Date().toISOString() + */extension
fs.writeFileSync(file,
const file = filename + "_" + /*new Date().toISOString() + */ extension
fs.writeFileSync(
file,
JSON.stringify(
extension === ".geojson" ? {
type: "FeatureCollection",
features
} : features,
extension === ".geojson"
? {
type: "FeatureCollection",
features,
}
: features,
null,
" "
)
@ -45,10 +48,12 @@ class VeloParkToGeojson extends Script {
}
addTo[k].add(data[k])
}
}
private static async downloadDataFor(url: string, allProperties: Record<string, Set<string>>): Promise<Feature[]> {
private static async downloadDataFor(
url: string,
allProperties: Record<string, Set<string>>
): Promise<Feature[]> {
const cachePath = "/home/pietervdvn/data/velopark_cache/" + url.replace(/[/:.]/g, "_")
if (!fs.existsSync(cachePath)) {
const data = await Utils.downloadJson(url)
@ -75,7 +80,7 @@ class VeloParkToGeojson extends Script {
console.log("Downloading velopark data")
// Download data for NIS-code 1000. 1000 means: all of belgium
const url = "https://www.velopark.be/api/parkings/1000"
const allVeloparkRaw: { url: string }[] = <{url: string}[]> await Utils.downloadJson(url)
const allVeloparkRaw: { url: string }[] = <{ url: string }[]>await Utils.downloadJson(url)
let failed = 0
console.log("Got", allVeloparkRaw.length, "items")
@ -85,15 +90,22 @@ class VeloParkToGeojson extends Script {
const f = allVeloparkRaw[i]
console.log("Handling", i + "/" + allVeloparkRaw.length)
try {
const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(f.url, allProperties)
const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(
f.url,
allProperties
)
allVelopark.push(...sections)
} catch (e) {
console.error("Loading ", f.url, " failed due to", e)
failed++
}
}
console.log("Fetching data done, got ", allVelopark.length + "/" + allVeloparkRaw.length, "failed:", failed)
console.log(
"Fetching data done, got ",
allVelopark.length + "/" + allVeloparkRaw.length,
"failed:",
failed
)
VeloParkToGeojson.exportGeojsonTo("velopark_all", allVelopark)
for (const k in allProperties) {
allProperties[k] = Array.from(allProperties[k])
@ -104,15 +116,15 @@ class VeloParkToGeojson extends Script {
return allVelopark
}
private static loadFromFile(maxCacheAgeSeconds = 24*60*60): Feature[] | null {
private static loadFromFile(maxCacheAgeSeconds = 24 * 60 * 60): Feature[] | null {
const path = "velopark_all.geojson"
if(!fs.existsSync(path)){
if (!fs.existsSync(path)) {
return null
}
// Millis since epoch
const mtime : number = fs.statSync(path).mtime.getTime()
const mtime: number = fs.statSync(path).mtime.getTime()
const stalenessSeconds = (new Date().getTime() - mtime) / 1000
if(stalenessSeconds > maxCacheAgeSeconds){
if (stalenessSeconds > maxCacheAgeSeconds) {
return null
}
@ -146,7 +158,7 @@ class VeloParkToGeojson extends Script {
private static async createDiff(allVelopark: Feature[]) {
const bboxBelgium = new BBox([
[2.51357303225, 49.5294835476],
[6.15665815596, 51.4750237087]
[6.15665815596, 51.4750237087],
])
const alreadyLinkedQuery = new Overpass(
@ -163,7 +175,9 @@ class VeloParkToGeojson extends Script {
this.exportGeojsonTo("osm_with_velopark_link", <Feature[]>alreadyLinkedFeatures.features)
console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")
const features: Feature[] = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))
const features: Feature[] = allVelopark.filter(
(f) => !seenIds.has(f.properties["ref:velopark"])
)
VeloParkToGeojson.exportGeojsonTo("velopark_nonsynced", features)
const allProperties = new Set<string>()
@ -178,16 +192,17 @@ class VeloParkToGeojson extends Script {
}
this.exportGeojsonTo("velopark_nonsynced_id_only", features)
}
async main(): Promise<void> {
const allVelopark =VeloParkToGeojson.loadFromFile() ?? await VeloParkToGeojson.downloadData()
const allVelopark =
VeloParkToGeojson.loadFromFile() ?? (await VeloParkToGeojson.downloadData())
console.log("Got", allVelopark.length, " items")
VeloParkToGeojson.exportExtraAmenities(allVelopark)
await VeloParkToGeojson.createDiff(allVelopark)
console.log("Use vite-node script/velopark/compare to compare the results and generate a diff file")
console.log(
"Use vite-node script/velopark/compare to compare the results and generate a diff file"
)
}
}