Chore: linting
This commit is contained in:
parent
4625ad9a5c
commit
097141f944
307 changed files with 5346 additions and 2147 deletions
|
@ -15,7 +15,7 @@ export default abstract class Script {
|
|||
args.splice(0, 2)
|
||||
const start = new Date()
|
||||
this.main(args)
|
||||
.then((_) =>{
|
||||
.then((_) => {
|
||||
const end = new Date()
|
||||
const millisNeeded = end.getTime() - start.getTime()
|
||||
|
||||
|
|
|
@ -154,10 +154,10 @@ export default class ScriptUtils {
|
|||
headers?: any
|
||||
): Promise<{ content: string } | { redirect: string }> {
|
||||
console.log("Fetching", url)
|
||||
const req = await fetch(url, {headers})
|
||||
const data= await req.text()
|
||||
console.log("Fetched", url,data)
|
||||
return {content: data}
|
||||
const req = await fetch(url, { headers })
|
||||
const data = await req.text()
|
||||
console.log("Fetched", url, data)
|
||||
return { content: data }
|
||||
}
|
||||
public static Download(
|
||||
url: string,
|
||||
|
@ -221,7 +221,13 @@ export default class ScriptUtils {
|
|||
}
|
||||
})
|
||||
const timeoutPromise = new Promise<any>((resolve, reject) => {
|
||||
setTimeout(() => timeoutSecs === undefined ? reject(new Error("Timout reached")) : resolve("timeout"), (timeoutSecs ?? 10) * 1000)
|
||||
setTimeout(
|
||||
() =>
|
||||
timeoutSecs === undefined
|
||||
? reject(new Error("Timout reached"))
|
||||
: resolve("timeout"),
|
||||
(timeoutSecs ?? 10) * 1000
|
||||
)
|
||||
})
|
||||
return Promise.race([requestPromise, timeoutPromise])
|
||||
}
|
||||
|
|
|
@ -10,12 +10,9 @@ export default class DownloadLinkedDataList extends Script {
|
|||
async main([url, noProxy]: string[]): Promise<void> {
|
||||
const useProxy = noProxy !== "--no-proxy"
|
||||
const data = await LinkedDataLoader.fetchJsonLd(url, {}, useProxy)
|
||||
const path = "linked_data_"+url.replace(/[^a-zA-Z0-9_]/g, "_")+".jsonld"
|
||||
writeFileSync(path,
|
||||
JSON.stringify(data),
|
||||
"utf8"
|
||||
)
|
||||
console.log("Written",path)
|
||||
const path = "linked_data_" + url.replace(/[^a-zA-Z0-9_]/g, "_") + ".jsonld"
|
||||
writeFileSync(path, JSON.stringify(data), "utf8")
|
||||
console.log("Written", path)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -95,7 +95,7 @@ export default class GenerateImageAnalysis extends Script {
|
|||
if (fs.existsSync(targetPath)) {
|
||||
return false
|
||||
}
|
||||
const attribution = await Imgur.singleton.DownloadAttribution({ url: image, })
|
||||
const attribution = await Imgur.singleton.DownloadAttribution({ url: image })
|
||||
|
||||
if ((attribution.artist ?? "") === "") {
|
||||
// This is an invalid attribution. We save the raw response as well
|
||||
|
@ -451,7 +451,7 @@ export default class GenerateImageAnalysis extends Script {
|
|||
args = args.filter((a) => a !== "--cached")
|
||||
const datapath = args[1] ?? "../../git/MapComplete-data/ImageLicenseInfo"
|
||||
const imageBackupPath = args[0]
|
||||
if(imageBackupPath === "" || imageBackupPath === undefined){
|
||||
if (imageBackupPath === "" || imageBackupPath === undefined) {
|
||||
throw "No imageBackup path specified"
|
||||
}
|
||||
await this.downloadData(datapath, cached)
|
||||
|
|
|
@ -104,9 +104,9 @@ class GenerateLayouts extends Script {
|
|||
if (!layout.icon.endsWith(".svg")) {
|
||||
console.warn(
|
||||
"Not creating a social image for " +
|
||||
layout.id +
|
||||
" as it is _not_ a .svg: " +
|
||||
layout.icon
|
||||
layout.id +
|
||||
" as it is _not_ a .svg: " +
|
||||
layout.icon
|
||||
)
|
||||
return undefined
|
||||
}
|
||||
|
@ -139,9 +139,9 @@ class GenerateLayouts extends Script {
|
|||
id: "icon",
|
||||
transform: `translate(${cx - r},${cy - r}) scale(${
|
||||
(r * 2) / Number(width)
|
||||
}) `
|
||||
}) `,
|
||||
},
|
||||
g: [svg]
|
||||
g: [svg],
|
||||
}
|
||||
},
|
||||
(mightBeTokenToReplace) => {
|
||||
|
@ -202,19 +202,19 @@ class GenerateLayouts extends Script {
|
|||
icons.push({
|
||||
src: name,
|
||||
sizes: size + "x" + size,
|
||||
type: "image/png"
|
||||
type: "image/png",
|
||||
})
|
||||
}
|
||||
icons.push({
|
||||
src: path,
|
||||
sizes: "513x513",
|
||||
type: "image/svg"
|
||||
type: "image/svg",
|
||||
})
|
||||
} else if (icon.endsWith(".png")) {
|
||||
icons.push({
|
||||
src: icon,
|
||||
sizes: "513x513",
|
||||
type: "image/png"
|
||||
type: "image/png",
|
||||
})
|
||||
} else {
|
||||
console.log(icon)
|
||||
|
@ -233,11 +233,11 @@ class GenerateLayouts extends Script {
|
|||
description: ogDescr,
|
||||
orientation: "portrait-primary, landscape-primary",
|
||||
icons: icons,
|
||||
categories: ["map", "navigation"]
|
||||
categories: ["map", "navigation"],
|
||||
}
|
||||
return {
|
||||
manifest,
|
||||
whiteIcons
|
||||
whiteIcons,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -261,7 +261,7 @@ class GenerateLayouts extends Script {
|
|||
const rasterLayers = [
|
||||
AvailableRasterLayers.defaultBackgroundLayer,
|
||||
...eli.features,
|
||||
...eli_global.layers.map((properties) => ({ properties }))
|
||||
...eli_global.layers.map((properties) => ({ properties })),
|
||||
]
|
||||
for (const feature of rasterLayers) {
|
||||
const f = <RasterLayerPolygon>feature
|
||||
|
@ -282,9 +282,9 @@ class GenerateLayouts extends Script {
|
|||
url = url.substring("pmtiles://".length)
|
||||
}
|
||||
const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
|
||||
Origin: "https://mapcomplete.org"
|
||||
Origin: "https://mapcomplete.org",
|
||||
})
|
||||
urls.push(...(f.properties["connect-src"]??[]))
|
||||
urls.push(...(f.properties["connect-src"] ?? []))
|
||||
for (const key of Object.keys(styleSpec?.sources ?? {})) {
|
||||
const url = styleSpec.sources[key].url
|
||||
if (!url) {
|
||||
|
@ -298,7 +298,7 @@ class GenerateLayouts extends Script {
|
|||
urls.push(url)
|
||||
if (urlClipped.endsWith(".json")) {
|
||||
const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
|
||||
Origin: "https://mapcomplete.org"
|
||||
Origin: "https://mapcomplete.org",
|
||||
})
|
||||
urls.push(tileInfo["tiles"] ?? [])
|
||||
}
|
||||
|
@ -326,7 +326,7 @@ class GenerateLayouts extends Script {
|
|||
"https://www.openstreetmap.org",
|
||||
"https://api.openstreetmap.org",
|
||||
"https://pietervdvn.goatcounter.com",
|
||||
"https://cache.mapcomplete.org"
|
||||
"https://cache.mapcomplete.org",
|
||||
].concat(...(await this.eliUrls()))
|
||||
|
||||
SpecialVisualizations.specialVisualizations.forEach((sv) => {
|
||||
|
@ -368,12 +368,14 @@ class GenerateLayouts extends Script {
|
|||
const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
|
||||
const vectorSources = vectorLayers.map((l) => l.properties.url)
|
||||
vectorSources.push(...vectorLayers.map((l) => l.properties.style))
|
||||
apiUrls.push(...vectorSources.map(url => {
|
||||
if (url?.startsWith("pmtiles://")) {
|
||||
return url.substring("pmtiles://".length)
|
||||
}
|
||||
return url
|
||||
}))
|
||||
apiUrls.push(
|
||||
...vectorSources.map((url) => {
|
||||
if (url?.startsWith("pmtiles://")) {
|
||||
return url.substring("pmtiles://".length)
|
||||
}
|
||||
return url
|
||||
})
|
||||
)
|
||||
}
|
||||
for (let connectSource of apiUrls.concat(geojsonSources)) {
|
||||
if (!connectSource) {
|
||||
|
@ -418,8 +420,8 @@ class GenerateLayouts extends Script {
|
|||
"script-src": [
|
||||
"'self'",
|
||||
"https://gc.zgo.at/count.js",
|
||||
...(options?.scriptSrcs?.map((s) => "'" + s + "'") ?? [])
|
||||
].join(" ")
|
||||
...(options?.scriptSrcs?.map((s) => "'" + s + "'") ?? []),
|
||||
].join(" "),
|
||||
}
|
||||
const content = Object.keys(csp)
|
||||
.map((k) => k + " " + csp[k])
|
||||
|
@ -427,7 +429,7 @@ class GenerateLayouts extends Script {
|
|||
|
||||
return [
|
||||
`<meta http-equiv ="Report-To" content='{"group":"csp-endpoint", "max_age": 86400,"endpoints": [\{"url": "https://report.mapcomplete.org/csp"}], "include_subdomains": true}'>`,
|
||||
`<meta http-equiv="Content-Security-Policy" content="${content}">`
|
||||
`<meta http-equiv="Content-Security-Policy" content="${content}">`,
|
||||
].join("\n")
|
||||
}
|
||||
|
||||
|
@ -439,12 +441,12 @@ class GenerateLayouts extends Script {
|
|||
) {
|
||||
Locale.language.setData(layout.language[0])
|
||||
const targetLanguage = layout.language[0]
|
||||
const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, "\\\"")
|
||||
const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"')
|
||||
const ogDescr = Translations.T(
|
||||
layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap"
|
||||
)
|
||||
.textFor(targetLanguage)
|
||||
.replace(/"/g, "\\\"")
|
||||
.replace(/"/g, '\\"')
|
||||
let ogImage = layout.socialImage
|
||||
let twitterImage = ogImage
|
||||
if (ogImage === LayoutConfig.defaultSocialImage && layout.official) {
|
||||
|
@ -503,7 +505,7 @@ class GenerateLayouts extends Script {
|
|||
og,
|
||||
customCss,
|
||||
`<link rel="icon" href="${icon}" sizes="any" type="image/svg+xml">`,
|
||||
...apple_icons
|
||||
...apple_icons,
|
||||
].join("\n")
|
||||
|
||||
let branchname = await this.getBranchName()
|
||||
|
@ -526,7 +528,7 @@ class GenerateLayouts extends Script {
|
|||
.replace(
|
||||
/<!-- CSP -->/,
|
||||
await this.generateCsp(layout, layoutJson, {
|
||||
scriptSrcs: [this.removeOtherLanguagesHash]
|
||||
scriptSrcs: [this.removeOtherLanguagesHash],
|
||||
})
|
||||
)
|
||||
.replace(
|
||||
|
@ -557,7 +559,7 @@ class GenerateLayouts extends Script {
|
|||
|
||||
const imports = [
|
||||
`import layout from "./src/assets/generated/themes/${theme.id}.json"`,
|
||||
`import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`
|
||||
`import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`,
|
||||
]
|
||||
for (const layerName of Constants.added_by_default) {
|
||||
imports.push(
|
||||
|
@ -604,7 +606,7 @@ class GenerateLayouts extends Script {
|
|||
"account",
|
||||
"openstreetmap",
|
||||
"custom",
|
||||
"theme"
|
||||
"theme",
|
||||
]
|
||||
// @ts-ignore
|
||||
const all: LayoutConfigJson[] = all_known_layouts.themes
|
||||
|
@ -656,7 +658,7 @@ class GenerateLayouts extends Script {
|
|||
startLon: 0,
|
||||
startZoom: 0,
|
||||
title: { en: "MapComplete" },
|
||||
description: { en: "A thematic map viewer and editor based on OpenStreetMap" }
|
||||
description: { en: "A thematic map viewer and editor based on OpenStreetMap" },
|
||||
}),
|
||||
alreadyWritten
|
||||
)
|
||||
|
|
|
@ -8,9 +8,11 @@ export class GenerateSunnyUnlabeled extends Script {
|
|||
}
|
||||
|
||||
async main(args: string[]): Promise<void> {
|
||||
const unlabeled = { "#":"AUTOMATICALLY GENERATED! Do not edit.", ...sunny }
|
||||
unlabeled.name = unlabeled.name+"-unlabeled"
|
||||
unlabeled.layers = sunny.layers.filter(l => l.type !== "symbol" || !l.layout["text-field"])
|
||||
const unlabeled = { "#": "AUTOMATICALLY GENERATED! Do not edit.", ...sunny }
|
||||
unlabeled.name = unlabeled.name + "-unlabeled"
|
||||
unlabeled.layers = sunny.layers.filter(
|
||||
(l) => l.type !== "symbol" || !l.layout["text-field"]
|
||||
)
|
||||
writeFileSync("public/assets/sunny-unlabeled.json", JSON.stringify(unlabeled, null, " "))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,13 +6,15 @@ import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator"
|
|||
// vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/
|
||||
class CompareWebsiteData extends Script {
|
||||
constructor() {
|
||||
super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir")
|
||||
super(
|
||||
"Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir"
|
||||
)
|
||||
}
|
||||
|
||||
private readonly urlFormatter = new UrlValidator()
|
||||
async getWithCache(cachedir : string, url: string): Promise<any>{
|
||||
const filename= cachedir+"/"+encodeURIComponent(url)
|
||||
if(fs.existsSync(filename)){
|
||||
async getWithCache(cachedir: string, url: string): Promise<any> {
|
||||
const filename = cachedir + "/" + encodeURIComponent(url)
|
||||
if (fs.existsSync(filename)) {
|
||||
return JSON.parse(fs.readFileSync(filename, "utf-8"))
|
||||
}
|
||||
const jsonLd = await LinkedDataLoader.fetchJsonLd(url, undefined, true)
|
||||
|
@ -20,25 +22,24 @@ class CompareWebsiteData extends Script {
|
|||
fs.writeFileSync(filename, JSON.stringify(jsonLd))
|
||||
return jsonLd
|
||||
}
|
||||
async handleEntry(line: string, cachedir: string, targetfile: string) : Promise<boolean>{
|
||||
async handleEntry(line: string, cachedir: string, targetfile: string): Promise<boolean> {
|
||||
const id = JSON.parse(line.split(",")[0])
|
||||
let tags = line.substring(line.indexOf("{") - 1)
|
||||
tags = tags.substring(1, tags.length - 1)
|
||||
tags = tags.replace(/""/g, "\"")
|
||||
tags = tags.replace(/""/g, '"')
|
||||
const data = JSON.parse(tags)
|
||||
|
||||
try{
|
||||
|
||||
const website = this.urlFormatter.reformat(data.website)
|
||||
console.log(website)
|
||||
const jsonld = await this.getWithCache(cachedir, website)
|
||||
if(Object.keys(jsonld).length === 0){
|
||||
return false
|
||||
}
|
||||
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
|
||||
fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n\n")
|
||||
return true
|
||||
}catch (e) {
|
||||
try {
|
||||
const website = this.urlFormatter.reformat(data.website)
|
||||
console.log(website)
|
||||
const jsonld = await this.getWithCache(cachedir, website)
|
||||
if (Object.keys(jsonld).length === 0) {
|
||||
return false
|
||||
}
|
||||
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
|
||||
fs.appendFileSync(targetfile, id + ", " + JSON.stringify(diff) + "\n\n")
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error("Could not download ", data.website)
|
||||
}
|
||||
}
|
||||
|
@ -48,7 +49,6 @@ class CompareWebsiteData extends Script {
|
|||
throw "Not enough arguments"
|
||||
}
|
||||
|
||||
|
||||
const readInterface = readline.createInterface({
|
||||
input: fs.createReadStream(args[0]),
|
||||
})
|
||||
|
@ -59,20 +59,19 @@ class CompareWebsiteData extends Script {
|
|||
fs.writeFileSync(targetfile, "id, diff-json\n")
|
||||
for await (const line of readInterface) {
|
||||
try {
|
||||
if(line.startsWith("\"id\"")){
|
||||
if (line.startsWith('"id"')) {
|
||||
continue
|
||||
}
|
||||
const madeComparison = await this.handleEntry(line, args[1], targetfile)
|
||||
handled ++
|
||||
handled++
|
||||
diffed = diffed + (madeComparison ? 1 : 0)
|
||||
if(handled % 1000 == 0){
|
||||
console.log("Handled ",handled," got ",diffed,"diff results")
|
||||
if (handled % 1000 == 0) {
|
||||
console.log("Handled ", handled, " got ", diffed, "diff results")
|
||||
}
|
||||
} catch (e) {
|
||||
// console.error(e)
|
||||
// console.error(e)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -115,7 +115,7 @@ class OsmPoiDatabase {
|
|||
meta[property] = value
|
||||
}
|
||||
this.metaCacheDate = now
|
||||
this.metaCache = <PoiDatabaseMeta> meta
|
||||
this.metaCache = <PoiDatabaseMeta>meta
|
||||
return this.metaCache
|
||||
}
|
||||
}
|
||||
|
@ -161,10 +161,11 @@ class CachedSqlCount {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
class TileCountServer extends Script {
|
||||
constructor() {
|
||||
super("Starts the tilecount server which calculates summary for a given tile, based on the database. Usage: [db-connection-string] [port=2345]")
|
||||
super(
|
||||
"Starts the tilecount server which calculates summary for a given tile, based on the database. Usage: [db-connection-string] [port=2345]"
|
||||
)
|
||||
}
|
||||
|
||||
async main(args: string[]): Promise<void> {
|
||||
|
@ -226,7 +227,6 @@ class TileCountServer extends Script {
|
|||
[3.2839964396059145, 51.172701162680994],
|
||||
])
|
||||
)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -8,8 +8,8 @@ export class Server {
|
|||
},
|
||||
handle: {
|
||||
mustMatch: string | RegExp
|
||||
mimetype: string,
|
||||
addHeaders?: Record<string, string>,
|
||||
mimetype: string
|
||||
addHeaders?: Record<string, string>
|
||||
handle: (path: string, queryParams: URLSearchParams) => Promise<string>
|
||||
}[]
|
||||
) {
|
||||
|
@ -91,11 +91,18 @@ export class Server {
|
|||
|
||||
try {
|
||||
const result = await handler.handle(path, url.searchParams)
|
||||
if(typeof result !== "string"){
|
||||
console.error("Internal server error: handling", url,"resulted in a ",typeof result," instead of a string:", result)
|
||||
if (typeof result !== "string") {
|
||||
console.error(
|
||||
"Internal server error: handling",
|
||||
url,
|
||||
"resulted in a ",
|
||||
typeof result,
|
||||
" instead of a string:",
|
||||
result
|
||||
)
|
||||
}
|
||||
const extraHeaders = handler.addHeaders ?? {}
|
||||
res.writeHead(200, { "Content-Type": handler.mimetype , ...extraHeaders})
|
||||
res.writeHead(200, { "Content-Type": handler.mimetype, ...extraHeaders })
|
||||
res.write(result)
|
||||
res.end()
|
||||
} catch (e) {
|
||||
|
|
|
@ -16,7 +16,7 @@ class ServerLdScrape extends Script {
|
|||
mustMatch: "extractgraph",
|
||||
mimetype: "application/ld+json",
|
||||
addHeaders: {
|
||||
"Cache-control":"max-age=3600, public"
|
||||
"Cache-control": "max-age=3600, public",
|
||||
},
|
||||
async handle(content, searchParams: URLSearchParams) {
|
||||
const url = searchParams.get("url")
|
||||
|
@ -30,19 +30,24 @@ class ServerLdScrape extends Script {
|
|||
return JSON.stringify(contents)
|
||||
}
|
||||
}
|
||||
let dloaded: { content: string } | { redirect: string } | "timeout" = { redirect: url }
|
||||
let dloaded: { content: string } | { redirect: string } | "timeout" = {
|
||||
redirect: url,
|
||||
}
|
||||
do {
|
||||
|
||||
dloaded = await ScriptUtils.Download(dloaded["redirect"], {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
|
||||
}, 10)
|
||||
dloaded = await ScriptUtils.Download(
|
||||
dloaded["redirect"],
|
||||
{
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36", // MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete",
|
||||
},
|
||||
10
|
||||
)
|
||||
if (dloaded === "timeout") {
|
||||
return "{\"#\":\"timout reached\"}"
|
||||
return '{"#":"timout reached"}'
|
||||
}
|
||||
} while (dloaded["redirect"])
|
||||
|
||||
if(dloaded["content"].startsWith("{")){
|
||||
if (dloaded["content"].startsWith("{")) {
|
||||
// This is probably a json
|
||||
const snippet = JSON.parse(dloaded["content"])
|
||||
console.log("Snippet is", snippet)
|
||||
|
|
|
@ -30,7 +30,7 @@ class Compare extends Script {
|
|||
Object.keys(osmParking.properties).concat(Object.keys(veloParking.properties))
|
||||
)
|
||||
for (const key of allKeys) {
|
||||
if(["name","numberOfLevels"].indexOf(key) >= 0){
|
||||
if (["name", "numberOfLevels"].indexOf(key) >= 0) {
|
||||
continue // We don't care about these tags
|
||||
}
|
||||
if (osmParking.properties[key] === veloParking.properties[key]) {
|
||||
|
@ -45,10 +45,12 @@ class Compare extends Script {
|
|||
diffs.push({
|
||||
key,
|
||||
osm: osmParking.properties[key],
|
||||
velopark: veloParking.properties[key]
|
||||
velopark: veloParking.properties[key],
|
||||
})
|
||||
}
|
||||
let osmid = osmParking.properties["@id"] ?? osmParking["id"] /*Not in the properties, that is how overpass returns it*/
|
||||
let osmid =
|
||||
osmParking.properties["@id"] ??
|
||||
osmParking["id"] /*Not in the properties, that is how overpass returns it*/
|
||||
if (!osmid.startsWith("http")) {
|
||||
osmid = "https://openstreetmap.org/" + osmid
|
||||
}
|
||||
|
@ -57,7 +59,7 @@ class Compare extends Script {
|
|||
ref: veloId,
|
||||
osmid,
|
||||
distance,
|
||||
diffs
|
||||
diffs,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -82,13 +84,16 @@ class Compare extends Script {
|
|||
}
|
||||
diffs.push(this.compare(veloId, parking, veloparking))
|
||||
}
|
||||
console.log("Found ", diffs.length, " items with differences between OSM and the provided data")
|
||||
console.log(
|
||||
"Found ",
|
||||
diffs.length,
|
||||
" items with differences between OSM and the provided data"
|
||||
)
|
||||
|
||||
|
||||
const maxDistance = Math.max(...diffs.map(d => d.distance))
|
||||
const maxDistance = Math.max(...diffs.map((d) => d.distance))
|
||||
const distanceBins = []
|
||||
const binSize = 5
|
||||
for (let i = 0; i < Math.ceil(maxDistance / binSize) ; i++) {
|
||||
for (let i = 0; i < Math.ceil(maxDistance / binSize); i++) {
|
||||
distanceBins.push(0)
|
||||
}
|
||||
for (const diff of diffs) {
|
||||
|
|
|
@ -17,13 +17,16 @@ class VeloParkToGeojson extends Script {
|
|||
}
|
||||
|
||||
private static exportGeojsonTo(filename: string, features: Feature[], extension = ".geojson") {
|
||||
const file = filename + "_" + /*new Date().toISOString() + */extension
|
||||
fs.writeFileSync(file,
|
||||
const file = filename + "_" + /*new Date().toISOString() + */ extension
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
JSON.stringify(
|
||||
extension === ".geojson" ? {
|
||||
type: "FeatureCollection",
|
||||
features
|
||||
} : features,
|
||||
extension === ".geojson"
|
||||
? {
|
||||
type: "FeatureCollection",
|
||||
features,
|
||||
}
|
||||
: features,
|
||||
null,
|
||||
" "
|
||||
)
|
||||
|
@ -45,10 +48,12 @@ class VeloParkToGeojson extends Script {
|
|||
}
|
||||
addTo[k].add(data[k])
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static async downloadDataFor(url: string, allProperties: Record<string, Set<string>>): Promise<Feature[]> {
|
||||
private static async downloadDataFor(
|
||||
url: string,
|
||||
allProperties: Record<string, Set<string>>
|
||||
): Promise<Feature[]> {
|
||||
const cachePath = "/home/pietervdvn/data/velopark_cache/" + url.replace(/[/:.]/g, "_")
|
||||
if (!fs.existsSync(cachePath)) {
|
||||
const data = await Utils.downloadJson(url)
|
||||
|
@ -75,7 +80,7 @@ class VeloParkToGeojson extends Script {
|
|||
console.log("Downloading velopark data")
|
||||
// Download data for NIS-code 1000. 1000 means: all of belgium
|
||||
const url = "https://www.velopark.be/api/parkings/1000"
|
||||
const allVeloparkRaw: { url: string }[] = <{url: string}[]> await Utils.downloadJson(url)
|
||||
const allVeloparkRaw: { url: string }[] = <{ url: string }[]>await Utils.downloadJson(url)
|
||||
|
||||
let failed = 0
|
||||
console.log("Got", allVeloparkRaw.length, "items")
|
||||
|
@ -85,15 +90,22 @@ class VeloParkToGeojson extends Script {
|
|||
const f = allVeloparkRaw[i]
|
||||
console.log("Handling", i + "/" + allVeloparkRaw.length)
|
||||
try {
|
||||
const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(f.url, allProperties)
|
||||
const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(
|
||||
f.url,
|
||||
allProperties
|
||||
)
|
||||
allVelopark.push(...sections)
|
||||
} catch (e) {
|
||||
console.error("Loading ", f.url, " failed due to", e)
|
||||
failed++
|
||||
|
||||
}
|
||||
}
|
||||
console.log("Fetching data done, got ", allVelopark.length + "/" + allVeloparkRaw.length, "failed:", failed)
|
||||
console.log(
|
||||
"Fetching data done, got ",
|
||||
allVelopark.length + "/" + allVeloparkRaw.length,
|
||||
"failed:",
|
||||
failed
|
||||
)
|
||||
VeloParkToGeojson.exportGeojsonTo("velopark_all", allVelopark)
|
||||
for (const k in allProperties) {
|
||||
allProperties[k] = Array.from(allProperties[k])
|
||||
|
@ -104,15 +116,15 @@ class VeloParkToGeojson extends Script {
|
|||
return allVelopark
|
||||
}
|
||||
|
||||
private static loadFromFile(maxCacheAgeSeconds = 24*60*60): Feature[] | null {
|
||||
private static loadFromFile(maxCacheAgeSeconds = 24 * 60 * 60): Feature[] | null {
|
||||
const path = "velopark_all.geojson"
|
||||
if(!fs.existsSync(path)){
|
||||
if (!fs.existsSync(path)) {
|
||||
return null
|
||||
}
|
||||
// Millis since epoch
|
||||
const mtime : number = fs.statSync(path).mtime.getTime()
|
||||
const mtime: number = fs.statSync(path).mtime.getTime()
|
||||
const stalenessSeconds = (new Date().getTime() - mtime) / 1000
|
||||
if(stalenessSeconds > maxCacheAgeSeconds){
|
||||
if (stalenessSeconds > maxCacheAgeSeconds) {
|
||||
return null
|
||||
}
|
||||
|
||||
|
@ -146,7 +158,7 @@ class VeloParkToGeojson extends Script {
|
|||
private static async createDiff(allVelopark: Feature[]) {
|
||||
const bboxBelgium = new BBox([
|
||||
[2.51357303225, 49.5294835476],
|
||||
[6.15665815596, 51.4750237087]
|
||||
[6.15665815596, 51.4750237087],
|
||||
])
|
||||
|
||||
const alreadyLinkedQuery = new Overpass(
|
||||
|
@ -163,7 +175,9 @@ class VeloParkToGeojson extends Script {
|
|||
this.exportGeojsonTo("osm_with_velopark_link", <Feature[]>alreadyLinkedFeatures.features)
|
||||
console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")
|
||||
|
||||
const features: Feature[] = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))
|
||||
const features: Feature[] = allVelopark.filter(
|
||||
(f) => !seenIds.has(f.properties["ref:velopark"])
|
||||
)
|
||||
VeloParkToGeojson.exportGeojsonTo("velopark_nonsynced", features)
|
||||
|
||||
const allProperties = new Set<string>()
|
||||
|
@ -178,16 +192,17 @@ class VeloParkToGeojson extends Script {
|
|||
}
|
||||
|
||||
this.exportGeojsonTo("velopark_nonsynced_id_only", features)
|
||||
|
||||
|
||||
}
|
||||
|
||||
async main(): Promise<void> {
|
||||
const allVelopark =VeloParkToGeojson.loadFromFile() ?? await VeloParkToGeojson.downloadData()
|
||||
const allVelopark =
|
||||
VeloParkToGeojson.loadFromFile() ?? (await VeloParkToGeojson.downloadData())
|
||||
console.log("Got", allVelopark.length, " items")
|
||||
VeloParkToGeojson.exportExtraAmenities(allVelopark)
|
||||
await VeloParkToGeojson.createDiff(allVelopark)
|
||||
console.log("Use vite-node script/velopark/compare to compare the results and generate a diff file")
|
||||
console.log(
|
||||
"Use vite-node script/velopark/compare to compare the results and generate a diff file"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue