midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Velopark: first decent, working version

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-04-05 17:49:31 +02:00
parent 890816d2dd
commit 5b6cd1d2ae
18 changed files with 7054 additions and 21769 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/ScriptUtils.ts
View file

@ -176,7 +176,7 @@ export default class ScriptUtils {
const requestPromise = new Promise((resolve, reject) => {
try {
headers = headers ?? {}
headers.accept = "application/json"
headers.accept ??= "application/json"
console.log(" > ScriptUtils.Download(", url, ")")
const urlObj = new URL(url)
const request = https.get(

22
scripts/downloadLinkedDataList.ts Normal file
View file

@ -0,0 +1,22 @@
import Script from "./Script"
import LinkedDataLoader from "../src/Logic/Web/LinkedDataLoader"
import { writeFileSync } from "fs"
export default class DownloadLinkedDataList extends Script {
constructor() {
super("Downloads the localBusinesses from the given location. Usage: url [--no-proxy]")
}
async main([url, noProxy]: string[]): Promise<void> {
const useProxy = noProxy !== "--no-proxy"
const data = await LinkedDataLoader.fetchJsonLd(url, {}, useProxy)
const path = "linked_data_"+url.replace(/[^a-zA-Z0-9_]/g, "_")+".jsonld"
writeFileSync(path,
JSON.stringify(data),
"utf8"
)
console.log("Written",path)
}
}
new DownloadLinkedDataList().run()

2
scripts/importscripts/compareWebsiteData.ts
View file

@ -15,7 +15,7 @@ class CompareWebsiteData extends Script {
if(fs.existsSync(filename)){
return JSON.parse(fs.readFileSync(filename, "utf-8"))
}
const jsonLd = await LinkedDataLoader.fetchJsonLdWithProxy(url)
const jsonLd = await LinkedDataLoader.fetchJsonLd(url, undefined, true)
console.log("Got:", jsonLd)
fs.writeFileSync(filename, JSON.stringify(jsonLd))
return jsonLd

23
scripts/server.ts
View file

@ -8,7 +8,8 @@ export class Server {
},
handle: {
mustMatch: string | RegExp
mimetype: string
mimetype: string,
addHeaders?: Record<string, string>,
handle: (path: string, queryParams: URLSearchParams) => Promise<string>
}[]
) {
@ -30,18 +31,18 @@ export class Server {
})
http.createServer(async (req: http.IncomingMessage, res) => {
try {
console.log(
req.method + " " + req.url,
"from:",
req.headers.origin,
new Date().toISOString()
)
const url = new URL(`http://127.0.0.1/` + req.url)
let path = url.pathname
while (path.startsWith("/")) {
path = path.substring(1)
}
console.log(
req.method + " " + req.url,
"from:",
req.headers.origin,
new Date().toISOString(),
path
)
if (options?.ignorePathPrefix) {
for (const toIgnore of options.ignorePathPrefix) {
if (path.startsWith(toIgnore)) {
@ -90,7 +91,11 @@ export class Server {
try {
const result = await handler.handle(path, url.searchParams)
res.writeHead(200, { "Content-Type": handler.mimetype })
if(typeof result !== "string"){
console.error("Internal server error: handling", url,"resulted in a ",typeof result," instead of a string:", result)
}
const extraHeaders = handler.addHeaders ?? {}
res.writeHead(200, { "Content-Type": handler.mimetype , ...extraHeaders})
res.write(result)
res.end()
} catch (e) {

13
scripts/serverLdScrape.ts
View file

@ -15,8 +15,12 @@ class ServerLdScrape extends Script {
{
mustMatch: "extractgraph",
mimetype: "application/ld+json",
addHeaders: {
"Cache-control":"max-age=3600, public"
},
async handle(content, searchParams: URLSearchParams) {
const url = searchParams.get("url")
console.log("URL", url)
if (cache[url] !== undefined) {
const { date, contents } = cache[url]
console.log(">>>", date, contents)
@ -37,6 +41,15 @@ class ServerLdScrape extends Script {
return "{\"#\":\"timout reached\"}"
}
} while (dloaded["redirect"])
if(dloaded["content"].startsWith("{")){
// This is probably a json
const snippet = JSON.parse(dloaded["content"])
console.log("Snippet is", snippet)
cache[url] = { contents: snippet, date: new Date() }
return JSON.stringify(snippet)
}
const parsed = parse(dloaded["content"])
const scripts = Array.from(parsed.getElementsByTagName("script"))
for (const script of scripts) {

26
scripts/velopark/compare.ts
View file

@ -2,8 +2,8 @@ import Script from "../Script"
import fs from "fs"
import { Feature, FeatureCollection } from "geojson"
import { GeoOperations } from "../../src/Logic/GeoOperations"
import * as os from "os"
// vite-node scripts/velopark/compare.ts -- scripts/velopark/velopark_all_2024-02-14T12\:18\:41.772Z.geojson ~/Projecten/OSM/Fietsberaad/2024-02-02\ Fietsenstallingen_OSM_met_velopark_ref.geojson
// vite-node scripts/velopark/compare.ts -- scripts/velopark/velopark_all.geojson osm_with_velopark_link_.geojson
class Compare extends Script {
compare(
veloId: string,
@ -30,6 +30,9 @@ class Compare extends Script {
Object.keys(osmParking.properties).concat(Object.keys(veloParking.properties))
)
for (const key of allKeys) {
if(["name","numberOfLevels"].indexOf(key) >= 0){
continue // We don't care about these tags
}
if (osmParking.properties[key] === veloParking.properties[key]) {
continue
}
@ -42,16 +45,22 @@ class Compare extends Script {
diffs.push({
key,
osm: osmParking.properties[key],
velopark: veloParking.properties[key],
velopark: veloParking.properties[key]
})
}
let osmid = osmParking.properties["@id"] ?? osmParking["id"] /*Not in the properties, that is how overpass returns it*/
if (!osmid.startsWith("http")) {
osmid = "https://openstreetmap.org/" + osmid
}
return {
ref: veloId,
osmid: osmParking.properties["@id"],
osmid,
distance,
diffs,
diffs
}
}
async main(args: string[]): Promise<void> {
let [velopark, osm, key] = args
key ??= "ref:velopark"
@ -60,7 +69,7 @@ class Compare extends Script {
const veloparkById: Record<string, Feature> = {}
for (const parking of veloparkData.features) {
veloparkById[parking.properties[key]] = parking
veloparkById[parking.properties[key] ?? parking.properties.url] = parking
}
const diffs = []
@ -73,9 +82,12 @@ class Compare extends Script {
}
diffs.push(this.compare(veloId, parking, veloparking))
}
console.log("Found ", diffs.length, " items with differences between OSM and the provided data")
fs.writeFileSync("report_diff.json", JSON.stringify(diffs))
fs.writeFileSync("report_diff.json", JSON.stringify(diffs, null, " "))
console.log("Written report_diff.json")
}
constructor() {
super(
"Compares a velopark geojson with OSM geojson. Usage: `compare velopark.geojson osm.geojson [key-to-compare-on]`. If key-to-compare-on is not given, `ref:velopark` will be used"

152
scripts/velopark/veloParkToGeojson.ts
View file

@ -1,75 +1,175 @@
import Script from "../Script"
import fs from "fs"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
import { Utils } from "../../src/Utils"
import { Feature } from "geojson"
import { BBox } from "../../src/Logic/BBox"
import { Overpass } from "../../src/Logic/Osm/Overpass"
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import Constants from "../../src/Models/Constants"
import { ImmutableStore } from "../../src/Logic/UIEventSource"
import { BBox } from "../../src/Logic/BBox"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
import Constants from "../../src/Models/Constants"
class VeloParkToGeojson extends Script {
constructor() {
super(
"Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory",
"Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory"
)
}
exportTo(filename: string, features) {
features = features.slice(0,25) // TODO REMOVE
const file = filename + "_" + /*new Date().toISOString() + */".geojson"
private static exportGeojsonTo(filename: string, features: Feature[], extension = ".geojson") {
const file = filename + "_" + /*new Date().toISOString() + */extension
fs.writeFileSync(file,
JSON.stringify(
{
extension === ".geojson" ? {
type: "FeatureCollection",
"#":"Only 25 features are shown!", // TODO REMOVE
features,
},
features
} : features,
null,
" ",
),
" "
)
)
console.log("Written",file)
console.log("Written", file, "("+features.length," features)")
}
async main(args: string[]): Promise<void> {
public static sumProperties(data: object, addTo: Record<string, Set<string>>) {
delete data["@context"]
for (const k in data) {
if (k === "@graph") {
for (const obj of data["@graph"]) {
this.sumProperties(obj, addTo)
}
continue
}
if (addTo[k] === undefined) {
addTo[k] = new Set<string>()
}
addTo[k].add(data[k])
}
}
private static async downloadData() {
console.log("Downloading velopark data")
// Download data for NIS-code 1000. 1000 means: all of belgium
const url = "https://www.velopark.be/api/parkings/1000"
const allVelopark = await LinkedDataLoader.fetchJsonLd(url, { country: "be" })
this.exportTo("velopark_all", allVelopark)
const allVeloparkRaw: { url: string }[] = await Utils.downloadJson(url)
let failed = 0
console.log("Got", allVeloparkRaw.length, "items")
const allVelopark: Feature[] = []
const allProperties = {}
for (let i = 0; i < allVeloparkRaw.length; i++) {
const f = allVeloparkRaw[i]
console.log("Handling", i + "/" + allVeloparkRaw.length)
try {
const cachePath = "/home/pietervdvn/data/velopark_cache/" + f.url.replace(/[/:.]/g, "_")
if (!fs.existsSync(cachePath)) {
const data = await Utils.downloadJson(f.url)
fs.writeFileSync(cachePath, JSON.stringify(data), "utf-8")
console.log("Saved a backup to", cachePath)
}
this.sumProperties(JSON.parse(fs.readFileSync(cachePath, "utf-8")), allProperties)
const linkedData = await LinkedDataLoader.fetchVeloparkEntry(f.url)
for (const sectionId in linkedData) {
const sectionInfo = linkedData[sectionId]
if (Object.keys(sectionInfo).length === 0) {
console.warn("No result for", f.url)
}
sectionInfo["ref:velopark"] = [sectionId ?? f.url]
allVelopark.push(sectionInfo)
}
} catch (e) {
console.error("Loading ", f.url, " failed due to", e)
failed++
}
}
console.log("Fetching data done, got ", allVelopark.length + "/" + allVeloparkRaw.length, "failed:", failed)
VeloParkToGeojson.exportGeojsonTo("velopark_all.geojson", allVelopark)
for (const k in allProperties) {
allProperties[k] = Array.from(allProperties[k])
}
fs.writeFileSync("all_properties_mashup.json", JSON.stringify(allProperties, null, " "))
return allVelopark
}
private static loadFromFile(): Feature[] {
return JSON.parse(fs.readFileSync("velopark_all.geojson", "utf-8")).features
}
private static exportExtraAmenities(allVelopark: Feature[]) {
const amenities: Record<string, Feature[]> = {}
for (const bikeparking of allVelopark) {
const props = bikeparking.properties
if (!props["fixme_nearby_amenity"]) {
continue
}
if (props["fixme_nearby_amenity"]?.endsWith("CameraSurveillance")) {
delete props["fixme_nearby_amenity"]
continue
}
const amenity = props["fixme_nearby_amenity"].split("#")[1]
if (!amenities[amenity]) {
amenities[amenity] = []
}
amenities[amenity].push(bikeparking)
}
for (const k in amenities) {
this.exportGeojsonTo("velopark_amenity_" + k + ".geojson", amenities[k])
}
}
private static async createDiff(allVelopark: Feature[]) {
const bboxBelgium = new BBox([
[2.51357303225, 49.5294835476],
[6.15665815596, 51.4750237087],
[6.15665815596, 51.4750237087]
])
const alreadyLinkedQuery = new Overpass(
new RegexTag("ref:velopark", /.+/),
[],
Constants.defaultOverpassUrls[0],
new ImmutableStore(60 * 5),
false,
false
)
const alreadyLinkedFeatures = await alreadyLinkedQuery.queryGeoJson(bboxBelgium)
const alreadyLinkedFeatures = (await alreadyLinkedQuery.queryGeoJson(bboxBelgium))[0]
const seenIds = new Set<string>(
alreadyLinkedFeatures[0].features.map((f) => f.properties["ref:velopark"]),
alreadyLinkedFeatures.features.map((f) => f.properties?.["ref:velopark"])
)
this.exportGeojsonTo("osm_with_velopark_link", <Feature[]> alreadyLinkedFeatures.features)
console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")
const features = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))
const features: Feature[] = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))
VeloParkToGeojson.exportGeojsonTo("velopark_nonsynced", features)
const allProperties = new Set<string>()
for (const feature of features) {
Object.keys(feature.properties).forEach((k) => allProperties.add(k))
Object.keys(feature).forEach((k) => allProperties.add(k))
}
this.exportTo("velopark_noncynced", features)
allProperties.delete("ref:velopark")
for (const feature of features) {
allProperties.forEach((k) => {
delete feature.properties[k]
delete feature[k]
})
}
this.exportTo("velopark_nonsynced_id_only", features)
this.exportGeojsonTo("velopark_nonsynced_id_only", features)
}
async main(args: string[]): Promise<void> {
const allVelopark = VeloParkToGeojson.loadFromFile() // VeloParkToGeojson.downloadData()
console.log("Got", allVelopark.length, " items")
// VeloParkToGeojson.exportExtraAmenities(allVelopark)
await VeloParkToGeojson.createDiff(allVelopark)
console.log("Use vite-node script/velopark/compare to compare the results and generate a diff file")
}
}