diff --git a/scripts/velopark/compare.ts b/scripts/velopark/compare.ts index 947e9b580..cea83eb63 100644 --- a/scripts/velopark/compare.ts +++ b/scripts/velopark/compare.ts @@ -65,6 +65,10 @@ class Compare extends Script { async main(args: string[]): Promise { let [velopark, osm, key] = args + if(velopark === undefined || osm === undefined){ + console.log("Needed argument: velopark.geojson osm.geojson [key]\nThe key is optional and will be `ref:velopark` by default\nUse overpass to get a geojson with ref:velopark") + return + } key ??= "ref:velopark" const veloparkData: FeatureCollection = JSON.parse(fs.readFileSync(velopark, "utf-8")) const osmData: FeatureCollection = JSON.parse(fs.readFileSync(osm, "utf-8")) @@ -82,6 +86,7 @@ class Compare extends Script { console.error("No velopark entry found for", veloId) continue } + console.log("Veloparking is", veloparking) diffs.push(this.compare(veloId, parking, veloparking)) } console.log( diff --git a/scripts/velopark/diffToCsv.ts b/scripts/velopark/diffToCsv.ts new file mode 100644 index 000000000..d0031ae8f --- /dev/null +++ b/scripts/velopark/diffToCsv.ts @@ -0,0 +1,63 @@ +import Script from "../Script" +import { readFileSync, writeFileSync } from "fs" +import { OsmId } from "../../src/Models/OsmFeature" +import { Utils } from "../../src/Utils" + +interface DiffItem { + /** + * Velopark-id + */ + "ref": string, + "osmid": OsmId, + "distance": number, + "diffs": { + key: string, + /** + * The value in OpenStreetMap + * Might be undefined if OSM doesn't have an appropriate value + */ + osm?: string, + velopark: string | number } [] +} + +export class DiffToCsv extends Script { + constructor() { + super("Converts a 'report.diff' to a CSV file for people who prefer LibreOffice Calc (or other Spreadsheet Software)") + } + + async main(args: string[]): Promise { + const file = args[0] ?? "report_diff.json" + const json = <{diffs:DiffItem[], distanceBinds: number[]}> JSON.parse(readFileSync(file, "utf8")) + const diffs = json.diffs + const allKeys = Utils.Dedup(diffs.flatMap(item => item.diffs.map(d => d.key))) + allKeys.sort() + + const header = ["osm_id","velopark_id", "distance",...allKeys.flatMap(k => ["osm:"+k, "velopark:"+k])] + const lines = [header] + for (const diffItem of diffs) { + const line = [] + lines.push(line) + line.push(diffItem.osmid) + line.push(diffItem.ref) + line.push(diffItem.distance) + + const d = diffItem.diffs + for (const k of allKeys) { + const found = d.find(i => i.key === k) + if(!found){ + line.push("","") + continue + } + line.push(found.osm, found.velopark) + } + + } + const path = "report_diff.csv" + writeFileSync(path, + lines.map(l => l.join(",")).join("\n") + ,"utf8") + console.log("Written", path) + } +} + +new DiffToCsv().run() diff --git a/scripts/velopark/veloParkToGeojson.ts b/scripts/velopark/veloParkToGeojson.ts index acbc6d0eb..839155079 100644 --- a/scripts/velopark/veloParkToGeojson.ts +++ b/scripts/velopark/veloParkToGeojson.ts @@ -23,9 +23,9 @@ class VeloParkToGeojson extends Script { JSON.stringify( extension === ".geojson" ? { - type: "FeatureCollection", - features, - } + type: "FeatureCollection", + features + } : features, null, " " @@ -34,34 +34,14 @@ class VeloParkToGeojson extends Script { console.log("Written", file, "(" + features.length, " features)") } - public static sumProperties(data: object, addTo: Record>) { - delete data["@context"] - for (const k in data) { - if (k === "@graph") { - for (const obj of data["@graph"]) { - this.sumProperties(obj, addTo) - } - continue - } - if (addTo[k] === undefined) { - addTo[k] = new Set() - } - addTo[k].add(data[k]) - } - } - private static async downloadDataFor( - url: string, - allProperties: Record> + url: string ): Promise { - const cachePath = "/home/pietervdvn/data/velopark_cache/" + url.replace(/[/:.]/g, "_") - if (!fs.existsSync(cachePath)) { - const data = await Utils.downloadJson(url) - fs.writeFileSync(cachePath, JSON.stringify(data), "utf-8") - console.log("Saved a backup to", cachePath) + const cachePath = "/home/pietervdvn/data/velopark_cache_refined/" + url.replace(/[/:.]/g, "_") + if (fs.existsSync(cachePath)) { + return JSON.parse(fs.readFileSync(cachePath, "utf8")) } - - this.sumProperties(JSON.parse(fs.readFileSync(cachePath, "utf-8")), allProperties) + console.log("Fetching data for", url) const linkedData = await LinkedDataLoader.fetchVeloparkEntry(url) const allVelopark: Feature[] = [] @@ -70,9 +50,12 @@ class VeloParkToGeojson extends Script { if (Object.keys(sectionInfo).length === 0) { console.warn("No result for", url) } - sectionInfo["ref:velopark"] = [sectionId ?? url] + if(!sectionInfo.geometry?.coordinates){ + throw "Invalid properties!" + } allVelopark.push(sectionInfo) } + fs.writeFileSync(cachePath, JSON.stringify(allVelopark), "utf8") return allVelopark } @@ -85,20 +68,24 @@ class VeloParkToGeojson extends Script { let failed = 0 console.log("Got", allVeloparkRaw.length, "items") const allVelopark: Feature[] = [] - const allProperties = {} - for (let i = 0; i < allVeloparkRaw.length; i++) { - const f = allVeloparkRaw[i] - console.log("Handling", i + "/" + allVeloparkRaw.length) - try { - const sections: Feature[] = await VeloParkToGeojson.downloadDataFor( - f.url, - allProperties - ) - allVelopark.push(...sections) - } catch (e) { - console.error("Loading ", f.url, " failed due to", e) - failed++ - } + const batchSize = 50 + for (let i = 0; i < allVeloparkRaw.length; i += batchSize) { + await Promise.all(Utils.TimesT(batchSize, j => j).map( + async j => { + const f = allVeloparkRaw[i + j] + if (!f) { + return + } + try { + const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(f.url) + allVelopark.push(...sections) + } catch (e) { + console.error("Loading ", f.url, " failed due to", e) + failed++ + } + } + )) + } console.log( "Fetching data done, got ", @@ -107,11 +94,6 @@ class VeloParkToGeojson extends Script { failed ) VeloParkToGeojson.exportGeojsonTo("velopark_all", allVelopark) - for (const k in allProperties) { - allProperties[k] = Array.from(allProperties[k]) - } - - fs.writeFileSync("all_properties_mashup.json", JSON.stringify(allProperties, null, " ")) return allVelopark } @@ -158,7 +140,7 @@ class VeloParkToGeojson extends Script { private static async createDiff(allVelopark: Feature[]) { const bboxBelgium = new BBox([ [2.51357303225, 49.5294835476], - [6.15665815596, 51.4750237087], + [6.15665815596, 51.4750237087] ]) const alreadyLinkedQuery = new Overpass( @@ -201,7 +183,7 @@ class VeloParkToGeojson extends Script { VeloParkToGeojson.exportExtraAmenities(allVelopark) await VeloParkToGeojson.createDiff(allVelopark) console.log( - "Use vite-node script/velopark/compare to compare the results and generate a diff file" + "Use vite-node scripts/velopark/compare.ts to compare the results and generate a diff file" ) } } diff --git a/src/Logic/Web/LinkedDataLoader.ts b/src/Logic/Web/LinkedDataLoader.ts index 7bf03953e..97636ec3b 100644 --- a/src/Logic/Web/LinkedDataLoader.ts +++ b/src/Logic/Web/LinkedDataLoader.ts @@ -316,14 +316,16 @@ export default class LinkedDataLoader { input: Record> ): Record { const output: Record = {} - console.log("Input for patchVelopark:", input) for (const k in input) { output[k] = Array.from(input[k]) } - if (output["type"][0] === "https://data.velopark.be/openvelopark/terms#BicycleLocker") { + if (output["type"]?.[0] === "https://data.velopark.be/openvelopark/terms#BicycleLocker") { output["bicycle_parking"] = ["lockers"] } + if(output["type"] === undefined){ + console.error("No type given for", output) + } delete output["type"] function on(key: string, applyF: (s: string) => string) { @@ -506,7 +508,6 @@ export default class LinkedDataLoader { " ?parking a ", "?parking " + property + " " + (variable ?? "") ) - console.log("Fetching a velopark property gave", property, results) return results }