Conflation script
This commit is contained in:
parent
6f7437aa46
commit
8eda65a24f
1 changed files with 211 additions and 0 deletions
211
scripts/conflate.ts
Normal file
211
scripts/conflate.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
import Script from "./Script"
|
||||
import fs from "fs"
|
||||
import { Feature } from "geojson"
|
||||
import { GeoOperations } from "../Logic/GeoOperations"
|
||||
import { Utils } from "../Utils"
|
||||
import { OsmObject } from "../Logic/Osm/OsmObject"
|
||||
|
||||
export class Conflate extends Script {
|
||||
constructor() {
|
||||
super(
|
||||
[
|
||||
"Conflation script",
|
||||
"",
|
||||
"This script is meant to be used to prepare imports. It takes one 'OSM'-dataset and one external dataset and tries to find an OSM-id for every external item.",
|
||||
"",
|
||||
"Arguments:",
|
||||
"osm_file.geojson external_file.geojson [search_range]",
|
||||
"- osm_file.geojson: a file exported from overpass, including meta (note: filename MUST contain either OSM or OpenStreetMap)",
|
||||
"- external_file.geojson: the data to import. Tags should be prepared to have an OSM-name",
|
||||
"- search_range: max distance at which a match will occur",
|
||||
].join("\n")
|
||||
)
|
||||
}
|
||||
|
||||
private async findTimeFork(
|
||||
externalName: string,
|
||||
osmName: string,
|
||||
osmId: string
|
||||
): Promise<{ earliestDateOfImport; latestDateOfImport }> {
|
||||
const history = await OsmObject.DownloadHistory(osmId).AsPromise((h) => h.length > 0)
|
||||
let earliest: Date = undefined
|
||||
let latest: Date = undefined
|
||||
for (const historyElement of history) {
|
||||
const csTime = new Date(historyElement.tags["_last_edit:timestamp"])
|
||||
if (isNaN(csTime.getTime())) {
|
||||
console.error("Could not parse" + historyElement.tags["_last_edit:timestamp"])
|
||||
return undefined
|
||||
}
|
||||
const nameIdentical = historyElement.tags.name === externalName
|
||||
if (nameIdentical) {
|
||||
if (earliest == undefined) {
|
||||
earliest = csTime
|
||||
}
|
||||
latest = csTime
|
||||
}
|
||||
}
|
||||
|
||||
if (history.at(-1).tags.name === externalName) {
|
||||
// Not changed yet, so no actual hint about when this import could have happened
|
||||
latest = new Date()
|
||||
}
|
||||
|
||||
if (this.earliestDate === undefined || earliest?.getTime() > this.earliestDate?.getTime()) {
|
||||
this.earliestDate = earliest
|
||||
}
|
||||
if (this.latestDate === undefined || latest?.getTime() < this.latestDate?.getTime()) {
|
||||
this.latestDate = latest
|
||||
}
|
||||
|
||||
return { earliestDateOfImport: earliest, latestDateOfImport: latest }
|
||||
}
|
||||
|
||||
private earliestDate: Date = undefined
|
||||
private latestDate: Date = undefined
|
||||
|
||||
async main(args: string[]): Promise<void> {
|
||||
const [osm_file_path, external_file_path] = args
|
||||
let max_range = 50
|
||||
if (args.length === 3) {
|
||||
max_range = Number(args[2])
|
||||
}
|
||||
if (
|
||||
osm_file_path.toLowerCase().indexOf("osm") < 0 &&
|
||||
osm_file_path.toLowerCase().indexOf("openstreetmap") < 0
|
||||
) {
|
||||
throw "OSM File path must contain 'osm' or 'openStreetMap'"
|
||||
}
|
||||
|
||||
if (
|
||||
external_file_path.toLowerCase().indexOf("osm") >= 0 ||
|
||||
external_file_path.toLowerCase().indexOf("openstreetmap") >= 0
|
||||
) {
|
||||
throw "External File path may not contain 'osm' or 'openStreetMap'"
|
||||
}
|
||||
|
||||
const external_features: Feature[] = JSON.parse(
|
||||
fs.readFileSync(external_file_path, { encoding: "utf-8" })
|
||||
).features
|
||||
const osm_features: Feature[] = JSON.parse(
|
||||
fs.readFileSync(osm_file_path, { encoding: "utf-8" })
|
||||
).features
|
||||
|
||||
const match_lengths: (string | number)[][] = [
|
||||
[
|
||||
"osm_id",
|
||||
"external_index",
|
||||
"match_distance",
|
||||
"name_levenshtein_distance",
|
||||
"osm_data",
|
||||
"external_data",
|
||||
"status",
|
||||
],
|
||||
]
|
||||
for (let i = 0; i < external_features.length; i++) {
|
||||
// console.log("Inspecting " + (i + 1) + "/" + external_features.length)
|
||||
const externalFeature = external_features[i]
|
||||
const possibleMatches: number[] = []
|
||||
for (const osmFeature of osm_features) {
|
||||
const d = GeoOperations.distanceBetween(
|
||||
GeoOperations.centerpointCoordinates(externalFeature),
|
||||
GeoOperations.centerpointCoordinates(osmFeature)
|
||||
)
|
||||
|
||||
if (d === 0) {
|
||||
console.log(
|
||||
"Found an exact match (name match: ",
|
||||
osmFeature.properties.name === externalFeature.properties.name,
|
||||
osmFeature.properties.name,
|
||||
externalFeature.properties.name
|
||||
)
|
||||
continue
|
||||
}
|
||||
continue
|
||||
if (d < max_range) {
|
||||
console.log("Found a match")
|
||||
match_lengths.push([
|
||||
osmFeature.properties["@id"],
|
||||
(i + " " + possibleMatches.join(",")).trim(),
|
||||
d,
|
||||
this.levenshteinDistancePharmacy(
|
||||
externalFeature.properties.name,
|
||||
osmFeature.properties.name
|
||||
),
|
||||
externalFeature.properties.status,
|
||||
...this.conflate(osmFeature.properties, externalFeature.properties),
|
||||
])
|
||||
possibleMatches.push(osmFeature.properties["@id"])
|
||||
/*
|
||||
possibleMatches.push({
|
||||
osmFeature,
|
||||
d,
|
||||
nameDist: Utils.levenshteinDistance(
|
||||
osmFeature.properties.name,
|
||||
externalFeature.properties.name
|
||||
),
|
||||
})//*/
|
||||
}
|
||||
// possibleMatches.sort((a, b) => b.d - a.d)
|
||||
}
|
||||
}
|
||||
match_lengths.sort((a, b) => <number>b[1] - <number>a[1])
|
||||
console.log(
|
||||
"The import probably happened between ",
|
||||
this.earliestDate?.toISOString(),
|
||||
"and",
|
||||
this.latestDate?.toISOString()
|
||||
)
|
||||
fs.writeFileSync(
|
||||
"../onwheels-data-prep/match_lengths.tsv",
|
||||
match_lengths.map((l) => l.join("\t")).join("\n")
|
||||
)
|
||||
console.log(match_lengths)
|
||||
}
|
||||
|
||||
private levenshteinDistancePharmacy(a?: string, b?: string) {
|
||||
a ??= ""
|
||||
b ??= ""
|
||||
a = a.toLowerCase()
|
||||
b = b.toLowerCase()
|
||||
return Math.min(
|
||||
...["", "pharmacie", "apotheek", "pharmacie de", "apotheke"].map((prefix) =>
|
||||
Math.min(
|
||||
Utils.levenshteinDistance(a, prefix + b),
|
||||
Utils.levenshteinDistance(prefix + a, b)
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private conflate(
|
||||
osmFeature: Record<string, string>,
|
||||
externalFeature: Record<string, string>
|
||||
): string[] {
|
||||
const r: string[] = []
|
||||
|
||||
for (const externalFeatureKey in externalFeature) {
|
||||
if (
|
||||
[
|
||||
"status",
|
||||
"healthcare",
|
||||
"unmeasurable_reason",
|
||||
"timestamp_created",
|
||||
"timestamp_last_modified",
|
||||
].indexOf(externalFeatureKey) >= 0
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const v = externalFeature[externalFeatureKey]
|
||||
const osmV = osmFeature[externalFeatureKey]
|
||||
if (osmV === undefined) {
|
||||
r.push("+" + externalFeatureKey + "=" + v)
|
||||
} else if (osmV !== v) {
|
||||
r.push("~" + externalFeatureKey + "=" + v + " (osm: " + osmV + ")")
|
||||
}
|
||||
}
|
||||
|
||||
return r.map((l) => l.replace(/\n/g, "\\n"))
|
||||
}
|
||||
}
|
||||
|
||||
new Conflate().run()
|
||||
Loading…
Add table
Reference in a new issue