Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add metadata to query, move metatagging of metadata to metatagger, tweaks to the generate caching so that the cached data becomes more stable (and is GIT-friendlier)

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-04-25 13:25:03 +02:00
parent 591ab349a6
commit 54f01ba554
7 changed files with 129 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Logic/Actors/UpdateTagsFromOsmAPI.ts
View file

@ -1,22 +1,10 @@
import {UIEventSource} from "../UIEventSource";
import {ElementStorage} from "../ElementStorage";
import {OsmObject, OsmObjectMeta} from "../Osm/OsmObject";
import SimpleMetaTagger from "../SimpleMetaTagger";
export default class UpdateTagsFromOsmAPI {
public static readonly metaTagger = new SimpleMetaTagger(
["_last_edit:contributor",
"_last_edit:contributor:uid",
"_last_edit:changeset",
"_last_edit:timestamp",
"_version_number"],
"Information about the last edit of this object. \n\nIMPORTANT: this data is _only_ loaded when the popup is added. This means it should _not_ be used to render icons!",
() => {/*Do nothing - this is only added for documentation reasons*/
}
)
/***
* This actor downloads the element from the OSM-API and updates the corresponding tags in the UI-updater.
*/

4
Logic/FeatureSource/GeoJsonSource.ts
View file

@ -182,8 +182,8 @@ export default class GeoJsonSource implements FeatureSource {
self.seenids.add(feature.properties.id)
let freshness: Date = time;
if (feature["_timestamp"] !== undefined) {
freshness = new Date(feature["_timestamp"])
if (feature.properties["_last_edit:timestamp"] !== undefined) {
freshness = new Date(feature["_last_edit:timestamp"])
}
newFeatures.push({feature: feature, freshness: freshness})

9
Logic/MetaTagging.ts
View file

@ -3,6 +3,7 @@ import SimpleMetaTagger from "./SimpleMetaTagger";
import {ExtraFunction} from "./ExtraFunction";
import State from "../State";
import {Relation} from "./Osm/ExtractRelations";
import {meta} from "@turf/turf";
interface Params {
@ -23,9 +24,15 @@ export default class MetaTagging {
* The features are a list of geojson-features, with a "properties"-field and geometry
*/
static addMetatags(features: { feature: any; freshness: Date }[],
relations: Map<string, { role: string, relation: Relation }[]>, layers: LayerConfig[]) {
relations: Map<string, { role: string, relation: Relation }[]>,
layers: LayerConfig[],
includeDates = true) {
for (const metatag of SimpleMetaTagger.metatags) {
if(metatag.includesDates && !includeDates){
// We do not add dated entries
continue;
}
try {
metatag.addMetaTags(features);
} catch (e) {

6
Logic/Osm/Overpass.ts
View file

@ -11,10 +11,12 @@ export class Overpass {
public static testUrl: string = null
private _filter: TagsFilter
private readonly _extraScripts: string[];
private _includeMeta: boolean;
constructor(filter: TagsFilter, extraScripts: string[]) {
constructor(filter: TagsFilter, extraScripts: string[], includeMeta = true) {
this._filter = filter
this._extraScripts = extraScripts;
this._includeMeta = includeMeta;
}
queryGeoJson(bounds: Bounds, continuation: ((any, date: Date) => void), onFail: ((reason) => void)): void {
@ -58,7 +60,7 @@ export class Overpass {
filter += '(' + extraScript + ');';
}
const query =
'[out:json][timeout:25]' + bbox + ';(' + filter + ');out body;>;out skel qt;'
`[out:json][timeout:25]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;`
return "https://overpass-api.de/api/interpreter?data=" + encodeURIComponent(query)
}
}

128
Logic/SimpleMetaTagger.ts
View file

@ -11,30 +11,36 @@ import UpdateTagsFromOsmAPI from "./Actors/UpdateTagsFromOsmAPI";
export default class SimpleMetaTagger {
public readonly keys: string[];
public readonly doc: string;
private readonly _f: (feature: any, index: number, freshness: Date) => void;
constructor(keys: string[], doc: string, f: ((feature: any, index: number, freshness: Date) => void)) {
this.keys = keys;
this.doc = doc;
this._f = f;
for (const key of keys) {
if (!key.startsWith('_')) {
throw `Incorrect metakey ${key}: it should start with underscore (_)`
}
}
}
addMetaTags(features: { feature: any, freshness: Date }[]) {
for (let i = 0; i < features.length; i++) {
let feature = features[i];
this._f(feature.feature, i, feature.freshness);
}
}
static coder: any;
private static latlon = new SimpleMetaTagger(["_lat", "_lon"], "The latitude and longitude of the point (or centerpoint in the case of a way/area)",
public static readonly objectMetaInfo = new SimpleMetaTagger(
{
keys: ["_last_edit:contributor",
"_last_edit:contributor:uid",
"_last_edit:changeset",
"_last_edit:timestamp",
"_version_number"],
doc: "Information about the last edit of this object."
},
(feature) => {/*Note: also handled by 'UpdateTagsFromOsmAPI'*/
const tgs = feature.properties;
tgs["_last_edit:contributor"] = tgs["user"]
tgs["_last_edit:contributor:uid"] = tgs["uid"]
tgs["_last_edit:changeset"] = tgs["changeset"]
tgs["_last_edit:timestamp"] = tgs["timestamp"]
tgs["_version_number"] = tgs["version"]
delete tgs["timestamp"]
delete tgs["version"]
delete tgs["changeset"]
delete tgs["user"]
delete tgs["uid"]
}
)
private static latlon = new SimpleMetaTagger({
keys: ["_lat", "_lon"],
doc: "The latitude and longitude of the point (or centerpoint in the case of a way/area)"
},
(feature => {
const centerPoint = GeoOperations.centerpoint(feature);
const lat = centerPoint.geometry.coordinates[1];
@ -46,7 +52,10 @@ export default class SimpleMetaTagger {
})
);
private static surfaceArea = new SimpleMetaTagger(
["_surface", "_surface:ha"], "The surface area of the feature, in square meters and in hectare. Not set on points and ways",
{
keys: ["_surface", "_surface:ha"],
doc: "The surface area of the feature, in square meters and in hectare. Not set on points and ways"
},
(feature => {
const sqMeters = GeoOperations.surfaceAreaInSqMeters(feature);
feature.properties["_surface"] = "" + sqMeters;
@ -54,20 +63,24 @@ export default class SimpleMetaTagger {
feature.area = sqMeters;
})
);
private static lngth = new SimpleMetaTagger(
["_length", "_length:km"], "The total length of a feature in meters (and in kilometers, rounded to one decimal for '_length:km'). For a surface, the length of the perimeter",
{
keys: ["_length", "_length:km"],
doc: "The total length of a feature in meters (and in kilometers, rounded to one decimal for '_length:km'). For a surface, the length of the perimeter"
},
(feature => {
const l = GeoOperations.lengthInMeters(feature)
feature.properties["_length"] = "" + l
const km = Math.floor(l / 1000)
const kmRest = Math.round((l - km * 1000) / 100)
feature.properties["_length:km"] = "" + km+ "." + kmRest
feature.properties["_length:km"] = "" + km + "." + kmRest
})
)
private static country = new SimpleMetaTagger(
["_country"], "The country code of the property (with latlon2country)",
{
keys: ["_country"],
doc: "The country code of the property (with latlon2country)"
},
feature => {
@ -87,15 +100,18 @@ export default class SimpleMetaTagger {
}
)
private static isOpen = new SimpleMetaTagger(
["_isOpen", "_isOpen:description"],
"If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')",
{
keys: ["_isOpen", "_isOpen:description"],
doc: "If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')",
includesDates: true
},
(feature => {
if(Utils.runningFromConsole){
if (Utils.runningFromConsole) {
// We are running from console, thus probably creating a cache
// isOpen is irrelevant
return
}
const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id);
tagsSource.addCallbackAndRun(tags => {
if (tags.opening_hours === undefined || tags._country === undefined) {
@ -155,7 +171,10 @@ export default class SimpleMetaTagger {
})
)
private static directionSimplified = new SimpleMetaTagger(
["_direction:simplified", "_direction:leftright"], "_direction:simplified turns 'camera:direction' and 'direction' into either 0, 45, 90, 135, 180, 225, 270 or 315, whichever is closest. _direction:leftright is either 'left' or 'right', which is left-looking on the map or 'right-looking' on the map",
{
keys: ["_direction:simplified", "_direction:leftright"],
doc: "_direction:simplified turns 'camera:direction' and 'direction' into either 0, 45, 90, 135, 180, 225, 270 or 315, whichever is closest. _direction:leftright is either 'left' or 'right', which is left-looking on the map or 'right-looking' on the map"
},
(feature => {
const tags = feature.properties;
const direction = tags["camera:direction"] ?? tags["direction"];
@ -178,8 +197,10 @@ export default class SimpleMetaTagger {
})
)
private static carriageWayWidth = new SimpleMetaTagger(
["_width:needed", "_width:needed:no_pedestrians", "_width:difference"],
"Legacy for a specific project calculating the needed width for safe traffic on a road. Only activated if 'width:carriageway' is present",
{
keys: ["_width:needed", "_width:needed:no_pedestrians", "_width:difference"],
doc: "Legacy for a specific project calculating the needed width for safe traffic on a road. Only activated if 'width:carriageway' is present"
},
(feature: any, index: number) => {
const properties = feature.properties;
@ -286,8 +307,11 @@ export default class SimpleMetaTagger {
}
);
private static currentTime = new SimpleMetaTagger(
["_now:date", "_now:datetime", "_loaded:date", "_loaded:_datetime"],
"Adds the time that the data got loaded - pretty much the time of downloading from overpass. The format is YYYY-MM-DD hh:mm, aka 'sortable' aka ISO-8601-but-not-entirely",
{
keys: ["_now:date", "_now:datetime", "_loaded:date", "_loaded:_datetime"],
doc: "Adds the time that the data got loaded - pretty much the time of downloading from overpass. The format is YYYY-MM-DD hh:mm, aka 'sortable' aka ISO-8601-but-not-entirely",
includesDates: true
},
(feature, _, freshness) => {
const now = new Date();
@ -318,9 +342,26 @@ export default class SimpleMetaTagger {
SimpleMetaTagger.isOpen,
SimpleMetaTagger.carriageWayWidth,
SimpleMetaTagger.directionSimplified,
SimpleMetaTagger.currentTime
SimpleMetaTagger.currentTime,
SimpleMetaTagger.objectMetaInfo
];
public readonly keys: string[];
public readonly doc: string;
public readonly includesDates: boolean
private readonly _f: (feature: any, index: number, freshness: Date) => void;
constructor(docs: { keys: string[], doc: string, includesDates?: boolean }, f: ((feature: any, index: number, freshness: Date) => void)) {
this.keys = docs.keys;
this.doc = docs.doc;
this._f = f;
this.includesDates = docs.includesDates ?? false;
for (const key of docs.keys) {
if (!key.startsWith('_')) {
throw `Incorrect metakey ${key}: it should start with underscore (_)`
}
}
}
static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) {
SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback)
@ -337,7 +378,7 @@ export default class SimpleMetaTagger {
];
for (const metatag of SimpleMetaTagger.metatags.concat(UpdateTagsFromOsmAPI.metaTagger)) {
for (const metatag of SimpleMetaTagger.metatags) {
subElements.push(
new Combine([
"<h3>", metatag.keys.join(", "), "</h3>",
@ -349,5 +390,12 @@ export default class SimpleMetaTagger {
return new Combine(subElements)
}
addMetaTags(features: { feature: any, freshness: Date }[]) {
for (let i = 0; i < features.length; i++) {
let feature = features[i];
this._f(feature.feature, i, feature.freshness);
}
}
}

1
Logic/Web/Hash.ts
View file

@ -26,6 +26,7 @@ export default class Hash {
}
const hash = new UIEventSource<string>(window.location.hash.substr(1));
hash.addCallback(h => {
console.trace("Hash was changed into ",h)
if (h === "undefined") {
console.warn("Got a literal 'undefined' as hash, ignoring")
h = undefined;

35
scripts/generateCache.ts
View file

@ -14,6 +14,8 @@ import ScriptUtils from "./ScriptUtils";
import ExtractRelations from "../Logic/Osm/ExtractRelations";
import * as OsmToGeoJson from "osmtogeojson";
import MetaTagging from "../Logic/MetaTagging";
import LayerConfig from "../Customizations/JSON/LayerConfig";
import {GeoOperations} from "../Logic/GeoOperations";
function createOverpassObject(theme: LayoutConfig) {
@ -114,8 +116,6 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
await ScriptUtils.sleep(1000)
}
}
}
}
@ -124,6 +124,7 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) {
let processed = 0;
const layerIndex = theme.LayerIndex();
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
processed++;
@ -150,7 +151,6 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
}
}
const featuresFreshness = geojson.features.map(feature => {
feature["_timestamp"] = rawOsm.osm3s.timestamp_osm_base;
return ({
freshness: osmTime,
feature: feature
@ -158,7 +158,25 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
});
// Extract the relationship information
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers, false);
for (const feature of geojson.features) {
const layer = layerIndex.get(feature["_matching_layer_id"])
if (layer === undefined) {
console.error("Didn't find a layer for " + feature["_matching_layer_id"])
continue
}
if (layer.wayHandling == LayerConfig.WAYHANDLING_CENTER_ONLY) {
const centerpoint = GeoOperations.centerpointCoordinates(feature)
feature.geometry.type = "Point"
feature.geometry["coordinates"] = centerpoint;
}
}
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " "))
@ -168,7 +186,6 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
}
async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
let processed = 0;
const z = r.zoomlevel;
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
@ -177,14 +194,14 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
for (const layer of theme.layers) {
const geojson = JSON.parse(file)
geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id)
if(geojson.features.length == 0){
if (geojson.features.length == 0) {
continue;
}
const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z);
const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z);
writeFileSync(new_path, JSON.stringify(geojson, null, " "))
}
}
}
}