Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Refactoring of metatagging and extrafunctions to splice out the relation memberships, add calculatedTags and metatags into cache

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-04-22 13:30:00 +02:00
parent 362abbf079
commit 0dec1d0f75
10 changed files with 98 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

29
Logic/ExtraFunction.ts
View file

@ -2,6 +2,7 @@ import {GeoOperations} from "./GeoOperations";
import {UIElement} from "../UI/UIElement";
import Combine from "../UI/Base/Combine";
import State from "../State";
import {Relation} from "./Osm/ExtractRelations";
export class ExtraFunction {
@ -40,11 +41,11 @@ Some advanced functions are available on <b>feat</b> as well:
"overlapWith",
"Gives a list of features from the specified layer which this feature overlaps with, the amount of overlap in m². The returned value is <b>{ feat: GeoJSONFeature, overlap: number}</b>",
["...layerIds - one or more layer ids of the layer from which every feature is checked for overlap)"],
(featuresPerLayer, feat) => {
(params, feat) => {
return (...layerIds: string[]) => {
const result = []
for (const layerId of layerIds) {
const otherLayer = featuresPerLayer.get(layerId);
const otherLayer = params.featuresPerLayer.get(layerId);
if (otherLayer === undefined) {
continue;
}
@ -80,10 +81,10 @@ Some advanced functions are available on <b>feat</b> as well:
"closest",
"Given either a list of geojson features or a single layer name, gives the single object which is nearest to the feature. In the case of ways/polygons, only the centerpoint is considered.",
["list of features"],
(featuresPerLayer, feature) => {
(params, feature) => {
return (features) => {
if (typeof features === "string") {
features = featuresPerLayer.get(features)
features = params.featuresPerLayer.get(features)
}
let closestFeature = undefined;
let closestDistance = undefined;
@ -118,11 +119,8 @@ Some advanced functions are available on <b>feat</b> as well:
"memberships",
"Gives a list of {role: string, relation: Relation}-objects, containing all the relations that this feature is part of. \n\nFor example: `_part_of_walking_routes=feat.memberships().map(r => r.relation.tags.name).join(';')`",
[],
(featuresPerLayer, feature) => {
return () => {
return State.state.knownRelations.data?.get(feature.id) ?? [];
}
(params, feature) => {
return () => params.relations ?? [];
}
)
@ -130,9 +128,9 @@ Some advanced functions are available on <b>feat</b> as well:
private readonly _name: string;
private readonly _args: string[];
private readonly _doc: string;
private readonly _f: (featuresPerLayer: Map<string, any[]>, feat: any) => any;
private readonly _f: (params: {featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[]}, feat: any) => any;
constructor(name: string, doc: string, args: string[], f: ((featuresPerLayer: Map<string, any[]>, feat: any) => any)) {
constructor(name: string, doc: string, args: string[], f: ((params: {featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[]}, feat: any) => any)) {
this._name = name;
this._doc = doc;
this._args = args;
@ -140,9 +138,9 @@ Some advanced functions are available on <b>feat</b> as well:
}
public static FullPatchFeature(featuresPerLayer: Map<string, any[]>, feature) {
public static FullPatchFeature(featuresPerLayer: Map<string, any[]>,relations: {role: string, relation: Relation}[], feature) {
for (const func of ExtraFunction.allFuncs) {
func.PatchFeature(featuresPerLayer, feature);
func.PatchFeature(featuresPerLayer, relations, feature);
}
}
@ -168,7 +166,8 @@ Some advanced functions are available on <b>feat</b> as well:
]);
}
public PatchFeature(featuresPerLayer: Map<string, any[]>, feature: any) {
feature[this._name] = this._f(featuresPerLayer, feature);
public PatchFeature(featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[], feature: any) {
feature[this._name] = this._f({featuresPerLayer: featuresPerLayer, relations: relations}, feature);
}
}

4
Logic/FeatureSource/FeatureDuplicatorPerLayer.ts
View file

@ -11,6 +11,10 @@ import LayerConfig from "../../Customizations/JSON/LayerConfig";
export default class FeatureDuplicatorPerLayer implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public static GetMatchingLayerId(){
}
constructor(layers: UIEventSource<{ layerDef: LayerConfig }[]>, upstream: FeatureSource) {
this.features = upstream.features.map(features => {

2
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -35,7 +35,7 @@ export default class FeaturePipeline implements FeatureSource {
const geojsonSources: FeatureSource [] = GeoJsonSource
.ConstructMultiSource(flayers.data, locationControl)
.map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
.map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
const amendedLocalStorageSource =
new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new LocalStorageSource(layout))

3
Logic/FeatureSource/MetaTaggingFeatureSource.ts
View file

@ -3,6 +3,7 @@ import {UIEventSource} from "../UIEventSource";
import State from "../../State";
import Hash from "../Web/Hash";
import MetaTagging from "../MetaTagging";
import ExtractRelations from "../Osm/ExtractRelations";
export default class MetaTaggingFeatureSource implements FeatureSource {
features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{feature: any; freshness: Date}[]>(undefined);
@ -21,7 +22,7 @@ export default class MetaTaggingFeatureSource implements FeatureSource {
}
})
MetaTagging.addMetatags(featuresFreshness, State.state.layoutToUse.data.layers);
MetaTagging.addMetatags(featuresFreshness, State.state.knownRelations.data, State.state.layoutToUse.data.layers);
self.features.setData(featuresFreshness);
});
}

29
Logic/MetaTagging.ts
View file

@ -1,6 +1,14 @@
import LayerConfig from "../Customizations/JSON/LayerConfig";
import SimpleMetaTagger from "./SimpleMetaTagger";
import {ExtraFunction} from "./ExtraFunction";
import State from "../State";
import {Relation} from "./Osm/ExtractRelations";
interface Params {
featuresPerLayer: Map<string, any[]>,
memberships: Map<string, { role: string, relation: Relation }[]>
}
/**
* Metatagging adds various tags to the elements, e.g. lat, lon, surface area, ...
@ -14,7 +22,8 @@ export default class MetaTagging {
* An actor which adds metatags on every feature in the given object
* The features are a list of geojson-features, with a "properties"-field and geometry
*/
static addMetatags(features: { feature: any; freshness: Date }[], layers: LayerConfig[]) {
static addMetatags(features: { feature: any; freshness: Date }[],
relations: Map<string, { role: string, relation: Relation }[]>, layers: LayerConfig[]) {
for (const metatag of SimpleMetaTagger.metatags) {
try {
@ -26,7 +35,7 @@ export default class MetaTagging {
}
// The functions - per layer - which add the new keys
const layerFuncs = new Map<string, ((featursPerLayer: Map<string, any[]>, feature: any) => void)>();
const layerFuncs = new Map<string, ((params: Params, feature: any) => void)>();
for (const layer of layers) {
layerFuncs.set(layer.id, this.createRetaggingFunc(layer));
}
@ -48,27 +57,26 @@ export default class MetaTagging {
if (f === undefined) {
continue;
}
f(featuresPerLayer, feature.feature)
f({featuresPerLayer: featuresPerLayer, memberships: relations}, feature.feature)
}
}
private static createRetaggingFunc(layer: LayerConfig): ((featuresPerLayer: Map<string, any[]>, feature: any) => void) {
private static createRetaggingFunc(layer: LayerConfig):
((params: Params, feature: any) => void) {
const calculatedTags: [string, string][] = layer.calculatedTags;
if (calculatedTags === undefined) {
return undefined;
}
const functions: ((featuresPerLayer: Map<string, any[]>, feature: any) => void)[] = [];
const functions: ((params: Params, feature: any) => void)[] = [];
for (const entry of calculatedTags) {
const key = entry[0]
const code = entry[1];
if (code === undefined) {
continue;
}
const func = new Function("feat", "return " + code + ";");
const f = (featuresPerLayer, feature: any) => {
@ -76,16 +84,17 @@ export default class MetaTagging {
}
functions.push(f)
}
return (featuresPerLayer: Map<string, any[]>, feature) => {
return (params: Params, feature) => {
const tags = feature.properties
if (tags === undefined) {
return;
}
ExtraFunction.FullPatchFeature(featuresPerLayer, feature);
const relations = params.memberships.get(feature.properties.id)
ExtraFunction.FullPatchFeature(params.featuresPerLayer, relations, feature);
try {
for (const f of functions) {
f(featuresPerLayer, feature);
f(params, feature);
}
} catch (e) {
console.error("While calculating a tag value: ", e)

3
Logic/Osm/ExtractRelations.ts
View file

@ -17,7 +17,6 @@ export default class ExtractRelations {
public static RegisterRelations(overpassJson: any) : void{
const memberships = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(overpassJson))
console.log("Assigned memberships: ", memberships)
State.state.knownRelations.setData(memberships)
}
@ -40,7 +39,7 @@ export default class ExtractRelations {
* @param relations
* @constructor
*/
private static BuildMembershipTable(relations: Relation[]): Map<string, { role: string, relation: Relation, }[]> {
public static BuildMembershipTable(relations: Relation[]): Map<string, { role: string, relation: Relation }[]> {
const memberships = new Map<string, { role: string, relation: Relation }[]>()
for (const relation of relations) {

10
Logic/SimpleMetaTagger.ts
View file

@ -9,6 +9,7 @@ import {UIElement} from "../UI/UIElement";
import Combine from "../UI/Base/Combine";
import UpdateTagsFromOsmAPI from "./Actors/UpdateTagsFromOsmAPI";
export default class SimpleMetaTagger {
public readonly keys: string[];
public readonly doc: string;
@ -89,7 +90,12 @@ export default class SimpleMetaTagger {
["_isOpen", "_isOpen:description"],
"If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')",
(feature => {
if(Utils.runningFromConsole){
// We are running from console, thus probably creating a cache
// isOpen is irrelevant
return
}
const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id);
tagsSource.addCallbackAndRun(tags => {
if (tags.opening_hours === undefined || tags._country === undefined) {
@ -317,7 +323,7 @@ export default class SimpleMetaTagger {
];
static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) {
SimpleMetaTagger.coder.GetCountryCodeFor(lon, lat, callback)
SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback)
}
static HelpText(): UIElement {

18
assets/layers/slow_roads/slow_roads.json
View file

@ -159,32 +159,34 @@
"width": {
"render": "4"
},
"dashArray": "2 10 6 10",
"color": {
"render": "#bb2",
"dashArray": {
"render": "",
"mappings": [
{
"if": "highway=cycleway",
"then": "#00c"
"then": ""
},
{
"if": "highway=path",
"then": "#bb2"
"then": "10 3"
},
{
"if": "highway=footway",
"then": "#c30"
"then": "10 10"
},
{
"if": "highway=pedestrian",
"then": "#3c3"
"then": "10 10"
},
{
"if": "highway=living_street",
"then": "#ccc"
"then": "10 5 3 5"
}
]
},
"color": {
"render": "#eaba2a"
},
"presets": [
]
}

2
package.json
View file

@ -15,7 +15,7 @@
"generate:translations": "ts-node scripts/generateTranslations.ts",
"generate:layouts": "ts-node scripts/generateLayouts.ts",
"generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts",
"generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ./cache/speelplekken 51.2003 4.3925 51.1058 4.5087",
"generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.37 51.11 4.51",
"generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail",
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
"validate:layeroverview": "ts-node scripts/generateLayerOverview.ts --report",

52
scripts/generateCache.ts
View file

@ -14,6 +14,8 @@ import ScriptUtils from "./ScriptUtils";
import ExtractRelations from "../Logic/Osm/ExtractRelations";
import * as OsmToGeoJson from "osmtogeojson";
import {Script} from "vm";
import MetaTagging from "../Logic/MetaTagging";
import State from "../State";
function createOverpassObject(theme: LayoutConfig) {
let filters: TagsFilter[] = [];
@ -26,8 +28,8 @@ function createOverpassObject(theme: LayoutConfig) {
continue;
}
if (layer.source.geojsonSource !== undefined) {
// Not our responsibility to download this layer!
continue;
// We download these anyway - we are building the cache after all!
//continue;
}
@ -68,17 +70,17 @@ function metaJsonName(targetDir: string, x: number, y: number, z: number): strin
return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json"
}
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) {
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) {
let downloaded = 0
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
downloaded++;
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
if (existsSync(filename)) {
console.log("Already exists: ", filename)
continue;
}
console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y)
const bounds = {
@ -87,28 +89,35 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)
east: Math.max(boundsArr[0][1], boundsArr[1][1]),
west: Math.min(boundsArr[0][1], boundsArr[1][1])
}
console.log("Downloading tile", r.zoomlevel, x, y, "with bounds", bounds)
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
let gotResponse = false
ScriptUtils.DownloadJSON(url,
chunks => {
gotResponse = true;
saveResponse(chunks, filename)
})
await ScriptUtils.sleep(10000)
console.debug("Waking up")
while (!gotResponse) {
await ScriptUtils.sleep(10000)
console.debug("Waking up")
if (!gotResponse) {
console.log("Didn't get an answer yet - waiting more")
}
}
}
}
}
async function postProcess(targetdir: string, r: TileRange) {
async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) {
let processed = 0;
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
processed++;
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
console.log(" Post processing", processed, "/",r. total, filename)
console.log(" Post processing", processed, "/", r.total, filename)
if (!existsSync(filename)) {
throw "Not found - and not downloaded. Run this script again!: " + filename
}
@ -118,11 +127,26 @@ async function postProcess(targetdir: string, r: TileRange) {
// Create and save the geojson file - which is the main chunk of the data
const geojson = OsmToGeoJson.default(rawOsm);
const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
for (const feature of geojson.features) {
for (const layer of theme.layers) {
if (layer.source.osmTags.matchesProperties(feature.properties)) {
feature["_matching_layer_id"] = layer.id;
break;
}
}
}
const featuresFreshness = geojson.features.map(feature => ({
freshness: osmTime,
feature: feature
}));
// Extract the relationship information
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
// Extract the relationship information
const relations = ExtractRelations.GetRelationElements(rawOsm)
const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
const meta = {
freshness: osmTime,
@ -145,7 +169,7 @@ async function main(args: string[]) {
}
const themeName = args[0]
const zoomlevel = Number(args[1])
const targetdir = args[2]
const targetdir = args[2] + "/" + themeName
const lat0 = Number(args[3])
const lon0 = Number(args[4])
const lat1 = Number(args[5])
@ -167,7 +191,7 @@ async function main(args: string[]) {
await downloadRaw(targetdir, tileRange, overpass)
await postProcess(targetdir, tileRange)
await postProcess(targetdir, tileRange, theme)
}