contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Optimize queries to overpass

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-03-13 01:27:19 +01:00
parent fbcb72df7a
commit 9008e333ac
15 changed files with 787 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Logic/Osm/Overpass.ts
View file

@ -25,7 +25,12 @@ export class Overpass {
includeMeta = true) {
this._timeout = timeout;
this._interpreterUrl = interpreterUrl;
this._filter = filter
const optimized = filter.optimize()
if(optimized === true || optimized === false){
throw "Invalid filter: optimizes to true of false"
}
this._filter = optimized
console.log("Overpass filter is",this._filter)
this._extraScripts = extraScripts;
this._includeMeta = includeMeta;
this._relationTracker = relationTracker

91
Logic/Tags/And.ts
View file

@ -1,4 +1,6 @@
import {TagsFilter} from "./TagsFilter";
import {Or} from "./Or";
import {TagUtils} from "./TagUtils";
export class And extends TagsFilter {
public and: TagsFilter[]
@ -109,6 +111,10 @@ export class And extends TagsFilter {
usedKeys(): string[] {
return [].concat(...this.and.map(subkeys => subkeys.usedKeys()));
}
usedTags(): { key: string; value: string }[] {
return [].concat(...this.and.map(subkeys => subkeys.usedTags()));
}
asChange(properties: any): { k: string; v: string }[] {
const result = []
@ -123,4 +129,89 @@ export class And extends TagsFilter {
and: this.and.map(a => a.AsJson())
}
}
optimize(): TagsFilter | boolean {
if(this.and.length === 0){
return true
}
const optimized = this.and.map(t => t.optimize())
const newAnds : TagsFilter[] = []
let containedOrs : Or[] = []
for (const tf of optimized) {
if(tf === false){
return false
}
if(tf === true){
continue
}
if(tf instanceof And){
newAnds.push(...tf.and)
}else if(tf instanceof Or){
containedOrs.push(tf)
} else {
newAnds.push(tf)
}
}
containedOrs = containedOrs.filter(ca => {
for (const element of ca.or) {
if(optimized.some(opt => typeof opt !== "boolean" && element.isEquivalent(opt) )){
// At least one part of the 'OR' is matched by the outer or, so this means that this OR isn't needed at all
// XY & (XY | AB) === XY
return false
}
}
return true;
})
// Extract common keys from the OR
if(containedOrs.length === 1){
newAnds.push(containedOrs[0])
}
if(containedOrs.length > 1){
let commonValues : TagsFilter [] = containedOrs[0].or
for (let i = 1; i < containedOrs.length && commonValues.length > 0; i++){
const containedOr = containedOrs[i];
commonValues = commonValues.filter(cv => containedOr.or.some(candidate => candidate.isEquivalent(cv)))
}
if(commonValues.length === 0){
newAnds.push(...containedOrs)
}else{
const newOrs: TagsFilter[] = []
for (const containedOr of containedOrs) {
const elements = containedOr.or
.filter(candidate => !commonValues.some(cv => cv.isEquivalent(candidate)))
const or = new Or(elements).optimize()
if(or === true){
// neutral element
continue
}
if(or === false){
return false
}
newOrs.push(or)
}
commonValues.push(new And(newOrs))
const result = new Or(commonValues).optimize()
if(result === false){
return false
}else if(result === true){
// neutral element: skip
}else{
newAnds.push(result)
}
}
}
if(newAnds.length === 1){
return newAnds[0]
}
TagUtils.sortFilters(newAnds, true)
return new And(newAnds)
}
}

7
Logic/Tags/ComparingTag.ts
View file

@ -38,9 +38,16 @@ export default class ComparingTag implements TagsFilter {
usedKeys(): string[] {
return [this._key];
}
usedTags(): { key: string; value: string }[] {
return [];
}
AsJson() {
return this.asHumanString(false, false, {})
}
optimize(): TagsFilter | boolean {
return this;
}
}

83
Logic/Tags/Or.ts
View file

@ -1,4 +1,6 @@
import {TagsFilter} from "./TagsFilter";
import {TagUtils} from "./TagUtils";
import {And} from "./And";
export class Or extends TagsFilter {
@ -58,6 +60,10 @@ export class Or extends TagsFilter {
return [].concat(...this.or.map(subkeys => subkeys.usedKeys()));
}
usedTags(): { key: string; value: string }[] {
return [].concat(...this.or.map(subkeys => subkeys.usedTags()));
}
asChange(properties: any): { k: string; v: string }[] {
const result = []
for (const tagsFilter of this.or) {
@ -71,6 +77,83 @@ export class Or extends TagsFilter {
or: this.or.map(o => o.AsJson())
}
}
optimize(): TagsFilter | boolean {
if(this.or.length === 0){
return false;
}
const optimized = this.or.map(t => t.optimize())
const newOrs : TagsFilter[] = []
let containedAnds : And[] = []
for (const tf of optimized) {
if(tf === true){
return true
}
if(tf === false){
continue
}
if(tf instanceof Or){
newOrs.push(...tf.or)
}else if(tf instanceof And){
containedAnds.push(tf)
} else {
newOrs.push(tf)
}
}
containedAnds = containedAnds.filter(ca => {
for (const element of ca.and) {
if(optimized.some(opt => typeof opt !== "boolean" && element.isEquivalent(opt) )){
// At least one part of the 'AND' is matched by the outer or, so this means that this OR isn't needed at all
// XY | (XY & AB) === XY
return false
}
}
return true;
})
// Extract common keys from the ANDS
if(containedAnds.length === 1){
newOrs.push(containedAnds[0])
} else if(containedAnds.length > 1){
let commonValues : TagsFilter [] = containedAnds[0].and
for (let i = 1; i < containedAnds.length && commonValues.length > 0; i++){
const containedAnd = containedAnds[i];
commonValues = commonValues.filter(cv => containedAnd.and.some(candidate => candidate.isEquivalent(cv)))
}
if(commonValues.length === 0){
newOrs.push(...containedAnds)
}else{
const newAnds: TagsFilter[] = []
for (const containedAnd of containedAnds) {
const elements = containedAnd.and.filter(candidate => !commonValues.some(cv => cv.isEquivalent(candidate)))
newAnds.push(new And(elements))
}
commonValues.push(new Or(newAnds))
const result = new And(commonValues).optimize()
if(result === true){
return true
}else if(result === false){
// neutral element: skip
}else{
newOrs.push(new And(commonValues))
}
}
}
if(newOrs.length === 1){
return newOrs[0]
}
TagUtils.sortFilters(newOrs, false)
return new Or(newOrs)
}
}

28
Logic/Tags/RegexTag.ts
View file

@ -43,10 +43,24 @@ export class RegexTag extends TagsFilter {
}
asOverpass(): string[] {
if (typeof this.key === "string") {
return [`["${this.key}"${this.invert ? "!" : ""}~"${RegexTag.source(this.value)}"]`];
const inv =this.invert ? "!" : ""
if (typeof this.key !== "string") {
// The key is a regex too
return [`[~"${this.key.source}"${inv}~"${RegexTag.source(this.value)}"]`];
}
return [`[~"${this.key.source}"${this.invert ? "!" : ""}~"${RegexTag.source(this.value)}"]`];
if(this.value instanceof RegExp){
const src =this.value.source
if(src === "^..*$"){
// anything goes
return [`[${inv}"${this.key}"]`]
}
return [`["${this.key}"${inv}~"${src}"]`]
}else{
// Normal key and normal value
return [`["${this.key}"${inv}="${this.value}"]`];
}
}
isUsableAsAnswer(): boolean {
@ -99,6 +113,10 @@ export class RegexTag extends TagsFilter {
}
throw "Key cannot be determined as it is a regex"
}
usedTags(): { key: string; value: string }[] {
return [];
}
asChange(properties: any): { k: string; v: string }[] {
if (this.invert) {
@ -120,4 +138,8 @@ export class RegexTag extends TagsFilter {
AsJson() {
return this.asHumanString()
}
optimize(): TagsFilter | boolean {
return this;
}
}

8
Logic/Tags/SubstitutingTag.ts
View file

@ -59,6 +59,10 @@ export default class SubstitutingTag implements TagsFilter {
return [this._key];
}
usedTags(): { key: string; value: string }[] {
return []
}
asChange(properties: any): { k: string; v: string }[] {
if (this._invert) {
throw "An inverted substituting tag can not be used to create a change"
@ -73,4 +77,8 @@ export default class SubstitutingTag implements TagsFilter {
AsJson() {
return this._key + (this._invert ? '!' : '') + "=" + this._value
}
optimize(): TagsFilter | boolean {
return this;
}
}

11
Logic/Tags/Tag.ts
View file

@ -80,6 +80,13 @@ export class Tag extends TagsFilter {
return [this.key];
}
usedTags(): { key: string; value: string }[] {
if(this.value == ""){
return []
}
return [this]
}
asChange(properties: any): { k: string; v: string }[] {
return [{k: this.key, v: this.value}];
}
@ -87,4 +94,8 @@ export class Tag extends TagsFilter {
AsJson() {
return this.asHumanString(false, false)
}
optimize(): TagsFilter | boolean {
return this;
}
}

86
Logic/Tags/TagUtils.ts
View file

@ -8,8 +8,10 @@ import SubstitutingTag from "./SubstitutingTag";
import {Or} from "./Or";
import {AndOrTagConfigJson} from "../../Models/ThemeConfig/Json/TagConfigJson";
import {isRegExp} from "util";
import * as key_counts from "../../assets/key_totals.json"
export class TagUtils {
private static keyCounts : {keys: any, tags: any} = key_counts["default"] ?? key_counts
private static comparators
: [string, (a: number, b: number) => boolean][]
= [
@ -174,6 +176,29 @@ export class TagUtils {
}
}
/**
* INLINE sort of the given list
*/
public static sortFilters(filters: TagsFilter [], usePopularity: boolean): void {
filters.sort((a,b) => TagUtils.order(a, b, usePopularity))
}
public static toString(f: TagsFilter, toplevel = true): string {
let r: string
if (f instanceof Or) {
r = TagUtils.joinL(f.or, "|", toplevel)
} else if (f instanceof And) {
r = TagUtils.joinL(f.and, "&", toplevel)
} else {
r = f.asHumanString(false, false, {})
}
if(toplevel){
r = r.trim()
}
return r
}
private static TagUnsafe(json: AndOrTagConfigJson | string, context: string = ""): TagsFilter {
if (json === undefined) {
@ -285,10 +310,11 @@ export class TagUtils {
throw `Error while parsing tag '${tag}' in ${context}: no key part and value part were found`
}
if(json.and !== undefined && json.or !== undefined){
throw `Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined`}
if (json.and !== undefined && json.or !== undefined) {
throw `Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined`
}
if (json.and !== undefined) {
return new And(json.and.map(t => TagUtils.Tag(t, context)));
}
@ -296,4 +322,56 @@ export class TagUtils {
return new Or(json.or.map(t => TagUtils.Tag(t, context)));
}
}
private static GetCount(key: string, value?: string) {
if(key === undefined) {
return undefined
}
const tag = TagUtils.keyCounts.tags[key]
if(tag !== undefined && tag[value] !== undefined) {
return tag[value]
}
return TagUtils.keyCounts.keys[key]
}
private static order(a: TagsFilter, b: TagsFilter, usePopularity: boolean): number {
const rta = a instanceof RegexTag
const rtb = b instanceof RegexTag
if(rta !== rtb) {
// Regex tags should always go at the end: these use a lot of computation at the overpass side, avoiding it is better
if(rta) {
return 1 // b < a
}else {
return -1
}
}
if (a["key"] !== undefined && b["key"] !== undefined) {
if(usePopularity) {
const countA = TagUtils.GetCount(a["key"], a["value"])
const countB = TagUtils.GetCount(b["key"], b["value"])
if(countA !== undefined && countB !== undefined) {
return countA - countB
}
}
if (a["key"] === b["key"]) {
return 0
}
if (a["key"] < b["key"]) {
return -1
}
return 1
}
return 0
}
private static joinL(tfs: TagsFilter[], seperator: string, toplevel: boolean) {
const joined = tfs.map(e => TagUtils.toString(e, false)).join(seperator)
if (toplevel) {
return joined
}
return " (" + joined + ") "
}
}

13
Logic/Tags/TagsFilter.ts
View file

@ -12,6 +12,12 @@ export abstract class TagsFilter {
abstract usedKeys(): string[];
/**
* Returns all normal key/value pairs
* Regex tags, substitutions, comparisons, ... are exempt
*/
abstract usedTags(): {key: string, value: string}[];
/**
* Converts the tagsFilter into a list of key-values that should be uploaded to OSM.
* Throws an error if not applicable.
@ -21,4 +27,11 @@ export abstract class TagsFilter {
abstract asChange(properties: any): { k: string, v: string }[]
abstract AsJson() ;
/**
* Returns an optimized version (or self) of this tagsFilter
*/
abstract optimize(): TagsFilter | boolean;
}