diff --git a/Logic/Actors/OverpassFeatureSource.ts b/Logic/Actors/OverpassFeatureSource.ts index 2487039542..fa5de3eed3 100644 --- a/Logic/Actors/OverpassFeatureSource.ts +++ b/Logic/Actors/OverpassFeatureSource.ts @@ -2,7 +2,6 @@ import {UIEventSource} from "../UIEventSource"; import Loc from "../../Models/Loc"; import {Or} from "../Tags/Or"; import {Overpass} from "../Osm/Overpass"; -import Bounds from "../../Models/Bounds"; import FeatureSource from "../FeatureSource/FeatureSource"; import {Utils} from "../../Utils"; import {TagsFilter} from "../Tags/TagsFilter"; @@ -36,16 +35,17 @@ export default class OverpassFeatureSource implements FeatureSource { * If the map location changes, we check for each layer if it is loaded: * we start checking the bounds at the first zoom level the layer might operate. If in bounds - no reload needed, otherwise we continue walking down */ - private readonly _previousBounds: Map = new Map(); + private readonly _previousBounds: Map = new Map(); private readonly state: { readonly locationControl: UIEventSource, readonly layoutToUse: LayoutConfig, - readonly overpassUrl: UIEventSource; + readonly overpassUrl: UIEventSource; readonly overpassTimeout: UIEventSource; - readonly currentBounds :UIEventSource + readonly currentBounds: UIEventSource } private readonly _isActive: UIEventSource; private _onUpdated?: (bbox: BBox, dataFreshness: Date) => void; + /** * The most important layer should go first, as that one gets first pick for the questions */ @@ -53,20 +53,20 @@ export default class OverpassFeatureSource implements FeatureSource { state: { readonly locationControl: UIEventSource, readonly layoutToUse: LayoutConfig, - readonly overpassUrl: UIEventSource; + readonly overpassUrl: UIEventSource; readonly overpassTimeout: UIEventSource; readonly overpassMaxZoom: UIEventSource, - readonly currentBounds :UIEventSource - }, - - options?: { + readonly currentBounds: UIEventSource + }, + options?: { isActive?: UIEventSource, - onUpdated?: (bbox: BBox, freshness: Date) => void, - relationTracker: RelationsTracker}) { + onUpdated?: (bbox: BBox, freshness: Date) => void, + relationTracker: RelationsTracker + }) { this.state = state this._isActive = options.isActive; - this._onUpdated =options. onUpdated; + this._onUpdated = options.onUpdated; this.relationsTracker = options.relationTracker const location = state.locationControl const self = this; @@ -79,14 +79,14 @@ export default class OverpassFeatureSource implements FeatureSource { location.addCallback(() => { self.update() }); - + state.currentBounds.addCallback(_ => { self.update() }) - + } - private GetFilter(): Overpass { + private GetFilter(interpreterUrl: string): Overpass { let filters: TagsFilter[] = []; let extraScripts: string[] = []; for (const layer of this.state.layoutToUse.layers) { @@ -113,7 +113,7 @@ export default class OverpassFeatureSource implements FeatureSource { continue; } for (const previousLoadedBound of previousLoadedBounds) { - previouslyLoaded = previouslyLoaded || this.IsInBounds(previousLoadedBound); + previouslyLoaded = previouslyLoaded || this.state.currentBounds.data.isContainedIn(previousLoadedBound); if (previouslyLoaded) { break; } @@ -133,16 +133,16 @@ export default class OverpassFeatureSource implements FeatureSource { if (filters.length + extraScripts.length === 0) { return undefined; } - return new Overpass(new Or(filters), extraScripts, this.state.overpassUrl, this.state.overpassTimeout, this.relationsTracker); + return new Overpass(new Or(filters), extraScripts, interpreterUrl, this.state.overpassTimeout, this.relationsTracker); } private update() { - if(!this._isActive.data){ + if (!this._isActive.data) { return; } const self = this this.updateAsync().then(bboxAndDate => { - if(bboxAndDate === undefined || self._onUpdated === undefined){ + if (bboxAndDate === undefined || self._onUpdated === undefined) { return; } const [bbox, date] = bboxAndDate @@ -162,51 +162,54 @@ export default class OverpassFeatureSource implements FeatureSource { } const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(14); - + if (bounds === undefined) { return undefined; } - - const n = Math.min(90, bounds.getNorth()); - const e = Math.min(180, bounds.getEast()); - const s = Math.max(-90, bounds.getSouth()); - const w = Math.max(-180, bounds.getWest()); - const queryBounds = {north: n, east: e, south: s, west: w}; - - const self = this; - const overpass = this.GetFilter(); - - if (overpass === undefined) { - return undefined; - } - this.runningQuery.setData(true); let data: any = undefined let date: Date = undefined + const overpassUrls = self.state.overpassUrl.data + let lastUsed = 0; do { - try { - [data, date] = await overpass.queryGeoJson(queryBounds) + const overpass = this.GetFilter(overpassUrls[lastUsed]); + + if (overpass === undefined) { + return undefined; + } + this.runningQuery.setData(true); + + [data, date] = await overpass.queryGeoJson(bounds) console.log("Querying overpass is done", data) } catch (e) { self.retries.data++; self.retries.ping(); - console.error(`QUERY FAILED (retrying in ${5 * self.retries.data} sec) due to`, e); + console.error(`QUERY FAILED due to`, e); - self.timeout.setData(self.retries.data * 5); - - while (self.timeout.data > 0) { - await Utils.waitFor(1000) - self.timeout.data-- - self.timeout.ping(); + await Utils.waitFor(1000) + + if (lastUsed + 1 < overpassUrls.length) { + lastUsed++ + console.log("Trying next time with", overpassUrls[lastUsed]) + } else { + lastUsed = 0 + self.timeout.setData(self.retries.data * 5); + + while (self.timeout.data > 0) { + await Utils.waitFor(1000) + console.log(self.timeout.data) + self.timeout.data-- + self.timeout.ping(); + } } } } while (data === undefined); const z = Math.floor(this.state.locationControl.data.zoom ?? 0); - self._previousBounds.get(z).push(queryBounds); + self._previousBounds.get(z).push(bounds); self.retries.setData(0); try { @@ -215,25 +218,10 @@ export default class OverpassFeatureSource implements FeatureSource { return [bounds, date]; } catch (e) { console.error("Got the overpass response, but could not process it: ", e, e.stack) - }finally { + } finally { self.runningQuery.setData(false); } - } - - private IsInBounds(bounds: Bounds): boolean { - if (this._previousBounds === undefined) { - return false; - } - - const b = this.state.currentBounds.data; - return b.getSouth() >= bounds.south && - b.getNorth() <= bounds.north && - b.getEast() <= bounds.east && - b.getWest() >= bounds.west; - } - - } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index e8d270f0b6..3db3a039d4 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -30,10 +30,10 @@ import {Tiles} from "../../Models/TileRange"; export default class FeaturePipeline { public readonly sufficientlyZoomed: UIEventSource; - + public readonly runningQuery: UIEventSource; public readonly timeout: UIEventSource; - + public readonly somethingLoaded: UIEventSource = new UIEventSource(false) public readonly newDataLoadedSignal: UIEventSource = new UIEventSource(undefined) @@ -50,7 +50,7 @@ export default class FeaturePipeline { readonly changes: Changes, readonly layoutToUse: LayoutConfig, readonly leafletMap: any, - readonly overpassUrl: UIEventSource; + readonly overpassUrl: UIEventSource; readonly overpassTimeout: UIEventSource; readonly overpassMaxZoom: UIEventSource; readonly osmConnection: OsmConnection @@ -67,6 +67,7 @@ export default class FeaturePipeline { const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12)) this.relationTracker = new RelationsTracker() + console.log("Tilefreshnesses are", tileFreshnesses.data) const oldestAllowedDate = new Date(new Date().getTime() - (60 * 60 * 24 * 30 * 1000)); const neededTilesFromOsm = state.currentBounds.map(bbox => { @@ -90,7 +91,7 @@ export default class FeaturePipeline { }) return tileIndexes }, [tileFreshnesses]) - + const updater = new OverpassFeatureSource(state, { relationTracker: this.relationTracker, @@ -105,8 +106,10 @@ export default class FeaturePipeline { } }); - + this.overpassUpdater = updater; + this.timeout = updater.timeout + this.sufficientlyZoomed = state.locationControl.map(location => { if (location?.zoom === undefined) { return false; @@ -115,10 +118,10 @@ export default class FeaturePipeline { return location.zoom >= minzoom; } ); - + this.timeout = updater.timeout - - + + // Register everything in the state' 'AllElements' new RegisteringAllFromFeatureSourceActor(updater) @@ -203,9 +206,8 @@ export default class FeaturePipeline { } } - - const osmFeatureSource = new OsmFeatureSource({ + const osmFeatureSource = new OsmFeatureSource({ isActive: useOsmApi, neededTiles: neededTilesFromOsm, handleTile: tile => { diff --git a/Logic/Osm/Overpass.ts b/Logic/Osm/Overpass.ts index 2bf1b4e705..47b92e278a 100644 --- a/Logic/Osm/Overpass.ts +++ b/Logic/Osm/Overpass.ts @@ -1,9 +1,9 @@ import * as OsmToGeoJson from "osmtogeojson"; -import Bounds from "../../Models/Bounds"; import {TagsFilter} from "../Tags/TagsFilter"; import RelationsTracker from "./RelationsTracker"; import {Utils} from "../../Utils"; import {UIEventSource} from "../UIEventSource"; +import {BBox} from "../BBox"; /** * Interfaces overpass to get all the latest data @@ -11,7 +11,7 @@ import {UIEventSource} from "../UIEventSource"; export class Overpass { public static testUrl: string = null private _filter: TagsFilter - private readonly _interpreterUrl: UIEventSource; + private readonly _interpreterUrl: string; private readonly _timeout: UIEventSource; private readonly _extraScripts: string[]; private _includeMeta: boolean; @@ -19,7 +19,7 @@ export class Overpass { constructor(filter: TagsFilter, extraScripts: string[], - interpreterUrl: UIEventSource, + interpreterUrl: string, timeout: UIEventSource, relationTracker: RelationsTracker, includeMeta = true) { @@ -31,9 +31,9 @@ export class Overpass { this._relationTracker = relationTracker } - public async queryGeoJson(bounds: Bounds): Promise<[any, Date]> { + public async queryGeoJson(bounds: BBox): Promise<[any, Date]> { - let query = this.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]") + let query = this.buildQuery("[bbox:" + bounds.getSouth() + "," + bounds.getWest() + "," + bounds.getNorth() + "," + bounds.getEast() + "]") if (Overpass.testUrl !== null) { console.log("Using testing URL") @@ -68,6 +68,6 @@ export class Overpass { } const query = `[out:json][timeout:${this._timeout.data}]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;` - return `${this._interpreterUrl.data}?data=${encodeURIComponent(query)}` + return `${this._interpreterUrl}?data=${encodeURIComponent(query)}` } } diff --git a/Models/Bounds.ts b/Models/Bounds.ts deleted file mode 100644 index 3a993c8e06..0000000000 --- a/Models/Bounds.ts +++ /dev/null @@ -1,6 +0,0 @@ -export default interface Bounds { - north: number, - east: number, - south: number, - west: number -} \ No newline at end of file diff --git a/Models/Constants.ts b/Models/Constants.ts index a7f79bf39f..b9855a2930 100644 --- a/Models/Constants.ts +++ b/Models/Constants.ts @@ -4,6 +4,16 @@ export default class Constants { public static vNumber = "0.10.0-alpha-4"; public static ImgurApiKey = '7070e7167f0a25a' + public static defaultOverpassUrls = [ + // The official instance, 10000 queries per day per project allowed + "https://overpass-api.de/api/interpreter", + // 'Fair usage' + "https://overpass.kumi.systems/api/interpreter", + // "https://overpass.nchc.org.tw/api/interpreter", + "https://overpass.openstreetmap.ru/cgi/interpreter", + // The french api, only 1000 per day per project allowed, so we put it as last resort + "https://overpass.openstreetmap.fr/api/interpreter" + ] // The user journey states thresholds when a new feature gets unlocked public static userJourney = { diff --git a/Models/ThemeConfig/Json/LayoutConfigJson.ts b/Models/ThemeConfig/Json/LayoutConfigJson.ts index 887649fc4d..73caee7929 100644 --- a/Models/ThemeConfig/Json/LayoutConfigJson.ts +++ b/Models/ThemeConfig/Json/LayoutConfigJson.ts @@ -263,9 +263,9 @@ export interface LayoutConfigJson { enablePdfDownload?: boolean; /** - * Set a different overpass URL. Default: https://overpass-api.de/api/interpreter + * Set one or more overpass URLs to use for this theme.. */ - overpassUrl?: string; + overpassUrl?: string | string[]; /** * Set a different timeout for overpass queries - in seconds. Default: 30s */ diff --git a/Models/ThemeConfig/LayoutConfig.ts b/Models/ThemeConfig/LayoutConfig.ts index 9e63250390..4f85516325 100644 --- a/Models/ThemeConfig/LayoutConfig.ts +++ b/Models/ThemeConfig/LayoutConfig.ts @@ -6,6 +6,7 @@ import AllKnownLayers from "../../Customizations/AllKnownLayers"; import {Utils} from "../../Utils"; import LayerConfig from "./LayerConfig"; import {LayerConfigJson} from "./Json/LayerConfigJson"; +import Constants from "../Constants"; export default class LayoutConfig { public readonly id: string; @@ -50,7 +51,7 @@ export default class LayoutConfig { How long is the cache valid, in seconds? */ public readonly cacheTimeout?: number; - public readonly overpassUrl: string; + public readonly overpassUrl: string[]; public readonly overpassTimeout: number; public readonly official: boolean; @@ -157,7 +158,14 @@ export default class LayoutConfig { this.enablePdfDownload = json.enablePdfDownload ?? false; this.customCss = json.customCss; this.cacheTimeout = json.cacheTimout ?? (60 * 24 * 60 * 60) - this.overpassUrl = json.overpassUrl ?? "https://overpass-api.de/api/interpreter" + this.overpassUrl = Constants.defaultOverpassUrls + if(json.overpassUrl !== undefined){ + if(typeof json.overpassUrl === "string"){ + this.overpassUrl = [json.overpassUrl] + }else{ + this.overpassUrl = json.overpassUrl + } + } this.overpassTimeout = json.overpassTimeout ?? 30 } diff --git a/State.ts b/State.ts index 0af8e5a648..00dac3bb50 100644 --- a/State.ts +++ b/State.ts @@ -81,7 +81,7 @@ export default class State { public readonly featureSwitchEnableExport: UIEventSource; public readonly featureSwitchFakeUser: UIEventSource; public readonly featureSwitchExportAsPdf: UIEventSource; - public readonly overpassUrl: UIEventSource; + public readonly overpassUrl: UIEventSource; public readonly overpassTimeout: UIEventSource; @@ -321,9 +321,9 @@ export default class State { ); this.overpassUrl = QueryParameters.GetQueryParameter("overpassUrl", - layoutToUse?.overpassUrl, + layoutToUse?.overpassUrl.join(","), "Point mapcomplete to a different overpass-instance. Example: https://overpass-api.de/api/interpreter" - ) + ).map(param => param.split(","), [], urls => urls.join(",")) this.overpassTimeout = QueryParameters.GetQueryParameter("overpassTimeout", "" + layoutToUse?.overpassTimeout, diff --git a/Utils.ts b/Utils.ts index 424784205f..bd5cd62a5a 100644 --- a/Utils.ts +++ b/Utils.ts @@ -320,6 +320,7 @@ export class Utils { } xhr.send(); + xhr.onerror = reject } ) } diff --git a/assets/themes/toerisme_vlaanderen/license_info.json b/assets/themes/toerisme_vlaanderen/license_info.json new file mode 100644 index 0000000000..374abea1e6 --- /dev/null +++ b/assets/themes/toerisme_vlaanderen/license_info.json @@ -0,0 +1,12 @@ +[ + { + "path": "logo.png", + "license": "Logo (all rights reserved)", + "authors": [ + "Toerisme Vlaanderen" + ], + "sources": [ + "https://www.toerismevlaanderen.be/" + ] + } +] \ No newline at end of file diff --git a/assets/themes/toerisme_vlaanderen/logo.png b/assets/themes/toerisme_vlaanderen/logo.png new file mode 100644 index 0000000000..6d0d16dd84 Binary files /dev/null and b/assets/themes/toerisme_vlaanderen/logo.png differ