Merge develop

This commit is contained in:
Pieter Vander Vennet 2025-01-21 21:14:11 +01:00
commit 4fdae55c97
206 changed files with 8760 additions and 3718 deletions

View file

@ -88,7 +88,7 @@ export default class InitialMapPositioning {
return
}
const targetLayer = layoutToUse.getMatchingLayer(osmObject.tags)
if(targetLayer){
if (targetLayer) {
this.zoom.setData(Math.max(this.zoom.data, targetLayer.minzoom))
}
const [lat, lon] = osmObject.centerpoint()

View file

@ -89,11 +89,11 @@ export default class DetermineTheme {
if (themes.length == 0) {
throw "Build failed or running, no layouts are known at all"
}
const themeInfo = themes.find(th => th.id === id)
const themeInfo = themes.find((th) => th.id === id)
if (themeInfo === undefined) {
const alternatives = Utils.sortedByLevenshteinDistance(
id,
themes.map(th => th.id),
themes.map((th) => th.id),
(i) => i
).slice(0, 3)
const msg = `No builtin map theme with name ${layoutId} exists. Perhaps you meant one of ${alternatives.join(
@ -103,7 +103,10 @@ export default class DetermineTheme {
}
// Actually fetch the theme
const config = await Utils.downloadJsonCached<ThemeConfigJson>("./assets/generated/themes/"+id+".json", 1000*60*60*60)
const config = await Utils.downloadJsonCached<ThemeConfigJson>(
"./assets/generated/themes/" + id + ".json",
1000 * 60 * 60 * 60
)
return new ThemeConfig(config, true)
}

View file

@ -22,7 +22,7 @@ export default class AllImageProviders {
...WikimediaImageProvider.commonsPrefixes,
...Mapillary.valuePrefixes,
...AllImageProviders.dontLoadFromPrefixes,
"Category:",
"Category:"
])
private static ImageAttributionSource: ImageProvider[] = [
@ -31,7 +31,7 @@ export default class AllImageProviders {
WikidataImageProvider.singleton,
WikimediaImageProvider.singleton,
Panoramax.singleton,
AllImageProviders.genericImageProvider,
AllImageProviders.genericImageProvider
]
public static apiUrls: string[] = [].concat(
...AllImageProviders.ImageAttributionSource.map((src) => src.apiUrls())
@ -44,7 +44,7 @@ export default class AllImageProviders {
mapillary: Mapillary.singleton,
wikidata: WikidataImageProvider.singleton,
wikimedia: WikimediaImageProvider.singleton,
panoramax: Panoramax.singleton,
panoramax: Panoramax.singleton
}
public static byName(name: string) {
@ -67,10 +67,33 @@ export default class AllImageProviders {
}
private static readonly _cachedImageStores: Record<string, Store<ProvidedImage[]>> = {}
/**
* Does a guess on the number of images that are probably there.
* Will simply count all image tags
*
* AllImageProviders.estimateNumberOfImages({image:"abc", "mapillary": "123", "panoramax:0": "xyz"}) // => 3
*
*/
public static estimateNumberOfImages(tags: Record<string, string>, prefixes: string[] = undefined): number {
let count = 0
const allPrefixes = Utils.Dedup(prefixes ?? [].concat(...AllImageProviders.ImageAttributionSource.map(s => s.defaultKeyPrefixes)))
for (const prefix of allPrefixes) {
for (const k in tags) {
if (k === prefix || k.startsWith(prefix + ":")) {
count++
continue
}
}
}
return count
}
/**
* Tries to extract all image data for this image. Cached on tags?.data?.id
*/
public static LoadImagesFor(
public static loadImagesFor(
tags: Store<Record<string, string>>,
tagKey?: string[]
): Store<ProvidedImage[]> {
@ -108,11 +131,11 @@ export default class AllImageProviders {
*/
public static loadImagesFrom(urls: string[]): Store<ProvidedImage[]> {
const tags = {
id: urls.join(";"),
id: urls.join(";")
}
for (let i = 0; i < urls.length; i++) {
tags["image:" + i] = urls[i]
}
return this.LoadImagesFor(new ImmutableStore(tags))
return this.loadImagesFor(new ImmutableStore(tags))
}
}

View file

@ -149,7 +149,7 @@ export default class PanoramaxImageProvider extends ImageProvider {
)
}
Stores.Chronic(1500, () => hasLoading(source.data)).addCallback(() => {
Stores.Chronic(5000, () => hasLoading(source.data)).addCallback(() => {
super.getRelevantUrlsFor(tags, prefixes).then((data) => {
source.set(data)
return !hasLoading(data)

View file

@ -610,8 +610,10 @@ export class OsmConnection {
if (!(this.apiIsOnline.data === "unreachable" || this.apiIsOnline.data === "offline")) {
return
}
if (!this.isLoggedIn.data) {
return
}
try {
console.log("Api is offline - trying to reconnect...")
this.AttemptLogin()
} catch (e) {
console.log("Could not login due to", e)

View file

@ -1,8 +1,8 @@
import { Store, UIEventSource } from "../UIEventSource"
import { OsmConnection } from "./OsmConnection"
import { LocalStorageSource } from "../Web/LocalStorageSource"
import OSMAuthInstance = OSMAuth.osmAuth
import { Utils } from "../../Utils"
import OSMAuthInstance = OSMAuth.osmAuth
export class OsmPreferences {
/**

View file

@ -124,7 +124,7 @@ export class And extends TagsFilter {
* t0.shadows(t0) // => true
* t1.shadows(t1) // => true
* t2.shadows(t2) // => true
* t0.shadows(t1) // => false
* t0.shadows(t1) // => true
* t0.shadows(t2) // => false
* t1.shadows(t0) // => false
* t1.shadows(t2) // => false
@ -135,33 +135,34 @@ export class And extends TagsFilter {
* const t1 = new And([new Tag("shop","clothes"), new Or([new Tag("brand","XYZ"),new Tag("brand:wikidata","Q1234")])])
* const t2 = new And([new RegexTag("shop","mall",true), new Or([TagUtils.Tag("shop~*"), new Tag("craft","shoemaker")])])
* t1.shadows(t2) // => true
*
* const t1 = new Tag("a","b")
* const t2 = new And([new Tag("x","y"), new Tag("a","b")])
* t2.shadows(t1) // => true
* t1.shadows(t2) // => false
*/
shadows(other: TagsFilter): boolean {
const phrases: TagsFilter[] = other instanceof And ? other.and : [other];
// The phrases of the _other_ and
const phrases: readonly TagsFilter[] = other instanceof And ? other.and : [other]
// A phrase might be shadowed by a certain subsection. We keep track of this here
const shadowedOthers = phrases.map(() => false)
for (const selfTag of this.and) {
let shadowsSome = false;
let shadowsAll = true;
for (let i = 0; i < phrases.length; i++){
let shadowsAll = true
for (let i = 0; i < phrases.length; i++) {
const otherTag = phrases[i]
const doesShadow = selfTag.shadows(otherTag)
if(doesShadow){
shadowedOthers[i] = true;
if (doesShadow) {
shadowedOthers[i] = true
}
shadowsSome ||= doesShadow;
shadowsAll &&= doesShadow;
shadowsAll &&= doesShadow
}
// If A => X and A => Y, then
// A&B implies X&Y. We discovered an A that implies all needed values
if (shadowsAll) {
return true;
}
if (!shadowsSome) {
return false;
return true
}
}
return !shadowedOthers.some(v => !v);
return !shadowedOthers.some((v) => !v)
}
usedKeys(): string[] {

View file

@ -83,7 +83,6 @@ export class Or extends TagsFilter {
return false
}
shadows(other: TagsFilter): boolean {
if (other instanceof Or) {
for (const selfTag of this.or) {

View file

@ -150,11 +150,11 @@ export class Tag extends TagsFilter {
return other.matchesProperties({ [this.key]: this.value })
}
}
if(other instanceof Or){
return other.or.some(other => this.shadows(other))
if (other instanceof Or) {
return other.or.some((other) => this.shadows(other))
}
if(other instanceof And){
return !other.and.some(other => !this.shadows(other))
if (other instanceof And) {
return !other.and.some((other) => !this.shadows(other))
}
return false
}

View file

@ -133,11 +133,11 @@ export class TagUtils {
"\n" +
"```json\n" +
"{\n" +
" \"mappings\": [\n" +
' "mappings": [\n' +
" {\n" +
" \"if\":\"key:={some_other_key}\",\n" +
" \"then\": \"...\",\n" +
" \"hideInAnswer\": \"some_other_key=\"\n" +
' "if":"key:={some_other_key}",\n' +
' "then": "...",\n' +
' "hideInAnswer": "some_other_key="\n' +
" }\n" +
" ]\n" +
"}\n" +
@ -175,10 +175,10 @@ export class TagUtils {
"\n" +
"```json\n" +
"{\n" +
" \"osmTags\": {\n" +
" \"or\": [\n" +
" \"amenity=school\",\n" +
" \"amenity=kindergarten\"\n" +
' "osmTags": {\n' +
' "or": [\n' +
' "amenity=school",\n' +
' "amenity=kindergarten"\n' +
" ]\n" +
" }\n" +
"}\n" +
@ -194,7 +194,7 @@ export class TagUtils {
"If the schema-files note a type [`TagConfigJson`](https://github.com/pietervdvn/MapComplete/blob/develop/src/Models/ThemeConfig/Json/TagConfigJson.ts), you can use one of these values.\n" +
"\n" +
"In some cases, not every type of tags-filter can be used. For example, _rendering_ an option with a regex is\n" +
"fine (`\"if\": \"brand~[Bb]randname\", \"then\":\" The brand is Brandname\"`); but this regex can not be used to write a value\n" +
'fine (`"if": "brand~[Bb]randname", "then":" The brand is Brandname"`); but this regex can not be used to write a value\n' +
"into the database. The theme loader will however refuse to work with such inconsistencies and notify you of this while\n" +
"you are building your theme.\n" +
"\n" +
@ -205,18 +205,18 @@ export class TagUtils {
"\n" +
"```json\n" +
"{\n" +
" \"and\": [\n" +
" \"key=value\",\n" +
' "and": [\n' +
' "key=value",\n' +
" {\n" +
" \"or\": [\n" +
" \"other_key=value\",\n" +
" \"other_key=some_other_value\"\n" +
' "or": [\n' +
' "other_key=value",\n' +
' "other_key=some_other_value"\n' +
" ]\n" +
" },\n" +
" \"key_which_should_be_missing=\",\n" +
" \"key_which_should_have_a_value~*\",\n" +
" \"key~.*some_regex_a*_b+_[a-z]?\",\n" +
" \"height<1\"\n" +
' "key_which_should_be_missing=",\n' +
' "key_which_should_have_a_value~*",\n' +
' "key~.*some_regex_a*_b+_[a-z]?",\n' +
' "height<1"\n' +
" ]\n" +
"}\n" +
"```\n" +
@ -246,7 +246,7 @@ export class TagUtils {
static asProperties(
tags: TagsFilter | TagsFilter[],
baseproperties: Record<string, string> = {},
baseproperties: Record<string, string> = {}
) {
if (Array.isArray(tags)) {
tags = new And(tags)
@ -274,11 +274,11 @@ export class TagUtils {
static SplitKeysRegex(tagsFilters: UploadableTag[], allowRegex: false): Record<string, string[]>
static SplitKeysRegex(
tagsFilters: UploadableTag[],
allowRegex: boolean,
allowRegex: boolean
): Record<string, (string | RegexTag)[]>
static SplitKeysRegex(
tagsFilters: UploadableTag[],
allowRegex: boolean,
allowRegex: boolean
): Record<string, (string | RegexTag)[]> {
const keyValues: Record<string, (string | RegexTag)[]> = {}
tagsFilters = [...tagsFilters] // copy all, use as queue
@ -307,7 +307,7 @@ export class TagUtils {
if (typeof key !== "string") {
console.error(
"Invalid type to flatten the multiAnswer: key is a regex too",
tagsFilter,
tagsFilter
)
throw "Invalid type to FlattenMultiAnswer: key is a regex too"
}
@ -508,7 +508,7 @@ export class TagUtils {
public static Tag(json: TagConfigJson, context?: string | ConversionContext): TagsFilterClosed
public static Tag(
json: TagConfigJson,
context: string | ConversionContext = "",
context: string | ConversionContext = ""
): TagsFilterClosed {
try {
const ctx = typeof context === "string" ? context : context.path.join(".")
@ -540,7 +540,7 @@ export class TagUtils {
throw `Error at ${context}: detected a non-uploadable tag at a location where this is not supported: ${t.asHumanString(
false,
false,
{},
{}
)}`
})
@ -661,7 +661,7 @@ export class TagUtils {
*/
public static removeShadowedElementsFrom(
blacklist: TagsFilter[],
listToFilter: TagsFilter[],
listToFilter: TagsFilter[]
): TagsFilter[] {
return listToFilter.filter((tf) => !blacklist.some((guard) => guard.shadows(tf)))
}
@ -690,15 +690,22 @@ export class TagUtils {
return result
}
public static removeKnownParts(tag: TagsFilter, known: TagsFilter, valueOfKnown = true): TagsFilter | boolean{
/**
* TagUtils.removeKnownParts(TagUtils.Tag({and: ["vending=excrement_bag"}),TagUtils.Tag({and: ["amenity=waste_basket", "vending=excrement_bag"]}), true) // => true
*/
public static removeKnownParts(
tag: TagsFilter,
known: TagsFilter,
valueOfKnown = true
): TagsFilter | boolean {
const tagOrBool = And.construct([tag]).optimize()
if(tagOrBool === true || tagOrBool === false){
if (tagOrBool === true || tagOrBool === false) {
return tagOrBool
}
if(tagOrBool instanceof And){
if (tagOrBool instanceof And) {
return tagOrBool.removePhraseConsideredKnown(known, valueOfKnown)
}
return tagOrBool
return new And([tagOrBool]).removePhraseConsideredKnown(known, valueOfKnown)
}
/**
@ -710,7 +717,7 @@ export class TagUtils {
*/
public static containsEquivalents(
guards: ReadonlyArray<TagsFilter>,
listToFilter: ReadonlyArray<TagsFilter>,
listToFilter: ReadonlyArray<TagsFilter>
): boolean {
return listToFilter.some((tf) => guards.some((guard) => guard.shadows(tf)))
}
@ -754,7 +761,7 @@ export class TagUtils {
values.push(i + "")
}
return values
}),
})
)
return Utils.NoNull(spec)
}
@ -762,13 +769,13 @@ export class TagUtils {
private static ParseTagUnsafe(json: TagConfigJson, context: string = ""): TagsFilterClosed {
if (json === undefined) {
throw new Error(
`Error while parsing a tag: 'json' is undefined in ${context}. Make sure all the tags are defined and at least one tag is present in a complex expression`,
`Error while parsing a tag: 'json' is undefined in ${context}. Make sure all the tags are defined and at least one tag is present in a complex expression`
)
}
if (typeof json != "string") {
if (json["and"] !== undefined && json["or"] !== undefined) {
throw `${context}: Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined. Did you override a value? Perhaps use \`"=parent": { ... }\` instead of "parent": {...}\` to trigger a replacement and not a fuse of values. The value is ${JSON.stringify(
json,
json
)}`
}
if (json["and"] !== undefined) {
@ -850,13 +857,13 @@ export class TagUtils {
return new RegexTag(
withRegex.key,
new RegExp(".+", "si" + withRegex.modifier),
withRegex.invert,
withRegex.invert
)
}
return new RegexTag(
withRegex.key,
new RegExp("^(" + value + ")$", "s" + withRegex.modifier),
withRegex.invert,
withRegex.invert
)
}
@ -978,16 +985,15 @@ export class TagUtils {
return ["", "## `" + mode + "` " + doc.name, "", doc.docs, "", ""].join("\n")
}),
"## " +
TagUtils.comparators.map((comparator) => "`" + comparator[0] + "`").join(" ") +
" Logical comparators",
TagUtils.comparators.map((comparator) => "`" + comparator[0] + "`").join(" ") +
" Logical comparators",
TagUtils.numberAndDateComparisonDocs,
TagUtils.logicalOperator,
].join("\n")
}
static fromProperties(tags: Record<string, string>): TagConfigJson | boolean {
const opt = new And(Object.keys(tags).map(k => new Tag(k, tags[k]))).optimize()
const opt = new And(Object.keys(tags).map((k) => new Tag(k, tags[k]))).optimize()
if (opt === true || opt === false) {
return opt
}

View file

@ -49,12 +49,11 @@ export interface NSIItem {
include: string[]
exclude: string[]
}
readonly tags: Readonly<Record<string, string>>
readonly fromTemplate?: boolean
readonly tags: Readonly<Record<string, string>>
readonly fromTemplate?: boolean
}
export default class NameSuggestionIndex {
public static readonly supportedTypes = ["brand", "flag", "operator", "transit"] as const
private readonly nsiFile: Readonly<NSIFile>
private readonly nsiWdFile: Readonly<
@ -80,7 +79,7 @@ export default class NameSuggestionIndex {
}
>
>,
features: Readonly<FeatureCollection>,
features: Readonly<FeatureCollection>
) {
this.nsiFile = nsiFile
this.nsiWdFile = nsiWdFile
@ -94,11 +93,17 @@ export default class NameSuggestionIndex {
return NameSuggestionIndex.inited
}
const [nsi, nsiWd, features] = await Promise.all(
["./assets/data/nsi/nsi.min.json", "./assets/data/nsi/wikidata.min.json", "./assets/data/nsi/featureCollection.min.json"].map((url) =>
Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30),
),
[
"./assets/data/nsi/nsi.min.json",
"./assets/data/nsi/wikidata.min.json",
"./assets/data/nsi/featureCollection.min.json",
].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30))
)
NameSuggestionIndex.inited = new NameSuggestionIndex(
<any>nsi,
<any>nsiWd["wikidata"],
<any>features
)
NameSuggestionIndex.inited = new NameSuggestionIndex(<any>nsi, <any>nsiWd["wikidata"], <any>features)
return NameSuggestionIndex.inited
}
@ -129,13 +134,13 @@ export default class NameSuggestionIndex {
try {
return Utils.downloadJsonCached<Record<string, number>>(
`./assets/data/nsi/stats/${type}.${c.toUpperCase()}.json`,
24 * 60 * 60 * 1000,
24 * 60 * 60 * 1000
)
} catch (e) {
console.error("Could not fetch " + type + " statistics due to", e)
return undefined
}
}),
})
)
stats = Utils.NoNull(stats)
if (stats.length === 1) {
@ -183,10 +188,13 @@ export default class NameSuggestionIndex {
* If set, sort by frequency instead of alphabetically
*/
sortByFrequency: boolean
},
}
): Promise<Mapping[]> {
const mappings: (Mapping & { frequency: number })[] = []
const frequencies = country !== undefined ? await NameSuggestionIndex.fetchFrequenciesFor(type, country) : {}
const frequencies =
country !== undefined
? await NameSuggestionIndex.fetchFrequenciesFor(type, country)
: {}
for (const key in tags) {
if (key.startsWith("_")) {
continue
@ -197,7 +205,7 @@ export default class NameSuggestionIndex {
key,
value,
country.join(";"),
location,
location
)
if (!actualBrands) {
continue
@ -242,7 +250,7 @@ export default class NameSuggestionIndex {
}
public supportedTags(
type: "operator" | "brand" | "flag" | "transit" | string,
type: "operator" | "brand" | "flag" | "transit" | string
): Record<string, string[]> {
const tags: Record<string, string[]> = {}
const keys = Object.keys(this.nsiFile.nsi)
@ -287,10 +295,10 @@ export default class NameSuggestionIndex {
type: string,
tags: { key: string; value: string }[],
country: string = undefined,
location: [number, number] = undefined,
location: [number, number] = undefined
): NSIItem[] {
return tags.flatMap((tag) =>
this.getSuggestionsForKV(type, tag.key, tag.value, country, location),
this.getSuggestionsForKV(type, tag.key, tag.value, country, location)
)
}
@ -313,7 +321,7 @@ export default class NameSuggestionIndex {
key: string,
value: string,
country: string = undefined,
location: [number, number] = undefined,
location: [number, number] = undefined
): NSIItem[] {
const path = `${type}s/${key}/${value}`
const entry = this.nsiFile.nsi[path]
@ -353,8 +361,7 @@ export default class NameSuggestionIndex {
}
const key = i.locationSet.include?.join(";") + "-" + i.locationSet.exclude?.join(";")
const fromCache = NameSuggestionIndex.resolvedSets[key]
const resolvedSet =
fromCache ?? this.loco.resolveLocationSet(i.locationSet)
const resolvedSet = fromCache ?? this.loco.resolveLocationSet(i.locationSet)
if (!fromCache) {
NameSuggestionIndex.resolvedSets[key] = resolvedSet
}
@ -377,13 +384,12 @@ export default class NameSuggestionIndex {
center: [number, number],
options: {
sortByFrequency: boolean
},
}
): Promise<Mapping[]> {
const nsi = await NameSuggestionIndex.getNsiIndex()
return nsi.generateMappings(key, tags, country, center, options)
}
/**
* Where can we find the URL on the world wide web?
* Probably facebook! Don't use in the website, might expose people
@ -402,7 +408,7 @@ export default class NameSuggestionIndex {
}
return icon
}
private static readonly brandPrefix = ["name", "alt_name", "operator","brand"] as const
private static readonly brandPrefix = ["name", "alt_name", "operator", "brand"] as const
/**
* An NSI-item might have tags such as `name=X`, `alt_name=brand X`, `brand=X`, `brand:wikidata`, `shop=Y`, `service:abc=yes`
@ -412,12 +418,14 @@ export default class NameSuggestionIndex {
*
* (More of an extension method on NSIItem)
*/
static asFilterTags(item: NSIItem): string | { and: TagConfigJson[] } | { or: TagConfigJson[] } {
static asFilterTags(
item: NSIItem
): string | { and: TagConfigJson[] } | { or: TagConfigJson[] } {
let brandDetection: string[] = []
let required: string[] = []
const tags: Record<string, string> = item.tags
for (const k in tags) {
if (NameSuggestionIndex.brandPrefix.some(br => k === br || k.startsWith(br + ":"))) {
if (NameSuggestionIndex.brandPrefix.some((br) => k === br || k.startsWith(br + ":"))) {
brandDetection.push(k + "=" + tags[k])
} else {
required.push(k + "=" + tags[k])

View file

@ -40,7 +40,7 @@ export default class TagInfo {
let url: string
if (value) {
url = `${this._backend}api/4/tag/stats?key=${encodeURIComponent(
key,
key
)}&value=${encodeURIComponent(value)}`
} else {
url = `${this._backend}api/4/key/stats?key=${encodeURIComponent(key)}`
@ -70,10 +70,10 @@ export default class TagInfo {
}
const countriesFC: FeatureCollection = await Utils.downloadJsonCached<FeatureCollection>(
"https://download.geofabrik.de/index-v1-nogeom.json",
24 * 1000 * 60 * 60,
24 * 1000 * 60 * 60
)
TagInfo._geofabrikCountries = countriesFC.features.map(
(f) => <GeofabrikCountryProperties>f.properties,
(f) => <GeofabrikCountryProperties>f.properties
)
return TagInfo._geofabrikCountries
}
@ -99,7 +99,7 @@ export default class TagInfo {
private static async getDistributionsFor(
countryCode: string,
key: string,
value?: string,
value?: string
): Promise<TagInfoStats> {
if (!countryCode) {
return undefined
@ -111,7 +111,16 @@ export default class TagInfo {
try {
return await ti.getStats(key, value)
} catch (e) {
console.warn("Could not fetch info from taginfo for", countryCode, key, value, "due to", e, "Taginfo country specific instance is ", ti._backend)
console.warn(
"Could not fetch info from taginfo for",
countryCode,
key,
value,
"due to",
e,
"Taginfo country specific instance is ",
ti._backend
)
return undefined
}
}
@ -127,9 +136,9 @@ export default class TagInfo {
*/
public static async getGlobalDistributionsFor<T>(
writeInto: Record<string, T>,
f: ((stats: TagInfoStats) => T),
f: (stats: TagInfoStats) => T,
key: string,
value?: string,
value?: string
): Promise<number> {
const countriesAll = await this.geofabrikCountries()
const countries = countriesAll
@ -138,14 +147,14 @@ export default class TagInfo {
let downloaded = 0
for (const country of countries) {
if(writeInto[country] !== undefined){
if (writeInto[country] !== undefined) {
continue
}
const r = await TagInfo.getDistributionsFor(country, key, value)
if(r === undefined){
if (r === undefined) {
continue
}
downloaded ++
downloaded++
writeInto[country] = f(r)
}
return downloaded