Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

merge develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-04-09 17:18:30 +02:00
commit 3e4708b0b9
506 changed files with 7945 additions and 74587 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/Logic/ImageProviders/AllImageProviders.ts
View file

@ -2,7 +2,7 @@ import { Mapillary } from "./Mapillary"
import { WikimediaImageProvider } from "./WikimediaImageProvider"
import { Imgur } from "./Imgur"
import GenericImageProvider from "./GenericImageProvider"
import { ImmutableStore, Store, UIEventSource } from "../UIEventSource"
import { ImmutableStore, Store, Stores } from "../UIEventSource"
import ImageProvider, { ProvidedImage } from "./ImageProvider"
import { WikidataImageProvider } from "./WikidataImageProvider"
import Panoramax from "./Panoramax"
@ -36,8 +36,8 @@ export default class AllImageProviders {
public static apiUrls: string[] = [].concat(
...AllImageProviders.imageAttributionSources.map((src) => src.apiUrls())
)
public static defaultKeys = [].concat(
AllImageProviders.imageAttributionSources.map((provider) => provider.defaultKeyPrefixes)
public static defaultKeys: string[] = [].concat(
...AllImageProviders.imageAttributionSources.map((provider) => provider.defaultKeyPrefixes)
)
private static providersByName = {
imgur: Imgur.singleton,
@ -122,7 +122,6 @@ export default class AllImageProviders {
return this._cachedImageStores[cachekey]
}
const source = new UIEventSource([])
const allSources: Store<ProvidedImage[]>[] = []
for (const imageProvider of AllImageProviders.imageAttributionSources) {
/*
@ -132,12 +131,11 @@ export default class AllImageProviders {
const prefixes = tagKey ?? imageProvider.defaultKeyPrefixes
const singleSource = tags.bindD((tags) => imageProvider.getRelevantUrls(tags, prefixes))
allSources.push(singleSource)
singleSource.addCallbackAndRunD((_) => {
const all: ProvidedImage[] = [].concat(...allSources.map((source) => source.data))
const dedup = Utils.DedupOnId(all, (i) => i?.id ?? i?.url)
source.set(dedup)
})
}
const source = Stores.fromStoresArray(allSources).map(result => {
const all = [].concat(...result)
return Utils.DedupOnId(all, (i) => i?.id ?? i?.url)
})
this._cachedImageStores[cachekey] = source
return source
}

341
src/Logic/ImageProviders/ImageUploadManager.ts
View file

@ -1,34 +1,53 @@
import { ImageUploader, UploadResult } from "./ImageUploader"
import LinkImageAction from "../Osm/Actions/LinkImageAction"
import FeaturePropertiesStore from "../FeatureSource/Actors/FeaturePropertiesStore"
import { OsmId, OsmTags } from "../../Models/OsmFeature"
import { NoteId, OsmId, OsmTags } from "../../Models/OsmFeature"
import ThemeConfig from "../../Models/ThemeConfig/ThemeConfig"
import { Store, UIEventSource } from "../UIEventSource"
import { OsmConnection } from "../Osm/OsmConnection"
import { Changes } from "../Osm/Changes"
import Translations from "../../UI/i18n/Translations"
import { Translation } from "../../UI/i18n/Translation"
import { IndexedFeatureSource } from "../FeatureSource/FeatureSource"
import { GeoOperations } from "../GeoOperations"
import { Feature } from "geojson"
import ImageUploadQueue, { ImageUploadArguments } from "./ImageUploadQueue"
import { GeoOperations } from "../GeoOperations"
import NoteCommentElement from "../../UI/Popup/Notes/NoteCommentElement"
import OsmObjectDownloader from "../Osm/OsmObjectDownloader"
/**
* The ImageUploadManager has a
*/
export class ImageUploadManager {
private readonly _queue: ImageUploadQueue = ImageUploadQueue.singleton
private readonly _uploader: ImageUploader
private readonly _featureProperties: FeaturePropertiesStore
private readonly _theme: ThemeConfig
private readonly _indexedFeatures: IndexedFeatureSource
private readonly _gps: Store<GeolocationCoordinates | undefined>
private readonly _uploadStarted: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadFinished: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadFailed: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadRetried: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadRetriedSuccess: Map<string, UIEventSource<number>> = new Map()
private readonly _osmConnection: OsmConnection
private readonly _changes: Changes
public readonly isUploading: Store<boolean>
/**
* Keeps track of the _features_ for which an upload was successfull. Only used to give an indication.
* Every time an image is uploaded, the featureID is added to the list. Not persisted (and should not be)
*/
private readonly _successfull: UIEventSource<string[]> = new UIEventSource<string[]>([])
public readonly successfull: Store<string[]> = this._successfull
/**
* Keeps track of the _features_ for which an upload failed. Only used to give an indication to the user.
* Every time an image upload fails, the featureID is added to the list. Not persisted (and should not be)
*/
private readonly _fails: UIEventSource<ImageUploadArguments[]> = new UIEventSource<ImageUploadArguments[]>([])
public readonly fails: Store<string[]> = this._fails.map(args => args.map(a => a.featureId))
/**
* FeatureIDs of queued items
*/
public readonly queued: Store<string[]> = this._queue.imagesInQueue.map(queue => queue.map(q => q.featureId))
public readonly queuedArgs = this._queue.imagesInQueue
/**
* The feature for which an upload is currently running
*/
public readonly _isUploading: UIEventSource<string | undefined> = new UIEventSource(undefined)
public readonly isUploading: Store<string | undefined> = this._isUploading
private readonly _reportError: (
message: string | Error | XMLHttpRequest,
extramessage?: string
@ -41,7 +60,6 @@ export class ImageUploadManager {
osmConnection: OsmConnection,
changes: Changes,
gpsLocation: Store<GeolocationCoordinates | undefined>,
allFeatures: IndexedFeatureSource,
reportError: (
message: string | Error | XMLHttpRequest,
extramessage?: string
@ -52,41 +70,8 @@ export class ImageUploadManager {
this._theme = layout
this._osmConnection = osmConnection
this._changes = changes
this._indexedFeatures = allFeatures
this._gps = gpsLocation
this._reportError = reportError
const failed = this.getCounterFor(this._uploadFailed, "*")
const done = this.getCounterFor(this._uploadFinished, "*")
this.isUploading = this.getCounterFor(this._uploadStarted, "*").map(
(startedCount) => {
return startedCount > failed.data + done.data
},
[failed, done]
)
}
/**
* Gets various counters.
* Note that counters can only increase
* If a retry was a success, both 'retrySuccess' _and_ 'uploadFinished' will be increased
* @param featureId the id of the feature you want information for. '*' has a global counter
*/
public getCountsFor(featureId: string | "*"): {
retried: Store<number>
uploadStarted: Store<number>
retrySuccess: Store<number>
failed: Store<number>
uploadFinished: Store<number>
} {
return {
uploadStarted: this.getCounterFor(this._uploadStarted, featureId),
uploadFinished: this.getCounterFor(this._uploadFinished, featureId),
retried: this.getCounterFor(this._uploadRetried, featureId),
failed: this.getCounterFor(this._uploadFailed, featureId),
retrySuccess: this.getCounterFor(this._uploadRetriedSuccess, featureId),
}
}
public canBeUploaded(file: File): true | { error: Translation } {
@ -94,162 +79,230 @@ export class ImageUploadManager {
if (sizeInBytes > this._uploader.maxFileSizeInMegabytes * 1000000) {
const error = Translations.t.image.toBig.Subs({
actual_size: Math.floor(sizeInBytes / 1000000) + "MB",
max_size: this._uploader.maxFileSizeInMegabytes + "MB",
max_size: this._uploader.maxFileSizeInMegabytes + "MB"
})
return { error }
}
const ext = file.name.split(".").at(-1).toLowerCase()
if (ext !== "jpg" && ext !== "jpeg") {
return { error: new Translation({ en: "Only JPG-files are allowed" }) }
}
return true
}
/**
* Uploads the given image, applies the correct title and license for the known user.
* Will then add this image to the OSM-feature or the OSM-note
* Will then add this image to the OSM-feature or the OSM-note automatically, based on the ID of the feature.
* Note: the image will actually be added to the queue. If the image-upload fails, this will be attempted when visiting MC again
* @param file a jpg file to upload
* @param tagsStore The tags of the feature
* @param targetKey Use this key to save the attribute under. Default: 'image'
* @param noblur if true, then the api call will indicate that the image is already blurred. The server won't apply blurring in this case
* @param feature the feature this image is about. Will be used as fallback to get the GPS-coordinates
*/
public async uploadImageAndApply(
public uploadImageAndApply(
file: File,
tagsStore: UIEventSource<OsmTags>,
targetKey: string,
noblur: boolean,
feature: Feature
): Promise<void> {
feature: Feature,
options: {
ignoreGPS: boolean | false
}
): void {
const canBeUploaded = this.canBeUploaded(file)
if (canBeUploaded !== true) {
throw canBeUploaded.error
}
const tags = tagsStore.data
const tags: OsmTags = tagsStore.data
const featureId = <OsmId | NoteId>tags.id
const featureId = <OsmId>tags.id
const author = this._osmConnection.userDetails.data.name
const author = this._osmConnection?.userDetails?.data?.name ?? "Anonymous" // Might be a note upload
const uploadResult = await this.uploadImageWithLicense(
featureId,
author,
file,
targetKey,
noblur,
feature
)
if (!uploadResult) {
return
/**
* The location to upload the image with.
* Note that EXIF-data will always be trusted _more_ by the uploader
*/
let location: [number, number] = GeoOperations.centerpointCoordinates(feature)
if (this._gps.data && !options?.ignoreGPS) {
location = [this._gps.data.longitude, this._gps.data.latitude]
}
const properties = this._featureProperties.getStore(featureId)
const action = new LinkImageAction(
featureId,
uploadResult.key,
uploadResult.value,
properties,
{
theme: tags?.data?.["_orig_theme"] ?? this._theme.id,
changeType: "add-image",
}
)
const args: ImageUploadArguments = {
location,
date: new Date().getTime(),
layoutId: this._theme.id,
author, blob: file, featureId, noblur, targetKey
}
console.log("Args are", args)
this._queue.add(args)
this.uploadQueue()
await this._changes.applyAction(action)
}
public async uploadImageWithLicense(
featureId: string,
author: string,
blob: File,
targetKey: string | undefined,
noblur: boolean,
feature: Feature,
ignoreGps: boolean = false
): Promise<UploadResult> {
this.increaseCountFor(this._uploadStarted, featureId)
/**
* Attempts to upload all items in the queue
*/
private uploadingAll = false
public async uploadQueue() {
if (this.uploadingAll) {
return
}
const queue = this._queue.imagesInQueue.data ?? []
if (queue.length === 0) {
return
}
console.log("Checking image upload queue and uploading if needed")
this.uploadingAll = true
try {
for (const imageToUpload of queue) {
await this.handleQueueItem(imageToUpload)
}
} catch (e) {
console.error("Error while handling the queue:", e)
await this._reportError("Image Upload Manager: queue stopped working:", e)
} finally {
this.uploadingAll = false
}
}
/**
* Handles a queue item:
* - starts upload
* - indicates that the upload is busy
* - Applies the action to the correct element
* - indicates failure
* @private
*/
private async handleQueueItem(args: ImageUploadArguments): Promise<void> {
console.log("Handling queue item", args)
if (!args) {
return
}
this._isUploading.set(args.featureId)
let result: UploadResult = undefined
let attempts = 2
while (attempts > 0 && result === undefined) {
attempts--
const doReport = attempts == 0
result = await this.attemptSingleUpload(args, doReport)
if (!result) {
console.log("Upload attempt failed, attempts left:", attempts)
}
}
this._isUploading.set(undefined)
if (result === undefined) {
this._fails.data.push(args)
this._fails.ping()
return
}
this._fails.set(this._fails.data.filter(a => a !== args))
let properties: UIEventSource<Record<string, string>> = this._featureProperties.getStore(args.featureId)
if (args.featureId.startsWith("note/")) {
// This is an OSM-note
const url = result.absoluteUrl
await this._osmConnection.addCommentToNote(args.featureId, url)
const properties: UIEventSource<Record<string, string>> = this._featureProperties.getStore(args.featureId)
if (properties) {
// Properties will not be defined if the note isn't loaded, but that is no problem as the below code is only relevant if the note is shown
NoteCommentElement.addCommentTo(url, properties, {
osmConnection: this._osmConnection
})
}
} else {
if (properties === undefined) {
const downloaded = await new OsmObjectDownloader(this._osmConnection.Backend(), this._changes).DownloadObjectAsync(args.featureId)
if (downloaded === "deleted") {
this._queue.delete(args)
return
}
this._featureProperties.trackFeature(downloaded.asGeoJson())
properties = this._featureProperties.getStore(args.featureId)
}
const action = new LinkImageAction(
args.featureId,
result.key,
result.value,
properties,
{
theme: properties?.data?.["_orig_theme"] ?? this._theme.id,
changeType: "add-image"
}
)
await this._changes.applyAction(action)
await this._changes.flushChanges("Image upload completed")
}
this._queue.delete(args)
}
/**
* Attempts to upload the image (once).
* Returns 'undefined' if failed
* @param featureId
* @param author
* @param blob
* @param targetKey
* @param noblur
* @param lastGpsLocation
* @param ignoreGps
* @param layoutId
* @param date
* @param reportOnFail If set, reports an error to the mapcomplete server so that pietervdvn can fix it
* @private
*/
private async attemptSingleUpload(
{
featureId,
author,
blob,
targetKey,
noblur,
location
}: ImageUploadArguments,
reportOnFail: boolean
): Promise<UploadResult | undefined> {
let key: string
let value: string
let absoluteUrl: string
let location: [number, number] = undefined
if (this._gps.data && !ignoreGps) {
location = [this._gps.data.longitude, this._gps.data.latitude]
}
{
feature ??= this._indexedFeatures.featuresById.data.get(featureId)
if (feature === undefined) {
throw "ImageUploadManager: no feature given and no feature found in the indexedFeature. Cannot upload this image"
}
const featureCenterpoint = GeoOperations.centerpointCoordinates(feature)
if (
location === undefined ||
location?.some((l) => l === undefined) ||
GeoOperations.distanceBetween(location, featureCenterpoint) > 150
) {
/* GPS location is either unknown or very far away from the photographed location.
* Default to the centerpoint
*/
location = featureCenterpoint
}
}
try {
;({ key, value, absoluteUrl } = await this._uploader.uploadImage(
({ key, value, absoluteUrl } = await this._uploader.uploadImage(
blob,
location,
author,
noblur
))
} catch (e) {
this.increaseCountFor(this._uploadRetried, featureId)
console.error("Could not upload image, trying again:", e)
try {
;({ key, value, absoluteUrl } = await this._uploader.uploadImage(
blob,
location,
author,
noblur
))
this.increaseCountFor(this._uploadRetriedSuccess, featureId)
} catch (e) {
console.error("Could again not upload image due to", e)
this.increaseCountFor(this._uploadFailed, featureId)
console.error("Could again not upload image due to", e)
if (reportOnFail) {
await this._reportError(
e,
JSON.stringify({
ctx: "While uploading an image in the Image Upload Manager",
featureId,
author,
targetKey,
targetKey
})
)
return undefined
}
return undefined
}
console.log("Uploading image done, creating action for", featureId)
key = targetKey ?? key
if (targetKey) {
if (targetKey && targetKey.indexOf(key) < 0) {
// This is a non-standard key, so we use the image link directly
value = absoluteUrl
}
this.increaseCountFor(this._uploadFinished, featureId)
return { key, absoluteUrl, value }
}
private getCounterFor(collection: Map<string, UIEventSource<number>>, key: string | "*") {
if (this._featureProperties.aliases.has(key)) {
key = this._featureProperties.aliases.get(key)
}
if (!collection.has(key)) {
collection.set(key, new UIEventSource<number>(0))
}
return collection.get(key)
}
private increaseCountFor(collection: Map<string, UIEventSource<number>>, key: string | "*") {
{
const counter = this.getCounterFor(collection, key)
counter.setData(counter.data + 1)
}
{
const global = this.getCounterFor(collection, "*")
global.setData(global.data + 1)
}
}
}

59
src/Logic/ImageProviders/ImageUploadQueue.ts Normal file
View file

@ -0,0 +1,59 @@
import { IdbLocalStorage } from "../Web/IdbLocalStorage"
import { Store, UIEventSource } from "../UIEventSource"
export interface ImageUploadArguments {
featureId: string,
readonly author: string,
readonly blob: File,
readonly targetKey: string | undefined,
readonly noblur: boolean,
readonly location: [number, number],
readonly layoutId: string
readonly date: number
}
/**
* The 'imageUploadQueue' keeps track of all images that should still be uploaded.
* It is backed up in the indexedDB as to not drop images in case of connection problems
*/
export default class ImageUploadQueue {
public static readonly singleton = new ImageUploadQueue()
private readonly _imagesInQueue: UIEventSource<ImageUploadArguments[]>
public readonly imagesInQueue: Store<ImageUploadArguments[]>
private constructor() {
this._imagesInQueue = IdbLocalStorage.Get<ImageUploadArguments[]>("failed-images-backup", { defaultValue: [] })
this.imagesInQueue = this._imagesInQueue
}
public add(args: ImageUploadArguments) {
this._imagesInQueue.data.push(args)
console.log("Got args", args)
this._imagesInQueue.ping()
}
public delete(img: ImageUploadArguments) {
const index = this._imagesInQueue.data.indexOf(img)
if (index < 0) {
return
}
this._imagesInQueue.data.splice(index, 1)
this._imagesInQueue.ping()
}
applyRemapping(oldId: string, newId: string) {
let hasChange = false
for (const img of this._imagesInQueue.data) {
if (img.featureId === oldId) {
img.featureId = newId
hasChange = true
}
}
if (hasChange) {
this._imagesInQueue.ping()
}
}
}

14
src/Logic/ImageProviders/Mapillary.ts
View file

@ -83,6 +83,8 @@ export class Mapillary extends ImageProvider {
/**
* Returns the correct key for API v4.0
*
* Mapillary.ExtractKeyFromURL("999924810651016") // => 999924810651016
*/
private static ExtractKeyFromURL(value: string): number {
let key: string
@ -180,7 +182,9 @@ export class Mapillary extends ImageProvider {
mapillaryId +
"?fields=thumb_1024_url,thumb_original_url,captured_at,creator&access_token=" +
Constants.mapillary_client_token_v4
const response = await Utils.downloadJsonCached(metadataUrl, 60 * 60)
const response = await Utils.downloadJsonCached<{
thumb_1024_url: string, thumb_original_url: string, captured_at, creator: string
}>(metadataUrl, 60 * 60)
const license = new LicenseInfo()
license.artist = response["creator"]["username"]
@ -200,9 +204,13 @@ export class Mapillary extends ImageProvider {
const metadataUrl =
"https://graph.mapillary.com/" +
mapillaryId +
"?fields=thumb_1024_url,thumb_original_url,captured_at,compass_angle,geometry,creator&access_token=" +
"?fields=thumb_1024_url,thumb_original_url,captured_at,compass_angle,geometry,creator,camera_type&access_token=" +
Constants.mapillary_client_token_v4
const response = await Utils.downloadJsonCached(metadataUrl, 60 * 60)
const response = await Utils.downloadJsonCached<{
thumb_1024_url: string, thumb_original_url: string, captured_at,
compass_angle: number,
creator: string
}>(metadataUrl, 60 * 60)
const url = <string>response["thumb_1024_url"]
const url_hd = <string>response["thumb_original_url"]
const date = new Date()

5
src/Logic/ImageProviders/Panoramax.ts
View file

@ -132,7 +132,8 @@ export default class PanoramaxImageProvider extends ImageProvider {
}
public async getInfo(hash: string): Promise<ProvidedImage> {
return await this.getInfoFor(hash).then((r) => this.featureToImage(<any>r))
const r: { data: ImageData; url: string } = await this.getInfoFor(hash)
return this.featureToImage(r)
}
getRelevantUrls(tags: Record<string, string>, prefixes: string[]): Store<ProvidedImage[]> {
@ -278,7 +279,7 @@ export class PanoramaxUploader implements ImageUploader {
}
console.log("Tags are", tags)
} catch (e) {
console.error("Could not read EXIF-tags")
console.warn("Could not read EXIF-tags")
}
const p = this.panoramax

3
src/Logic/ImageProviders/WikidataImageProvider.ts
View file

@ -5,7 +5,6 @@ import Wikidata from "../Web/Wikidata"
import SvelteUIElement from "../../UI/Base/SvelteUIElement"
import * as Wikidata_icon from "../../assets/svg/Wikidata.svelte"
import { Utils } from "../../Utils"
import { ImmutableStore, Store, Stores, UIEventSource } from "../UIEventSource"
export class WikidataImageProvider extends ImageProvider {
public static readonly singleton = new WikidataImageProvider()
@ -64,7 +63,7 @@ export class WikidataImageProvider extends ImageProvider {
return [].concat(...resolved)
}
public DownloadAttribution(_): Promise<any> {
public DownloadAttribution(): Promise<undefined> {
throw new Error("Method not implemented; shouldn't be needed!")
}
}