Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Feature: image upload now uses the indexedDB-backed queue (formerly known as EmergencyBackup), rework (and simplify) counter logic (fix #2186; fix #1942; helps #2022)

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-04-07 02:53:21 +02:00
parent 55c015ad84
commit 3d3a72a70a
19 changed files with 402 additions and 503 deletions
Download patch file Download diff file Expand all files Collapse all files

339
src/Logic/ImageProviders/ImageUploadManager.ts
View file

@ -1,35 +1,53 @@
import { ImageUploader, UploadResult } from "./ImageUploader"
import LinkImageAction from "../Osm/Actions/LinkImageAction"
import FeaturePropertiesStore from "../FeatureSource/Actors/FeaturePropertiesStore"
import { OsmId, OsmTags } from "../../Models/OsmFeature"
import { NoteId, OsmId, OsmTags } from "../../Models/OsmFeature"
import ThemeConfig from "../../Models/ThemeConfig/ThemeConfig"
import { Store, UIEventSource } from "../UIEventSource"
import { OsmConnection } from "../Osm/OsmConnection"
import { Changes } from "../Osm/Changes"
import Translations from "../../UI/i18n/Translations"
import { Translation } from "../../UI/i18n/Translation"
import { IndexedFeatureSource } from "../FeatureSource/FeatureSource"
import { GeoOperations } from "../GeoOperations"
import { Feature } from "geojson"
import EmergencyImageBackup from "./EmergencyImageBackup"
import ImageUploadQueue, { ImageUploadArguments } from "./ImageUploadQueue"
import { GeoOperations } from "../GeoOperations"
import NoteCommentElement from "../../UI/Popup/Notes/NoteCommentElement"
import OsmObjectDownloader from "../Osm/OsmObjectDownloader"
/**
* The ImageUploadManager has a
*/
export class ImageUploadManager {
private readonly _queue: ImageUploadQueue = ImageUploadQueue.singleton
private readonly _uploader: ImageUploader
private readonly _featureProperties: FeaturePropertiesStore
private readonly _theme: ThemeConfig
private readonly _indexedFeatures: IndexedFeatureSource
private readonly _gps: Store<GeolocationCoordinates | undefined>
private readonly _uploadStarted: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadFinished: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadFailed: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadRetried: Map<string, UIEventSource<number>> = new Map()
private readonly _uploadRetriedSuccess: Map<string, UIEventSource<number>> = new Map()
private readonly _osmConnection: OsmConnection
private readonly _changes: Changes
public readonly isUploading: Store<boolean>
/**
* Keeps track of the _features_ for which an upload was successfull. Only used to give an indication.
* Every time an image is uploaded, the featureID is added to the list. Not persisted (and should not be)
*/
private readonly _successfull: UIEventSource<string[]> = new UIEventSource<string[]>([])
public readonly successfull: Store<string[]> = this._successfull
/**
* Keeps track of the _features_ for which an upload failed. Only used to give an indication to the user.
* Every time an image upload fails, the featureID is added to the list. Not persisted (and should not be)
*/
private readonly _fails: UIEventSource<ImageUploadArguments[]> = new UIEventSource<ImageUploadArguments[]>([])
public readonly fails: Store<string[]> = this._fails.map(args => args.map(a => a.featureId))
/**
* FeatureIDs of queued items
*/
public readonly queued: Store<string[]> = this._queue.imagesInQueue.map(queue => queue.map(q => q.featureId))
public readonly queuedArgs = this._queue.imagesInQueue
/**
* The feature for which an upload is currently running
*/
public readonly _isUploading: UIEventSource<string | undefined> = new UIEventSource(undefined)
public readonly isUploading: Store<string | undefined> = this._isUploading
private readonly _reportError: (
message: string | Error | XMLHttpRequest,
extramessage?: string
@ -42,7 +60,6 @@ export class ImageUploadManager {
osmConnection: OsmConnection,
changes: Changes,
gpsLocation: Store<GeolocationCoordinates | undefined>,
allFeatures: IndexedFeatureSource,
reportError: (
message: string | Error | XMLHttpRequest,
extramessage?: string
@ -53,41 +70,8 @@ export class ImageUploadManager {
this._theme = layout
this._osmConnection = osmConnection
this._changes = changes
this._indexedFeatures = allFeatures
this._gps = gpsLocation
this._reportError = reportError
const failed = this.getCounterFor(this._uploadFailed, "*")
const done = this.getCounterFor(this._uploadFinished, "*")
this.isUploading = this.getCounterFor(this._uploadStarted, "*").map(
(startedCount) => {
return startedCount > failed.data + done.data
},
[failed, done]
)
}
/**
* Gets various counters.
* Note that counters can only increase
* If a retry was a success, both 'retrySuccess' _and_ 'uploadFinished' will be increased
* @param featureId the id of the feature you want information for. '*' has a global counter
*/
public getCountsFor(featureId: string | "*"): {
retried: Store<number>
uploadStarted: Store<number>
retrySuccess: Store<number>
failed: Store<number>
uploadFinished: Store<number>
} {
return {
uploadStarted: this.getCounterFor(this._uploadStarted, featureId),
uploadFinished: this.getCounterFor(this._uploadFinished, featureId),
retried: this.getCounterFor(this._uploadRetried, featureId),
failed: this.getCounterFor(this._uploadFailed, featureId),
retrySuccess: this.getCounterFor(this._uploadRetriedSuccess, featureId)
}
}
public canBeUploaded(file: File): true | { error: Translation } {
@ -108,110 +92,187 @@ export class ImageUploadManager {
/**
* Uploads the given image, applies the correct title and license for the known user.
* Will then add this image to the OSM-feature or the OSM-note
* Will then add this image to the OSM-feature or the OSM-note automatically, based on the ID of the feature.
* Note: the image will actually be added to the queue. If the image-upload fails, this will be attempted when visiting MC again
* @param file a jpg file to upload
* @param tagsStore The tags of the feature
* @param targetKey Use this key to save the attribute under. Default: 'image'
* @param noblur if true, then the api call will indicate that the image is already blurred. The server won't apply blurring in this case
* @param feature the feature this image is about. Will be used as fallback to get the GPS-coordinates
*/
public async uploadImageAndApply(
public uploadImageAndApply(
file: File,
tagsStore: UIEventSource<OsmTags>,
targetKey: string,
noblur: boolean,
feature: Feature
): Promise<void> {
feature: Feature,
options: {
ignoreGPS: boolean | false
}
): void {
const canBeUploaded = this.canBeUploaded(file)
if (canBeUploaded !== true) {
throw canBeUploaded.error
}
const tags = tagsStore.data
const tags: OsmTags = tagsStore.data
const featureId = <OsmId | NoteId>tags.id
const featureId = <OsmId>tags.id
const author = this._osmConnection.userDetails.data.name
const author = this._osmConnection?.userDetails?.data?.name ?? "Anonymous" // Might be a note upload
const uploadResult = await this.uploadImageWithLicense(
featureId,
author,
file,
targetKey,
noblur,
feature
)
if (!uploadResult) {
return
/**
* The location to upload the image with.
* Note that EXIF-data will always be trusted _more_ by the uploader
*/
let location: [number, number] = GeoOperations.centerpointCoordinates(feature)
if (this._gps.data && !options?.ignoreGPS) {
location = [this._gps.data.longitude, this._gps.data.latitude]
}
const properties = this._featureProperties.getStore(featureId)
const action = new LinkImageAction(
featureId,
uploadResult.key,
uploadResult.value,
properties,
{
theme: tags?.data?.["_orig_theme"] ?? this._theme.id,
changeType: "add-image"
}
)
const args: ImageUploadArguments = {
location,
date: new Date().getTime(),
layoutId: this._theme.id,
author, blob: file, featureId, noblur, targetKey
}
console.log("Args are", args)
this._queue.add(args)
this.uploadQueue()
await this._changes.applyAction(action)
}
/**
* Uploads an image; returns undefined if the image upload failed.
* Errors are handled internally
* Attempts to upload all items in the queue
*/
private uploadingAll = false
public async uploadQueue() {
if (this.uploadingAll) {
return
}
const queue = this._queue.imagesInQueue.data ?? []
if (queue.length === 0) {
return
}
console.log("Checking image upload queue and uploading if needed")
this.uploadingAll = true
try {
for (const imageToUpload of queue) {
await this.handleQueueItem(imageToUpload)
}
} catch (e) {
console.error("Error while handling the queue:", e)
await this._reportError("Image Upload Manager: queue stopped working:", e)
} finally {
this.uploadingAll = false
}
}
/**
* Handles a queue item:
* - starts upload
* - indicates that the upload is busy
* - Applies the action to the correct element
* - indicates failure
* @private
*/
private async handleQueueItem(args: ImageUploadArguments): Promise<void> {
console.log("Handling queue item", args)
if (!args) {
return
}
this._isUploading.set(args.featureId)
let result: UploadResult = undefined
let attempts = 2
while (attempts > 0 && result === undefined) {
attempts--
const doReport = attempts == 0
result = await this.attemptSingleUpload(args, doReport)
if (!result) {
console.log("Upload attempt failed, attempts left:", attempts)
}
}
this._isUploading.set(undefined)
if (result === undefined) {
this._fails.data.push(args)
this._fails.ping()
return
}
this._fails.set(this._fails.data.filter(a => a !== args))
let properties: UIEventSource<Record<string, string>> = this._featureProperties.getStore(args.featureId)
if (args.featureId.startsWith("note/")) {
// This is an OSM-note
const url = result.absoluteUrl
await this._osmConnection.addCommentToNote(args.featureId, url)
const properties: UIEventSource<Record<string, string>> = this._featureProperties.getStore(args.featureId)
if (properties) {
// Properties will not be defined if the note isn't loaded, but that is no problem as the below code is only relevant if the note is shown
NoteCommentElement.addCommentTo(url, properties, {
osmConnection: this._osmConnection
})
}
} else {
if (properties === undefined) {
const downloaded = await new OsmObjectDownloader(this._osmConnection.Backend(), this._changes).DownloadObjectAsync(args.featureId)
if (downloaded === "deleted") {
this._queue.delete(args)
return
}
properties = new UIEventSource(downloaded.tags)
}
const action = new LinkImageAction(
args.featureId,
result.key,
result.value,
properties,
{
theme: properties?.data?.["_orig_theme"] ?? this._theme.id,
changeType: "add-image"
}
)
await this._changes.applyAction(action)
await this._changes.flushChanges("Image upload completed")
}
this._queue.delete(args)
}
/**
* Attempts to upload the image (once).
* Returns 'undefined' if failed
* @param featureId
* @param author
* @param blob
* @param targetKey
* @param noblur
* @param feature
* @param options
* @param lastGpsLocation
* @param ignoreGps
* @param layoutId
* @param date
* @param reportOnFail If set, reports an error to the mapcomplete server so that pietervdvn can fix it
* @private
*/
public async uploadImageWithLicense(
featureId: string,
author: string,
blob: File,
targetKey: string | undefined,
noblur: boolean,
feature: Feature,
options?: {
ignoreGps?: boolean,
noBackup?: boolean,
overwriteGps?: GeolocationCoordinates
}
private async attemptSingleUpload(
{
featureId,
author,
blob,
targetKey,
noblur,
location
}: ImageUploadArguments,
reportOnFail: boolean
): Promise<UploadResult | undefined> {
this.increaseCountFor(this._uploadStarted, featureId)
let key: string
let value: string
let absoluteUrl: string
let location: [number, number] = undefined
if (options?.overwriteGps) {
location = [options.overwriteGps.longitude, options.overwriteGps.latitude]
} else if (this._gps.data && !options?.ignoreGps) {
location = [this._gps.data.longitude, this._gps.data.latitude]
}
{
feature ??= this._indexedFeatures.featuresById.data.get(featureId)
if (feature === undefined) {
throw "ImageUploadManager: no feature given and no feature found in the indexedFeature. Cannot upload this image"
}
const featureCenterpoint = GeoOperations.centerpointCoordinates(feature)
if (
location === undefined ||
location?.some((l) => l === undefined) ||
GeoOperations.distanceBetween(location, featureCenterpoint) > 150
) {
/* GPS location is either unknown or very far away from the photographed location.
* Default to the centerpoint
*/
location = featureCenterpoint
}
}
try {
({ key, value, absoluteUrl } = await this._uploader.uploadImage(
blob,
@ -220,27 +281,9 @@ export class ImageUploadManager {
noblur
))
} catch (e) {
this.increaseCountFor(this._uploadRetried, featureId)
console.error("Could not upload image, trying again:", e)
try {
({ key, value, absoluteUrl } = await this._uploader.uploadImage(
blob,
location,
author,
noblur
))
this.increaseCountFor(this._uploadRetriedSuccess, featureId)
} catch (e) {
console.error("Could again not upload image due to", e)
this.increaseCountFor(this._uploadFailed, featureId)
if (!options?.noBackup) {
EmergencyImageBackup.singleton.addFailedImage({
blob, author, noblur, featureId, targetKey, ignoreGps: options?.ignoreGps,
layoutId: this._theme.id,
lastGpsLocation: this._gps.data,
date: new Date().getTime()
})
}
console.error("Could again not upload image due to", e)
if (reportOnFail) {
await this._reportError(
e,
JSON.stringify({
@ -250,37 +293,15 @@ export class ImageUploadManager {
targetKey
})
)
return undefined
}
return undefined
}
console.log("Uploading image done, creating action for", featureId, " targetkey is", targetKey, "key is", key)
key = targetKey ?? key
if (targetKey && targetKey.indexOf(key) < 0) {
// This is a non-standard key, so we use the image link directly
value = absoluteUrl
}
this.increaseCountFor(this._uploadFinished, featureId)
return { key, absoluteUrl, value }
}
private getCounterFor(collection: Map<string, UIEventSource<number>>, key: string | "*") {
if (this._featureProperties.aliases.has(key)) {
key = this._featureProperties.aliases.get(key)
}
if (!collection.has(key)) {
collection.set(key, new UIEventSource<number>(0))
}
return collection.get(key)
}
private increaseCountFor(collection: Map<string, UIEventSource<number>>, key: string | "*") {
{
const counter = this.getCounterFor(collection, key)
counter.setData(counter.data + 1)
}
{
const global = this.getCounterFor(collection, "*")
global.setData(global.data + 1)
}
}
}