midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge latest develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-12-10 01:42:00 +01:00
commit 17450deb82
386 changed files with 12073 additions and 25528 deletions
Download patch file Download diff file Expand all files Collapse all files

313
scripts/ImgurToPanoramax.ts
View file

@ -4,7 +4,7 @@ import { RegexTag } from "../src/Logic/Tags/RegexTag"
import Constants from "../src/Models/Constants"
import { BBox } from "../src/Logic/BBox"
import { existsSync, readFileSync, writeFileSync } from "fs"
import { PanoramaxUploader } from "../src/Logic/ImageProviders/Panoramax"
import PanoramaxImageProvider, { PanoramaxUploader } from "../src/Logic/ImageProviders/Panoramax"
import { Feature } from "geojson"
import { LicenseInfo } from "../src/Logic/ImageProviders/LicenseInfo"
import { GeoOperations } from "../src/Logic/GeoOperations"
@ -16,25 +16,31 @@ import { Changes } from "../src/Logic/Osm/Changes"
import { ChangeDescription } from "../src/Logic/Osm/Actions/ChangeDescription"
import OsmObjectDownloader from "../src/Logic/Osm/OsmObjectDownloader"
import { OsmObject } from "../src/Logic/Osm/OsmObject"
import { createReadStream } from "node:fs"
import { File } from "buffer"
import { open } from "node:fs/promises"
import { UploadableTag } from "../src/Logic/Tags/TagTypes"
import { Imgur } from "../src/Logic/ImageProviders/Imgur"
import { Or } from "../src/Logic/Tags/Or"
import ScriptUtils from "./ScriptUtils"
import { ImmutableStore } from "../src/Logic/UIEventSource"
export class ImgurToPanoramax extends Script {
private readonly panoramax = new PanoramaxUploader(
Constants.panoramax.url,
Constants.panoramax.token
)
private licenseChecker = new PanoramaxImageProvider()
private readonly alreadyUploaded: Record<string, string> = {}
private readonly alreadyUploaded: Record<string, string> = this.readAlreadyUploaded()
private readonly alreadyUploadedInv: Record<string, string> = Utils.transposeMapSimple(
this.alreadyUploaded
)
private _imageDirectory: string
private _licenseDirectory: string
private readonly sequenceIds = {
test: "7f34cf53-27ff-46c9-ac22-78511fa8457a",
cc0: "f0d6f78a-ff95-4db1-8494-6eb44a17bb37",
cc0: "e9bcb8c0-8ade-4ac9-bc9f-cfa464221fd6", // "1de6f4a1-73ac-4c75-ab7f-2a2aabddf50a", // "f0d6f78a-ff95-4db1-8494-6eb44a17bb37",
ccby: "288a8052-b475-422c-811a-4f6f1a00015e",
ccbysa: "f3d02893-b4c1-4cd6-8b27-e27ab57eb59a",
} as const
@ -45,27 +51,110 @@ export class ImgurToPanoramax extends Script {
)
}
async uploadImage(
key: string,
feat: Feature,
sequences: {
id: string
"stats:items": { count: number }
}[]
): Promise<UploadableTag | undefined> {
private async getRawInfo(imgurUrl): Promise<{ description?: string; datetime: number }> {
const fallbackpath =
this._licenseDirectory + "/raw/" + imgurUrl.replaceAll(/[^a-zA-Z0-9]/g, "_") + ".json"
if (existsSync(fallbackpath)) {
console.log("Loaded raw info from fallback path")
return JSON.parse(readFileSync(fallbackpath, "utf8"))["data"]
}
// No local data available; lets ask imgur themselves
return new Promise((resolve) => {
Imgur.singleton.DownloadAttribution({ url: imgurUrl }, (raw) => {
console.log("Writing fallback to", fallbackpath, "(via raw)")
writeFileSync(fallbackpath, JSON.stringify(raw), "utf8")
resolve(raw["data"])
})
})
}
private async getLicenseFor(imgurUrl: string): Promise<LicenseInfo> {
const imageName = imgurUrl.split("/").at(-1)
const licensePath: string = this._licenseDirectory + "/" + imageName
if (existsSync(licensePath)) {
const rawText = readFileSync(licensePath, "utf8")
if (rawText?.toLowerCase() === "cc0" || rawText?.toLowerCase().startsWith("cc0")) {
return { licenseShortName: "CC0", artist: "Unknown" }
}
try {
const licenseText: LicenseInfo = JSON.parse(rawText)
if (licenseText.licenseShortName) {
return licenseText
}
console.log("<<< No valid license found in text", rawText)
return undefined
} catch (e) {
console.error(
"Could not read ",
rawText.slice(0, 20),
"as json for image",
imgurUrl,
"from",
licensePath
)
}
}
// We didn't find the expected license in the expected location; search for the fallback (raw) license
const fallbackpath =
this._licenseDirectory + "/raw/" + imgurUrl.replaceAll(/[^a-zA-Z0-9]/g, "_") + ".json"
if (existsSync(fallbackpath)) {
const fallbackRaw: string = JSON.parse(readFileSync(fallbackpath, "utf8"))["data"]
?.description
if (
fallbackRaw?.toLowerCase()?.startsWith("cc0") ||
fallbackRaw?.toLowerCase()?.indexOf("#cc0") >= 0
) {
return { licenseShortName: "CC0", artist: "Unknown" }
}
const license = Imgur.parseLicense(fallbackRaw)
if (license) {
return license
}
console.log(
"No (fallback) license found for (but file exists), not uploading",
imgurUrl,
fallbackRaw
)
return undefined
}
// No local data available; lets ask imgur themselves
const attr = await Imgur.singleton.DownloadAttribution({ url: imgurUrl }, (raw) => {
console.log("Writing fallback to", fallbackpath)
writeFileSync(fallbackpath, JSON.stringify(raw), "utf8")
})
console.log("Got license via API:", attr?.licenseShortName)
await ScriptUtils.sleep(500)
if (attr?.licenseShortName) {
return attr
}
return undefined
}
async uploadImage(key: string, feat: Feature): Promise<UploadableTag | undefined> {
const v = feat.properties[key]
if (!v) {
return undefined
}
const imageHash = v.split("/").at(-1).split(".").at(0)
const isPng = v.endsWith(".png")
if (this.alreadyUploaded[imageHash]) {
const imageHash = v.split("/").at(-1).split(".").at(0)
{
const panohash = this.alreadyUploaded[imageHash]
return new And([new Tag(key.replace("image", panohash), panohash), new Tag(key, "")])
if (panohash) {
console.log("Already uploaded", panohash)
return new And([
new Tag(key.replace("image", "panoramax"), panohash),
new Tag(key, ""),
])
}
}
let path: string = undefined
if (existsSync(this._imageDirectory + "/" + imageHash + ".jpg")) {
if (isPng) {
path = this._imageDirectory + "/../imgur_png_images/jpg/" + imageHash + ".jpg"
} else if (existsSync(this._imageDirectory + "/" + imageHash + ".jpg")) {
path = this._imageDirectory + "/" + imageHash + ".jpg"
} else if (existsSync(this._imageDirectory + "/" + imageHash + ".jpeg")) {
path = this._imageDirectory + "/" + imageHash + ".jpeg"
@ -73,22 +162,26 @@ export class ImgurToPanoramax extends Script {
if (!path) {
return undefined
}
const licensePath =
this._licenseDirectory + "/" + v.replaceAll(/[^a-zA-Z0-9]/g, "_") + ".json"
if (!existsSync(licensePath)) {
let license: LicenseInfo
try {
license = await this.getLicenseFor(v)
} catch (e) {
console.error("Could not fetch license due to", e)
if (e === 404) {
console.log("NOT FOUND")
return new Tag(key, "")
}
throw e
}
if (license === undefined) {
return undefined
}
const licenseText: LicenseInfo = JSON.parse(readFileSync(licensePath, "utf8"))
if (!licenseText.licenseShortName) {
console.log("No license found for", path, licenseText)
const sequence = this.sequenceIds[license.licenseShortName?.toLowerCase()]
console.log("Reading ", path)
if (!existsSync(path)) {
return undefined
}
const license = licenseText.licenseShortName.toLowerCase().split(" ")[0].replace(/-/g, "")
const sequence = this.sequenceIds[license]
const author = licenseText.artist
const handle = await open(path)
const stat = await handle.stat()
class MyFile extends File {
@ -104,71 +197,164 @@ export class ImgurToPanoramax extends Script {
return handle.readableWebStream()
}
const licenseRaw = await this.getRawInfo(v)
const date = new Date(licenseRaw.datetime * 1000)
console.log("Uploading", imageHash, sequence)
const result = await this.panoramax.uploadImage(
<any>file,
GeoOperations.centerpointCoordinates(feat),
author,
license.artist,
true,
sequence
sequence,
date.toISOString()
)
this.alreadyUploaded[imageHash] = result.value
await handle.close()
this.alreadyUploaded[imageHash] = result.value
this.writeAlreadyUploaded()
return new And([new Tag(key.replace("image", result.key), result.value), new Tag(key, "")])
}
private writeAlreadyUploaded() {
writeFileSync("uploaded_images.json", JSON.stringify(this.alreadyUploaded))
}
private readAlreadyUploaded() {
const uploaded = JSON.parse(readFileSync("uploaded_images.json", "utf8"))
console.log("Detected ", Object.keys(uploaded).length, "previously uploaded images")
return uploaded
}
private async patchDate(panokey: string) {
const imgurkey = this.alreadyUploadedInv[panokey]
const license = await this.getRawInfo("https://i.imgur.com/" + imgurkey + ".jpg")
const date = new Date(license.datetime * 1000)
const panolicense = await this.panoramax.panoramax.search({
ids: [panokey],
})
const panodata = panolicense[0]
const collection: string = panodata.collection
console.log({ imgurkey, date, panodata, datetime: license.datetime })
const p = this.panoramax.panoramax
const url = p.host + "/collections/" + collection + "/items/" + panokey
const result = await p.fetch(url, {
method: "PATCH",
headers: { "content-type": "application/json" },
body: JSON.stringify({
ts: date.getTime(),
}),
})
console.log(
"Patched date of ",
p.createViewLink({
imageId: panokey,
}),
url,
"result is",
result.status,
await result.text()
)
}
async main(args: string[]): Promise<void> {
this._imageDirectory = args[0] ?? "/home/pietervdvn/data/imgur-image-backup"
this._licenseDirectory = args[1] ?? "/home/pietervdvn/git/MapComplete-data/ImageLicenseInfo"
const bounds = new BBox([
[3.6984301050112833, 51.06715570450848],
[3.7434328399847914, 51.039379568816145],
])
const maxcount = 500
const filter = new RegexTag("image", /^https:\/\/i.imgur.com\/.*/)
const overpass = new Overpass(filter, [], Constants.defaultOverpassUrls[0])
const features = (await overpass.queryGeoJson(bounds))[0].features
// await this.panoramax.panoramax.createCollection("CC0 - part 2")
// return
/* for (const panohash in this.alreadyUploadedInv) {
await this.patchDate(panohash)
break
}*/
const bounds = new BBox([
[-180, -90],
[180, 90],
])
const maxcount = 10000
const overpassfilters: RegexTag[] = []
const r = /^https:\/\/i.imgur.com\/.*/
for (const k of ["image", "image:menu", "image:streetsign"]) {
overpassfilters.push(new RegexTag(k, r))
for (let i = 0; i < 20; i++) {
overpassfilters.push(new RegexTag(k + ":" + i, r))
}
}
const overpass = new Overpass(
new Or(overpassfilters),
[],
Constants.defaultOverpassUrls[0],
new ImmutableStore(500)
)
const features = (await overpass.queryGeoJson(bounds))[0].features
const featuresCopy = [...features]
let converted = 0
const pano = this.panoramax.panoramax
const sequences = await pano.mySequences()
const total = features.length
const changes: ChangeDescription[] = []
do {
const f = features.shift()
if (!f) {
break
}
if (converted % 100 === 0) {
console.log(
"Converted:",
converted,
"total:",
total,
"progress:",
Math.round((converted * 100) / total) + "%"
)
}
const changedTags: (UploadableTag | undefined)[] = []
let changedTags: (UploadableTag | undefined)[] = []
console.log(converted + "/" + total, " handling " + f.properties.id)
for (const k of ["image", "image:menu", "image:streetsign"]) {
changedTags.push(await this.uploadImage(k, f, sequences))
changedTags.push(await this.uploadImage(k, f))
for (let i = 0; i < 20; i++) {
changedTags.push(await this.uploadImage(k + ":" + i, f, sequences))
changedTags.push(await this.uploadImage(k + ":" + i, f))
}
}
const action = new ChangeTagAction(
f.properties.id,
new And(Utils.NoNull(changedTags)),
f.properties,
{
theme: "image-mover",
changeType: "link-image",
}
)
changes.push(...(await action.CreateChangeDescriptions()))
changedTags = Utils.NoNull(changedTags)
if (changedTags.length > 0) {
const action = new ChangeTagAction(
f.properties.id,
new And(changedTags),
f.properties,
{
theme: "image-mover",
changeType: "link-image",
}
)
changes.push(...(await action.CreateChangeDescriptions()))
}
converted++
} while (converted < maxcount)
console.log("Uploaded images for", converted, "items; now creating the changeset")
const modif: string[] = Utils.Dedup(changes.map((ch) => ch.type + "/" + ch.id))
const modifiedObjectsFresh = <OsmObject[]>(
(
await Promise.all(
modif.map((id) => new OsmObjectDownloader().DownloadObjectAsync(id))
const modifiedObjectsFresh: OsmObject[] = []
const dloader = new OsmObjectDownloader()
for (let i = 0; i < modif.length; i++) {
if (i % 100 === 0) {
console.log(
"Downloaded osm object",
i,
"/",
modif.length,
"(" + Math.round((i * 100) / modif.length) + "%)"
)
).filter((m) => m !== "deleted")
)
}
const id = modif[i]
const obj = await dloader.DownloadObjectAsync(id)
if (obj === "deleted") {
continue
}
modifiedObjectsFresh.push(obj)
}
const modifiedObjects = Changes.createChangesetObjectsStatic(
changes,
modifiedObjectsFresh,
@ -177,6 +363,13 @@ export class ImgurToPanoramax extends Script {
)
const cs = Changes.buildChangesetXML("0", modifiedObjects)
writeFileSync("imgur_to_panoramax.osc", cs, "utf8")
const usernames = featuresCopy.map((f) => f.properties.user)
const hist: Record<string, number> = {}
for (const username of usernames) {
hist[username] = (hist[username] ?? 0) + 1
}
console.log(hist)
}
}

2
scripts/generateImageAnalysis.ts
View file

@ -88,7 +88,7 @@ export default class GenerateImageAnalysis extends Script {
if (image === undefined) {
return false
}
if (!image.match(/https:\/\/i\.imgur\.com\/[a-zA-Z0-9]+\.jpg/)) {
if (!image.match(/https:\/\/i\.imgur\.com\/[a-zA-Z0-9]+(\.jpe?g)|(\.png)/)) {
return false
}
const filename = image.replace(/[\/:.\-%]/g, "_") + ".json"

14
scripts/generateLayerOverview.ts
View file

@ -643,11 +643,15 @@ class LayerOverviewUtils extends Script {
LayerOverviewUtils.layerPath +
sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/"))
if (!forceReload && !this.shouldBeUpdated(sharedLayerPath, targetPath)) {
const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
sharedLayers.set(sharedLayer.id, sharedLayer)
skippedLayers.push(sharedLayer.id)
ScriptUtils.erasableLog("Loaded " + sharedLayer.id)
continue
try{
const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
sharedLayers.set(sharedLayer.id, sharedLayer)
skippedLayers.push(sharedLayer.id)
ScriptUtils.erasableLog("Loaded " + sharedLayer.id)
continue
}catch (e) {
throw "Could not parse "+targetPath+" : "+e
}
}
}

2
scripts/generateTaginfoProjectFiles.ts
View file

@ -206,7 +206,7 @@ function main() {
if (layout.hideFromOverview) {
continue
}
if(layout.id === "personal"){
if (layout.id === "personal") {
continue
}
files.push(generateTagInfoEntry(layout))

5
scripts/generateTranslations.ts
View file

@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { Utils } from "../src/Utils"
import ScriptUtils from "./ScriptUtils"
import Script from "./Script"
import Constants from "../src/Models/Constants"
const knownLanguages = ["en", "nl", "de", "fr", "es", "gl", "ca"]
const ignoreTerms = ["searchTerms"]
@ -262,7 +263,9 @@ class TranslationPart {
lang = weblatepart
weblatepart = "core"
}
const fixLink = `Fix it on https://hosted.weblate.org/translate/mapcomplete/${weblatepart}/${lang}/?offset=1&q=context%3A%3D%22${encodeURIComponent(
const fixLink = `Fix it on ${
Constants.weblate
}translate/mapcomplete/${weblatepart}/${lang}/?offset=1&q=context%3A%3D%22${encodeURIComponent(
path.join(".")
)}%22`
let subparts: string[] = value.match(/{[^}]*}/g)