Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Fix(linkeddata): velopark deals with sections, fix image loading

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-12-17 03:31:28 +01:00
parent 7a06bb9930
commit ef1d2c9f56
8 changed files with 242 additions and 123 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/Logic/Maproulette.ts
View file

@ -6,6 +6,17 @@ export interface MaprouletteTask {
description: string
instruction: string
}
export const maprouletteStatus = ["Open",
"Fixed",
"False_positive",
"Skipped",
"Deleted",
"Already fixed",
"Too_Hard",
"Disabled",
] as const
export type MaprouletteStatus = typeof maprouletteStatus[number]
export default class Maproulette {
public static readonly defaultEndpoint = "https://maproulette.org/api/v2"
@ -19,16 +30,7 @@ export default class Maproulette {
public static readonly STATUS_TOO_HARD = 6
public static readonly STATUS_DISABLED = 9
public static readonly STATUS_MEANING = {
0: "Open",
1: "Fixed",
2: "False_positive",
3: "Skipped",
4: "Deleted",
5: "Already fixed",
6: "Too_Hard",
9: "Disabled"
}
public static singleton = new Maproulette()
/*
* The API endpoint to use
@ -62,12 +64,11 @@ export default class Maproulette {
if (code === "Created") {
return Maproulette.STATUS_OPEN
}
for (let i = 0; i < 9; i++) {
if (Maproulette.STATUS_MEANING["" + i] === code) {
return i
}
const i = maprouletteStatus.findIndex(<any> code)
if(i < 0){
return undefined
}
return undefined
return i
}
/**
@ -87,7 +88,7 @@ export default class Maproulette {
tags?: string
requestReview?: boolean
completionResponses?: Record<string, string>
}
},
): Promise<void> {
console.log("Maproulette: setting", `${this.endpoint}/task/${taskId}/${status}`, options)
options ??= {}
@ -97,9 +98,9 @@ export default class Maproulette {
method: "PUT",
headers: {
"Content-Type": "application/json",
apiKey: this.apiKey
apiKey: this.apiKey,
},
body: JSON.stringify(options)
body: JSON.stringify(options),
})
if (response.status !== 204) {
console.log(`Failed to close task: ${response.status}`)

48
src/Logic/State/UserSettingsMetaTagging.ts
View file

@ -1,42 +1,14 @@
import { Utils } from "../../Utils"
/** This code is autogenerated - do not edit. Edit ./assets/layers/usersettings/usersettings.json instead */
export class ThemeMetaTagging {
public static readonly themeName = "usersettings"
public static readonly themeName = "usersettings"
public metaTaggging_for_usersettings(feat: { properties: Record<string, string> }) {
Utils.AddLazyProperty(feat.properties, "_mastodon_candidate_md", () =>
feat.properties._description
.match(/\[[^\]]*\]\((.*(mastodon|en.osm.town).*)\).*/)
?.at(1)
)
Utils.AddLazyProperty(
feat.properties,
"_d",
() => feat.properties._description?.replace(/&lt;/g, "<")?.replace(/&gt;/g, ">") ?? ""
)
Utils.AddLazyProperty(feat.properties, "_mastodon_candidate_a", () =>
((feat) => {
const e = document.createElement("div")
e.innerHTML = feat.properties._d
return Array.from(e.getElementsByTagName("a")).filter(
(a) => a.href.match(/mastodon|en.osm.town/) !== null
)[0]?.href
})(feat)
)
Utils.AddLazyProperty(feat.properties, "_mastodon_link", () =>
((feat) => {
const e = document.createElement("div")
e.innerHTML = feat.properties._d
return Array.from(e.getElementsByTagName("a")).filter(
(a) => a.getAttribute("rel")?.indexOf("me") >= 0
)[0]?.href
})(feat)
)
Utils.AddLazyProperty(
feat.properties,
"_mastodon_candidate",
() => feat.properties._mastodon_candidate_md ?? feat.properties._mastodon_candidate_a
)
feat.properties["__current_backgroun"] = "initial_value"
}
}
public metaTaggging_for_usersettings(feat: {properties: Record<string, string>}) {
Utils.AddLazyProperty(feat.properties, '_mastodon_candidate_md', () => feat.properties._description.match(/\[[^\]]*\]\((.*(mastodon|en.osm.town).*)\).*/)?.at(1) )
Utils.AddLazyProperty(feat.properties, '_d', () => feat.properties._description?.replace(/&lt;/g,'<')?.replace(/&gt;/g,'>') ?? '' )
Utils.AddLazyProperty(feat.properties, '_mastodon_candidate_a', () => (feat => {const e = document.createElement('div');e.innerHTML = feat.properties._d;return Array.from(e.getElementsByTagName("a")).filter(a => a.href.match(/mastodon|en.osm.town/) !== null)[0]?.href }) (feat) )
Utils.AddLazyProperty(feat.properties, '_mastodon_link', () => (feat => {const e = document.createElement('div');e.innerHTML = feat.properties._d;return Array.from(e.getElementsByTagName("a")).filter(a => a.getAttribute("rel")?.indexOf('me') >= 0)[0]?.href})(feat) )
Utils.AddLazyProperty(feat.properties, '_mastodon_candidate', () => feat.properties._mastodon_candidate_md ?? feat.properties._mastodon_candidate_a )
feat.properties['__current_backgroun'] = 'initial_value'
}
}

131
src/Logic/Web/LinkedDataLoader.ts
View file

@ -68,7 +68,7 @@ export default class LinkedDataLoader {
coors
.trim()
.split(" ")
.map((n) => Number(n))
.map((n) => Number(n)),
),
],
}
@ -156,7 +156,7 @@ export default class LinkedDataLoader {
}
const compacted = await jsonld.compact(
openingHoursSpecification,
<any>LinkedDataLoader.COMPACTING_CONTEXT_OH
<any>LinkedDataLoader.COMPACTING_CONTEXT_OH,
)
const spec: object = compacted["@graph"]
if (!spec) {
@ -190,12 +190,12 @@ export default class LinkedDataLoader {
const compacted = await jsonld.compact(data, <any>LinkedDataLoader.COMPACTING_CONTEXT)
compacted["opening_hours"] = await LinkedDataLoader.ohToOsmFormat(
compacted["opening_hours"]
compacted["opening_hours"],
)
if (compacted["openingHours"]) {
const ohspec: string[] = <any>compacted["openingHours"]
compacted["opening_hours"] = OH.simplify(
ohspec.map((r) => LinkedDataLoader.ohStringToOsmFormat(r)).join("; ")
ohspec.map((r) => LinkedDataLoader.ohStringToOsmFormat(r)).join("; "),
)
delete compacted["openingHours"]
}
@ -236,7 +236,7 @@ export default class LinkedDataLoader {
static async fetchJsonLd(
url: string,
options?: JsonLdLoaderOptions,
mode?: "fetch-lod" | "fetch-raw" | "proxy"
mode?: "fetch-lod" | "fetch-raw" | "proxy",
): Promise<object> {
mode ??= "fetch-lod"
if (mode === "proxy") {
@ -251,7 +251,7 @@ export default class LinkedDataLoader {
const div = document.createElement("div")
div.innerHTML = htmlContent
const script = Array.from(div.getElementsByTagName("script")).find(
(script) => script.type === "application/ld+json"
(script) => script.type === "application/ld+json",
)
const snippet = JSON.parse(script.textContent)
@ -266,7 +266,7 @@ export default class LinkedDataLoader {
*/
static removeDuplicateData(
externalData: Record<string, string>,
currentData: Record<string, string>
currentData: Record<string, string>,
): Record<string, string> {
const d = { ...externalData }
delete d["@context"]
@ -332,7 +332,7 @@ export default class LinkedDataLoader {
}
private static patchVeloparkProperties(
input: Record<string, Set<string>>
input: Record<string, Set<string>>,
): Record<string, string[]> {
const output: Record<string, string[]> = {}
for (const k in input) {
@ -472,7 +472,7 @@ export default class LinkedDataLoader {
audience,
"for",
input["ref:velopark"],
" assuming yes"
" assuming yes",
)
return "yes"
})
@ -516,8 +516,11 @@ export default class LinkedDataLoader {
private static async fetchVeloparkProperty<T extends string, G extends T>(
url: string,
property: string,
variable?: string
variable?: string,
): Promise<SparqlResult<T, G>> {
if(property === "schema:photos"){
console.log(">> Getting photos")
}
const results = await new TypedSparql().typedSparql<T, G>(
{
schema: "http://schema.org/",
@ -529,17 +532,25 @@ export default class LinkedDataLoader {
[url],
undefined,
" ?parking a <http://schema.mobivoc.org/BicycleParkingStation>",
"?parking " + property + " " + (variable ?? "")
"?parking " + property + " " + (variable ?? ""),
)
return results
}
/**
*
* @param url
* @param property
* @param subExpr
* @private
*/
private static async fetchVeloparkGraphProperty<T extends string>(
url: string,
property: string,
subExpr?: string
subExpr?: string,
): Promise<SparqlResult<T, "g">> {
return await new TypedSparql().typedSparql<T, "g">(
const result = await new TypedSparql().typedSparql<T, "g">(
{
schema: "http://schema.org/",
mv: "http://schema.mobivoc.org/",
@ -551,8 +562,10 @@ export default class LinkedDataLoader {
"g",
" ?parking a <http://schema.mobivoc.org/BicycleParkingStation>",
S.graph("g", "?section " + property + " " + (subExpr ?? ""), "?section a ?type")
S.graph("g", "?section " + property + " " + (subExpr ?? ""), "?section a ?type", "BIND(STR(?section) AS ?id)"),
)
return result
}
/**
@ -569,26 +582,67 @@ export default class LinkedDataLoader {
continue
}
for (const sectionKey in subResult) {
if (!r[sectionKey]) {
r[sectionKey] = {}
}
const section = subResult[sectionKey]
for (const key in section) {
r[sectionKey][key] ??= section[key]
if (sectionKey === "default") {
r["default"] ??= {}
const section = subResult["default"]
for (const key in section) {
r["default"][key] ??= section[key]
}
} else {
const section = subResult[sectionKey]
const actualId = Array.from(section["id"] ?? [])[0] ?? sectionKey
r[actualId] ??= {}
for (const key in section) {
r[actualId][key] ??= section[key]
}
}
}
}
if (r["default"] !== undefined && Object.keys(r).length > 1) {
/**
* Copy all values from the section with name "key" into the other sections,
* remove section "key" afterwards
* @param key
*/
function spreadSection(key: string){
for (const section in r) {
if (section === "default") {
if (section === key) {
continue
}
for (const k in r.default) {
r[section][k] ??= r.default[k]
for (const k in r[key]) {
r[section][k] ??= r[key][k]
}
}
delete r.default
delete r[key]
}
// The "default" part of the result contains all general info
// The other 'sections' need to get those copied! Then, we delete the "default"-section
if (r["default"] !== undefined && Object.keys(r).length > 1) {
spreadSection("default")
}
if (Object.keys(r).length > 1) {
// This result has multiple sections
// We should check that the naked URL got distributed and scrapped
const keys = Object.keys(r)
if (Object.keys(r).length > 2) {
console.log("Multiple sections detected: ", JSON.stringify(keys))
}
const shortestKeyLength: number = Math.min(...keys.map(k => k.length))
const key = keys.find(k => k.length === shortestKeyLength)
if (keys.some(k => !k.startsWith(key))) {
throw "Invalid multi-object: the shortest key is not the start of all the others: " + JSON.stringify(keys)
}
spreadSection(key)
}
if (Object.keys(r).length == 1) {
const key = Object.keys(r)[0]
if(key.indexOf("#")>0){
const newKey = key.split("#")[0]
r[newKey] = r[key]
delete r[key]
}
}
return r
}
@ -597,7 +651,7 @@ export default class LinkedDataLoader {
directUrl: string,
propertiesWithoutGraph: PropertiesSpec<T>,
propertiesInGraph: PropertiesSpec<T>,
extra?: string[]
extra?: string[],
): Promise<SparqlResult<T, string>> {
const allPartialResults: SparqlResult<T, string>[] = []
for (const propertyName in propertiesWithoutGraph) {
@ -607,7 +661,7 @@ export default class LinkedDataLoader {
const result = await this.fetchVeloparkProperty(
directUrl,
propertyName,
"?" + variableName
"?" + variableName,
)
allPartialResults.push(result)
} else {
@ -616,7 +670,7 @@ export default class LinkedDataLoader {
const result = await this.fetchVeloparkProperty(
directUrl,
propertyName,
`[${subProperty} ?${variableName}] `
`[${subProperty} ?${variableName}] `,
)
allPartialResults.push(result)
}
@ -634,7 +688,7 @@ export default class LinkedDataLoader {
const result = await this.fetchVeloparkGraphProperty(
directUrl,
propertyName,
variableName
variableName,
)
allPartialResults.push(result)
}
@ -646,7 +700,7 @@ export default class LinkedDataLoader {
const result = await this.fetchVeloparkGraphProperty(
directUrl,
propertyName,
variableName
variableName,
)
allPartialResults.push(result)
} else {
@ -655,7 +709,7 @@ export default class LinkedDataLoader {
const result = await this.fetchVeloparkGraphProperty(
directUrl,
propertyName,
`[${subProperty} ?${variableName}] `
`[${subProperty} ?${variableName}] `,
)
allPartialResults.push(result)
}
@ -675,16 +729,18 @@ export default class LinkedDataLoader {
/**
* Fetches all data relevant to velopark.
* The id will be saved as `ref:velopark`
* If the entry has multiple sections, this will return multiple items
* @param url
*/
public static async fetchVeloparkEntry(
url: string,
includeExtras: boolean = false
includeExtras: boolean = false,
): Promise<Feature[]> {
const cacheKey = includeExtras + url
if (this.veloparkCache[cacheKey]) {
return this.veloparkCache[cacheKey]
}
// Note: the proxy doesn't make any changes in this case
const withProxyUrl = Constants.linkedDataProxy.replace("{url}", encodeURIComponent(url))
const optionalPaths: Record<string, string | Record<string, string>> = {
"schema:interactionService": {
@ -697,6 +753,7 @@ export default class LinkedDataLoader {
"schema:email": "email",
"schema:telephone": "phone",
},
// "schema:photos": "images",
"schema:dateModified": "_last_edit_timestamp",
}
if (includeExtras) {
@ -738,8 +795,16 @@ export default class LinkedDataLoader {
withProxyUrl,
optionalPaths,
graphOptionalPaths,
extra
extra,
)
for (const unpatchedKey in unpatched) {
// Dirty hack
const rawData = await Utils.downloadJsonCached<object>(url, 1000*60*60)
const images = rawData["photos"].map(ph => <string> ph.image)
unpatched[unpatchedKey].images = new Set<string>(images)
}
console.log("Got unpatched:", unpatched)
const patched: Feature[] = []
for (const section in unpatched) {
const p = LinkedDataLoader.patchVeloparkProperties(unpatched[section])

6
src/Logic/Web/TypedSparql.ts
View file

@ -67,13 +67,11 @@ export default class TypedSparql {
bindings.forEach((item) => {
const result = <Record<VARS | G, Set<string>>>{}
item.forEach((value, key) => {
if (!result[key.value]) {
result[key.value] = new Set()
}
result[key.value] ??= new Set()
result[key.value].add(value.value)
})
if (graphVariable && result[graphVariable]?.size > 0) {
const id = Array.from(result[graphVariable])?.[0] ?? "default"
const id: string = (<string> Array.from(result["id"] ?? [])?.[0] ?? Array.from(result[graphVariable] ?? [])?.[0]) ?? "default"
resultAllGraphs[id] = result
} else {
resultAllGraphs["default"] = result