Merge develop
This commit is contained in:
commit
f0823f4c4d
524 changed files with 18747 additions and 8546 deletions
|
|
@ -39,7 +39,7 @@ export default class ScriptUtils {
|
|||
|
||||
public static DownloadFileTo(url, targetFilePath: string): Promise<void> {
|
||||
ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath)
|
||||
return new Promise<void>((resolve, err) => {
|
||||
return new Promise<void>((resolve) => {
|
||||
https.get(url, (res) => {
|
||||
const filePath = fs.createWriteStream(targetFilePath)
|
||||
res.pipe(filePath)
|
||||
|
|
|
|||
|
|
@ -18,11 +18,12 @@ fi
|
|||
export NODE_OPTIONS=--max-old-space-size=16000
|
||||
which vite
|
||||
vite --version
|
||||
vite build --sourcemap
|
||||
vite build --sourcemap || { echo 'Vite build failed' ; exit 1; }
|
||||
# Copy the layer files, as these might contain assets (e.g. svgs)
|
||||
cp -r assets/layers/ dist/assets/layers/
|
||||
cp -r assets/themes/ dist/assets/themes/
|
||||
cp -r assets/svg/ dist/assets/svg/
|
||||
cp -r assets/png/ dist/assets/png/
|
||||
mkdir dist/assets/langs
|
||||
mkdir dist/assets/langs/layers
|
||||
cp -r langs/layers/ dist/assets/langs/
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ export class Conflate extends Script {
|
|||
|
||||
const changedObjects: OsmObject[] = []
|
||||
for (const { match, replayed } of bestMatches) {
|
||||
const { external_feature, d, osm_feature } = match
|
||||
const { d, osm_feature } = match
|
||||
const { possibly_imported, certainly_imported, resting_properties } = replayed
|
||||
const status = resting_properties["status"]
|
||||
delete resting_properties["status"]
|
||||
|
|
|
|||
296
scripts/downloadCommons.ts
Normal file
296
scripts/downloadCommons.ts
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
/**
|
||||
* Script to download images from Wikimedia Commons, and save them together with license information.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs"
|
||||
import { unescape } from "querystring"
|
||||
import SmallLicense from "../src/Models/smallLicense"
|
||||
|
||||
interface ExtMetadataProp {
|
||||
value: string
|
||||
source: string
|
||||
hidden: string
|
||||
}
|
||||
|
||||
interface ImageQueryAPIResponse {
|
||||
continue: {
|
||||
iistart: string
|
||||
continue: string
|
||||
}
|
||||
query: {
|
||||
normalized?: {
|
||||
from: string
|
||||
to: string
|
||||
}[]
|
||||
pages: {
|
||||
[key: string]: {
|
||||
pageid: number
|
||||
ns: number
|
||||
title: string
|
||||
imagerepository: string
|
||||
imageinfo?: {
|
||||
user: string
|
||||
url: string
|
||||
descriptionurl: string
|
||||
descriptionshorturl: string
|
||||
extmetadata?: {
|
||||
DateTime: ExtMetadataProp
|
||||
ObjectName: ExtMetadataProp
|
||||
CommonsMetadataExtension?: ExtMetadataProp
|
||||
Categories?: ExtMetadataProp
|
||||
Assessments?: ExtMetadataProp
|
||||
ImageDescription?: ExtMetadataProp
|
||||
DateTimeOriginal?: ExtMetadataProp
|
||||
Credit?: ExtMetadataProp
|
||||
Artist?: ExtMetadataProp
|
||||
LicenseShortName?: ExtMetadataProp
|
||||
UsageTerms?: ExtMetadataProp
|
||||
AttributionRequired?: ExtMetadataProp
|
||||
Copyrighted?: ExtMetadataProp
|
||||
Restrictions?: ExtMetadataProp
|
||||
License?: ExtMetadataProp
|
||||
}
|
||||
}[]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface CategoryMember {
|
||||
pageid: number
|
||||
ns: number
|
||||
title: string
|
||||
}
|
||||
|
||||
interface CategoryQueryAPIResponse {
|
||||
batchcomplete: string
|
||||
query: {
|
||||
categorymembers: CategoryMember[]
|
||||
}
|
||||
}
|
||||
|
||||
interface TemplateQueryAPIResponse {
|
||||
batchcomplete: string
|
||||
query: {
|
||||
normalized?: {
|
||||
from: string
|
||||
to: string
|
||||
}[]
|
||||
pages: {
|
||||
[key: string]: {
|
||||
pageid: number
|
||||
ns: number
|
||||
title: string
|
||||
templates?: {
|
||||
ns: number
|
||||
title: string
|
||||
}[]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Map license names of Wikimedia Commons to different names
|
||||
const licenseMapping = {}
|
||||
|
||||
// Map template names to license names
|
||||
const templateMapping = {
|
||||
"Template:PD": "Public Domain",
|
||||
}
|
||||
|
||||
async function main(args: string[]) {
|
||||
if (args.length < 2) {
|
||||
console.log("Usage: downloadCommons.ts <output folder> <url> <?url> <?url> .. ")
|
||||
console.log(
|
||||
"Example: npx vite-node downloadCommons.ts -- assets/svg https://commons.wikimedia.org/wiki/File:Example.jpg"
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
const [outputFolder, ...urls] = args
|
||||
|
||||
for (const url of urls) {
|
||||
// Download details from the API
|
||||
const commonsFileNamePath = url.split("/").pop()
|
||||
if (commonsFileNamePath !== undefined) {
|
||||
const commonsFileName = commonsFileNamePath.split("?").shift()
|
||||
|
||||
if (commonsFileName !== undefined) {
|
||||
console.log(`Processing ${commonsFileName}...`)
|
||||
|
||||
const baseUrl = url.split("/").slice(0, 3).join("/")
|
||||
|
||||
// Check if it is a file or a category
|
||||
if (url.includes("Category:")) {
|
||||
// Download all files in the category
|
||||
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&list=categorymembers&cmtitle=${commonsFileName}&cmlimit=250&cmtype=file`
|
||||
const response = await fetch(apiUrl)
|
||||
const apiDetails: CategoryQueryAPIResponse = await response.json()
|
||||
for (const member of apiDetails.query.categorymembers) {
|
||||
await downloadImage(member.title, outputFolder, baseUrl)
|
||||
}
|
||||
} else {
|
||||
await downloadImage(commonsFileName, outputFolder, baseUrl)
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
"\x1b[31m%s\x1b[0m",
|
||||
`URL ${url} doesn't seem to contain a filename or category! Skipping...`
|
||||
)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
"\x1b[31m%s\x1b[0m",
|
||||
`URL ${url} doesn't seem to be a valid URL! Skipping...`
|
||||
)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadImage(filename: string, outputFolder: string, baseUrl: string) {
|
||||
const apiUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=imageinfo&iiprop=url|extmetadata|user&iimetadataversion=latest&titles=${filename}`
|
||||
const response = await fetch(apiUrl)
|
||||
const apiDetails: ImageQueryAPIResponse = await response.json()
|
||||
const missingPage = apiDetails.query.pages["-1"]
|
||||
|
||||
// Check if the file exists, locally or externally
|
||||
if (missingPage !== undefined) {
|
||||
// Image does not exist locally, check if it exists externally
|
||||
if (
|
||||
apiDetails.query.pages["-1"].imagerepository !== "local" &&
|
||||
apiDetails.query.pages["-1"].imagerepository !== ""
|
||||
) {
|
||||
// Check if we actually have image info
|
||||
if (missingPage.imageinfo?.length !== undefined && missingPage.imageinfo.length > 0) {
|
||||
const externalUrl = missingPage.imageinfo[0].descriptionurl
|
||||
const externalBase = externalUrl.split("/").slice(0, 3).join("/")
|
||||
|
||||
const externalFilenamePath = externalUrl.split("/").pop()
|
||||
if (externalFilenamePath !== undefined) {
|
||||
const externalFilename = externalFilenamePath.split("?").shift()
|
||||
console.log(
|
||||
`\x1b[33m%s\x1b[0m`,
|
||||
`${filename} is external, re-running with ${externalUrl}...`
|
||||
)
|
||||
if (externalFilename !== undefined) {
|
||||
await downloadImage(externalFilename, outputFolder, externalBase)
|
||||
return
|
||||
} else {
|
||||
// Edge case
|
||||
console.log(
|
||||
`\x1b[33m%s\x1b[0m`,
|
||||
`External URL ${externalUrl} doesn't seem to contain a filename or category! Skipping...`
|
||||
)
|
||||
}
|
||||
} else {
|
||||
// Edge case
|
||||
console.log(
|
||||
`\x1b[33m%s\x1b[0m`,
|
||||
`External URL ${externalUrl} doesn't seem to be a valid URL! Skipping...`
|
||||
)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
`\x1b[33m%s\x1b[0m`,
|
||||
`${filename} does not have image info!, skipping...`
|
||||
)
|
||||
}
|
||||
}
|
||||
console.log(`\x1b[33m%s\x1b[0m`, `${filename} does not exist!, skipping...`)
|
||||
} else {
|
||||
// Harvest useful information
|
||||
const wikiPage = apiDetails.query.pages[Object.keys(apiDetails.query.pages)[0]]
|
||||
|
||||
// Check if we actually have image info
|
||||
if (wikiPage.imageinfo?.length !== undefined && wikiPage.imageinfo.length > 0) {
|
||||
const wikiUrl = wikiPage.imageinfo[0].descriptionurl
|
||||
const fileUrl = wikiPage.imageinfo[0].url
|
||||
const author =
|
||||
wikiPage.imageinfo[0].extmetadata?.Artist?.value || wikiPage.imageinfo[0].user
|
||||
let license = wikiPage.imageinfo[0].extmetadata?.LicenseShortName?.value || null
|
||||
|
||||
// Check if the output folder exists
|
||||
if (!existsSync(outputFolder)) {
|
||||
const parts = outputFolder.split("/")
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const part = parts.slice(0, i + 1).join("/")
|
||||
if (!existsSync(part)) {
|
||||
console.log(`Creating folder ${part}`)
|
||||
mkdirSync(part)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the license is present
|
||||
if (!license) {
|
||||
console.log(
|
||||
`${filename} does not have a license, falling back to checking template...`
|
||||
)
|
||||
const templateUrl = `${baseUrl}/w/api.php?action=query&format=json&prop=templates&titles=${filename}&tllimit=500`
|
||||
const templateResponse = await fetch(templateUrl)
|
||||
const templateDetails: TemplateQueryAPIResponse = await templateResponse.json()
|
||||
|
||||
// Loop through all templates and check if one of them is a license
|
||||
const wikiPage =
|
||||
templateDetails.query.pages[Object.keys(templateDetails.query.pages)[0]]
|
||||
if (wikiPage.templates) {
|
||||
for (const template of wikiPage.templates) {
|
||||
if (templateMapping[template.title]) {
|
||||
console.log(
|
||||
`Found license ${templateMapping[template.title]} for ${filename}`
|
||||
)
|
||||
license = templateMapping[template.title]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no license was found, skip the file
|
||||
if (!license) {
|
||||
// Log in yellow
|
||||
console.log(
|
||||
`\x1b[33m%s\x1b[0m`,
|
||||
`No license found for ${filename}, skipping...`
|
||||
)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Download the file and save it
|
||||
const cleanFileName = unescape(filename).replace("File:", "")
|
||||
console.log(
|
||||
`Downloading ${cleanFileName} from ${fileUrl} and saving it to ${outputFolder}/${cleanFileName}...`
|
||||
)
|
||||
const fileResponse = await fetch(fileUrl)
|
||||
const fileBuffer = await fileResponse.arrayBuffer()
|
||||
const file = Buffer.from(fileBuffer)
|
||||
const filePath = `${outputFolder}/${cleanFileName}`
|
||||
writeFileSync(filePath, file)
|
||||
|
||||
// Save the license information
|
||||
const licenseInfo: SmallLicense = {
|
||||
path: cleanFileName,
|
||||
license: licenseMapping[license] || license,
|
||||
authors: [author],
|
||||
sources: [wikiUrl],
|
||||
}
|
||||
|
||||
const licensePath = `${outputFolder}/license_info.json`
|
||||
if (!existsSync(licensePath)) {
|
||||
// Create the file if it doesn't exist
|
||||
writeFileSync(licensePath, JSON.stringify([licenseInfo], null, 2))
|
||||
} else {
|
||||
// Append to the file if it does exist
|
||||
const licenseFile = await readFileSync(licensePath, "utf8")
|
||||
const licenseData = JSON.parse(licenseFile)
|
||||
licenseData.push(licenseInfo)
|
||||
writeFileSync(licensePath, JSON.stringify(licenseData, null, 2))
|
||||
}
|
||||
} else {
|
||||
console.log(`\x1b[33m%s\x1b[0m`, `${filename} does not have image info!, skipping...`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main(process.argv.slice(2))
|
||||
|
|
@ -2,11 +2,13 @@ import Script from "./Script"
|
|||
import { Utils } from "../src/Utils"
|
||||
import { Eli, EliEntry } from "./@types/eli"
|
||||
import fs from "fs"
|
||||
import { BingRasterLayer } from "../src/UI/Map/BingRasterLayer"
|
||||
|
||||
class DownloadEli extends Script {
|
||||
constructor() {
|
||||
super("Downloads a fresh copy of the editor layer index, removes all unnecessary data.")
|
||||
}
|
||||
|
||||
async main(args: string[]): Promise<void> {
|
||||
const url = "https://osmlab.github.io/editor-layer-index/imagery.geojson"
|
||||
// Target should use '.json' instead of '.geojson', as the latter cannot be imported by the build systems
|
||||
|
|
@ -20,7 +22,18 @@ class DownloadEli extends Script {
|
|||
|
||||
if (props.type === "bing") {
|
||||
// A lot of work to implement - see https://github.com/pietervdvn/MapComplete/issues/648
|
||||
continue
|
||||
try {
|
||||
const bing = await BingRasterLayer.get()
|
||||
if (bing === "error") {
|
||||
continue
|
||||
}
|
||||
delete props.default
|
||||
props.category = "photo"
|
||||
props.url = bing.properties.url.replace("%7Bquadkey%7D", "{quadkey}")
|
||||
} catch (e) {
|
||||
console.error("Could not fetch URL for bing", e)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if (props.id === "MAPNIK") {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ function genImages(dryrun = false) {
|
|||
"add",
|
||||
"addSmall",
|
||||
"back",
|
||||
"circle",
|
||||
"blocked",
|
||||
"brick_wall",
|
||||
"brick_wall_raw",
|
||||
|
|
@ -33,6 +34,7 @@ function genImages(dryrun = false) {
|
|||
"duplicate",
|
||||
"elevator",
|
||||
"elevator_wheelchair",
|
||||
"envelope",
|
||||
"eye",
|
||||
"filter",
|
||||
"filter_disable",
|
||||
|
|
@ -65,6 +67,7 @@ function genImages(dryrun = false) {
|
|||
"mapillary_black",
|
||||
"mastodon",
|
||||
"min",
|
||||
"move",
|
||||
"move-arrows",
|
||||
"move_confirm",
|
||||
"move_not_allowed",
|
||||
|
|
@ -72,23 +75,33 @@ function genImages(dryrun = false) {
|
|||
"osm_logo_us",
|
||||
"osm-logo-us",
|
||||
"party",
|
||||
"pencil",
|
||||
"person",
|
||||
"pin",
|
||||
"plantnet_logo",
|
||||
"plus",
|
||||
"reload",
|
||||
"resolved",
|
||||
"ring",
|
||||
"robot",
|
||||
"scissors",
|
||||
"search",
|
||||
"search_disable",
|
||||
"share",
|
||||
"SocialImageForeground",
|
||||
"speech_bubble",
|
||||
"speech_bubble_black_outline",
|
||||
"square",
|
||||
"square_rounded",
|
||||
"star",
|
||||
"star_half",
|
||||
"star_outline",
|
||||
"teardrop",
|
||||
"teardrop_with_hole_green",
|
||||
"statistics",
|
||||
"translate",
|
||||
"triangle",
|
||||
"up",
|
||||
"Upload",
|
||||
"wikidata",
|
||||
"wikimedia-commons-white",
|
||||
|
|
|
|||
|
|
@ -9,12 +9,17 @@ import {
|
|||
DoesImageExist,
|
||||
PrevalidateTheme,
|
||||
ValidateLayer,
|
||||
ValidateThemeAndLayers, ValidateThemeEnsemble,
|
||||
ValidateThemeAndLayers,
|
||||
ValidateThemeEnsemble,
|
||||
} from "../src/Models/ThemeConfig/Conversion/Validation"
|
||||
import { Translation } from "../src/UI/i18n/Translation"
|
||||
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
|
||||
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
|
||||
import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
|
||||
import {
|
||||
Conversion,
|
||||
DesugaringContext,
|
||||
DesugaringStep,
|
||||
} from "../src/Models/ThemeConfig/Conversion/Conversion"
|
||||
import { Utils } from "../src/Utils"
|
||||
import Script from "./Script"
|
||||
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
|
||||
|
|
@ -49,7 +54,7 @@ class ParseLayer extends Conversion<
|
|||
|
||||
convert(
|
||||
path: string,
|
||||
context: ConversionContext,
|
||||
context: ConversionContext
|
||||
): {
|
||||
parsed: LayerConfig
|
||||
raw: LayerConfigJson
|
||||
|
|
@ -65,7 +70,7 @@ class ParseLayer extends Conversion<
|
|||
try {
|
||||
parsed = JSON.parse(fileContents)
|
||||
} catch (e) {
|
||||
context.err("Could not parse file as JSON")
|
||||
context.err("Could not parse file as JSON: " + e)
|
||||
return undefined
|
||||
}
|
||||
if (parsed === undefined) {
|
||||
|
|
@ -104,7 +109,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
|
|||
const fixed = json.raw
|
||||
const layerConfig = json.parsed
|
||||
const pointRendering: PointRenderingConfig = layerConfig.mapRendering.find((pr) =>
|
||||
pr.location.has("point"),
|
||||
pr.location.has("point")
|
||||
)
|
||||
const defaultTags = layerConfig.GetBaseTags()
|
||||
fixed["_layerIcon"] = Utils.NoNull(
|
||||
|
|
@ -119,13 +124,12 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
|
|||
result["color"] = c
|
||||
}
|
||||
return result
|
||||
}),
|
||||
})
|
||||
)
|
||||
return { raw: fixed, parsed: layerConfig }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class LayerOverviewUtils extends Script {
|
||||
public static readonly layerPath = "./src/assets/generated/layers/"
|
||||
public static readonly themePath = "./src/assets/generated/themes/"
|
||||
|
|
@ -142,7 +146,7 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
private static extractLayerIdsFrom(
|
||||
themeFile: LayoutConfigJson,
|
||||
includeInlineLayers = true,
|
||||
includeInlineLayers = true
|
||||
): string[] {
|
||||
const publicLayerIds = []
|
||||
if (!Array.isArray(themeFile.layers)) {
|
||||
|
|
@ -205,10 +209,10 @@ class LayerOverviewUtils extends Script {
|
|||
| LayerConfigJson
|
||||
| string
|
||||
| {
|
||||
builtin
|
||||
}
|
||||
)[]
|
||||
}[],
|
||||
builtin
|
||||
}
|
||||
)[]
|
||||
}[]
|
||||
) {
|
||||
const perId = new Map<string, any>()
|
||||
for (const theme of themes) {
|
||||
|
|
@ -249,7 +253,7 @@ class LayerOverviewUtils extends Script {
|
|||
writeFileSync(
|
||||
"./src/assets/generated/theme_overview.json",
|
||||
JSON.stringify(sorted, null, " "),
|
||||
{ encoding: "utf8" },
|
||||
{ encoding: "utf8" }
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -261,7 +265,7 @@ class LayerOverviewUtils extends Script {
|
|||
writeFileSync(
|
||||
`${LayerOverviewUtils.themePath}${theme.id}.json`,
|
||||
JSON.stringify(theme, null, " "),
|
||||
{ encoding: "utf8" },
|
||||
{ encoding: "utf8" }
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -272,13 +276,13 @@ class LayerOverviewUtils extends Script {
|
|||
writeFileSync(
|
||||
`${LayerOverviewUtils.layerPath}${layer.id}.json`,
|
||||
JSON.stringify(layer, null, " "),
|
||||
{ encoding: "utf8" },
|
||||
{ encoding: "utf8" }
|
||||
)
|
||||
}
|
||||
|
||||
getSharedTagRenderings(
|
||||
doesImageExist: DoesImageExist,
|
||||
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null,
|
||||
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null
|
||||
): Map<string, QuestionableTagRenderingConfigJson> {
|
||||
const prepareLayer = new PrepareLayer({
|
||||
tagRenderings: bootstrapTagRenderings,
|
||||
|
|
@ -339,8 +343,8 @@ class LayerOverviewUtils extends Script {
|
|||
if (contents.indexOf("<text") > 0) {
|
||||
console.warn(
|
||||
"The SVG at " +
|
||||
path +
|
||||
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path",
|
||||
path +
|
||||
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path"
|
||||
)
|
||||
errCount++
|
||||
}
|
||||
|
|
@ -352,11 +356,19 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
async main(args: string[]) {
|
||||
console.log("Generating layer overview...")
|
||||
const themeWhitelist = new Set(args.find(a => a.startsWith("--themes="))
|
||||
?.substring("--themes=".length)?.split(",") ?? [])
|
||||
const themeWhitelist = new Set(
|
||||
args
|
||||
.find((a) => a.startsWith("--themes="))
|
||||
?.substring("--themes=".length)
|
||||
?.split(",") ?? []
|
||||
)
|
||||
|
||||
const layerWhitelist = new Set(args.find(a => a.startsWith("--layers="))
|
||||
?.substring("--layers=".length)?.split(",") ?? [])
|
||||
const layerWhitelist = new Set(
|
||||
args
|
||||
.find((a) => a.startsWith("--layers="))
|
||||
?.substring("--layers=".length)
|
||||
?.split(",") ?? []
|
||||
)
|
||||
|
||||
const forceReload = args.some((a) => a == "--force")
|
||||
|
||||
|
|
@ -384,19 +396,19 @@ class LayerOverviewUtils extends Script {
|
|||
sharedLayers,
|
||||
recompiledThemes,
|
||||
forceReload,
|
||||
themeWhitelist,
|
||||
themeWhitelist
|
||||
)
|
||||
|
||||
new ValidateThemeEnsemble().convertStrict(
|
||||
Array.from(sharedThemes.values()).map(th => new LayoutConfig(th, true)))
|
||||
|
||||
Array.from(sharedThemes.values()).map((th) => new LayoutConfig(th, true))
|
||||
)
|
||||
|
||||
if (recompiledThemes.length > 0) {
|
||||
writeFileSync(
|
||||
"./src/assets/generated/known_layers.json",
|
||||
JSON.stringify({
|
||||
layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"),
|
||||
}),
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -417,7 +429,7 @@ class LayerOverviewUtils extends Script {
|
|||
const proto: LayoutConfigJson = JSON.parse(
|
||||
readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", {
|
||||
encoding: "utf8",
|
||||
}),
|
||||
})
|
||||
)
|
||||
const protolayer = <LayerConfigJson>(
|
||||
proto.layers.filter((l) => l["id"] === "mapcomplete-changes")[0]
|
||||
|
|
@ -434,12 +446,12 @@ class LayerOverviewUtils extends Script {
|
|||
layers: ScriptUtils.getLayerFiles().map((f) => f.parsed),
|
||||
themes: ScriptUtils.getThemeFiles().map((f) => f.parsed),
|
||||
},
|
||||
ConversionContext.construct([], []),
|
||||
ConversionContext.construct([], [])
|
||||
)
|
||||
|
||||
for (const [_, theme] of sharedThemes) {
|
||||
theme.layers = theme.layers.filter(
|
||||
(l) => Constants.added_by_default.indexOf(l["id"]) < 0,
|
||||
(l) => Constants.added_by_default.indexOf(l["id"]) < 0
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -448,21 +460,19 @@ class LayerOverviewUtils extends Script {
|
|||
"./src/assets/generated/known_themes.json",
|
||||
JSON.stringify({
|
||||
themes: Array.from(sharedThemes.values()),
|
||||
}),
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
|
||||
console.error(
|
||||
"This was a bootstrapping-run. Run generate layeroverview again!"
|
||||
)
|
||||
console.error("This was a bootstrapping-run. Run generate layeroverview again!")
|
||||
}
|
||||
}
|
||||
|
||||
private parseLayer(
|
||||
doesImageExist: DoesImageExist,
|
||||
prepLayer: PrepareLayer,
|
||||
sharedLayerPath: string,
|
||||
sharedLayerPath: string
|
||||
): {
|
||||
raw: LayerConfigJson
|
||||
parsed: LayerConfig
|
||||
|
|
@ -473,7 +483,7 @@ class LayerOverviewUtils extends Script {
|
|||
const parsed = parser.convertStrict(sharedLayerPath, context)
|
||||
const result = AddIconSummary.singleton.convertStrict(
|
||||
parsed,
|
||||
context.inOperation("AddIconSummary"),
|
||||
context.inOperation("AddIconSummary")
|
||||
)
|
||||
return { ...result, context }
|
||||
}
|
||||
|
|
@ -481,7 +491,7 @@ class LayerOverviewUtils extends Script {
|
|||
private buildLayerIndex(
|
||||
doesImageExist: DoesImageExist,
|
||||
forceReload: boolean,
|
||||
whitelist: Set<string>,
|
||||
whitelist: Set<string>
|
||||
): Map<string, LayerConfigJson> {
|
||||
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
|
||||
// At the same time, an index of available layers is built.
|
||||
|
|
@ -501,7 +511,10 @@ class LayerOverviewUtils extends Script {
|
|||
for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
|
||||
if (whitelist.size > 0) {
|
||||
const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0]
|
||||
if (Constants.priviliged_layers.indexOf(<any>idByPath) < 0 && !whitelist.has(idByPath)) {
|
||||
if (
|
||||
Constants.priviliged_layers.indexOf(<any>idByPath) < 0 &&
|
||||
!whitelist.has(idByPath)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
|
@ -533,17 +546,17 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
console.log(
|
||||
"Recompiled layers " +
|
||||
recompiledLayers.join(", ") +
|
||||
" and skipped " +
|
||||
skippedLayers.length +
|
||||
" layers. Detected " +
|
||||
warningCount +
|
||||
" warnings",
|
||||
recompiledLayers.join(", ") +
|
||||
" and skipped " +
|
||||
skippedLayers.length +
|
||||
" layers. Detected " +
|
||||
warningCount +
|
||||
" warnings"
|
||||
)
|
||||
// We always need the calculated tags of 'usersettings', so we export them separately
|
||||
this.extractJavascriptCodeForLayer(
|
||||
state.sharedLayers.get("usersettings"),
|
||||
"./src/Logic/State/UserSettingsMetaTagging.ts",
|
||||
"./src/Logic/State/UserSettingsMetaTagging.ts"
|
||||
)
|
||||
|
||||
return sharedLayers
|
||||
|
|
@ -560,8 +573,8 @@ class LayerOverviewUtils extends Script {
|
|||
private extractJavascriptCode(themeFile: LayoutConfigJson) {
|
||||
const allCode = [
|
||||
"import {Feature} from 'geojson'",
|
||||
"import { ExtraFuncType } from \"../../../Logic/ExtraFunctions\";",
|
||||
"import { Utils } from \"../../../Utils\"",
|
||||
'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
|
||||
'import { Utils } from "../../../Utils"',
|
||||
"export class ThemeMetaTagging {",
|
||||
" public static readonly themeName = " + JSON.stringify(themeFile.id),
|
||||
"",
|
||||
|
|
@ -573,8 +586,8 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
allCode.push(
|
||||
" public metaTaggging_for_" +
|
||||
id +
|
||||
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {",
|
||||
id +
|
||||
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
|
||||
)
|
||||
allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions")
|
||||
for (const line of code) {
|
||||
|
|
@ -585,10 +598,10 @@ class LayerOverviewUtils extends Script {
|
|||
if (!isStrict) {
|
||||
allCode.push(
|
||||
" Utils.AddLazyProperty(feat.properties, '" +
|
||||
attributeName +
|
||||
"', () => " +
|
||||
expression +
|
||||
" ) ",
|
||||
attributeName +
|
||||
"', () => " +
|
||||
expression +
|
||||
" ) "
|
||||
)
|
||||
} else {
|
||||
attributeName = attributeName.substring(0, attributeName.length - 1).trim()
|
||||
|
|
@ -633,7 +646,7 @@ class LayerOverviewUtils extends Script {
|
|||
const code = l.calculatedTags ?? []
|
||||
|
||||
allCode.push(
|
||||
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {",
|
||||
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {"
|
||||
)
|
||||
for (const line of code) {
|
||||
const firstEq = line.indexOf("=")
|
||||
|
|
@ -643,10 +656,10 @@ class LayerOverviewUtils extends Script {
|
|||
if (!isStrict) {
|
||||
allCode.push(
|
||||
" Utils.AddLazyProperty(feat.properties, '" +
|
||||
attributeName +
|
||||
"', () => " +
|
||||
expression +
|
||||
" ) ",
|
||||
attributeName +
|
||||
"', () => " +
|
||||
expression +
|
||||
" ) "
|
||||
)
|
||||
} else {
|
||||
attributeName = attributeName.substring(0, attributeName.length - 2).trim()
|
||||
|
|
@ -671,20 +684,20 @@ class LayerOverviewUtils extends Script {
|
|||
sharedLayers: Map<string, LayerConfigJson>,
|
||||
recompiledThemes: string[],
|
||||
forceReload: boolean,
|
||||
whitelist: Set<string>,
|
||||
whitelist: Set<string>
|
||||
): Map<string, LayoutConfigJson> {
|
||||
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
|
||||
const themeFiles = ScriptUtils.getThemeFiles()
|
||||
const fixed = new Map<string, LayoutConfigJson>()
|
||||
|
||||
const publicLayers = LayerOverviewUtils.publicLayerIdsFrom(
|
||||
themeFiles.map((th) => th.parsed),
|
||||
themeFiles.map((th) => th.parsed)
|
||||
)
|
||||
|
||||
const convertState: DesugaringContext = {
|
||||
sharedLayers,
|
||||
tagRenderings: this.getSharedTagRenderings(
|
||||
new DoesImageExist(licensePaths, existsSync),
|
||||
new DoesImageExist(licensePaths, existsSync)
|
||||
),
|
||||
publicLayers,
|
||||
}
|
||||
|
|
@ -717,15 +730,15 @@ class LayerOverviewUtils extends Script {
|
|||
LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
|
||||
|
||||
const usedLayers = Array.from(
|
||||
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false),
|
||||
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
|
||||
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
|
||||
|
||||
if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
|
||||
fixed.set(
|
||||
themeFile.id,
|
||||
JSON.parse(
|
||||
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8"),
|
||||
),
|
||||
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8")
|
||||
)
|
||||
)
|
||||
ScriptUtils.erasableLog("Skipping", themeFile.id)
|
||||
skippedThemes.push(themeFile.id)
|
||||
|
|
@ -736,23 +749,23 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
new PrevalidateTheme().convertStrict(
|
||||
themeFile,
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"]),
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"])
|
||||
)
|
||||
try {
|
||||
themeFile = new PrepareTheme(convertState, {
|
||||
skipDefaultLayers: true,
|
||||
}).convertStrict(
|
||||
themeFile,
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"]),
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"])
|
||||
)
|
||||
new ValidateThemeAndLayers(
|
||||
new DoesImageExist(licensePaths, existsSync, knownTagRenderings),
|
||||
themePath,
|
||||
true,
|
||||
knownTagRenderings,
|
||||
knownTagRenderings
|
||||
).convertStrict(
|
||||
themeFile,
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"]),
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"])
|
||||
)
|
||||
|
||||
if (themeFile.icon.endsWith(".svg")) {
|
||||
|
|
@ -807,16 +820,16 @@ class LayerOverviewUtils extends Script {
|
|||
.OnEveryLanguage((s) => parse_html(s).textContent).translations,
|
||||
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
|
||||
}
|
||||
}),
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
console.log(
|
||||
"Recompiled themes " +
|
||||
recompiledThemes.join(", ") +
|
||||
" and skipped " +
|
||||
skippedThemes.length +
|
||||
" themes",
|
||||
recompiledThemes.join(", ") +
|
||||
" and skipped " +
|
||||
skippedThemes.length +
|
||||
" themes"
|
||||
)
|
||||
|
||||
return fixed
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ import * as crypto from "crypto"
|
|||
import * as eli from "../src/assets/editor-layer-index.json"
|
||||
import * as eli_global from "../src/assets/global-raster-layers.json"
|
||||
import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils"
|
||||
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
|
||||
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
|
||||
|
||||
const sharp = require("sharp")
|
||||
const template = readFileSync("theme.html", "utf8")
|
||||
|
|
@ -271,7 +273,6 @@ async function generateCsp(
|
|||
}
|
||||
): Promise<string> {
|
||||
const apiUrls: string[] = [
|
||||
"'self'",
|
||||
...Constants.defaultOverpassUrls,
|
||||
Constants.countryCoderEndpoint,
|
||||
Constants.nominatimEndpoint,
|
||||
|
|
@ -282,19 +283,24 @@ async function generateCsp(
|
|||
|
||||
SpecialVisualizations.specialVisualizations.forEach((sv) => {
|
||||
if (typeof sv.needsUrls === "function") {
|
||||
// Handled below
|
||||
return
|
||||
}
|
||||
apiUrls.push(...(sv.needsUrls ?? []))
|
||||
})
|
||||
|
||||
const usedSpecialVisualisations = ValidationUtils.getSpecialVisualisationsWithArgs(layoutJson)
|
||||
const usedSpecialVisualisations = [].concat(...layoutJson.layers.map(l => ValidationUtils.getAllSpecialVisualisations(<QuestionableTagRenderingConfigJson[]> (<LayerConfigJson>l).tagRenderings ?? [])))
|
||||
for (const usedSpecialVisualisation of usedSpecialVisualisations) {
|
||||
if (typeof usedSpecialVisualisation === "string") {
|
||||
continue
|
||||
}
|
||||
const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
|
||||
if (typeof neededUrls === "function") {
|
||||
apiUrls.push(...neededUrls(usedSpecialVisualisation.args))
|
||||
let needed: string | string[] = neededUrls(usedSpecialVisualisation.args)
|
||||
if(typeof needed === "string"){
|
||||
needed = [needed]
|
||||
}
|
||||
apiUrls.push(...needed)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -306,11 +312,14 @@ async function generateCsp(
|
|||
const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
|
||||
const vectorSources = vectorLayers.map((l) => l.properties.url)
|
||||
apiUrls.push(...vectorSources)
|
||||
for (const connectSource of apiUrls.concat(geojsonSources)) {
|
||||
for (let connectSource of apiUrls.concat(geojsonSources)) {
|
||||
if (!connectSource) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
if(!connectSource.startsWith("http")){
|
||||
connectSource = "https://"+connectSource
|
||||
}
|
||||
const url = new URL(connectSource)
|
||||
hosts.add("https://" + url.host)
|
||||
} catch (e) {
|
||||
|
|
@ -340,7 +349,7 @@ async function generateCsp(
|
|||
"default-src": "'self'",
|
||||
"child-src": "'self' blob: ",
|
||||
"img-src": "* data:", // maplibre depends on 'data:' to load
|
||||
"connect-src": connectSrc.join(" "),
|
||||
"connect-src": "'self' "+connectSrc.join(" "),
|
||||
"report-to": "https://report.mapcomplete.org/csp",
|
||||
"worker-src": "'self' blob:", // Vite somehow loads the worker via a 'blob'
|
||||
"style-src": "'self' 'unsafe-inline'", // unsafe-inline is needed to change the default background pin colours
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import { readFileSync, writeFileSync } from "fs"
|
|||
import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
|
||||
import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
|
||||
import { Utils } from "../src/Utils"
|
||||
import TagRenderingConfig from "../src/Models/ThemeConfig/TagRenderingConfig"
|
||||
|
||||
/**
|
||||
* Generates all the files in "Docs/TagInfo". These are picked up by the taginfo project, showing a link to the mapcomplete theme if the key is used
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@
|
|||
|
||||
cp config.json config.json.bu &&
|
||||
cp ./scripts/hetzner/config.json . && # Copy the config _before_ building, as the config might contain some needed URLs
|
||||
# npm run reset:layeroverview
|
||||
# npm run test &&
|
||||
# npm run prepare-deploy &&
|
||||
npm run reset:layeroverview
|
||||
npm run test &&
|
||||
npm run prepare-deploy &&
|
||||
zip dist.zip -r dist/* &&
|
||||
mv config.json.bu config.json &&
|
||||
scp ./scripts/hetzner/config/* hetzner:/root/ &&
|
||||
|
|
|
|||
|
|
@ -30,17 +30,6 @@ t.OnEveryLanguage((txt, ln) => {
|
|||
return txt
|
||||
})
|
||||
|
||||
const articles = {
|
||||
/* de: "eine",
|
||||
es: 'una',
|
||||
fr: 'une',
|
||||
it: 'una',
|
||||
nb_NO: 'en',
|
||||
nl: 'een',
|
||||
pt: 'uma',
|
||||
pt_BR : 'uma',//*/
|
||||
}
|
||||
|
||||
function reorder(object: object, order: string[]) {
|
||||
const allKeys = new Set<string>(Object.keys(object))
|
||||
const copy = {}
|
||||
|
|
@ -54,38 +43,6 @@ function reorder(object: object, order: string[]) {
|
|||
return copy
|
||||
}
|
||||
|
||||
function addArticleToPresets(layerConfig: { presets?: { title: any }[] }) {
|
||||
/*
|
||||
if(layerConfig.presets === undefined){
|
||||
return
|
||||
}
|
||||
for (const preset of layerConfig.presets) {
|
||||
preset.title = new Translation(preset.title, "autofix")
|
||||
.OnEveryLanguage((txt, lang) => {
|
||||
let article = articles[lang]
|
||||
if(lang === "en"){
|
||||
if(["a","e","u","o","i"].some(vowel => txt.toLowerCase().startsWith(vowel))) {
|
||||
article = "an"
|
||||
}else{
|
||||
article = "a"
|
||||
}
|
||||
}
|
||||
if(article === undefined){
|
||||
return txt;
|
||||
}
|
||||
if(txt.startsWith(article+" ")){
|
||||
return txt;
|
||||
}
|
||||
if(txt.startsWith("an ")){
|
||||
return txt;
|
||||
}
|
||||
return article +" " + txt.toLowerCase();
|
||||
})
|
||||
.translations
|
||||
}
|
||||
//*/
|
||||
}
|
||||
|
||||
const layerFiles = ScriptUtils.getLayerFiles()
|
||||
for (const layerFile of layerFiles) {
|
||||
try {
|
||||
|
|
@ -95,7 +52,6 @@ for (const layerFile of layerFiles) {
|
|||
ConversionContext.construct([layerFile.path.split("/").at(-1)], ["update legacy"])
|
||||
)
|
||||
)
|
||||
addArticleToPresets(fixed)
|
||||
const reordered = reorder(fixed, layerAttributesOrder)
|
||||
writeFileSync(layerFile.path, JSON.stringify(reordered, null, " ") + "\n")
|
||||
} catch (e) {
|
||||
|
|
@ -110,11 +66,7 @@ for (const themeFile of themeFiles) {
|
|||
themeFile.parsed,
|
||||
ConversionContext.construct([themeFile.path.split("/").at(-1)], ["update legacy layer"])
|
||||
)
|
||||
for (const layer of fixed.layers) {
|
||||
if (layer["presets"] !== undefined) {
|
||||
addArticleToPresets(<any>layer)
|
||||
}
|
||||
}
|
||||
|
||||
// extractInlineLayer(fixed)
|
||||
const endsWithNewline = themeFile.raw.at(-1) === "\n"
|
||||
const ordered = reorder(fixed, themeAttributesOrder)
|
||||
|
|
|
|||
72
scripts/velopark/compare.ts
Normal file
72
scripts/velopark/compare.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import Script from "../Script"
|
||||
import fs from "fs"
|
||||
import { Feature, FeatureCollection } from "geojson"
|
||||
import { GeoOperations } from "../../src/Logic/GeoOperations"
|
||||
import * as os from "os"
|
||||
// vite-node scripts/velopark/compare.ts -- scripts/velopark/velopark_all_2024-02-14T12\:18\:41.772Z.geojson ~/Projecten/OSM/Fietsberaad/2024-02-02\ Fietsenstallingen_OSM_met_velopark_ref.geojson
|
||||
class Compare extends Script {
|
||||
|
||||
compare(veloId: string, osmParking: Feature, veloParking: Feature): {distance: number, ref: string, osmid: string, diffs: {
|
||||
osm: string, velopark: string, key: string
|
||||
}[] }{
|
||||
const osmCenterpoint = GeoOperations.centerpointCoordinates(osmParking)
|
||||
const veloparkCenterpoint = GeoOperations.centerpointCoordinates(veloParking)
|
||||
const distance = Math.round(GeoOperations.distanceBetween(osmCenterpoint, veloparkCenterpoint))
|
||||
const diffs: { osm: string, velopark: string, key: string}[] = []
|
||||
|
||||
const allKeys = new Set<string>(Object.keys(osmParking.properties).concat(Object.keys(veloParking.properties)))
|
||||
for (const key of allKeys) {
|
||||
if(osmParking.properties[key] === veloParking.properties[key]){
|
||||
continue
|
||||
}
|
||||
if(Number(osmParking.properties[key]) === veloParking.properties[key]){
|
||||
continue
|
||||
}
|
||||
if(veloParking.properties[key] === undefined){
|
||||
continue
|
||||
}
|
||||
diffs.push({
|
||||
key,
|
||||
osm: osmParking.properties[key],
|
||||
velopark: veloParking.properties[key]
|
||||
})
|
||||
}
|
||||
return {
|
||||
ref: veloId,
|
||||
osmid: osmParking.properties["@id"],
|
||||
distance, diffs
|
||||
}
|
||||
}
|
||||
async main(args: string[]): Promise<void> {
|
||||
let [velopark, osm, key] = args
|
||||
key ??= "ref:velopark"
|
||||
const veloparkData: FeatureCollection = JSON.parse(fs.readFileSync(velopark, "utf-8"))
|
||||
const osmData : FeatureCollection = JSON.parse(fs.readFileSync(osm, "utf-8"))
|
||||
|
||||
const veloparkById : Record<string, Feature> = {}
|
||||
for (const parking of veloparkData.features) {
|
||||
veloparkById[parking.properties[key]] = parking
|
||||
}
|
||||
|
||||
const diffs = []
|
||||
for (const parking of osmData.features) {
|
||||
const veloId = parking.properties[key]
|
||||
const veloparking = veloparkById[veloId]
|
||||
if(veloparking === undefined){
|
||||
console.error("No velopark entry found for", veloId)
|
||||
continue
|
||||
}
|
||||
diffs.push(this.compare(veloId, parking, veloparking))
|
||||
}
|
||||
|
||||
fs.writeFileSync("report_diff.json",JSON.stringify(diffs))
|
||||
|
||||
|
||||
}
|
||||
constructor() {
|
||||
super("Compares a velopark geojson with OSM geojson. Usage: `compare velopark.geojson osm.geojson [key-to-compare-on]`. If key-to-compare-on is not given, `ref:velopark` will be used")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
new Compare().run()
|
||||
75
scripts/velopark/veloParkToGeojson.ts
Normal file
75
scripts/velopark/veloParkToGeojson.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import Script from "../Script"
|
||||
import { Utils } from "../../src/Utils"
|
||||
import VeloparkLoader, { VeloparkData } from "../../src/Logic/Web/VeloparkLoader"
|
||||
import fs from "fs"
|
||||
import { Overpass } from "../../src/Logic/Osm/Overpass"
|
||||
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
|
||||
import Constants from "../../src/Models/Constants"
|
||||
import { ImmutableStore } from "../../src/Logic/UIEventSource"
|
||||
import { BBox } from "../../src/Logic/BBox"
|
||||
|
||||
class VeloParkToGeojson extends Script {
|
||||
constructor() {
|
||||
super(
|
||||
"Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory"
|
||||
)
|
||||
}
|
||||
|
||||
exportTo(filename: string, features){
|
||||
fs.writeFileSync(
|
||||
filename+"_" + new Date().toISOString() + ".geojson",
|
||||
JSON.stringify(
|
||||
{
|
||||
type: "FeatureCollection",
|
||||
features,
|
||||
},
|
||||
null,
|
||||
" "
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
async main(args: string[]): Promise<void> {
|
||||
console.log("Downloading velopark data")
|
||||
// Download data for NIS-code 1000. 1000 means: all of belgium
|
||||
const url = "https://www.velopark.be/api/parkings/1000"
|
||||
const data = <VeloparkData[]>await Utils.downloadJson(url)
|
||||
|
||||
const bboxBelgium = new BBox([
|
||||
[2.51357303225, 49.5294835476],
|
||||
[6.15665815596, 51.4750237087],
|
||||
])
|
||||
const alreadyLinkedQuery = new Overpass(
|
||||
new RegexTag("ref:velopark", /.+/),
|
||||
[],
|
||||
Constants.defaultOverpassUrls[0],
|
||||
new ImmutableStore(60 * 5),
|
||||
false
|
||||
)
|
||||
const alreadyLinkedFeatures = await alreadyLinkedQuery.queryGeoJson(bboxBelgium)
|
||||
const seenIds = new Set<string>(
|
||||
alreadyLinkedFeatures[0].features.map((f) => f.properties["ref:velopark"])
|
||||
)
|
||||
console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")
|
||||
const allVelopark = data.map((f) => VeloparkLoader.convert(f))
|
||||
this.exportTo("velopark_all", allVelopark)
|
||||
|
||||
const features = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))
|
||||
|
||||
const allProperties = new Set<string>()
|
||||
for (const feature of features) {
|
||||
Object.keys(feature.properties).forEach((k) => allProperties.add(k))
|
||||
}
|
||||
this.exportTo("velopark_noncynced",features)
|
||||
allProperties.delete("ref:velopark")
|
||||
for (const feature of features) {
|
||||
allProperties.forEach((k) => {
|
||||
delete feature.properties[k]
|
||||
})
|
||||
}
|
||||
|
||||
this.exportTo("velopark_nonsynced_id_only", features)
|
||||
}
|
||||
}
|
||||
|
||||
new VeloParkToGeojson().run()
|
||||
Loading…
Add table
Add a link
Reference in a new issue