From 35c31f9861ad0982c06d4feef84c0e0232f625d1 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Thu, 22 Feb 2024 18:58:34 +0100 Subject: [PATCH] Add linked data module which scrapes websites --- assets/layers/questions/questions.json | 9 + .../mapcomplete-changes.json | 116 +--- package-lock.json | 504 +++++++++++++++++- package.json | 6 +- scripts/serverLdScrape.ts | 16 +- src/Logic/Web/LinkedDataLoader.ts | 142 +++++ src/Models/Constants.ts | 1 + .../ThemeConfig/Conversion/PrepareLayer.ts | 3 + .../BigComponents/SelectedElementView.svelte | 5 +- src/UI/Image/ImageCarousel.ts | 2 +- .../InputElement/Validators/PhoneValidator.ts | 2 +- .../InputElement/Validators/UrlValidator.ts | 7 +- src/UI/LinkedDataDisplay.svelte | 90 ++++ src/UI/SpecialVisualizations.ts | 47 +- src/UI/Test.svelte | 50 +- 15 files changed, 870 insertions(+), 130 deletions(-) create mode 100644 src/Logic/Web/LinkedDataLoader.ts create mode 100644 src/UI/LinkedDataDisplay.svelte diff --git a/assets/layers/questions/questions.json b/assets/layers/questions/questions.json index e2f06be10..114241830 100644 --- a/assets/layers/questions/questions.json +++ b/assets/layers/questions/questions.json @@ -2620,6 +2620,15 @@ } ] }, + { + "id": "lod", + "labels": [ + "added_by_default" + ], + "render": { + "*": "{linked_data_from_website()}" + } + }, { "id": "qr_code", "labels": [ diff --git a/assets/themes/mapcomplete-changes/mapcomplete-changes.json b/assets/themes/mapcomplete-changes/mapcomplete-changes.json index 3d4f53082..591381bdf 100644 --- a/assets/themes/mapcomplete-changes/mapcomplete-changes.json +++ b/assets/themes/mapcomplete-changes/mapcomplete-changes.json @@ -1,19 +1,13 @@ { "id": "mapcomplete-changes", "title": { - "en": "Changes made with MapComplete", - "de": "Änderungen mit MapComplete", - "es": "Cambios hechos con MapComplete" + "en": "Changes made with MapComplete" }, "shortDescription": { - "en": "Shows changes made by MapComplete", - "de": "Änderungen von MapComplete anzeigen", - "es": "Muestra los cambios hechos por MapComplete" + "en": "Shows changes made by MapComplete" }, "description": { - "en": "This maps shows all the changes made with MapComplete", - "de": "Diese Karte zeigt alle mit MapComplete vorgenommenen Änderungen", - "es": "Este mapa muestra todos los cambios hechos con MapComplete" + "en": "This maps shows all the changes made with MapComplete" }, "icon": "./assets/svg/logo.svg", "hideFromOverview": true, @@ -26,9 +20,7 @@ { "id": "mapcomplete-changes", "name": { - "en": "Changeset centers", - "de": "Zentrum der Änderungssätze", - "es": "Centro del conjunto de cambios" + "en": "Changeset centers" }, "minzoom": 0, "source": { @@ -39,55 +31,41 @@ }, "title": { "render": { - "en": "Changeset for {theme}", - "de": "Änderungssatz für {theme}", - "es": "Conjunto de cambios para {theme}" + "en": "Changeset for {theme}" } }, "description": { - "en": "Shows all MapComplete changes", - "de": "Alle MapComplete-Änderungen anzeigen", - "es": "Muestra todos los cambios de MapComplete" + "en": "Shows all MapComplete changes" }, "tagRenderings": [ { "id": "show_changeset_id", "render": { - "en": "Changeset {id}", - "de": "Änderungssatz {id}", - "es": "Conjunto de cambios {id}" + "en": "Changeset {id}" } }, { "id": "contributor", "question": { - "en": "What contributor did make this change?", - "de": "Wer hat diese Änderung vorgenommen?", - "es": "¿Quién realizó este cambio?" + "en": "What contributor did make this change?" }, "freeform": { "key": "user" }, "render": { - "en": "Change made by {user}", - "de": "Änderung von {user}", - "es": "Cambio hecho por {user}" + "en": "Change made by {user}" } }, { "id": "theme-id", "question": { - "en": "What theme was used to make this change?", - "de": "Welches Thema wurde für die Änderung verwendet?", - "es": "¿Qué tema se utilizó para realizar este cambio?" + "en": "What theme was used to make this change?" }, "freeform": { "key": "theme" }, "render": { - "en": "Change with theme {theme}", - "de": "Geändert mit Thema {theme}", - "es": "Cambio con el tema {theme}" + "en": "Change with theme {theme}" } }, { @@ -96,27 +74,19 @@ "key": "locale" }, "question": { - "en": "What locale (language) was this change made in?", - "de": "In welcher Benutzersprache wurde die Änderung vorgenommen?", - "es": "¿En qué configuración regional (idioma) se realizó este cambio?" + "en": "What locale (language) was this change made in?" }, "render": { - "en": "User locale is {locale}", - "de": "Benutzersprache {locale}", - "es": "La configuración regional del usuario es {locale}" + "en": "User locale is {locale}" } }, { "id": "host", "render": { - "en": "Change with with {host}", - "de": "Änderung über {host}", - "es": "Cambio con {host}" + "en": "Change with with {host}" }, "question": { - "en": "What host (website) was this change made with?", - "de": "Über welchen Host (Webseite) wurde diese Änderung vorgenommen?", - "es": "¿Con qué host (página web) se realizó este cambio?" + "en": "What host (website) was this change made with?" }, "freeform": { "key": "host" @@ -137,14 +107,10 @@ { "id": "version", "question": { - "en": "What version of MapComplete was used to make this change?", - "de": "Mit welcher MapComplete Version wurde die Änderung vorgenommen?", - "es": "¿Qué versión de MapComplete se usó para realizar este cambio?" + "en": "What version of MapComplete was used to make this change?" }, "render": { - "en": "Made with {editor}", - "de": "Erstellt mit {editor}", - "es": "Hecho con {editor}" + "en": "Made with {editor}" }, "freeform": { "key": "editor" @@ -518,9 +484,7 @@ } ], "question": { - "en": "Themename contains {search}", - "de": "Themename enthält {search}", - "es": "El nombre del tema contiene {search}" + "en": "Themename contains {search}" } } ] @@ -536,9 +500,7 @@ } ], "question": { - "en": "Themename does not contain {search}", - "de": "Der Name enthält nicht {search}", - "es": "El nombre del tema no contiene {search}" + "en": "Themename does not contain {search}" } } ] @@ -554,9 +516,7 @@ } ], "question": { - "en": "Made by contributor {search}", - "de": "Erstellt vom Mitwirkenden {search}", - "es": "Hecho por el colaborador {search}" + "en": "Made by contributor {search}" } } ] @@ -572,9 +532,7 @@ } ], "question": { - "en": "Not made by contributor {search}", - "de": "Nicht erstellt von Mitwirkendem {search}", - "es": "No hecho por el colaborador {search}" + "en": "Not made by contributor {search}" } } ] @@ -591,9 +549,7 @@ } ], "question": { - "en": "Made before {search}", - "de": "Erstellt vor {search}", - "es": "Hecho antes de {search}" + "en": "Made before {search}" } } ] @@ -610,9 +566,7 @@ } ], "question": { - "en": "Made after {search}", - "de": "Erstellt nach {search}", - "es": "Hecho después de {search}" + "en": "Made after {search}" } } ] @@ -628,9 +582,7 @@ } ], "question": { - "en": "User language (iso-code) {search}", - "de": "Benutzersprache (ISO-Code) {search}", - "es": "Idioma del usuario (código ISO) {search}" + "en": "User language (iso-code) {search}" } } ] @@ -646,9 +598,7 @@ } ], "question": { - "en": "Made with host {search}", - "de": "Erstellt mit host {search}", - "es": "Hecho con el host {search}" + "en": "Made with host {search}" } } ] @@ -659,9 +609,7 @@ { "osmTags": "add-image>0", "question": { - "en": "Changeset added at least one image", - "de": "Im Änderungssatz wurde mindestens ein Bild hinzugefügt", - "es": "El conjunto de cambios ha añadido al menos una imagen" + "en": "Changeset added at least one image" } } ] @@ -672,9 +620,7 @@ { "osmTags": "theme!=grb", "question": { - "en": "Exclude GRB theme", - "de": "GRB-Thema ausschließen", - "es": "Excluir el tema del GRB" + "en": "Exclude GRB theme" } } ] @@ -685,9 +631,7 @@ { "osmTags": "theme!=etymology", "question": { - "en": "Exclude etymology theme", - "de": "Etymologie-Thema ausschließen", - "es": "Excluir el tema de la etimología" + "en": "Exclude etymology theme" } } ] @@ -702,9 +646,7 @@ { "id": "link_to_more", "render": { - "en": "More statistics can be found here", - "de": "Weitere Statistiken gibt es hier", - "es": "Puede encontrar más estadísticas aquí" + "en": "More statistics can be found here" } }, { diff --git a/package-lock.json b/package-lock.json index edabe369d..f99aba7e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mapcomplete", - "version": "0.38.0", + "version": "0.38.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mapcomplete", - "version": "0.38.0", + "version": "0.38.1", "license": "GPL-3.0-or-later", "dependencies": { "@rgossiaux/svelte-headlessui": "^1.0.2", @@ -38,6 +38,8 @@ "i18next-client": "^1.11.4", "idb-keyval": "^6.0.3", "jest-mock": "^29.4.1", + "jsonld": "^8.3.2", + "jsonld-request": "^2.0.1", "jspdf": "^2.5.1", "latlon2country": "^1.2.6", "libphonenumber-js": "^1.10.8", @@ -76,6 +78,7 @@ "@tsconfig/svelte": "^3.0.0", "@types/chai": "^4.3.0", "@types/geojson": "^7946.0.10", + "@types/jsonld": "^1.5.13", "@types/lz-string": "^1.3.34", "@types/mocha": "^10.0.1", "@types/node": "^18.11.18", @@ -1644,6 +1647,19 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@digitalbazaar/http-client": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@digitalbazaar/http-client/-/http-client-3.4.1.tgz", + "integrity": "sha512-Ahk1N+s7urkgj7WvvUND5f8GiWEPfUw0D41hdElaqLgu8wZScI8gdI0q+qWw5N1d35x7GCRH2uk9mi+Uzo9M3g==", + "dependencies": { + "ky": "^0.33.3", + "ky-universal": "^0.11.0", + "undici": "^5.21.2" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/@esbuild/android-arm": { "version": "0.18.20", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", @@ -2079,6 +2095,14 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fastify/busboy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", + "engines": { + "node": ">=14" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", @@ -4191,6 +4215,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/jsonld": { + "version": "1.5.13", + "resolved": "https://registry.npmjs.org/@types/jsonld/-/jsonld-1.5.13.tgz", + "integrity": "sha512-n7fUU6W4kSYK8VQlf/LsE9kddBHPKhODoVOjsZswmve+2qLwBy6naWxs/EiuSZN9NU0N06Ra01FR+j87C62T0A==", + "dev": true + }, "node_modules/@types/lz-string": { "version": "1.3.34", "resolved": "https://registry.npmjs.org/@types/lz-string/-/lz-string-1.3.34.tgz", @@ -4317,9 +4347,9 @@ } }, "node_modules/@types/yargs": { - "version": "17.0.22", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.22.tgz", - "integrity": "sha512-pet5WJ9U8yPVRhkwuEIp5ktAeAqRZOq4UdAyWLWzxbtpyXnzbtLdKiXAjJzi/KLmPGS9wk86lUFWZFN6sISo4g==", + "version": "17.0.32", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.32.tgz", + "integrity": "sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==", "dependencies": { "@types/yargs-parser": "*" } @@ -4723,6 +4753,17 @@ "optional": true, "peer": true }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.6.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", @@ -5376,6 +5417,11 @@ } ] }, + "node_modules/canonicalize": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/canonicalize/-/canonicalize-1.0.8.tgz", + "integrity": "sha512-0CNTVCLZggSh7bc5VkX5WWPWO+cyZbNd07IHIsSXLia/eAq+r836hgk+8BKoEh7949Mda87VUOitx5OddVj64A==" + }, "node_modules/canvg": { "version": "3.0.10", "resolved": "https://registry.npmjs.org/canvg/-/canvg-3.0.10.tgz", @@ -5699,6 +5745,14 @@ "quickselect": "^2.0.0" } }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/convert-source-map": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", @@ -5921,6 +5975,14 @@ "node": ">=0.10" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", @@ -6504,9 +6566,9 @@ } }, "node_modules/entities": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", - "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", "devOptional": true, "engines": { "node": ">=0.12" @@ -7038,6 +7100,14 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -7271,6 +7341,28 @@ "reusify": "^1.0.4" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fflate": { "version": "0.4.8", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz", @@ -7420,6 +7512,17 @@ "node": ">= 6" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/fs": { "version": "0.0.1-security", "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", @@ -7588,6 +7691,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-stdin": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", + "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -8254,6 +8368,11 @@ "node": ">=8" } }, + "node_modules/iri": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/iri/-/iri-1.3.1.tgz", + "integrity": "sha512-CpzxCf3ycTphqF2hQRQAp25yK7+XM1iMJHUoJbFiQDLt70hYC+vUlrgQXuhzPrLNMuejhV0VelFp8zQmtqgAxA==" + }, "node_modules/is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -8754,6 +8873,52 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/jsonld": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/jsonld/-/jsonld-8.3.2.tgz", + "integrity": "sha512-MwBbq95szLwt8eVQ1Bcfwmgju/Y5P2GdtlHE2ncyfuYjIdEhluUVyj1eudacf1mOkWIoS9GpDBTECqhmq7EOaA==", + "dependencies": { + "@digitalbazaar/http-client": "^3.4.1", + "canonicalize": "^1.0.1", + "lru-cache": "^6.0.0", + "rdf-canonize": "^3.4.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/jsonld-request": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jsonld-request/-/jsonld-request-2.0.1.tgz", + "integrity": "sha512-nV6iFlTHVrgwbnKFLpMJEcuEgwpf2hU8qH6wkcpIDHJKFy77WetKNcEr6M85Z6xN+9D6i6wOWNVDVlNn7749EQ==", + "dependencies": { + "@digitalbazaar/http-client": "^3.2.0", + "@xmldom/xmldom": "^0.8.2", + "content-type": "^1.0.4", + "get-stdin": "^9.0.0", + "jsonld": "^8.1.0", + "rdfa": "^0.0.10" + }, + "engines": { + "node": ">=14.13.1" + } + }, + "node_modules/jsonld/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jsonld/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/jsonparse": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-0.0.5.tgz", @@ -8873,6 +9038,58 @@ "integrity": "sha512-uMCj6+hZYDoffuvAJjFAPz56E9uoowFHmTkqRtRq5WyC5Q6Cu/fTZKNQpX/RbzChBYLLl3lo8CjFZBAZXq9qFg==", "dev": true }, + "node_modules/ky": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/ky/-/ky-0.33.3.tgz", + "integrity": "sha512-CasD9OCEQSFIam2U8efFK81Yeg8vNMTBUqtMOHlrcWQHqUX3HeCl9Dr31u4toV7emlH8Mymk5+9p0lL6mKb/Xw==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/ky?sponsor=1" + } + }, + "node_modules/ky-universal": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/ky-universal/-/ky-universal-0.11.0.tgz", + "integrity": "sha512-65KyweaWvk+uKKkCrfAf+xqN2/epw1IJDtlyCPxYffFCMR8u1sp2U65NtWpnozYfZxQ6IUzIlvUcw+hQ82U2Xw==", + "dependencies": { + "abort-controller": "^3.0.0", + "node-fetch": "^3.2.10" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/ky-universal?sponsor=1" + }, + "peerDependencies": { + "ky": ">=0.31.4", + "web-streams-polyfill": ">=3.2.1" + }, + "peerDependenciesMeta": { + "web-streams-polyfill": { + "optional": true + } + } + }, + "node_modules/ky-universal/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/latlon2country": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/latlon2country/-/latlon2country-1.2.6.tgz", @@ -9530,6 +9747,24 @@ "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", "dev": true }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-html-parser": { "version": "6.1.5", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz", @@ -10502,6 +10737,34 @@ "node": ">=0.10.0" } }, + "node_modules/rdf": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/rdf/-/rdf-4.1.1.tgz", + "integrity": "sha512-WEXBQmMXubplwnpcL6Wo184LkAXvTaMJYxxXoTlBr4SM7XhIx3BVjltrrbh8ARTmwBthuI7p2vkv6m3rKNucgw==", + "dependencies": { + "iri": "~1" + } + }, + "node_modules/rdf-canonize": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/rdf-canonize/-/rdf-canonize-3.4.0.tgz", + "integrity": "sha512-fUeWjrkOO0t1rg7B2fdyDTvngj+9RlUyL92vOdiB7c0FPguWVsniIMjEtHH+meLBO9rzkUlUzBVXgWrjI8P9LA==", + "dependencies": { + "setimmediate": "^1.0.5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/rdfa": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/rdfa/-/rdfa-0.0.10.tgz", + "integrity": "sha512-/n8BSzgcZpa1kmDZcWCJ/SPRdvKTVA8WpHye5QvkOfswe5wfJ4GfSHtQ/D2yeeV18hvZNWLnyot+9yYNNSp1zw==", + "dependencies": { + "iri": "^1.3.0", + "rdf": "^4.0.0" + } + }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -11080,6 +11343,11 @@ "node": ">=0.10.0" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, "node_modules/sharp": { "version": "0.32.6", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", @@ -13153,6 +13421,17 @@ "underscore": "1.x" } }, + "node_modules/undici": { + "version": "5.28.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz", + "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/unicode-canonical-property-names-ecmascript": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", @@ -13521,6 +13800,14 @@ "defaults": "^1.0.3" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -14964,6 +15251,16 @@ } } }, + "@digitalbazaar/http-client": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@digitalbazaar/http-client/-/http-client-3.4.1.tgz", + "integrity": "sha512-Ahk1N+s7urkgj7WvvUND5f8GiWEPfUw0D41hdElaqLgu8wZScI8gdI0q+qWw5N1d35x7GCRH2uk9mi+Uzo9M3g==", + "requires": { + "ky": "^0.33.3", + "ky-universal": "^0.11.0", + "undici": "^5.21.2" + } + }, "@esbuild/android-arm": { "version": "0.18.20", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", @@ -15169,6 +15466,11 @@ "integrity": "sha512-Ag+9YM4ocKQx9AarydN0KY2j0ErMHNIocPDrVo8zAE44xLTjEtz81OdR68/cydGtk6m6jDb5Za3r2useMzYmSw==", "dev": true }, + "@fastify/busboy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==" + }, "@humanwhocodes/config-array": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", @@ -16823,6 +17125,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "@types/jsonld": { + "version": "1.5.13", + "resolved": "https://registry.npmjs.org/@types/jsonld/-/jsonld-1.5.13.tgz", + "integrity": "sha512-n7fUU6W4kSYK8VQlf/LsE9kddBHPKhODoVOjsZswmve+2qLwBy6naWxs/EiuSZN9NU0N06Ra01FR+j87C62T0A==", + "dev": true + }, "@types/lz-string": { "version": "1.3.34", "resolved": "https://registry.npmjs.org/@types/lz-string/-/lz-string-1.3.34.tgz", @@ -16947,9 +17255,9 @@ } }, "@types/yargs": { - "version": "17.0.22", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.22.tgz", - "integrity": "sha512-pet5WJ9U8yPVRhkwuEIp5ktAeAqRZOq4UdAyWLWzxbtpyXnzbtLdKiXAjJzi/KLmPGS9wk86lUFWZFN6sISo4g==", + "version": "17.0.32", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.32.tgz", + "integrity": "sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==", "requires": { "@types/yargs-parser": "*" } @@ -17224,6 +17532,14 @@ "optional": true, "peer": true }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + }, "acorn": { "version": "8.6.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", @@ -17695,6 +18011,11 @@ "integrity": "sha512-+hVY9jE44uKLkH0SrUTqxjxqNTOWHsbnQDIKjwkZ3lNTzUUVdBLBGXtj/q5Mp5u98r3droaZAewQuEDzjQdZlQ==", "dev": true }, + "canonicalize": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/canonicalize/-/canonicalize-1.0.8.tgz", + "integrity": "sha512-0CNTVCLZggSh7bc5VkX5WWPWO+cyZbNd07IHIsSXLia/eAq+r836hgk+8BKoEh7949Mda87VUOitx5OddVj64A==" + }, "canvg": { "version": "3.0.10", "resolved": "https://registry.npmjs.org/canvg/-/canvg-3.0.10.tgz", @@ -17942,6 +18263,11 @@ } } }, + "content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" + }, "convert-source-map": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", @@ -18117,6 +18443,11 @@ "assert-plus": "^1.0.0" } }, + "data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" + }, "data-urls": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", @@ -18558,9 +18889,9 @@ } }, "entities": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", - "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", "devOptional": true }, "es6-object-assign": { @@ -18936,6 +19267,11 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, "expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -19132,6 +19468,15 @@ "reusify": "^1.0.4" } }, + "fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "requires": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + } + }, "fflate": { "version": "0.4.8", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz", @@ -19233,6 +19578,14 @@ "mime-types": "^2.1.12" } }, + "formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "requires": { + "fetch-blob": "^3.1.2" + } + }, "fs": { "version": "0.0.1-security", "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", @@ -19373,6 +19726,11 @@ "has-symbols": "^1.0.3" } }, + "get-stdin": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", + "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==" + }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -19875,6 +20233,11 @@ } } }, + "iri": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/iri/-/iri-1.3.1.tgz", + "integrity": "sha512-CpzxCf3ycTphqF2hQRQAp25yK7+XM1iMJHUoJbFiQDLt70hYC+vUlrgQXuhzPrLNMuejhV0VelFp8zQmtqgAxA==" + }, "is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -20230,6 +20593,45 @@ "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==", "dev": true }, + "jsonld": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/jsonld/-/jsonld-8.3.2.tgz", + "integrity": "sha512-MwBbq95szLwt8eVQ1Bcfwmgju/Y5P2GdtlHE2ncyfuYjIdEhluUVyj1eudacf1mOkWIoS9GpDBTECqhmq7EOaA==", + "requires": { + "@digitalbazaar/http-client": "^3.4.1", + "canonicalize": "^1.0.1", + "lru-cache": "^6.0.0", + "rdf-canonize": "^3.4.0" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, + "jsonld-request": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jsonld-request/-/jsonld-request-2.0.1.tgz", + "integrity": "sha512-nV6iFlTHVrgwbnKFLpMJEcuEgwpf2hU8qH6wkcpIDHJKFy77WetKNcEr6M85Z6xN+9D6i6wOWNVDVlNn7749EQ==", + "requires": { + "@digitalbazaar/http-client": "^3.2.0", + "@xmldom/xmldom": "^0.8.2", + "content-type": "^1.0.4", + "get-stdin": "^9.0.0", + "jsonld": "^8.1.0", + "rdfa": "^0.0.10" + } + }, "jsonparse": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-0.0.5.tgz", @@ -20328,6 +20730,32 @@ "integrity": "sha512-uMCj6+hZYDoffuvAJjFAPz56E9uoowFHmTkqRtRq5WyC5Q6Cu/fTZKNQpX/RbzChBYLLl3lo8CjFZBAZXq9qFg==", "dev": true }, + "ky": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/ky/-/ky-0.33.3.tgz", + "integrity": "sha512-CasD9OCEQSFIam2U8efFK81Yeg8vNMTBUqtMOHlrcWQHqUX3HeCl9Dr31u4toV7emlH8Mymk5+9p0lL6mKb/Xw==" + }, + "ky-universal": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/ky-universal/-/ky-universal-0.11.0.tgz", + "integrity": "sha512-65KyweaWvk+uKKkCrfAf+xqN2/epw1IJDtlyCPxYffFCMR8u1sp2U65NtWpnozYfZxQ6IUzIlvUcw+hQ82U2Xw==", + "requires": { + "abort-controller": "^3.0.0", + "node-fetch": "^3.2.10" + }, + "dependencies": { + "node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "requires": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + } + } + } + }, "latlon2country": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/latlon2country/-/latlon2country-1.2.6.tgz", @@ -20842,6 +21270,11 @@ "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", "dev": true }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" + }, "node-html-parser": { "version": "6.1.5", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.5.tgz", @@ -21490,6 +21923,31 @@ } } }, + "rdf": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/rdf/-/rdf-4.1.1.tgz", + "integrity": "sha512-WEXBQmMXubplwnpcL6Wo184LkAXvTaMJYxxXoTlBr4SM7XhIx3BVjltrrbh8ARTmwBthuI7p2vkv6m3rKNucgw==", + "requires": { + "iri": "~1" + } + }, + "rdf-canonize": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/rdf-canonize/-/rdf-canonize-3.4.0.tgz", + "integrity": "sha512-fUeWjrkOO0t1rg7B2fdyDTvngj+9RlUyL92vOdiB7c0FPguWVsniIMjEtHH+meLBO9rzkUlUzBVXgWrjI8P9LA==", + "requires": { + "setimmediate": "^1.0.5" + } + }, + "rdfa": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/rdfa/-/rdfa-0.0.10.tgz", + "integrity": "sha512-/n8BSzgcZpa1kmDZcWCJ/SPRdvKTVA8WpHye5QvkOfswe5wfJ4GfSHtQ/D2yeeV18hvZNWLnyot+9yYNNSp1zw==", + "requires": { + "iri": "^1.3.0", + "rdf": "^4.0.0" + } + }, "react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -21938,6 +22396,11 @@ "split-string": "^3.0.1" } }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, "sharp": { "version": "0.32.6", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", @@ -23532,6 +23995,14 @@ "integrity": "sha512-4OuSOlFNkiVFVc3khkeG112Pdu1gbitMj7t9B9ENb61uFmN70Jq7Iluhi3oflcSgexkKfDdJ5XAJET2gEq6ikA==", "requires": {} }, + "undici": { + "version": "5.28.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz", + "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==", + "requires": { + "@fastify/busboy": "^2.0.0" + } + }, "unicode-canonical-property-names-ecmascript": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", @@ -23760,6 +24231,11 @@ "defaults": "^1.0.3" } }, + "web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==" + }, "webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", diff --git a/package.json b/package.json index 69b394dcf..b0c8bb90e 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,8 @@ "https://overpass.openstreetmap.ru/cgi/interpreter" ], "country_coder_host": "https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/latlon2country", - "nominatimEndpoint": "https://geocoding.geofabrik.de/b75350b1cfc34962ac49824fe5b582dc/" + "nominatimEndpoint": "https://geocoding.geofabrik.de/b75350b1cfc34962ac49824fe5b582dc/", + "jsonld-proxy": "http://127.0.0.1:2346/extractgraph?url={url}" }, "scripts": { "start": "npm run generate:layeroverview && npm run strt", @@ -137,6 +138,8 @@ "i18next-client": "^1.11.4", "idb-keyval": "^6.0.3", "jest-mock": "^29.4.1", + "jsonld": "^8.3.2", + "jsonld-request": "^2.0.1", "jspdf": "^2.5.1", "latlon2country": "^1.2.6", "libphonenumber-js": "^1.10.8", @@ -175,6 +178,7 @@ "@tsconfig/svelte": "^3.0.0", "@types/chai": "^4.3.0", "@types/geojson": "^7946.0.10", + "@types/jsonld": "^1.5.13", "@types/lz-string": "^1.3.34", "@types/mocha": "^10.0.1", "@types/node": "^18.11.18", diff --git a/scripts/serverLdScrape.ts b/scripts/serverLdScrape.ts index 31d66b0e8..d1770a2b1 100644 --- a/scripts/serverLdScrape.ts +++ b/scripts/serverLdScrape.ts @@ -8,32 +8,40 @@ class ServerLdScrape extends Script { } async main(args: string[]): Promise { const port = Number(args[0] ?? 2346) + const cache: Record = [] new Server(port, {}, [ { mustMatch: "extractgraph", mimetype: "application/ld+json", async handle(content, searchParams: URLSearchParams) { const url = searchParams.get("url") + if (cache[url]) { + return JSON.stringify(cache[url]) + } const dloaded = await Utils.download(url, { "User-Agent": - "MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", // "MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://github.com/pietervdvn/MapComplete", }) + // return dloaded const parsed = parse(dloaded) const scripts = Array.from(parsed.getElementsByTagName("script")) - const snippets = [] for (const script of scripts) { const tp = script.attributes["type"] if (tp !== "application/ld+json") { continue } try { - snippets.push(JSON.parse(script.textContent)) + const snippet = JSON.parse(script.textContent) + snippet["@base"] = url + cache[url] = snippet + + return JSON.stringify(snippet) } catch (e) { console.error(e) } } - return JSON.stringify(snippets) + return JSON.stringify({}) }, }, ]) diff --git a/src/Logic/Web/LinkedDataLoader.ts b/src/Logic/Web/LinkedDataLoader.ts new file mode 100644 index 000000000..8b738983f --- /dev/null +++ b/src/Logic/Web/LinkedDataLoader.ts @@ -0,0 +1,142 @@ +import type { Geometry } from "geojson" +import jsonld from "jsonld" +import { OH, OpeningHour } from "../../UI/OpeningHours/OpeningHours" +import { Utils } from "../../Utils" +import PhoneValidator from "../../UI/InputElement/Validators/PhoneValidator" +import EmailValidator from "../../UI/InputElement/Validators/EmailValidator" +import { Validator } from "../../UI/InputElement/Validator" +import UrlValidator from "../../UI/InputElement/Validators/UrlValidator" + +export default class LinkedDataLoader { + private static readonly COMPACTING_CONTEXT = { + name: "http://schema.org/name", + website: { "@id": "http://schema.org/url", "@type": "@id" }, + phone: { "@id": "http://schema.org/telephone" }, + email: { "@id": "http://schema.org/email" }, + image: { "@id": "http://schema.org/image", "@type": "@id" }, + opening_hours: { "@id": "http://schema.org/openingHoursSpecification" }, + openingHours: { "@id": "http://schema.org/openingHours", "@container": "@set" }, + + geo: { "@id": "http://schema.org/geo" }, + } + private static COMPACTING_CONTEXT_OH = { + dayOfWeek: { "@id": "http://schema.org/dayOfWeek", "@container": "@set" }, + closes: { "@id": "http://schema.org/closes" }, + opens: { "@id": "http://schema.org/opens" }, + } + private static formatters: Record = { + phone: new PhoneValidator(), + email: new EmailValidator(), + website: new UrlValidator(undefined, undefined, true), + } + private static ignoreKeys = [ + "http://schema.org/logo", + "http://schema.org/address", + "@type", + "@id", + "@base", + "http://schema.org/contentUrl", + "http://schema.org/datePublished", + "http://schema.org/description", + "http://schema.org/hasMap", + "http://schema.org/priceRange", + "http://schema.org/contactPoint", + ] + + static async geoToGeometry(geo): Promise { + const context = { + lat: { + "@id": "http://schema.org/latitude", + }, + lon: { + "@id": "http://schema.org/longitude", // TODO formatting to decimal should be possible from this type? + }, + } + const flattened = await jsonld.compact(geo, context) + + return { + type: "Point", + coordinates: [Number(flattened.lon), Number(flattened.lat)], + } + } + + /** + * Parses http://schema.org/openingHours + * + * // Weird data format from C&A + * LinkedDataLoader.ohStringToOsmFormat("MO 09:30-18:00 TU 09:30-18:00 WE 09:30-18:00 TH 09:30-18:00 FR 09:30-18:00 SA 09:30-18:00") // => "Mo-Sa 09:30-18:00" + */ + static ohStringToOsmFormat(oh: string) { + oh = oh.toLowerCase() + if (oh === "mo-su") { + return "24/7" + } + const regex = /([a-z]+ [0-9:]+-[0-9:]+) (.*)/ + let match = oh.match(regex) + let parts: string[] = [] + while (match) { + parts.push(match[1]) + oh = match[2] + match = oh?.match(regex) + } + parts.push(oh) + + // actually the same as OSM-oh + return OH.simplify(parts.join(";")) + } + + static async ohToOsmFormat(openingHoursSpecification): Promise { + const compacted = await jsonld.flatten( + openingHoursSpecification, + LinkedDataLoader.COMPACTING_CONTEXT_OH + ) + const spec: any = compacted["@graph"] + let allRules: OpeningHour[] = [] + for (const rule of spec) { + const dow: string[] = rule.dayOfWeek.map((dow) => dow.toLowerCase().substring(0, 2)) + const opens: string = rule.opens + const closes: string = rule.closes === "23:59" ? "24:00" : rule.closes + allRules.push(...OH.ParseRule(dow + " " + opens + "-" + closes)) + } + + return OH.ToString(OH.MergeTimes(allRules)) + } + + static async fetchJsonLd(url: string, country?: string): Promise> { + const proxy = "http://127.0.0.1:2346/extractgraph" // "https://cache.mapcomplete.org/extractgraph" + const data = await Utils.downloadJson(`${proxy}?url=${url}`) + const compacted = await jsonld.compact(data, LinkedDataLoader.COMPACTING_CONTEXT) + compacted["opening_hours"] = await LinkedDataLoader.ohToOsmFormat( + compacted["opening_hours"] + ) + if (compacted["openingHours"]) { + const ohspec: string[] = compacted["openingHours"] + compacted["opening_hours"] = OH.simplify( + ohspec.map((r) => LinkedDataLoader.ohStringToOsmFormat(r)).join("; ") + ) + delete compacted["openingHours"] + } + if (compacted["geo"]) { + compacted["geo"] = await LinkedDataLoader.geoToGeometry(compacted["geo"]) + } + for (const k in compacted) { + if (compacted[k] === "") { + delete compacted[k] + continue + } + if (this.ignoreKeys.indexOf(k) >= 0) { + delete compacted[k] + continue + } + const formatter = LinkedDataLoader.formatters[k] + if (formatter) { + if (country) { + compacted[k] = formatter.reformat(compacted[k], () => country) + } else { + compacted[k] = formatter.reformat(compacted[k]) + } + } + } + return compacted + } +} diff --git a/src/Models/Constants.ts b/src/Models/Constants.ts index f815756d0..864e3c251 100644 --- a/src/Models/Constants.ts +++ b/src/Models/Constants.ts @@ -112,6 +112,7 @@ export default class Constants { public static countryCoderEndpoint: string = Constants.config.country_coder_host public static osmAuthConfig: AuthConfig = Constants.config.oauth_credentials public static nominatimEndpoint: string = Constants.config.nominatimEndpoint + public static linkedDataProxy: string = Constants.config["jsonld-proxy"] /** * These are the values that are allowed to use as 'backdrop' icon for a map pin */ diff --git a/src/Models/ThemeConfig/Conversion/PrepareLayer.ts b/src/Models/ThemeConfig/Conversion/PrepareLayer.ts index b2150de99..a95e0a9f3 100644 --- a/src/Models/ThemeConfig/Conversion/PrepareLayer.ts +++ b/src/Models/ThemeConfig/Conversion/PrepareLayer.ts @@ -610,6 +610,9 @@ export class AddEditingElements extends DesugaringStep { typeof sv === "string" ? undefined : sv.func.funcName ) ) + if (!allIds.has("lod")) { + json.tagRenderings.push(this._desugaring.tagRenderings.get("lod")) + } if (!usedSpecialFunctions.has("minimap")) { json.tagRenderings.push(this._desugaring.tagRenderings.get("minimap")) } diff --git a/src/UI/BigComponents/SelectedElementView.svelte b/src/UI/BigComponents/SelectedElementView.svelte index c9b1afd7b..ef76ae20a 100644 --- a/src/UI/BigComponents/SelectedElementView.svelte +++ b/src/UI/BigComponents/SelectedElementView.svelte @@ -14,7 +14,7 @@ export let selectedElement: Feature export let highlightedRendering: UIEventSource = undefined - export let tags: UIEventSource> = state.featureProperties.getStore( + export let tags: UIEventSource> = state?.featureProperties?.getStore( selectedElement.properties.id ) @@ -22,11 +22,14 @@ let stillMatches = tags.map(tags => !layer?.source?.osmTags || layer.source.osmTags?.matchesProperties(tags)) let _metatags: Record + if(state?.userRelatedState?.preferencesAsTags){ + onDestroy( state.userRelatedState.preferencesAsTags.addCallbackAndRun((tags) => { _metatags = tags }) ) + } let knownTagRenderings: Store = tags.mapD((tgs) => layer.tagRenderings.filter( diff --git a/src/UI/Image/ImageCarousel.ts b/src/UI/Image/ImageCarousel.ts index 4677f2082..0422031dd 100644 --- a/src/UI/Image/ImageCarousel.ts +++ b/src/UI/Image/ImageCarousel.ts @@ -30,7 +30,7 @@ export class ImageCarousel extends Toggle { try { let image: BaseUIElement = new SvelteUIElement(AttributedImage, { image: url, - previewedImage: state.previewedImage, + previewedImage: state?.previewedImage, }) if (url.key !== undefined) { diff --git a/src/UI/InputElement/Validators/PhoneValidator.ts b/src/UI/InputElement/Validators/PhoneValidator.ts index 6ef6a5027..79c16d18b 100644 --- a/src/UI/InputElement/Validators/PhoneValidator.ts +++ b/src/UI/InputElement/Validators/PhoneValidator.ts @@ -24,7 +24,7 @@ export default class PhoneValidator extends Validator { return generic } - public isValid(str, country: () => string): boolean { + public isValid(str: string, country?: () => string): boolean { if (str === undefined) { return false } diff --git a/src/UI/InputElement/Validators/UrlValidator.ts b/src/UI/InputElement/Validators/UrlValidator.ts index 3e54101b1..a17754ab6 100644 --- a/src/UI/InputElement/Validators/UrlValidator.ts +++ b/src/UI/InputElement/Validators/UrlValidator.ts @@ -1,13 +1,15 @@ import { Validator } from "../Validator" export default class UrlValidator extends Validator { - constructor(name?: string, explanation?: string) { + private readonly _forceHttps: boolean + constructor(name?: string, explanation?: string, forceHttps?: boolean) { super( name ?? "url", explanation ?? "The validatedTextField will format URLs to always be valid and have a https://-header (even though the 'https'-part will be hidden from the user. Furthermore, some tracking parameters will be removed", "url" ) + this._forceHttps = forceHttps ?? false } reformat(str: string): string { try { @@ -22,6 +24,9 @@ export default class UrlValidator extends Validator { } else { url = new URL(str) } + if (this._forceHttps) { + url.protocol = "https:" + } const blacklistedTrackingParams = [ "fbclid", // Oh god, how I hate the fbclid. Let it burn, burn in hell! "gclid", diff --git a/src/UI/LinkedDataDisplay.svelte b/src/UI/LinkedDataDisplay.svelte new file mode 100644 index 000000000..d7b106dbe --- /dev/null +++ b/src/UI/LinkedDataDisplay.svelte @@ -0,0 +1,90 @@ + +{#if $error} +
+ {$error} +
+{:else if $url} +
+ {#if $dataCleaned !== undefined && Object.keys($dataCleaned).length === 0} + No new data from website + {:else if !$data} + + {:else} + {$distanceToFeature} +
    + {#each Object.keys($dataCleaned) as k} +
  • + {k}: {JSON.stringify($dataCleaned[k])} {$tagsSource[k]} {($dataCleaned[k]) === $tagsSource[k]} +
    • + {/each} +
+ {/if} +
+{/if} diff --git a/src/UI/SpecialVisualizations.ts b/src/UI/SpecialVisualizations.ts index e353fcdd9..808f046fe 100644 --- a/src/UI/SpecialVisualizations.ts +++ b/src/UI/SpecialVisualizations.ts @@ -93,6 +93,7 @@ import SpecialVisualisationUtils from "./SpecialVisualisationUtils" import LoginButton from "./Base/LoginButton.svelte" import Toggle from "./Input/Toggle" import ImportReviewIdentity from "./Reviews/ImportReviewIdentity.svelte" +import LinkedDataDisplay from "./LinkedDataDisplay.svelte" class NearbyImageVis implements SpecialVisualization { // Class must be in SpecialVisualisations due to weird cyclical import that breaks the tests @@ -741,12 +742,20 @@ export default class SpecialVisualizations { { funcName: "import_mangrove_key", docs: "Only makes sense in the usersettings. Allows to import a mangrove public key and to use this to make reviews", - args: [{ - name: "text", - doc: "The text that is shown on the button", - }], + args: [ + { + name: "text", + doc: "The text that is shown on the button", + }, + ], needsUrls: [], - constr(state: SpecialVisualizationState, tagSource: UIEventSource>, argument: string[], feature: Feature, layer: LayerConfig): BaseUIElement { + constr( + state: SpecialVisualizationState, + tagSource: UIEventSource>, + argument: string[], + feature: Feature, + layer: LayerConfig + ): BaseUIElement { const [text] = argument return new SvelteUIElement(ImportReviewIdentity, { state, text }) }, @@ -1718,6 +1727,34 @@ export default class SpecialVisualizations { ) }, }, + { + funcName: "linked_data_from_website", + docs: "Attempts to load (via a proxy) the specified website and parsed ld+json from there. Suitable data will be offered to import into OSM", + args: [ + { + name: "key", + defaultValue: "website", + doc: "Attempt to load ld+json from the specified URL. This can be in an embedded - - +{#if $data} + +{/if}