Extract and validate images using a new conversion, regenerate docs

This commit is contained in:
Pieter Vander Vennet 2022-02-09 22:37:21 +01:00
parent 5198f5d310
commit b3c58ae82e
52 changed files with 8611 additions and 3408 deletions

View file

@ -0,0 +1,127 @@
import {Conversion, DesugaringStep} from "./Conversion";
import {LayoutConfigJson} from "../Json/LayoutConfigJson";
import {Utils} from "../../../Utils";
import * as metapaths from "../../../assets/layoutconfigmeta.json";
import * as tagrenderingmetapaths from "../../../assets/tagrenderingconfigmeta.json";
export class ExtractImages extends Conversion<LayoutConfigJson, string[]> {
constructor() {
super("Extract all images from a layoutConfig using the meta paths");
}
convert(json: LayoutConfigJson, context: string): { result: string[] } {
const paths = metapaths["default"] ?? metapaths
const trpaths = tagrenderingmetapaths["default"] ?? tagrenderingmetapaths
const allFoundImages = []
for (const metapath of paths) {
if (metapath.typeHint === undefined) {
continue
}
if (metapath.typeHint !== "image" && metapath.typeHint !== "icon") {
continue
}
const mightBeTr = Array.isArray(metapath.type) && metapath.type.some(t => t["$ref"] == "#/definitions/TagRenderingConfigJson")
const found = Utils.CollectPath(metapath.path, json)
if (mightBeTr) {
// We might have tagRenderingConfigs containing icons here
for (const foundImage of found) {
if (typeof foundImage === "string") {
allFoundImages.push(foundImage)
} else {
// This is a tagRendering where every rendered value might be an icon!
for (const trpath of trpaths) {
if (trpath.typeHint !== "rendered") {
continue
}
Utils.CollectPath(trpath.path, foundImage, allFoundImages)
}
}
}
} else {
allFoundImages.push(...found)
}
}
const splitParts = [].concat(...allFoundImages.map(img => img.split(";")))
.map(img => img.split(":")[0])
return {result: Utils.Dedup(splitParts)};
}
}
export class FixImages extends DesugaringStep<LayoutConfigJson> {
private readonly _knownImages: Set<string>;
constructor(knownImages: Set<string>) {
super("Walks over the entire theme and replaces images to the relative URL. Only works if the ID of the theme is an URL");
this._knownImages = knownImages;
}
convert(json: LayoutConfigJson, context: string): { result: LayoutConfigJson; errors?: string[]; warnings?: string[] } {
let url: URL;
try {
url = new URL(json.id)
} catch (e) {
// Not a URL, we don't rewrite
return {result: json}
}
const absolute = url.protocol + "//" + url.host
let relative = url.protocol + "//" + url.host + url.pathname
relative = relative.substring(0, relative.lastIndexOf("/"))
const self = this;
function replaceString(leaf: string) {
if (self._knownImages.has(leaf)) {
return leaf;
}
if (leaf.startsWith("./")) {
return relative + leaf.substring(1)
}
if (leaf.startsWith("/")) {
return absolute + leaf
}
return leaf;
}
json = Utils.Clone(json)
let paths = metapaths["default"] ?? metapaths
let trpaths = tagrenderingmetapaths["default"] ?? tagrenderingmetapaths
for (const metapath of paths) {
if (metapath.typeHint !== "image" && metapath.typeHint !== "icon") {
continue
}
const mightBeTr = Array.isArray(metapath.type) && metapath.type.some(t => t["$ref"] == "#/definitions/TagRenderingConfigJson")
Utils.WalkPath(metapath.path, json, leaf => {
if (typeof leaf === "string") {
return replaceString(leaf)
}
if (mightBeTr) {
// We might have reached a tagRenderingConfig containing icons
// lets walk every rendered value and fix the images in there
for (const trpath of trpaths) {
if (trpath.typeHint !== "rendered") {
continue
}
Utils.WalkPath(trpath.path, leaf, (rendered => {
return replaceString(rendered)
}))
}
}
return leaf;
})
}
return {
result: json
};
}
}

View file

@ -3,7 +3,6 @@ import {Utils} from "../../../Utils";
import LineRenderingConfigJson from "../Json/LineRenderingConfigJson";
import {LayerConfigJson} from "../Json/LayerConfigJson";
import {DesugaringStep, Fuse, OnEvery} from "./Conversion";
import * as metapaths from "../../../assets/layoutconfigmeta.json"
export class UpdateLegacyLayer extends DesugaringStep<LayerConfigJson | string | { builtin, override }> {
@ -160,101 +159,3 @@ export class FixLegacyTheme extends Fuse<LayoutConfigJson> {
}
export class FixImages extends DesugaringStep<LayoutConfigJson> {
private readonly _knownImages: Set<string>;
constructor(knownImages: Set<string>) {
super("Walks over the entire theme and replaces images to the relative URL. Only works if the ID of the theme is an URL");
this._knownImages = knownImages;
}
/**
* Walks the path into the object till the end.
*
* If a list is encountered, this is tranparently walked recursively on every object.
*
* The leaf objects are replaced
*/
private static WalkPath(path: string[], object: any, replaceLeaf: ((leaf: any) => any)) {
const head = path[0]
if (path.length === 1) {
// We have reached the leaf
const leaf = object[head];
if (leaf !== undefined) {
object[head] = replaceLeaf(leaf)
}
return
}
const sub = object[head]
if (sub === undefined) {
return;
}
if (typeof sub !== "object") {
return;
}
if (sub["forEach"] !== undefined) {
sub.forEach(el => FixImages.WalkPath(path.slice(1), el, replaceLeaf))
return;
}
FixImages.WalkPath(path.slice(1), sub, replaceLeaf)
}
convert(json: LayoutConfigJson, context: string): { result: LayoutConfigJson; errors?: string[]; warnings?: string[] } {
let url: URL;
console.log("Fixing images!")
try {
url = new URL(json.id)
} catch (e) {
// Not a URL, we don't rewrite
return {result: json}
}
const absolute = url.protocol +"//"+url.host
let relative = url.protocol +"//"+ url.host + url.pathname
relative = relative.substring(0, relative.lastIndexOf("/"))
const self = this;
function replaceString(leaf: string) {
if (self._knownImages.has(leaf)) {
return leaf;
}
if (leaf.startsWith("./")) {
return relative + leaf.substring(1)
}
if (leaf.startsWith("/")) {
return absolute + leaf
}
return leaf;
}
json = Utils.Clone(json)
let paths = metapaths["default"] ?? metapaths
for (const metapath of paths) {
if (metapath.typeHint !== "image" && metapath.typeHint !== "icon") {
continue
}
FixImages.WalkPath(metapath.path, json, leaf => {
console.log("Detected leaf: ", leaf)
if (typeof leaf === "string") {
return replaceString(leaf)
}
if (metapath.type["some"] !== undefined && (<any[]>metapath.type).some(t => t["$ref"] == "\"#/definitions/TagRenderingConfigJson\"")) {
console.log("Possibly found a tagrendering")
}
return leaf;
})
}
return {
result: json
};
}
}

View file

@ -8,6 +8,7 @@ import {LayoutConfigJson} from "../Json/LayoutConfigJson";
import LayoutConfig from "../LayoutConfig";
import {TagRenderingConfigJson} from "../Json/TagRenderingConfigJson";
import {TagUtils} from "../../../Logic/Tags/TagUtils";
import {ExtractImages} from "./FixImages";
class ValidateLanguageCompleteness extends DesugaringStep<any> {
@ -54,8 +55,9 @@ class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
this._isBuiltin = isBuiltin;
}
convert(json: LayoutConfigJson, context: string): { result: LayoutConfigJson; errors: string[] } {
convert(json: LayoutConfigJson, context: string): { result: LayoutConfigJson; errors: string[], warnings: string[] } {
const errors = []
const warnings = []
{
// Legacy format checks
if (this._isBuiltin) {
@ -67,7 +69,34 @@ class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
}
}
}
{
// Check for remote images
const images = new ExtractImages().convertStrict(json, "validation")
const remoteImages = images.filter(img => img.indexOf("http") == 0)
for (const remoteImage of remoteImages) {
errors.push("Found a remote image: " + remoteImage + " in theme " + json.id + ", please download it.")
}
for (const image of images) {
if (image.indexOf("{") >= 0) {
warnings.push("Ignoring image with { in the path: ", image)
continue
}
if(image === "assets/SocialImage.png"){
continue
}
if(image.match(/[a-z]*/)){
// This is a builtin img, e.g. 'checkmark' or 'crosshair'
continue;
}
if (this.knownImagePaths !== undefined && !this.knownImagePaths.has(image)) {
const ctx = context === undefined ? "" : ` in a layer defined in the theme ${context}`
errors.push(`Image with path ${image} not found or not attributed; it is used in ${json.id}${ctx}`)
}
}
}
try {
const theme = new LayoutConfig(json, true, "test")
if (theme.id !== theme.id.toLowerCase()) {
@ -97,7 +126,8 @@ class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
return {
result: json,
errors
errors,
warnings
};
}
}
@ -247,27 +277,6 @@ export class ValidateLayer extends DesugaringStep<LayerConfigJson> {
errors.push(context + ": layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'")
}
}
{
// Check for remote images
const layer = new LayerConfig(json, "test", true)
const images = Array.from(layer.ExtractImages())
const remoteImages = images.filter(img => img.indexOf("http") == 0)
for (const remoteImage of remoteImages) {
errors.push("Found a remote image: " + remoteImage + " in layer " + layer.id + ", please download it. You can use the fixTheme script to automate this")
}
for (const image of images) {
if (image.indexOf("{") >= 0) {
warnings.push("Ignoring image with { in the path: ", image)
continue
}
if (this.knownImagePaths !== undefined && !this.knownImagePaths.has(image)) {
const ctx = context === undefined ? "" : ` in a layer defined in the theme ${context}`
errors.push(`Image with path ${image} not found or not attributed; it is used in ${layer.id}${ctx}`)
}
}
}
{
// CHeck location of layer file
const expected: string = `assets/layers/${json.id}/${json.id}.json`

View file

@ -195,7 +195,7 @@ export interface LayerConfigJson {
/**
* Example images, which show real-life pictures of what such a feature might look like
*
* Type: image[]
* Type: image
*/
exampleImages?: string[]

View file

@ -30,6 +30,7 @@ export interface TagRenderingConfigJson {
* If neither 'textFieldQuestion' nor 'mappings' are defined, this text is simply shown as default value.
*
* Note that this is a HTML-interpreted value, so you can add links as e.g. '<a href='{website}'>{website}</a>' or include images such as `This is of type A <br><img src='typeA-icon.svg' />`
* type: rendered
*/
render?: string | any,
@ -114,6 +115,7 @@ export interface TagRenderingConfigJson {
/**
* If the condition `if` is met, the text `then` will be rendered.
* If not known yet, the user will be presented with `then` as an option
* type: rendered
*/
then: string | any,
/**

View file

@ -311,8 +311,11 @@ export default class LayerConfig extends WithContextLoader {
if (mapRendering === undefined) {
return undefined
}
const baseTags = TagUtils.changeAsProperties(this.source.osmTags.asChange({id: "node/-1"}))
return mapRendering.GetSimpleIcon(new UIEventSource(baseTags))
return mapRendering.GetBaseIcon(this.GetBaseTags())
}
public GetBaseTags(): any{
return TagUtils.changeAsProperties(this.source.osmTags.asChange({id: "node/-1"}))
}
public GenerateDocumentation(usedInThemes: string[], layerIsNeededBy: Map<string, string[]>, dependencies: {
@ -365,12 +368,6 @@ export default class LayerConfig extends WithContextLoader {
extraProps.push(new Combine(["This layer is needed as dependency for layer", new Link(revDep, "#" + revDep)]))
}
const icon = Array.from(this.mapRendering[0]?.icon?.ExtractImages(true) ?? [])[0]
let iconImg = ""
if (icon !== undefined) {
iconImg = `<img src='https://mapcomplete.osm.be/${icon}' height="100px"> `
}
let neededTags: TagsFilter[] = [this.source.osmTags]
if (this.source.osmTags["and"] !== undefined) {
neededTags = this.source.osmTags["and"]
@ -402,6 +399,16 @@ export default class LayerConfig extends WithContextLoader {
]).SetClass("flex-col flex")
}
const icon = this.mapRendering
.filter(mr => mr.location.has("point"))
.map(mr => mr.icon.render.txt)
.find(i => i !== undefined)
let iconImg = ""
if (icon !== undefined) {
// This is for the documentation, so we have to use raw HTML
iconImg = `<img src='https://mapcomplete.osm.be/${icon}' height="100px"> `
}
return new Combine([
new Combine([
new Title(this.id, 1),
@ -435,25 +442,6 @@ export default class LayerConfig extends WithContextLoader {
return Utils.NoNull([...this.tagRenderings, ...this.titleIcons, this.title, this.isShown])
}
public ExtractImages(): Set<string> {
const parts: Set<string>[] = [];
parts.push(...this.tagRenderings?.map((tr) => tr.ExtractImages(false)));
parts.push(...this.titleIcons?.map((tr) => tr.ExtractImages(true)));
for (const preset of this.presets) {
parts.push(new Set<string>(preset.description?.ExtractImages(false)));
parts.push(new Set(preset.exampleImages ?? []))
}
for (const pointRenderingConfig of this.mapRendering) {
parts.push(pointRenderingConfig.ExtractImages())
}
const allIcons = new Set<string>();
for (const part of parts) {
part?.forEach(allIcons.add, allIcons);
}
return allIcons;
}
public isLeftRightSensitive(): boolean {
return this.lineRendering.some(lr => lr.leftRightSensitive)
}

View file

@ -4,6 +4,7 @@ import LayerConfig from "./LayerConfig";
import {LayerConfigJson} from "./Json/LayerConfigJson";
import Constants from "../Constants";
import TilesourceConfig from "./TilesourceConfig";
import {ExtractImages} from "./Conversion/FixImages";
export default class LayoutConfig {
public readonly id: string;
@ -50,6 +51,8 @@ export default class LayoutConfig {
public readonly overpassMaxZoom: number
public readonly osmApiTileSize: number
public readonly official: boolean;
public readonly usedImages : string[]
constructor(json: LayoutConfigJson, official = true, context?: string) {
this.official = official;
@ -67,7 +70,7 @@ export default class LayoutConfig {
this.credits = json.credits;
this.version = json.version;
this.language = json.mustHaveLanguage ?? Array.from(Object.keys(json.title));
this.usedImages =Array.from( new ExtractImages().convertStrict(json, "while extracting the images")).sort()
{
if (typeof json.title === "string") {
throw `The title of a theme should always be a translation, as it sets the corresponding languages (${context}.title). The themenID is ${this.id}; the offending object is ${JSON.stringify(json.title)} which is a ${typeof json.title})`
@ -177,63 +180,6 @@ export default class LayoutConfig {
return custom;
}
public ExtractImages(): Set<string> {
const icons = new Set<string>()
for (const layer of this.layers) {
layer.ExtractImages().forEach(icons.add, icons)
}
icons.add(this.icon)
icons.add(this.socialImage)
return icons
}
/**
* Replaces all the relative image-urls with a fixed image url
* This is to fix loading from external sources
*
* It should be passed the location where the original theme file is hosted.
*
* If no images are rewritten, the same object is returned, otherwise a new copy is returned
*/
public patchImages(originalURL: string, originalJson: string): LayoutConfig {
const allImages = Array.from(this.ExtractImages())
const rewriting = new Map<string, string>()
// Needed for absolute urls: note that it doesn't contain a trailing slash
const origin = new URL(originalURL).origin
let path = new URL(originalURL).href
path = path.substring(0, path.lastIndexOf("/"))
for (const image of allImages) {
if (image == "" || image == undefined) {
continue
}
if (image.startsWith("http://") || image.startsWith("https://")) {
continue
}
if (image.startsWith("/")) {
// This is an absolute path
rewriting.set(image, origin + image)
} else if (image.startsWith("./assets/themes")) {
// Legacy workaround
rewriting.set(image, path + image.substring(image.lastIndexOf("/")))
} else if (image.startsWith("./")) {
// This is a relative url
rewriting.set(image, path + image.substring(1))
} else {
// This is a relative URL with only the path
rewriting.set(image, path + image)
}
}
if (rewriting.size == 0) {
return this;
}
rewriting.forEach((value, key) => {
console.log("Rewriting", key, "==>", value)
originalJson = originalJson.replace(new RegExp(key, "g"), value)
})
return new LayoutConfig(JSON.parse(originalJson), false, "Layout rewriting")
}
public isLeftRightSensitive() {
return this.layers.some(l => l.isLeftRightSensitive())
}

View file

@ -116,35 +116,23 @@ export default class PointRenderingConfig extends WithContextLoader {
}
}
public ExtractImages(): Set<string> {
const parts: Set<string>[] = [];
parts.push(this.icon?.ExtractImages(true));
parts.push(
...this.iconBadges?.map((overlay) => overlay.then.ExtractImages(true))
);
const allIcons = new Set<string>();
for (const part of parts) {
part?.forEach(allIcons.add, allIcons);
public GetBaseIcon(tags?: any): BaseUIElement {
tags = tags ?? {id: "node/-1"}
const rotation = Utils.SubstituteKeys(this.rotation?.GetRenderValue(tags)?.txt ?? "0deg", tags)
const htmlDefs = Utils.SubstituteKeys(this.icon.GetRenderValue(tags)?.txt, tags)
let defaultPin: BaseUIElement = undefined
if (this.label === undefined) {
defaultPin = Svg.teardrop_with_hole_green_svg()
}
return allIcons;
return PointRenderingConfig.FromHtmlMulti(htmlDefs, rotation, false, defaultPin)
}
public GetSimpleIcon(tags: UIEventSource<any>): BaseUIElement {
const self = this;
if (this.icon === undefined) {
return undefined;
}
return new VariableUiElement(tags.map(tags => {
const rotation = Utils.SubstituteKeys(self.rotation?.GetRenderValue(tags)?.txt ?? "0deg", tags)
const htmlDefs = Utils.SubstituteKeys(self.icon.GetRenderValue(tags)?.txt, tags)
let defaultPin: BaseUIElement = undefined
if (self.label === undefined) {
defaultPin = Svg.teardrop_with_hole_green_svg()
}
return PointRenderingConfig.FromHtmlMulti(htmlDefs, rotation, false, defaultPin)
})).SetClass("w-full h-full block")
return new VariableUiElement(tags.map(tags => self.GetBaseIcon(tags))).SetClass("w-full h-full block")
}
public GenerateLeafletStyle(

View file

@ -425,18 +425,6 @@ export default class TagRenderingConfig {
return translations;
}
public ExtractImages(isIcon: boolean): Set<string> {
const usedIcons = new Set<string>()
this.render?.ExtractImages(isIcon)?.forEach(usedIcons.add, usedIcons)
for (const mapping of this.mappings ?? []) {
mapping.then.ExtractImages(isIcon).forEach(usedIcons.add, usedIcons)
}
return usedIcons;
}
FreeformValues(): { key: string, type?: string, values?: string [] } {
try {