From 7c71e566e9b764378423a37f522aa8d4acfcb8fd Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 28 Apr 2024 03:48:07 +0200 Subject: [PATCH] Use libraries to generate MD, steps to factor out more of the UIElements --- package-lock.json | 35 ++++-- package.json | 2 +- scripts/generateDocs.ts | 101 ++++++++++------- src/Models/ThemeConfig/LayerConfig.ts | 3 +- src/UI/Base/TableOfContents.ts | 151 +++++++++++++------------- 5 files changed, 167 insertions(+), 125 deletions(-) diff --git a/package-lock.json b/package-lock.json index 782cf607c..2d1dae96c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,7 +40,6 @@ "fake-dom": "^1.0.4", "geojson2svg": "^1.3.3", "html-to-image": "^1.11.11", - "html-to-markdown": "^1.0.0", "i18next-client": "^1.11.4", "idb-keyval": "^6.0.3", "jest-mock": "^29.4.1", @@ -70,6 +69,7 @@ "tailwind-merge": "^1.13.1", "tailwindcss": "^3.1.8", "trap-focus-svelte": "^1.0.2", + "turndown": "^7.1.3", "vite-node": "^0.28.3", "vitest": "^0.28.3", "wikibase-sdk": "^7.14.0", @@ -8820,6 +8820,11 @@ "domelementtype": "1" } }, + "node_modules/domino": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz", + "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" + }, "node_modules/dompurify": { "version": "3.0.5", "license": "(MPL-2.0 OR Apache-2.0)" @@ -10294,10 +10299,6 @@ "version": "1.11.11", "license": "MIT" }, - "node_modules/html-to-markdown": { - "version": "1.0.0", - "license": "MIT" - }, "node_modules/html2canvas": { "version": "1.4.1", "license": "MIT", @@ -15923,6 +15924,14 @@ "turf-inside": "^3.0.12" } }, + "node_modules/turndown": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz", + "integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==", + "dependencies": { + "domino": "^2.1.6" + } + }, "node_modules/tweetnacl": { "version": "0.14.5", "license": "Unlicense" @@ -23072,6 +23081,11 @@ "domelementtype": "1" } }, + "domino": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz", + "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" + }, "dompurify": { "version": "3.0.5" }, @@ -24012,9 +24026,6 @@ "html-to-image": { "version": "1.11.11" }, - "html-to-markdown": { - "version": "1.0.0" - }, "html2canvas": { "version": "1.4.1", "optional": true, @@ -27705,6 +27716,14 @@ "turf-inside": "^3.0.12" } }, + "turndown": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz", + "integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==", + "requires": { + "domino": "^2.1.6" + } + }, "tweetnacl": { "version": "0.14.5" }, diff --git a/package.json b/package.json index d84fcfa44..171ae85d4 100644 --- a/package.json +++ b/package.json @@ -158,7 +158,6 @@ "fake-dom": "^1.0.4", "geojson2svg": "^1.3.3", "html-to-image": "^1.11.11", - "html-to-markdown": "^1.0.0", "i18next-client": "^1.11.4", "idb-keyval": "^6.0.3", "jest-mock": "^29.4.1", @@ -188,6 +187,7 @@ "tailwind-merge": "^1.13.1", "tailwindcss": "^3.1.8", "trap-focus-svelte": "^1.0.2", + "turndown": "^7.1.3", "vite-node": "^0.28.3", "vitest": "^0.28.3", "wikibase-sdk": "^7.14.0", diff --git a/scripts/generateDocs.ts b/scripts/generateDocs.ts index c6a968c3a..97dee3ff0 100644 --- a/scripts/generateDocs.ts +++ b/scripts/generateDocs.ts @@ -2,7 +2,6 @@ import Combine from "../src/UI/Base/Combine" import BaseUIElement from "../src/UI/BaseUIElement" import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs" import { AllKnownLayouts } from "../src/Customizations/AllKnownLayouts" -import TableOfContents from "../src/UI/Base/TableOfContents" import SimpleMetaTaggers from "../src/Logic/SimpleMetaTagger" import SpecialVisualizations from "../src/UI/SpecialVisualizations" import { ExtraFunctions } from "../src/Logic/ExtraFunctions" @@ -31,6 +30,7 @@ import { Utils } from "../src/Utils" import { TagUtils } from "../src/Logic/Tags/TagUtils" import Script from "./Script" import { Changes } from "../src/Logic/Osm/Changes" +import TableOfContents from "../src/UI/Base/TableOfContents" /** * Converts a markdown-file into a .json file, which a walkthrough/slideshow element can use @@ -56,15 +56,15 @@ class ToSlideshowJson { sections.push(currentSection) currentSection = [] } - line = line.replace('src="../../public/', 'src="./') - line = line.replace('src="../../', 'src="./') + line = line.replace("src=\"../../public/", "src=\"./") + line = line.replace("src=\"../../", "src=\"./") currentSection.push(line) } sections.push(currentSection) writeFileSync( this._target, JSON.stringify({ - sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0), + sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0) }) ) } @@ -83,7 +83,7 @@ class WikiPageGenerator { generate() { let wikiPage = - '{|class="wikitable sortable"\n' + + "{|class=\"wikitable sortable\"\n" + "! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" + "|-" @@ -141,7 +141,7 @@ export class GenerateDocs extends Script { } this.WriteFile("./Docs/Tags_format.md", TagUtils.generateDocs(), [ - "src/Logic/Tags/TagUtils.ts", + "src/Logic/Tags/TagUtils.ts" ]) new ToSlideshowJson( @@ -166,30 +166,33 @@ export class GenerateDocs extends Script { }) this.WriteFile("./Docs/SpecialRenderings.md", SpecialVisualizations.HelpMessage(), [ - "src/UI/SpecialVisualizations.ts", + "src/UI/SpecialVisualizations.ts" ]) this.WriteFile( "./Docs/CalculatedTags.md", new Combine([ new Title("Metatags", 1), SimpleMetaTaggers.HelpText(), - ExtraFunctions.HelpText(), + ExtraFunctions.HelpText() ]).SetClass("flex-col"), ["src/Logic/SimpleMetaTagger.ts", "src/Logic/ExtraFunctions.ts"] ) this.WriteFile("./Docs/SpecialInputElements.md", Validators.HelpText(), [ - "src/UI/InputElement/Validators.ts", + "src/UI/InputElement/Validators.ts" ]) this.WriteFile("./Docs/ChangesetMeta.md", Changes.getDocs(), [ "src/Logic/Osm/Changes.ts", - "src/Logic/Osm/ChangesetHandler.ts", + "src/Logic/Osm/ChangesetHandler.ts" ]) new WikiPageGenerator().generate() console.log("Generated docs") } + /** + * @deprecated + */ private WriteFile( filename, html: string | BaseUIElement, @@ -201,6 +204,29 @@ export class GenerateDocs extends Script { if (!html) { return } + + let md = new Combine([ + Translations.W(html), + "\n\nThis document is autogenerated from " + + autogenSource + .map( + (file) => + `[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})` + ) + .join(", ") + ]).AsMarkdown() + this.WriteMarkdownFile(filename, md, autogenSource, options) + } + + + private WriteMarkdownFile( + filename: string, + markdown: string, + autogenSource: string[], + options?: { + noTableOfContents: boolean + } + ): void { for (const source of autogenSource) { if (source.indexOf("*") > 0) { continue @@ -214,22 +240,12 @@ export class GenerateDocs extends Script { } } - if (html instanceof Combine && !options?.noTableOfContents) { - const toc = new TableOfContents(html) - const els = html.getElements() - html = new Combine([els.shift(), toc, ...els]).SetClass("flex flex-col") - } - let md = new Combine([ - Translations.W(html), - "\n\nThis document is autogenerated from " + - autogenSource - .map( - (file) => - `[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})` - ) - .join(", "), - ]).AsMarkdown() + let md = markdown + + if (options?.noTableOfContents !== false) { + md = TableOfContents.insertTocIntoMd(md) + } md.replace(/\n\n\n+/g, "\n\n") @@ -238,7 +254,7 @@ export class GenerateDocs extends Script { } const warnAutomated = - "[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)" + "[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)\n\n" writeFileSync(filename, warnAutomated + md) } @@ -278,7 +294,7 @@ export class GenerateDocs extends Script { } this.WriteFile("./Docs/builtin_units.md", new Combine([new Title("Units", 1), ...els]), [ - `assets/layers/unit/unit.json`, + `assets/layers/unit/unit.json` ]) } @@ -359,9 +375,7 @@ export class GenerateDocs extends Script { if (inlineSource !== undefined) { source = `assets/themes/${inlineSource}/${inlineSource}.json` } - this.WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source], { - noTableOfContents: true, - }) + this.WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source]) }) } @@ -405,14 +419,19 @@ export class GenerateDocs extends Script { builtinsPerLayer.set(layer.id, usedBuiltins) } - const docs = new Combine([ - new Title("Index of builtin TagRendering", 1), - new Title("Existing builtin tagrenderings", 2), - ...Array.from(layersUsingBuiltin.entries()).map(([builtin, usedByLayers]) => - new Combine([new Title(builtin), new List(usedByLayers)]).SetClass("flex flex-col") - ), - ]).SetClass("flex flex-col") - this.WriteFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"]) + let docs =` + # Index of builtin TagRenderings + ## Existing builtin tagrenderings + ` + + for (const [builtin, usedByLayers] of Array.from(layersUsingBuiltin.entries())) { + docs += ` + ### ${builtin} + + ${usedByLayers.map(item => " - "+item).join("\n")} + ` + } + this.WriteMarkdownFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"]) } private generateQueryParameterDocs() { @@ -448,7 +467,7 @@ export class GenerateDocs extends Script { theme.title, "(", new Link(theme.id, "https://mapcomplete.org/" + theme.id), - ")", + ")" ]), 2 ), @@ -460,7 +479,7 @@ export class GenerateDocs extends Script { .map((l) => new Link(l.id, "../Layers/" + l.id + ".md")) ), "Available languages:", - new List(theme.language.filter((ln) => ln !== "_context")), + new List(theme.language.filter((ln) => ln !== "_context")) ]).SetClass("flex flex-col") this.WriteFile( "./Docs/Themes/" + theme.id + ".md", @@ -538,7 +557,7 @@ export class GenerateDocs extends Script { Array.from(AllSharedLayers.sharedLayers.keys()).map( (id) => new Link(id, "./Layers/" + id + ".md") ) - ), + ) ]) this.WriteFile("./Docs/BuiltinLayers.md", el, ["src/Customizations/AllKnownLayouts.ts"]) } diff --git a/src/Models/ThemeConfig/LayerConfig.ts b/src/Models/ThemeConfig/LayerConfig.ts index e81140d7d..49cea46b7 100644 --- a/src/Models/ThemeConfig/LayerConfig.ts +++ b/src/Models/ThemeConfig/LayerConfig.ts @@ -368,7 +368,6 @@ export default class LayerConfig extends WithContextLoader { canBeIncluded = true ): BaseUIElement { const extraProps: (string | BaseUIElement)[] = [] - extraProps.push("This layer is shown at zoomlevel **" + this.minzoom + "** and higher") if (canBeIncluded) { @@ -424,7 +423,7 @@ export default class LayerConfig extends WithContextLoader { if (!addedByDefault) { if (usedInThemes?.length > 0) { usingLayer = [ - new Title("Themes using this layer", 4), + new Title("Themes using this layer", 2), new List( (usedInThemes ?? []).map( (id) => new Link(id, "https://mapcomplete.org/" + id) diff --git a/src/UI/Base/TableOfContents.ts b/src/UI/Base/TableOfContents.ts index 12dac51b1..14aed6b23 100644 --- a/src/UI/Base/TableOfContents.ts +++ b/src/UI/Base/TableOfContents.ts @@ -1,73 +1,21 @@ import Combine from "./Combine" import BaseUIElement from "../BaseUIElement" -import { Translation } from "../i18n/Translation" -import { FixedUiElement } from "./FixedUiElement" import Title from "./Title" import List from "./List" import Link from "./Link" +import { marked } from "marked" +import { parse as parse_html } from "node-html-parser" +import {default as turndown} from "turndown" import { Utils } from "../../Utils" -export default class TableOfContents extends Combine { - private readonly titles: Title[] +export default class TableOfContents { - constructor( - elements: Combine | Title[], - options?: { - noTopLevel: false | boolean - maxDepth?: number - } - ) { - let titles: Title[] - if (elements instanceof Combine) { - titles = TableOfContents.getTitles(elements.getElements()) ?? [] - } else { - titles = elements ?? [] - } - let els: { level: number; content: BaseUIElement }[] = [] - for (const title of titles) { - let content: BaseUIElement - if (title.title instanceof Translation) { - content = title.title.Clone() - } else if (title.title instanceof FixedUiElement) { - content = new FixedUiElement(title.title.content) - } else if (Utils.runningFromConsole) { - content = new FixedUiElement(title.AsMarkdown()) - } else if (title["title"] !== undefined) { - content = new FixedUiElement(title.title.ConstructElement().textContent) - } else { - console.log("Not generating a title for ", title) - continue - } - - const vis = new Link(content, "#" + title.id) - - els.push({ level: title.level, content: vis }) - } - const minLevel = Math.min(...els.map((e) => e.level)) - if (options?.noTopLevel) { - els = els.filter((e) => e.level !== minLevel) - } - - if (options?.maxDepth) { - els = els.filter((e) => e.level <= options.maxDepth + minLevel) - } - - super(TableOfContents.mergeLevel(els).map((el) => el.SetClass("mt-2"))) - this.SetClass("flex flex-col") - this.titles = titles - } - - private static getTitles(elements: BaseUIElement[]): Title[] { - const titles = [] - for (const uiElement of elements) { - if (uiElement instanceof Combine) { - titles.push(...TableOfContents.getTitles(uiElement.getElements())) - } else if (uiElement instanceof Title) { - titles.push(uiElement) - } - } - return titles + private static asLinkableId(text: string): string { + return text + ?.replace(/ /g, "-") + ?.replace(/[?#.;:/]/, "") + ?.toLowerCase() ?? "" } private static mergeLevel( @@ -88,7 +36,7 @@ export default class TableOfContents extends Combine { if (running.length !== undefined) { result.push({ content: new List(running), - level: maxLevel - 1, + level: maxLevel - 1 }) running = [] } @@ -97,24 +45,81 @@ export default class TableOfContents extends Combine { if (running.length !== undefined) { result.push({ content: new List(running), - level: maxLevel - 1, + level: maxLevel - 1 }) } return TableOfContents.mergeLevel(result) } - AsMarkdown(): string { - const depthIcons = ["1.", " -", " +", " *"] - const lines = ["## Table of contents\n"] - const minLevel = Math.min(...this.titles.map((t) => t.level)) - for (const title of this.titles) { - const prefix = depthIcons[title.level - minLevel] ?? " ~" - const text = title.title.AsMarkdown().replace("\n", "") - const link = title.id - lines.push(prefix + " [" + text + "](#" + link + ")") + public static insertTocIntoMd(md: string): string { + const htmlSource = marked.parse(md) + const el = parse_html(htmlSource) + const structure = TableOfContents.generateStructure(el) + let firstTitle = structure[1] + let minDepth = undefined + do { + minDepth = Math.min(...structure.map(s => s.depth)) + const minDepthCount = structure.filter(s => s.depth === minDepth) + if (minDepthCount.length > 1) { + break + } + // Erase a single top level heading + structure.splice(structure.findIndex(s => s.depth === minDepth), 1) + } while (structure.length > 0) + + if (structure.length <= 1) { + return md + } + const separators = { + 1: " -", + 2: " +", + 3: " *" } - return lines.join("\n") + "\n\n" + let toc = "" + let topLevelCount = 0 + for (const el of structure) { + const depthDiff = el.depth - minDepth + let link = `[${el.title}](#${TableOfContents.asLinkableId(el.title)})` + if (depthDiff === 0) { + topLevelCount++ + toc += `${topLevelCount}. ${link}\n` + } else if (depthDiff <= 3) { + toc += `${separators[depthDiff]} ${link}\n` + } + } + + const heading = Utils.Times(() => "#", firstTitle.depth) + toc = heading +" Table of contents\n\n"+toc + + const original = el.outerHTML + const firstTitleIndex = original.indexOf(firstTitle.el.outerHTML) + const tocHtml = (marked.parse(toc)) + const withToc = original.substring(0, firstTitleIndex) + tocHtml + original.substring(firstTitleIndex) + + const htmlToMd = new turndown() + return htmlToMd.turndown(withToc) + + + } + + public static generateStructure(html: Element): { depth: number, title: string, el: Element }[] { + if (html === undefined) { + return [] + } + return [].concat(...Array.from(html.childNodes ?? []).map( + child => { + const tag: string = child["tagName"]?.toLowerCase() + if (!tag) { + return [] + } + if (tag.match(/h[0-9]/)) { + const depth = Number(tag.substring(1)) + return [{ depth, title: child.textContent, el: child }] + } + return TableOfContents.generateStructure(child) + } + )) } }