Use libraries to generate MD, steps to factor out more of the UIElements

package-lock.json

@@ -40,7 +40,6 @@
         "fake-dom": "^1.0.4",
         "geojson2svg": "^1.3.3",
         "html-to-image": "^1.11.11",
-        "html-to-markdown": "^1.0.0",
         "i18next-client": "^1.11.4",
         "idb-keyval": "^6.0.3",
         "jest-mock": "^29.4.1",
@@ -70,6 +69,7 @@
         "tailwind-merge": "^1.13.1",
         "tailwindcss": "^3.1.8",
         "trap-focus-svelte": "^1.0.2",
+        "turndown": "^7.1.3",
         "vite-node": "^0.28.3",
         "vitest": "^0.28.3",
         "wikibase-sdk": "^7.14.0",
@@ -8820,6 +8820,11 @@
         "domelementtype": "1"
       }
     },
+    "node_modules/domino": {
+      "version": "2.1.6",
+      "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
+      "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
+    },
     "node_modules/dompurify": {
       "version": "3.0.5",
       "license": "(MPL-2.0 OR Apache-2.0)"
@@ -10294,10 +10299,6 @@
       "version": "1.11.11",
       "license": "MIT"
     },
-    "node_modules/html-to-markdown": {
-      "version": "1.0.0",
-      "license": "MIT"
-    },
     "node_modules/html2canvas": {
       "version": "1.4.1",
       "license": "MIT",
@@ -15923,6 +15924,14 @@
         "turf-inside": "^3.0.12"
       }
     },
+    "node_modules/turndown": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz",
+      "integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==",
+      "dependencies": {
+        "domino": "^2.1.6"
+      }
+    },
     "node_modules/tweetnacl": {
       "version": "0.14.5",
       "license": "Unlicense"
@@ -23072,6 +23081,11 @@
         "domelementtype": "1"
       }
     },
+    "domino": {
+      "version": "2.1.6",
+      "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
+      "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
+    },
     "dompurify": {
       "version": "3.0.5"
     },
@@ -24012,9 +24026,6 @@
     "html-to-image": {
       "version": "1.11.11"
     },
-    "html-to-markdown": {
-      "version": "1.0.0"
-    },
     "html2canvas": {
       "version": "1.4.1",
       "optional": true,
@@ -27705,6 +27716,14 @@
         "turf-inside": "^3.0.12"
       }
     },
+    "turndown": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz",
+      "integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==",
+      "requires": {
+        "domino": "^2.1.6"
+      }
+    },
     "tweetnacl": {
       "version": "0.14.5"
     },

package.json

@@ -158,7 +158,6 @@
     "fake-dom": "^1.0.4",
     "geojson2svg": "^1.3.3",
     "html-to-image": "^1.11.11",
-    "html-to-markdown": "^1.0.0",
     "i18next-client": "^1.11.4",
     "idb-keyval": "^6.0.3",
     "jest-mock": "^29.4.1",
@@ -188,6 +187,7 @@
     "tailwind-merge": "^1.13.1",
     "tailwindcss": "^3.1.8",
     "trap-focus-svelte": "^1.0.2",
+    "turndown": "^7.1.3",
     "vite-node": "^0.28.3",
     "vitest": "^0.28.3",
     "wikibase-sdk": "^7.14.0",

scripts/generateDocs.ts

@@ -2,7 +2,6 @@ import Combine from "../src/UI/Base/Combine"
 import BaseUIElement from "../src/UI/BaseUIElement"
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
 import { AllKnownLayouts } from "../src/Customizations/AllKnownLayouts"
-import TableOfContents from "../src/UI/Base/TableOfContents"
 import SimpleMetaTaggers from "../src/Logic/SimpleMetaTagger"
 import SpecialVisualizations from "../src/UI/SpecialVisualizations"
 import { ExtraFunctions } from "../src/Logic/ExtraFunctions"
@@ -31,6 +30,7 @@ import { Utils } from "../src/Utils"
 import { TagUtils } from "../src/Logic/Tags/TagUtils"
 import Script from "./Script"
 import { Changes } from "../src/Logic/Osm/Changes"
+import TableOfContents from "../src/UI/Base/TableOfContents"
 
 /**
  * Converts a markdown-file into a .json file, which a walkthrough/slideshow element can use
@@ -56,15 +56,15 @@ class ToSlideshowJson {
                 sections.push(currentSection)
                 currentSection = []
             }
-            line = line.replace('src="../../public/', 'src="./')
+            line = line.replace("src=\"../../public/", "src=\"./")
-            line = line.replace('src="../../', 'src="./')
+            line = line.replace("src=\"../../", "src=\"./")
             currentSection.push(line)
         }
         sections.push(currentSection)
         writeFileSync(
             this._target,
             JSON.stringify({
-                sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0),
+                sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0)
             })
         )
     }
@@ -83,7 +83,7 @@ class WikiPageGenerator {
 
     generate() {
         let wikiPage =
-            '{|class="wikitable sortable"\n' +
+            "{|class=\"wikitable sortable\"\n" +
             "! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" +
             "|-"
 
@@ -141,7 +141,7 @@ export class GenerateDocs extends Script {
         }
 
         this.WriteFile("./Docs/Tags_format.md", TagUtils.generateDocs(), [
-            "src/Logic/Tags/TagUtils.ts",
+            "src/Logic/Tags/TagUtils.ts"
         ])
 
         new ToSlideshowJson(
@@ -166,30 +166,33 @@ export class GenerateDocs extends Script {
         })
 
         this.WriteFile("./Docs/SpecialRenderings.md", SpecialVisualizations.HelpMessage(), [
-            "src/UI/SpecialVisualizations.ts",
+            "src/UI/SpecialVisualizations.ts"
         ])
         this.WriteFile(
             "./Docs/CalculatedTags.md",
             new Combine([
                 new Title("Metatags", 1),
                 SimpleMetaTaggers.HelpText(),
-                ExtraFunctions.HelpText(),
+                ExtraFunctions.HelpText()
             ]).SetClass("flex-col"),
             ["src/Logic/SimpleMetaTagger.ts", "src/Logic/ExtraFunctions.ts"]
         )
         this.WriteFile("./Docs/SpecialInputElements.md", Validators.HelpText(), [
-            "src/UI/InputElement/Validators.ts",
+            "src/UI/InputElement/Validators.ts"
         ])
 
         this.WriteFile("./Docs/ChangesetMeta.md", Changes.getDocs(), [
             "src/Logic/Osm/Changes.ts",
-            "src/Logic/Osm/ChangesetHandler.ts",
+            "src/Logic/Osm/ChangesetHandler.ts"
         ])
         new WikiPageGenerator().generate()
 
         console.log("Generated docs")
     }
 
+    /**
+     * @deprecated
+     */
     private WriteFile(
         filename,
         html: string | BaseUIElement,
@@ -201,6 +204,29 @@ export class GenerateDocs extends Script {
         if (!html) {
             return
         }
+
+        let md = new Combine([
+            Translations.W(html),
+            "\n\nThis document is autogenerated from " +
+            autogenSource
+                .map(
+                    (file) =>
+                        `[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})`
+                )
+                .join(", ")
+        ]).AsMarkdown()
+        this.WriteMarkdownFile(filename, md, autogenSource, options)
+    }
+
+
+    private WriteMarkdownFile(
+        filename: string,
+        markdown: string,
+        autogenSource: string[],
+        options?: {
+            noTableOfContents: boolean
+        }
+    ): void {
         for (const source of autogenSource) {
             if (source.indexOf("*") > 0) {
                 continue
@@ -214,22 +240,12 @@ export class GenerateDocs extends Script {
             }
         }
 
-        if (html instanceof Combine && !options?.noTableOfContents) {
-            const toc = new TableOfContents(html)
-            const els = html.getElements()
-            html = new Combine([els.shift(), toc, ...els]).SetClass("flex flex-col")
-        }
 
-        let md = new Combine([
+        let md = markdown
-            Translations.W(html),
+
-            "\n\nThis document is autogenerated from " +
+        if (options?.noTableOfContents !== false) {
-                autogenSource
+            md = TableOfContents.insertTocIntoMd(md)
-                    .map(
+        }
-                        (file) =>
-                            `[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})`
-                    )
-                    .join(", "),
-        ]).AsMarkdown()
 
         md.replace(/\n\n\n+/g, "\n\n")
 
@@ -238,7 +254,7 @@ export class GenerateDocs extends Script {
         }
 
         const warnAutomated =
-            "[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)"
+            "[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)\n\n"
 
         writeFileSync(filename, warnAutomated + md)
     }
@@ -278,7 +294,7 @@ export class GenerateDocs extends Script {
         }
 
         this.WriteFile("./Docs/builtin_units.md", new Combine([new Title("Units", 1), ...els]), [
-            `assets/layers/unit/unit.json`,
+            `assets/layers/unit/unit.json`
         ])
     }
 
@@ -359,9 +375,7 @@ export class GenerateDocs extends Script {
             if (inlineSource !== undefined) {
                 source = `assets/themes/${inlineSource}/${inlineSource}.json`
             }
-            this.WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source], {
+            this.WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source])
-                noTableOfContents: true,
-            })
         })
     }
 
@@ -405,14 +419,19 @@ export class GenerateDocs extends Script {
             builtinsPerLayer.set(layer.id, usedBuiltins)
         }
 
-        const docs = new Combine([
+        let docs =`
-            new Title("Index of builtin TagRendering", 1),
+            # Index of builtin TagRenderings
-            new Title("Existing builtin tagrenderings", 2),
+            ## Existing builtin tagrenderings
-            ...Array.from(layersUsingBuiltin.entries()).map(([builtin, usedByLayers]) =>
+        `
-                new Combine([new Title(builtin), new List(usedByLayers)]).SetClass("flex flex-col")
+
-            ),
+        for (const [builtin, usedByLayers] of Array.from(layersUsingBuiltin.entries())) {
-        ]).SetClass("flex flex-col")
+            docs += `
-        this.WriteFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"])
+            ### ${builtin}
+
+            ${usedByLayers.map(item => " - "+item).join("\n")}
+            `
+        }
+        this.WriteMarkdownFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"])
     }
 
     private generateQueryParameterDocs() {
@@ -448,7 +467,7 @@ export class GenerateDocs extends Script {
                     theme.title,
                     "(",
                     new Link(theme.id, "https://mapcomplete.org/" + theme.id),
-                    ")",
+                    ")"
                 ]),
                 2
             ),
@@ -460,7 +479,7 @@ export class GenerateDocs extends Script {
                     .map((l) => new Link(l.id, "../Layers/" + l.id + ".md"))
             ),
             "Available languages:",
-            new List(theme.language.filter((ln) => ln !== "_context")),
+            new List(theme.language.filter((ln) => ln !== "_context"))
         ]).SetClass("flex flex-col")
         this.WriteFile(
             "./Docs/Themes/" + theme.id + ".md",
@@ -538,7 +557,7 @@ export class GenerateDocs extends Script {
                 Array.from(AllSharedLayers.sharedLayers.keys()).map(
                     (id) => new Link(id, "./Layers/" + id + ".md")
                 )
-            ),
+            )
         ])
         this.WriteFile("./Docs/BuiltinLayers.md", el, ["src/Customizations/AllKnownLayouts.ts"])
     }

src/Models/ThemeConfig/LayerConfig.ts

@@ -368,7 +368,6 @@ export default class LayerConfig extends WithContextLoader {
         canBeIncluded = true
     ): BaseUIElement {
         const extraProps: (string | BaseUIElement)[] = []
-
         extraProps.push("This layer is shown at zoomlevel **" + this.minzoom + "** and higher")
 
         if (canBeIncluded) {
@@ -424,7 +423,7 @@ export default class LayerConfig extends WithContextLoader {
         if (!addedByDefault) {
             if (usedInThemes?.length > 0) {
                 usingLayer = [
-                    new Title("Themes using this layer", 4),
+                    new Title("Themes using this layer", 2),
                     new List(
                         (usedInThemes ?? []).map(
                             (id) => new Link(id, "https://mapcomplete.org/" + id)

src/UI/Base/TableOfContents.ts

@@ -1,73 +1,21 @@
 import Combine from "./Combine"
 import BaseUIElement from "../BaseUIElement"
-import { Translation } from "../i18n/Translation"
-import { FixedUiElement } from "./FixedUiElement"
 import Title from "./Title"
 import List from "./List"
 import Link from "./Link"
+import { marked } from "marked"
+import { parse as parse_html } from "node-html-parser"
+import {default as turndown} from "turndown"
 import { Utils } from "../../Utils"
 
-export default class TableOfContents extends Combine {
+export default class TableOfContents {
-    private readonly titles: Title[]
 
-    constructor(
-        elements: Combine | Title[],
-        options?: {
-            noTopLevel: false | boolean
-            maxDepth?: number
-        }
-    ) {
-        let titles: Title[]
-        if (elements instanceof Combine) {
-            titles = TableOfContents.getTitles(elements.getElements()) ?? []
-        } else {
-            titles = elements ?? []
-        }
 
-        let els: { level: number; content: BaseUIElement }[] = []
+    private static asLinkableId(text: string): string {
-        for (const title of titles) {
+        return text
-            let content: BaseUIElement
+            ?.replace(/ /g, "-")
-            if (title.title instanceof Translation) {
+            ?.replace(/[?#.;:/]/, "")
-                content = title.title.Clone()
+            ?.toLowerCase() ?? ""
-            } else if (title.title instanceof FixedUiElement) {
-                content = new FixedUiElement(title.title.content)
-            } else if (Utils.runningFromConsole) {
-                content = new FixedUiElement(title.AsMarkdown())
-            } else if (title["title"] !== undefined) {
-                content = new FixedUiElement(title.title.ConstructElement().textContent)
-            } else {
-                console.log("Not generating a title for ", title)
-                continue
-            }
-
-            const vis = new Link(content, "#" + title.id)
-
-            els.push({ level: title.level, content: vis })
-        }
-        const minLevel = Math.min(...els.map((e) => e.level))
-        if (options?.noTopLevel) {
-            els = els.filter((e) => e.level !== minLevel)
-        }
-
-        if (options?.maxDepth) {
-            els = els.filter((e) => e.level <= options.maxDepth + minLevel)
-        }
-
-        super(TableOfContents.mergeLevel(els).map((el) => el.SetClass("mt-2")))
-        this.SetClass("flex flex-col")
-        this.titles = titles
-    }
-
-    private static getTitles(elements: BaseUIElement[]): Title[] {
-        const titles = []
-        for (const uiElement of elements) {
-            if (uiElement instanceof Combine) {
-                titles.push(...TableOfContents.getTitles(uiElement.getElements()))
-            } else if (uiElement instanceof Title) {
-                titles.push(uiElement)
-            }
-        }
-        return titles
     }
 
     private static mergeLevel(
@@ -88,7 +36,7 @@ export default class TableOfContents extends Combine {
             if (running.length !== undefined) {
                 result.push({
                     content: new List(running),
-                    level: maxLevel - 1,
+                    level: maxLevel - 1
                 })
                 running = []
             }
@@ -97,24 +45,81 @@ export default class TableOfContents extends Combine {
         if (running.length !== undefined) {
             result.push({
                 content: new List(running),
-                level: maxLevel - 1,
+                level: maxLevel - 1
             })
         }
 
         return TableOfContents.mergeLevel(result)
     }
 
-    AsMarkdown(): string {
+    public static insertTocIntoMd(md: string): string {
-        const depthIcons = ["1.", "  -", "    +", "      *"]
+        const htmlSource = <string>marked.parse(md)
-        const lines = ["## Table of contents\n"]
+        const el = parse_html(htmlSource)
-        const minLevel = Math.min(...this.titles.map((t) => t.level))
+        const structure = TableOfContents.generateStructure(<any>el)
-        for (const title of this.titles) {
+        let firstTitle = structure[1]
-            const prefix = depthIcons[title.level - minLevel] ?? "        ~"
+        let minDepth = undefined
-            const text = title.title.AsMarkdown().replace("\n", "")
+        do {
-            const link = title.id
+            minDepth = Math.min(...structure.map(s => s.depth))
-            lines.push(prefix + " [" + text + "](#" + link + ")")
+            const minDepthCount = structure.filter(s => s.depth === minDepth)
+            if (minDepthCount.length > 1) {
+                break
+            }
+            // Erase a single top level heading
+            structure.splice(structure.findIndex(s => s.depth === minDepth), 1)
+        } while (structure.length > 0)
+
+        if (structure.length <= 1) {
+            return md
+        }
+        const separators = {
+            1: "  -",
+            2: "    +",
+            3: "       *"
         }
 
-        return lines.join("\n") + "\n\n"
+        let toc = ""
+        let topLevelCount = 0
+        for (const el of structure) {
+            const depthDiff = el.depth - minDepth
+            let link = `[${el.title}](#${TableOfContents.asLinkableId(el.title)})`
+            if (depthDiff === 0) {
+                topLevelCount++
+                toc += `${topLevelCount}. ${link}\n`
+            } else if (depthDiff <= 3) {
+                toc += `${separators[depthDiff]} ${link}\n`
+            }
+        }
+
+        const heading = Utils.Times(() => "#", firstTitle.depth)
+        toc = heading +" Table of contents\n\n"+toc
+
+        const original = el.outerHTML
+        const firstTitleIndex = original.indexOf(firstTitle.el.outerHTML)
+        const tocHtml = (<string>marked.parse(toc))
+        const withToc = original.substring(0, firstTitleIndex) + tocHtml + original.substring(firstTitleIndex)
+
+        const htmlToMd = new turndown()
+        return htmlToMd.turndown(withToc)
+
+
+    }
+
+    public static generateStructure(html: Element): { depth: number, title: string, el: Element }[] {
+        if (html === undefined) {
+            return []
+        }
+        return [].concat(...Array.from(html.childNodes ?? []).map(
+            child => {
+                const tag: string = child["tagName"]?.toLowerCase()
+                if (!tag) {
+                    return []
+                }
+                if (tag.match(/h[0-9]/)) {
+                    const depth = Number(tag.substring(1))
+                    return [{ depth, title: child.textContent, el: child }]
+                }
+                return TableOfContents.generateStructure(<Element>child)
+            }
+        ))
     }
 }