Use libraries to generate MD, steps to factor out more of the UIElements

This commit is contained in:
Pieter Vander Vennet 2024-04-28 03:48:07 +02:00
parent 015b9aebad
commit 7c71e566e9
5 changed files with 167 additions and 125 deletions

35
package-lock.json generated
View file

@ -40,7 +40,6 @@
"fake-dom": "^1.0.4",
"geojson2svg": "^1.3.3",
"html-to-image": "^1.11.11",
"html-to-markdown": "^1.0.0",
"i18next-client": "^1.11.4",
"idb-keyval": "^6.0.3",
"jest-mock": "^29.4.1",
@ -70,6 +69,7 @@
"tailwind-merge": "^1.13.1",
"tailwindcss": "^3.1.8",
"trap-focus-svelte": "^1.0.2",
"turndown": "^7.1.3",
"vite-node": "^0.28.3",
"vitest": "^0.28.3",
"wikibase-sdk": "^7.14.0",
@ -8820,6 +8820,11 @@
"domelementtype": "1"
}
},
"node_modules/domino": {
"version": "2.1.6",
"resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
"integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
},
"node_modules/dompurify": {
"version": "3.0.5",
"license": "(MPL-2.0 OR Apache-2.0)"
@ -10294,10 +10299,6 @@
"version": "1.11.11",
"license": "MIT"
},
"node_modules/html-to-markdown": {
"version": "1.0.0",
"license": "MIT"
},
"node_modules/html2canvas": {
"version": "1.4.1",
"license": "MIT",
@ -15923,6 +15924,14 @@
"turf-inside": "^3.0.12"
}
},
"node_modules/turndown": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz",
"integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==",
"dependencies": {
"domino": "^2.1.6"
}
},
"node_modules/tweetnacl": {
"version": "0.14.5",
"license": "Unlicense"
@ -23072,6 +23081,11 @@
"domelementtype": "1"
}
},
"domino": {
"version": "2.1.6",
"resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
"integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
},
"dompurify": {
"version": "3.0.5"
},
@ -24012,9 +24026,6 @@
"html-to-image": {
"version": "1.11.11"
},
"html-to-markdown": {
"version": "1.0.0"
},
"html2canvas": {
"version": "1.4.1",
"optional": true,
@ -27705,6 +27716,14 @@
"turf-inside": "^3.0.12"
}
},
"turndown": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz",
"integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==",
"requires": {
"domino": "^2.1.6"
}
},
"tweetnacl": {
"version": "0.14.5"
},

View file

@ -158,7 +158,6 @@
"fake-dom": "^1.0.4",
"geojson2svg": "^1.3.3",
"html-to-image": "^1.11.11",
"html-to-markdown": "^1.0.0",
"i18next-client": "^1.11.4",
"idb-keyval": "^6.0.3",
"jest-mock": "^29.4.1",
@ -188,6 +187,7 @@
"tailwind-merge": "^1.13.1",
"tailwindcss": "^3.1.8",
"trap-focus-svelte": "^1.0.2",
"turndown": "^7.1.3",
"vite-node": "^0.28.3",
"vitest": "^0.28.3",
"wikibase-sdk": "^7.14.0",

View file

@ -2,7 +2,6 @@ import Combine from "../src/UI/Base/Combine"
import BaseUIElement from "../src/UI/BaseUIElement"
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { AllKnownLayouts } from "../src/Customizations/AllKnownLayouts"
import TableOfContents from "../src/UI/Base/TableOfContents"
import SimpleMetaTaggers from "../src/Logic/SimpleMetaTagger"
import SpecialVisualizations from "../src/UI/SpecialVisualizations"
import { ExtraFunctions } from "../src/Logic/ExtraFunctions"
@ -31,6 +30,7 @@ import { Utils } from "../src/Utils"
import { TagUtils } from "../src/Logic/Tags/TagUtils"
import Script from "./Script"
import { Changes } from "../src/Logic/Osm/Changes"
import TableOfContents from "../src/UI/Base/TableOfContents"
/**
* Converts a markdown-file into a .json file, which a walkthrough/slideshow element can use
@ -56,15 +56,15 @@ class ToSlideshowJson {
sections.push(currentSection)
currentSection = []
}
line = line.replace('src="../../public/', 'src="./')
line = line.replace('src="../../', 'src="./')
line = line.replace("src=\"../../public/", "src=\"./")
line = line.replace("src=\"../../", "src=\"./")
currentSection.push(line)
}
sections.push(currentSection)
writeFileSync(
this._target,
JSON.stringify({
sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0),
sections: sections.map((s) => s.join("\n")).filter((s) => s.length > 0)
})
)
}
@ -83,7 +83,7 @@ class WikiPageGenerator {
generate() {
let wikiPage =
'{|class="wikitable sortable"\n' +
"{|class=\"wikitable sortable\"\n" +
"! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" +
"|-"
@ -141,7 +141,7 @@ export class GenerateDocs extends Script {
}
this.WriteFile("./Docs/Tags_format.md", TagUtils.generateDocs(), [
"src/Logic/Tags/TagUtils.ts",
"src/Logic/Tags/TagUtils.ts"
])
new ToSlideshowJson(
@ -166,30 +166,33 @@ export class GenerateDocs extends Script {
})
this.WriteFile("./Docs/SpecialRenderings.md", SpecialVisualizations.HelpMessage(), [
"src/UI/SpecialVisualizations.ts",
"src/UI/SpecialVisualizations.ts"
])
this.WriteFile(
"./Docs/CalculatedTags.md",
new Combine([
new Title("Metatags", 1),
SimpleMetaTaggers.HelpText(),
ExtraFunctions.HelpText(),
ExtraFunctions.HelpText()
]).SetClass("flex-col"),
["src/Logic/SimpleMetaTagger.ts", "src/Logic/ExtraFunctions.ts"]
)
this.WriteFile("./Docs/SpecialInputElements.md", Validators.HelpText(), [
"src/UI/InputElement/Validators.ts",
"src/UI/InputElement/Validators.ts"
])
this.WriteFile("./Docs/ChangesetMeta.md", Changes.getDocs(), [
"src/Logic/Osm/Changes.ts",
"src/Logic/Osm/ChangesetHandler.ts",
"src/Logic/Osm/ChangesetHandler.ts"
])
new WikiPageGenerator().generate()
console.log("Generated docs")
}
/**
* @deprecated
*/
private WriteFile(
filename,
html: string | BaseUIElement,
@ -201,6 +204,29 @@ export class GenerateDocs extends Script {
if (!html) {
return
}
let md = new Combine([
Translations.W(html),
"\n\nThis document is autogenerated from " +
autogenSource
.map(
(file) =>
`[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})`
)
.join(", ")
]).AsMarkdown()
this.WriteMarkdownFile(filename, md, autogenSource, options)
}
private WriteMarkdownFile(
filename: string,
markdown: string,
autogenSource: string[],
options?: {
noTableOfContents: boolean
}
): void {
for (const source of autogenSource) {
if (source.indexOf("*") > 0) {
continue
@ -214,22 +240,12 @@ export class GenerateDocs extends Script {
}
}
if (html instanceof Combine && !options?.noTableOfContents) {
const toc = new TableOfContents(html)
const els = html.getElements()
html = new Combine([els.shift(), toc, ...els]).SetClass("flex flex-col")
}
let md = new Combine([
Translations.W(html),
"\n\nThis document is autogenerated from " +
autogenSource
.map(
(file) =>
`[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})`
)
.join(", "),
]).AsMarkdown()
let md = markdown
if (options?.noTableOfContents !== false) {
md = TableOfContents.insertTocIntoMd(md)
}
md.replace(/\n\n\n+/g, "\n\n")
@ -238,7 +254,7 @@ export class GenerateDocs extends Script {
}
const warnAutomated =
"[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)"
"[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)\n\n"
writeFileSync(filename, warnAutomated + md)
}
@ -278,7 +294,7 @@ export class GenerateDocs extends Script {
}
this.WriteFile("./Docs/builtin_units.md", new Combine([new Title("Units", 1), ...els]), [
`assets/layers/unit/unit.json`,
`assets/layers/unit/unit.json`
])
}
@ -359,9 +375,7 @@ export class GenerateDocs extends Script {
if (inlineSource !== undefined) {
source = `assets/themes/${inlineSource}/${inlineSource}.json`
}
this.WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source], {
noTableOfContents: true,
})
this.WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source])
})
}
@ -405,14 +419,19 @@ export class GenerateDocs extends Script {
builtinsPerLayer.set(layer.id, usedBuiltins)
}
const docs = new Combine([
new Title("Index of builtin TagRendering", 1),
new Title("Existing builtin tagrenderings", 2),
...Array.from(layersUsingBuiltin.entries()).map(([builtin, usedByLayers]) =>
new Combine([new Title(builtin), new List(usedByLayers)]).SetClass("flex flex-col")
),
]).SetClass("flex flex-col")
this.WriteFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"])
let docs =`
# Index of builtin TagRenderings
## Existing builtin tagrenderings
`
for (const [builtin, usedByLayers] of Array.from(layersUsingBuiltin.entries())) {
docs += `
### ${builtin}
${usedByLayers.map(item => " - "+item).join("\n")}
`
}
this.WriteMarkdownFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"])
}
private generateQueryParameterDocs() {
@ -448,7 +467,7 @@ export class GenerateDocs extends Script {
theme.title,
"(",
new Link(theme.id, "https://mapcomplete.org/" + theme.id),
")",
")"
]),
2
),
@ -460,7 +479,7 @@ export class GenerateDocs extends Script {
.map((l) => new Link(l.id, "../Layers/" + l.id + ".md"))
),
"Available languages:",
new List(theme.language.filter((ln) => ln !== "_context")),
new List(theme.language.filter((ln) => ln !== "_context"))
]).SetClass("flex flex-col")
this.WriteFile(
"./Docs/Themes/" + theme.id + ".md",
@ -538,7 +557,7 @@ export class GenerateDocs extends Script {
Array.from(AllSharedLayers.sharedLayers.keys()).map(
(id) => new Link(id, "./Layers/" + id + ".md")
)
),
)
])
this.WriteFile("./Docs/BuiltinLayers.md", el, ["src/Customizations/AllKnownLayouts.ts"])
}

View file

@ -368,7 +368,6 @@ export default class LayerConfig extends WithContextLoader {
canBeIncluded = true
): BaseUIElement {
const extraProps: (string | BaseUIElement)[] = []
extraProps.push("This layer is shown at zoomlevel **" + this.minzoom + "** and higher")
if (canBeIncluded) {
@ -424,7 +423,7 @@ export default class LayerConfig extends WithContextLoader {
if (!addedByDefault) {
if (usedInThemes?.length > 0) {
usingLayer = [
new Title("Themes using this layer", 4),
new Title("Themes using this layer", 2),
new List(
(usedInThemes ?? []).map(
(id) => new Link(id, "https://mapcomplete.org/" + id)

View file

@ -1,73 +1,21 @@
import Combine from "./Combine"
import BaseUIElement from "../BaseUIElement"
import { Translation } from "../i18n/Translation"
import { FixedUiElement } from "./FixedUiElement"
import Title from "./Title"
import List from "./List"
import Link from "./Link"
import { marked } from "marked"
import { parse as parse_html } from "node-html-parser"
import {default as turndown} from "turndown"
import { Utils } from "../../Utils"
export default class TableOfContents extends Combine {
private readonly titles: Title[]
export default class TableOfContents {
constructor(
elements: Combine | Title[],
options?: {
noTopLevel: false | boolean
maxDepth?: number
}
) {
let titles: Title[]
if (elements instanceof Combine) {
titles = TableOfContents.getTitles(elements.getElements()) ?? []
} else {
titles = elements ?? []
}
let els: { level: number; content: BaseUIElement }[] = []
for (const title of titles) {
let content: BaseUIElement
if (title.title instanceof Translation) {
content = title.title.Clone()
} else if (title.title instanceof FixedUiElement) {
content = new FixedUiElement(title.title.content)
} else if (Utils.runningFromConsole) {
content = new FixedUiElement(title.AsMarkdown())
} else if (title["title"] !== undefined) {
content = new FixedUiElement(title.title.ConstructElement().textContent)
} else {
console.log("Not generating a title for ", title)
continue
}
const vis = new Link(content, "#" + title.id)
els.push({ level: title.level, content: vis })
}
const minLevel = Math.min(...els.map((e) => e.level))
if (options?.noTopLevel) {
els = els.filter((e) => e.level !== minLevel)
}
if (options?.maxDepth) {
els = els.filter((e) => e.level <= options.maxDepth + minLevel)
}
super(TableOfContents.mergeLevel(els).map((el) => el.SetClass("mt-2")))
this.SetClass("flex flex-col")
this.titles = titles
}
private static getTitles(elements: BaseUIElement[]): Title[] {
const titles = []
for (const uiElement of elements) {
if (uiElement instanceof Combine) {
titles.push(...TableOfContents.getTitles(uiElement.getElements()))
} else if (uiElement instanceof Title) {
titles.push(uiElement)
}
}
return titles
private static asLinkableId(text: string): string {
return text
?.replace(/ /g, "-")
?.replace(/[?#.;:/]/, "")
?.toLowerCase() ?? ""
}
private static mergeLevel(
@ -88,7 +36,7 @@ export default class TableOfContents extends Combine {
if (running.length !== undefined) {
result.push({
content: new List(running),
level: maxLevel - 1,
level: maxLevel - 1
})
running = []
}
@ -97,24 +45,81 @@ export default class TableOfContents extends Combine {
if (running.length !== undefined) {
result.push({
content: new List(running),
level: maxLevel - 1,
level: maxLevel - 1
})
}
return TableOfContents.mergeLevel(result)
}
AsMarkdown(): string {
const depthIcons = ["1.", " -", " +", " *"]
const lines = ["## Table of contents\n"]
const minLevel = Math.min(...this.titles.map((t) => t.level))
for (const title of this.titles) {
const prefix = depthIcons[title.level - minLevel] ?? " ~"
const text = title.title.AsMarkdown().replace("\n", "")
const link = title.id
lines.push(prefix + " [" + text + "](#" + link + ")")
public static insertTocIntoMd(md: string): string {
const htmlSource = <string>marked.parse(md)
const el = parse_html(htmlSource)
const structure = TableOfContents.generateStructure(<any>el)
let firstTitle = structure[1]
let minDepth = undefined
do {
minDepth = Math.min(...structure.map(s => s.depth))
const minDepthCount = structure.filter(s => s.depth === minDepth)
if (minDepthCount.length > 1) {
break
}
// Erase a single top level heading
structure.splice(structure.findIndex(s => s.depth === minDepth), 1)
} while (structure.length > 0)
if (structure.length <= 1) {
return md
}
const separators = {
1: " -",
2: " +",
3: " *"
}
return lines.join("\n") + "\n\n"
let toc = ""
let topLevelCount = 0
for (const el of structure) {
const depthDiff = el.depth - minDepth
let link = `[${el.title}](#${TableOfContents.asLinkableId(el.title)})`
if (depthDiff === 0) {
topLevelCount++
toc += `${topLevelCount}. ${link}\n`
} else if (depthDiff <= 3) {
toc += `${separators[depthDiff]} ${link}\n`
}
}
const heading = Utils.Times(() => "#", firstTitle.depth)
toc = heading +" Table of contents\n\n"+toc
const original = el.outerHTML
const firstTitleIndex = original.indexOf(firstTitle.el.outerHTML)
const tocHtml = (<string>marked.parse(toc))
const withToc = original.substring(0, firstTitleIndex) + tocHtml + original.substring(firstTitleIndex)
const htmlToMd = new turndown()
return htmlToMd.turndown(withToc)
}
public static generateStructure(html: Element): { depth: number, title: string, el: Element }[] {
if (html === undefined) {
return []
}
return [].concat(...Array.from(html.childNodes ?? []).map(
child => {
const tag: string = child["tagName"]?.toLowerCase()
if (!tag) {
return []
}
if (tag.match(/h[0-9]/)) {
const depth = Number(tag.substring(1))
return [{ depth, title: child.textContent, el: child }]
}
return TableOfContents.generateStructure(<Element>child)
}
))
}
}