Docs: improve documentation directory structure and scripts, remove some no longer needed documentation

scripts/generateDocs.ts

@@ -8,16 +8,13 @@ import ScriptUtils from "./ScriptUtils"
 import Translations from "../src/UI/i18n/Translations"
 import themeOverview from "../src/assets/generated/theme_overview.json"
 import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig"
-import bookcases from "../public/assets/generated/themes/bookcases.json"
 import fakedom from "fake-dom"
 import unit from "../public/assets/generated/layers/unit.json"
-import Hotkeys from "../src/UI/Base/Hotkeys"
 import { QueryParameters } from "../src/Logic/Web/QueryParameters"
 import Constants from "../src/Models/Constants"
 import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
 import DependencyCalculator from "../src/Models/ThemeConfig/DependencyCalculator"
 import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
-import ThemeViewState from "../src/Models/ThemeViewState"
 import Validators from "../src/UI/InputElement/Validators"
 import questions from "../public/assets/generated/layers/questions.json"
 import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
@@ -28,9 +25,7 @@ import TableOfContents from "../src/UI/Base/TableOfContents"
 import MarkdownUtils from "../src/Utils/MarkdownUtils"
 import { parse as parse_html } from "node-html-parser"
 import { AvailableRasterLayers } from "../src/Models/RasterLayers"
-import { ImmutableStore } from "../src/Logic/UIEventSource"
 import * as unitUsage from "../Docs/Schemas/UnitConfigJson.schema.json"
-import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
 import { ServerSourceInfo, SourceOverview } from "../src/Models/SourceOverview"
 import { Lists } from "../src/Utils/Lists"
 import { Translation, TypedTranslation } from "../src/UI/i18n/Translation"
@@ -141,89 +136,86 @@ export class GenerateDocs extends Script {
         console.log("Starting documentation generation...")
         ScriptUtils.fixUtils()
 
-        this.writeMarkdownFile(
-            "./Docs/SpecialRenderings.md",
-            SpecialVisualizations.HelpMessage(),
-            ["src/UI/SpecialVisualizations.ts"],
-            {
-                tocMaxDepth: 3,
+
+        {
+            // For studio: prepare slideshow
+            new ToSlideshowJson(
+                "./Docs/Studio/Introduction.md",
+                "./src/assets/studio_introduction.json",
+            ).convert()
+            new ToSlideshowJson(
+                "./Docs/Studio/TagRendering_How_to_work_with_TagRenderings.md",
+                "./src/assets/studio_tagrenderings_intro.json",
+            ).convert()
+        }
+
+        // Files for studio/for theme creators
+        {
+            this.generateSpecialLayerOverviewText()
+            this.generateBuiltinIndex()
+            this.generateBuiltinQuestions()
+            this.generateEliDocs()
+            this.generateBuiltinUnits()
+            this.writeMarkdownFile("./Docs/Studio/SpecialInputElements.md", Validators.HelpText(), [
+                "src/UI/InputElement/Validators.ts",
+            ])
+            this.writeMarkdownFile("./Docs/Studio/Tags_format.md", TagUtils.generateDocs(), [
+                "src/Logic/Tags/TagUtils.ts",
+            ])
+
+            this.writeMarkdownFile(
+                "./Docs/Studio/SpecialRenderings.md",
+                SpecialVisualizations.HelpMessage(),
+                ["src/UI/SpecialVisualizations.ts"],
+                {
+                    tocMaxDepth: 3,
+                },
+            )
+            this.writeMarkdownFile(
+                "./Docs/Studio/CalculatedTags.md",
+                ["# Metatags", SimpleMetaTaggers.HelpText(), ExtraFunctions.HelpText()].join("\n"),
+                ["src/Logic/SimpleMetaTagger.ts", "src/Logic/ExtraFunctions.ts"],
+                { noTableOfContents: false },
+            )
+        }
+
+
+
+        // For dev
+        {
+            this.generateQueryParameterDocs()
+            await this.generateSourcesOverview()
+        }
+
+        // For layer+theme overview
+        {
+            if (!existsSync("./Docs/Themes")) {
+                mkdirSync("./Docs/Themes")
+            }
+            this.generateOverviewsForAllSingleLayer()
+            this.generateNormalLayerOverview()
+            Array.from(AllKnownLayouts.allKnownLayouts.values()).map((theme) => {
+                this.generateForTheme(theme)
+                // this.generateForTheme(theme, { path: "./Docs/themes_nl", lang: "nl" })
+                ScriptUtils.erasableLog("Written docs for theme", theme.id)
+            })
+            /*
+            if (!existsSync("./Docs/themes_nl")) {
+                mkdirSync("./Docs/themes_nl")
             }
-        )
 
-        if (!existsSync("./Docs/Themes")) {
-            mkdirSync("./Docs/Themes")
+             this.generateOverviewsForAllSingleLayer("./Docs/layers_nl", "nl")
+            //*/
         }
 
-        this.writeMarkdownFile("./Docs/Tags_format.md", TagUtils.generateDocs(), [
-            "src/Logic/Tags/TagUtils.ts",
-        ])
-
-        new ToSlideshowJson(
-            "./Docs/Studio/Introduction.md",
-            "./src/assets/studio_introduction.json"
-        ).convert()
-        new ToSlideshowJson(
-            "./Docs/Studio/TagRenderingIntro.md",
-            "./src/assets/studio_tagrenderings_intro.json"
-        ).convert()
-
-        this.generateHotkeyDocs()
-        this.generateBuiltinIndex()
-        this.generateQueryParameterDocs()
-        this.generateBuiltinQuestions()
-        this.generateOverviewsForAllSingleLayer()
-        // this.generateOverviewsForAllSingleLayer("./Docs/layers_nl", "nl")
-
-        this.generateLayerOverviewText()
-        this.generateBuiltinUnits()
-        await this.generateSourcesOverview()
-
-        if (!existsSync("./Docs/themes_nl")) {
-            mkdirSync("./Docs/themes_nl")
-        }
-        Array.from(AllKnownLayouts.allKnownLayouts.values()).map((theme) => {
-            this.generateForTheme(theme)
-            // this.generateForTheme(theme, { path: "./Docs/themes_nl", lang: "nl" })
-
-            ScriptUtils.erasableLog("Written docs for theme", theme.id)
-        })
-
-        this.writeMarkdownFile(
-            "./Docs/CalculatedTags.md",
-            ["# Metatags", SimpleMetaTaggers.HelpText(), ExtraFunctions.HelpText()].join("\n"),
-            ["src/Logic/SimpleMetaTagger.ts", "src/Logic/ExtraFunctions.ts"],
-            { noTableOfContents: false }
-        )
-        this.writeMarkdownFile("./Docs/SpecialInputElements.md", Validators.HelpText(), [
-            "src/UI/InputElement/Validators.ts",
-        ])
-
         this.writeMarkdownFile("./Docs/ChangesetMeta.md", Changes.getDocs(), [
             "src/Logic/Osm/Changes.ts",
             "src/Logic/Osm/ChangesetHandler.ts",
         ])
-        const eli = AvailableRasterLayers.editorLayerIndex()
-        this.writeMarkdownFile(
-            "./Docs/ELI-overview.md",
-            [
-                "# Layers in the Editor Layer Index",
-                "This table gives a summary of ids, names and other metainformation. [See the online, interactive map here](https://osmlab.github.io/editor-layer-index/) or [visit the repository](https://github.com/osmlab/editor-layer-index)",
-                MarkdownUtils.table(
-                    ["id", "name", "category", "Best", "attribution"],
-                    eli.map((f) => [
-                        f.properties.id,
-                        f.properties.name,
-                        f.properties.category,
-                        f.properties.best ? "⭐" : "",
-                        f.properties.attribution?.html ?? f.properties.attribution?.text,
-                    ])
-                ),
-            ].join("\n\n"),
-            ["./public/assets/data/editor-layer-index.json"]
-        )
 
         new WikiPageGenerator().generate()
 
+        this.generateSidebar() // Must be last as it inspects the generated markdown files
         console.log("Generated docs")
     }
 
@@ -234,7 +226,8 @@ export class GenerateDocs extends Script {
         options?: {
             noTableOfContents?: boolean
             tocMaxDepth?: number
-            lang?: string
+            lang?: string,
+            noWarn?: boolean
         }
     ): void {
         const lang = options?.lang ?? "en"
@@ -258,12 +251,13 @@ export class GenerateDocs extends Script {
         }
 
         md = md.replace(/\n\n\n+/g, "\n\n")
+        md = md.replace("<script", "&lt;script")
 
         if (!md.endsWith("\n")) {
             md += "\n"
         }
 
-        const warnAutomated =
+        const warnAutomated = options?.noWarn ? "" :
             "[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)\n\n"
 
 
@@ -277,18 +271,32 @@ export class GenerateDocs extends Script {
             }).Subs({ sources }).textFor(lang)
 
 
-
-        writeFileSync(filename, warnAutomated + md + "\n\n" + generatedFrom + "\n")
+        writeFileSync(filename, warnAutomated + md + (options?.noWarn ? "" : "\n\n" + generatedFrom + "\n"))
     }
 
-    private generateHotkeyDocs() {
-        new ThemeViewState(
-            new ThemeConfig(<ThemeConfigJson>(<unknown>bookcases)),
-            new ImmutableStore(new Set())
+
+    private generateEliDocs() {
+        const eli = AvailableRasterLayers.editorLayerIndex()
+        this.writeMarkdownFile(
+            "./Docs/Studio/ELI-overview.md",
+            [
+                "# Layers in the Editor Layer Index",
+                "This table gives a summary of ids, names and other metainformation of background imagery that is available in MapComplete and that can be used as (default) map background." +
+                "These are sourced from [the Editor Layer Index](https://github.com/osmlab/editor-layer-index)", +
+                "\n[See the online, interactive map here](https://osmlab.github.io/editor-layer-index/)",
+                MarkdownUtils.table(
+                    ["id", "name", "category", "Best", "attribution"],
+                    eli.map((f) => [
+                        f.properties.id,
+                        f.properties.name,
+                        f.properties.category,
+                        f.properties.best ? "⭐" : "",
+                        f.properties.attribution?.html ?? f.properties.attribution?.text,
+                    ]),
+                ),
+            ].join("\n\n"),
+            ["./public/assets/data/editor-layer-index.json"],
         )
-        this.writeMarkdownFile("./Docs/Hotkeys.md", Hotkeys.generateDocumentation(), [
-            "src/UI/Base/Hotkeys.ts",
-        ])
     }
 
     private generateBuiltinUnits() {
@@ -336,7 +344,7 @@ export class GenerateDocs extends Script {
             }
         }
 
-        this.writeMarkdownFile("./Docs/builtin_units.md", els.join("\n\n"), [
+        this.writeMarkdownFile("./Docs/Studio/builtin_units.md", els.join("\n\n"), [
             `assets/layers/unit/unit.json`,
             `src/Models/ThemeConfig/Json/UnitConfigJson.ts`,
         ])
@@ -469,19 +477,18 @@ export class GenerateDocs extends Script {
             builtinsPerLayer.set(layer.id, usedBuiltins)
         }
 
-        let docs = `
-            # Index of builtin TagRenderings
-            ## Existing builtin tagrenderings
-        `
+        let docs: string[] = [
+            "# Which tagrendering is used where?", "",
+            "This document details where a tagRendering from one layer is reused in another layer, either by directly using it or by using a `{\"builtin\": id, \"override\": ...}` syntax",
+            "Having this overview supports e.g. refactoring efforts",
+            "## Existing builtin tagrenderings", ""]
 
         for (const [builtin, usedByLayers] of Array.from(layersUsingBuiltin.entries())) {
-            docs += `
-            ### ${builtin}
-
-            ${usedByLayers.map((item) => " - " + item).join("\n")}
-            `
+            docs.push(`### ${builtin}\n`)
+            docs.push(usedByLayers.length + " usages")
+            docs.push(`${usedByLayers.map((item) => ` - [${item}](./Docs/Layers/${item}.md)`).join("\n")}`)
         }
-        this.writeMarkdownFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"])
+        this.writeMarkdownFile("./Docs/Studio/TagRendering_reuse_overview.md", docs.join("\n"), ["assets/layers/*.json"])
     }
 
     private generateQueryParameterDocs() {
@@ -495,7 +502,7 @@ export class GenerateDocs extends Script {
         )
 
         this.writeMarkdownFile(
-            "./Docs/URL_Parameters.md",
+            "./Docs/Dev/URL_Parameters.md",
             QueryParameterDocumentation.GenerateQueryParameterDocs(),
             ["src/Logic/Web/QueryParameters.ts", "src/UI/QueryParameterDocumentation.ts"]
         )
@@ -512,7 +519,7 @@ export class GenerateDocs extends Script {
             Array.from(AllSharedLayers.sharedLayers.values())
         )
         const docs = qLayer.generateDocumentation({ reusedTagRenderings })
-        this.writeMarkdownFile("./Docs/BuiltinQuestions.md", docs, [
+        this.writeMarkdownFile("./Docs/Studio/BuiltinQuestions.md", docs, [
             "assets/layers/questions/questions.json",
         ])
     }
@@ -706,22 +713,112 @@ export class GenerateDocs extends Script {
         md.push(MarkdownUtils.list(urls))
 
         this.writeMarkdownFile(
-            "./Docs/OnlineServicesOverview.md",
+            "./Docs/Dev/OnlineServicesOverview.md",
             md.join("\n\n"),
             ["src/Models/SourceOverview.ts"],
             { tocMaxDepth: 2 }
         )
     }
 
+    /**
+     * Generates the '_sidebar.md' file that is used by docsify
+     */
+    private generateSidebar() {
+        const sidebar: string[] = [
+            "<a href='https://mapcomplete.org' class='back-to-mc'>Back to MapComplete</a>",
+        ]
+        const allFiles = ScriptUtils.readDirRecSync("./Docs")
+            .filter(path => path.endsWith(".md"))
+            .filter(path => !path.startsWith("_"))
+            .map(path => path.substring("./Docs/".length))
+
+        const perDirectory = new Map<string, string[]>()
+
+        function addFile(dir: string, path: string) {
+            const list = perDirectory.get(dir)
+            if (!list) {
+                perDirectory.set(dir, [path])
+            } else {
+                list.push(path)
+            }
+        }
+
+        for (const file of allFiles) {
+            const perDir = file.split("/")
+            if (perDir.length === 1) {
+                addFile("", perDir[0])
+            } else if (perDir.length === 2) {
+                const [dir, path] = perDir
+                addFile(dir, path)
+            } else {
+                const dir = perDir.shift()
+                const path = perDir.join("/")
+                addFile(dir, path)
+            }
+        }
+
+        // The directories to run over:
+        const directories = [
+            ["", ""],
+            ["Studio", "For theme builders"],
+            ["Dev", "For developers"],
+            ["Layers", "Overview of layers"], ["Themes", "Overview of map themes"],
+            ["UserTests", "Usability tests with users"]]
+
+
+        for (const [dir, title] of directories) {
+            const files = perDirectory.get(dir)
+            if (!files) {
+                console.error("No directory for " + dir)
+                continue
+            }
+            if (dir !== "") {
+                sidebar.push(`\n\n [**${title}**](${dir}/README.md)`)
+            }
+            for (const path of files) {
+                if (path.startsWith("_") || path.endsWith("README.md")) {
+                    continue
+                }
+                const shown = path.substring(0, path.length - 3).replaceAll("_", " ")
+                if (dir === "") {
+                    sidebar.push(`- [${shown}](${encodeURIComponent(path)})`)
+                } else {
+                    sidebar.push(`   + [${shown}](${encodeURIComponent(dir + "/" + path)})`)
+                }
+            }
+        }
+
+        this.writeMarkdownFile("./Docs/_sidebar.md", sidebar.join("\n"), [], {
+            noTableOfContents: true,
+            noWarn: true,
+        })
+    }
+
+    private generateNormalLayerOverview() {
+        const doc = ["# Layers",
+            `The following layers are available in MapComplete ${Constants.vNumber}:`,
+            MarkdownUtils.list(
+                Array.from(AllSharedLayers.sharedLayers.keys()).map(
+                    (id) => `[${id}](./Layers/${id}.md)`,
+                ),
+            )]
+        this.writeMarkdownFile("./Docs/Layers/README.md", doc.join("\n\n"), ["./assets/layers/*.json"])
+    }
+
     /**
      * Generates the documentation for the layers overview page
      * @constructor
      */
-    private generateLayerOverviewText(): void {
+    private generateSpecialLayerOverviewText(): void {
+        console.log("Generating the special layers overview")
+
         for (const id of Constants.priviliged_layers) {
+            if (id === "favourite") {
+                continue
+            }
             if (!AllSharedLayers.sharedLayers.has(id)) {
-                console.error("Priviliged layer definition not found: " + id)
-                return undefined
+                throw ("Privileged layer definition not found: " + id)
+
             }
         }
 
@@ -760,9 +857,10 @@ export class GenerateDocs extends Script {
         }
 
         const el = [
-            "# Special and other useful layers",
-            "MapComplete has a few data layers available in the theme which have special properties through builtin-hooks. Furthermore, there are some normal layers (which are built from normal Theme-config files) but are so general that they get a mention here.",
-            "# Priviliged layers",
+            "# Special and priviliged layers", "",
+            "MapComplete has a few data layers available which have special properties through builtin-hooks.",
+            "They perform various tasks, such as showing the GPS-location and track on the screen or help in special elements such as the 'cut way'-element.",
+            "As a theme builder, you can influence the behaviour of those layers by overriding them in your .json file",
             MarkdownUtils.list(
                 Constants.priviliged_layers.map((id) => "[" + id + "](#" + id + ")")
             ),
@@ -775,17 +873,10 @@ export class GenerateDocs extends Script {
                     dependencies: DependencyCalculator.getLayerDependencies(l),
                     addedByDefault: Constants.added_by_default.indexOf(<any>l.id) >= 0,
                     canBeIncluded: Constants.no_include.indexOf(<any>l.id) < 0,
-                })
+                }) + "\n"
             ),
-            "# Normal layers",
-            "The following layers are included in MapComplete:",
-            MarkdownUtils.list(
-                Array.from(AllSharedLayers.sharedLayers.keys()).map(
-                    (id) => `[${id}](./Layers/${id}.md)`
-                )
-            ),
-        ].join("\n\n")
-        this.writeMarkdownFile("./Docs/BuiltinLayers.md", el, [
+        ].join("\n")
+        this.writeMarkdownFile("./Docs/Studio/SpecialLayers.md", el, [
             "src/Customizations/AllKnownLayouts.ts",
         ])
     }