From 5ef9a57bb077858bd0df1eca08cf3f69780f99de Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Thu, 18 Aug 2022 23:37:44 +0200 Subject: [PATCH] More work on statistics --- Docs/Tools/GenerateSeries.ts | 61 +------ UI/BigComponents/TagRenderingChart.ts | 181 +++++++++++-------- UI/StatisticsGUI.ts | 246 ++++++++++++++++++++++---- Utils.ts | 7 + test.ts | 3 + 5 files changed, 325 insertions(+), 173 deletions(-) diff --git a/Docs/Tools/GenerateSeries.ts b/Docs/Tools/GenerateSeries.ts index 1b8a075023..07d4a8ab98 100644 --- a/Docs/Tools/GenerateSeries.ts +++ b/Docs/Tools/GenerateSeries.ts @@ -154,57 +154,7 @@ interface ChangeSetData { } } -const theme_remappings = { - "metamap": "maps", - "groen": "buurtnatuur", - "updaten van metadata met mapcomplete": "buurtnatuur", - "Toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur", - "wiki:mapcomplete/fritures": "fritures", - "wiki:MapComplete/Fritures": "fritures", - "lits": "lit", - "pomp": "cyclofix", - "wiki:user:joost_schouppe/campersite": "campersite", - "wiki-user-joost_schouppe-geveltuintjes": "geveltuintjes", - "wiki-user-joost_schouppe-campersite": "campersite", - "wiki-User-joost_schouppe-campersite": "campersite", - "wiki-User-joost_schouppe-geveltuintjes": "geveltuintjes", - "wiki:User:joost_schouppe/campersite": "campersite", - "arbres": "arbres_llefia", - "aed_brugge": "aed", - "https://llefia.org/arbres/mapcomplete.json": "arbres_llefia", - "https://llefia.org/arbres/mapcomplete1.json": "arbres_llefia", - "toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur", - "testing mapcomplete 0.0.0": "buurtnatuur", - "https://raw.githubusercontent.com/osmbe/play/master/mapcomplete/geveltuinen/geveltuinen.json": "geveltuintjes" -} -class ChangesetDataTools { - - public static cleanChangesetData(cs: ChangeSetData): ChangeSetData { - if (cs.properties.metadata.theme === undefined) { - cs.properties.metadata.theme = cs.properties.comment.substr(cs.properties.comment.lastIndexOf("#") + 1) - } - cs.properties.metadata.theme = cs.properties.metadata.theme.toLowerCase() - const remapped = theme_remappings[cs.properties.metadata.theme] - cs.properties.metadata.theme = remapped ?? cs.properties.metadata.theme - if (cs.properties.metadata.theme.startsWith("https://raw.githubusercontent.com/")) { - cs.properties.metadata.theme = "gh://" + cs.properties.metadata.theme.substr("https://raw.githubusercontent.com/".length) - } - if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) { - cs.properties.metadata.theme = "EMPTY CS" - } - try { - cs.properties.metadata.host = new URL(cs.properties.metadata.host).host - } catch (e) { - - } - if (cs.properties.metadata["answer"] > 100) { - console.log("Lots of answers for https://osm.org/changeset/" + cs.id) - } - return cs - } - -} interface PlotSpec { name: string, @@ -827,8 +777,7 @@ async function main(): Promise { const allPaths = readdirSync(targetDir) .filter(p => p.startsWith("stats.") && p.endsWith(".json")); let allFeatures: ChangeSetData[] = [].concat(...allPaths - .map(path => JSON.parse(readFileSync("Docs/Tools/stats/" + path, "utf-8")).features - .map(cs => ChangesetDataTools.cleanChangesetData(cs)))); + .map(path => JSON.parse(readFileSync("Docs/Tools/stats/" + path, "utf-8")).features)); allFeatures = allFeatures.filter(f => f.properties.editor === null || f.properties.editor.toLowerCase().startsWith("mapcomplete")) const emptyCS = allFeatures.filter(f => f.properties.metadata.theme === "EMPTY CS") @@ -843,18 +792,16 @@ async function main(): Promise { const allFiles = readdirSync("Docs/Tools/stats").filter(p => p.endsWith(".json")) writeFileSync("Docs/Tools/stats/file-overview.json", JSON.stringify(allFiles)) - /* await createMiscGraphs(allFeatures, emptyCS) const grbOnly = allFeatures.filter(f => f.properties.metadata.theme === "grb") allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "grb") - await createGraphs(allFeatures, "") - await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020") + await createGraphs(allFeatures, "") + /*await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020") await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021") await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022") await createGraphs(allFeatures.filter(f => f.properties.metadata.theme === "toerisme_vlaanderen"), " met pin je punt", 0) - await createGraphs(grbOnly, " with the GRB import tool", 0) -*/ + await createGraphs(grbOnly, " with the GRB import tool", 0)*/ } main().then(_ => console.log("All done!")) diff --git a/UI/BigComponents/TagRenderingChart.ts b/UI/BigComponents/TagRenderingChart.ts index f88cce188a..32aa49f0c3 100644 --- a/UI/BigComponents/TagRenderingChart.ts +++ b/UI/BigComponents/TagRenderingChart.ts @@ -1,10 +1,15 @@ import ChartJs from "../Base/ChartJs"; -import {OsmFeature} from "../../Models/OsmFeature"; import TagRenderingConfig from "../../Models/ThemeConfig/TagRenderingConfig"; import {ChartConfiguration} from 'chart.js'; import Combine from "../Base/Combine"; import {TagUtils} from "../../Logic/Tags/TagUtils"; +export interface TagRenderingChartOptions { + + groupToOtherCutoff?: 3 | number, + sort?: boolean +} + export default class TagRenderingChart extends Combine { private static readonly unkownColor = 'rgba(128, 128, 128, 0.2)' @@ -16,7 +21,7 @@ export default class TagRenderingChart extends Combine { private static readonly notApplicableBorderColor = 'rgba(255, 0, 0)' - private static readonly backgroundColors = [ + public static readonly backgroundColors = [ 'rgba(255, 99, 132, 0.2)', 'rgba(54, 162, 235, 0.2)', 'rgba(255, 206, 86, 0.2)', @@ -37,90 +42,27 @@ export default class TagRenderingChart extends Combine { /** * Creates a chart about this tagRendering for the given data */ - constructor(features: OsmFeature[], tagRendering: TagRenderingConfig, options?: { - chartclasses?: string, + constructor(features: { properties: Record }[], tagRendering: TagRenderingConfig, options?: TagRenderingChartOptions & { chartclasses?: string, chartstyle?: string, includeTitle?: boolean, - groupToOtherCutoff?: 3 | number - }) { - - const mappings = tagRendering.mappings ?? [] - if (mappings.length === 0 && tagRendering.freeform?.key === undefined) { + chartType?: "pie" | "bar" | "doughnut" }) { + if (tagRendering.mappings?.length === 0 && tagRendering.freeform?.key === undefined) { super([]) this.SetClass("hidden") return; } - let unknownCount = 0; - const categoryCounts = mappings.map(_ => 0) - const otherCounts: Record = {} - let notApplicable = 0; - let barchartMode = tagRendering.multiAnswer; - for (const feature of features) { - const props = feature.properties - if (tagRendering.condition !== undefined && !tagRendering.condition.matchesProperties(props)) { - notApplicable++; - continue; - } - if (!tagRendering.IsKnown(props)) { - unknownCount++; - continue; - } - let foundMatchingMapping = false; - if (!tagRendering.multiAnswer) { - for (let i = 0; i < mappings.length; i++) { - const mapping = mappings[i]; - if (mapping.if.matchesProperties(props)) { - categoryCounts[i]++ - foundMatchingMapping = true - break; - } - } - } else { - for (let i = 0; i < mappings.length; i++) { - const mapping = mappings[i]; - if (TagUtils.MatchesMultiAnswer( mapping.if, props)) { - categoryCounts[i]++ - foundMatchingMapping = true - } - } - } - if (!foundMatchingMapping) { - if (tagRendering.freeform?.key !== undefined && props[tagRendering.freeform.key] !== undefined) { - const otherValue = props[tagRendering.freeform.key] - otherCounts[otherValue] = (otherCounts[otherValue] ?? 0) + 1 - } else { - unknownCount++ - } - } - } - - if (unknownCount + notApplicable === features.length) { + const {labels, data} = TagRenderingChart.extractDataAndLabels(tagRendering, features, options) + if (labels === undefined || data === undefined) { super([]) this.SetClass("hidden") - return - } - - let otherGrouped = 0; - const otherLabels: string[] = [] - const otherData : number[] = [] - for (const v in otherCounts) { - const count = otherCounts[v] - if(count >= (options.groupToOtherCutoff ?? 3)){ - otherLabels.push(v) - otherData.push(otherCounts[v]) - }else{ - otherGrouped++; - } + return } - - const labels = ["Unknown", "Other", "Not applicable", ...mappings?.map(m => m.then.txt) ?? [], ...otherLabels] - const data = [unknownCount, otherGrouped, notApplicable, ...categoryCounts, ... otherData] + const borderColor = [TagRenderingChart.unkownBorderColor, TagRenderingChart.otherBorderColor, TagRenderingChart.notApplicableBorderColor] const backgroundColor = [TagRenderingChart.unkownColor, TagRenderingChart.otherColor, TagRenderingChart.notApplicableColor] - while (borderColor.length < data.length) { borderColor.push(...TagRenderingChart.borderColors) @@ -128,24 +70,27 @@ export default class TagRenderingChart extends Combine { } for (let i = data.length; i >= 0; i--) { - if (data[i] === 0) { + if (data[i]?.length === 0) { labels.splice(i, 1) data.splice(i, 1) borderColor.splice(i, 1) backgroundColor.splice(i, 1) } } - - if(labels.length > 9){ + + + + let barchartMode = tagRendering.multiAnswer; + if (labels.length > 9) { barchartMode = true; } const config = { - type: barchartMode ? 'bar' : 'doughnut', + type: options.chartType ?? (barchartMode ? 'bar' : 'doughnut'), data: { labels, datasets: [{ - data, + data: data.map(l => l.length), backgroundColor, borderColor, borderWidth: 1, @@ -169,11 +114,91 @@ export default class TagRenderingChart extends Combine { super([ - options?.includeTitle ? (tagRendering.question.Clone() ?? tagRendering.id) : undefined, + options?.includeTitle ? (tagRendering.question.Clone() ?? tagRendering.id) : undefined, chart]) this.SetClass("block") } + + public static extractDataAndLabels}>(tagRendering: TagRenderingConfig, features: T[], options?:TagRenderingChartOptions): {labels: string[], data: T[][]} { + const mappings = tagRendering.mappings ?? [] + options = options ?? {} + let unknownCount : T[] = []; + const categoryCounts : T[][]= mappings.map(_ => []) + const otherCounts: Record = {} + let notApplicable : T[] = []; + for (const feature of features) { + const props = feature.properties + if (tagRendering.condition !== undefined && !tagRendering.condition.matchesProperties(props)) { + notApplicable.push(feature); + continue; + } + + if (!tagRendering.IsKnown(props)) { + unknownCount.push(feature); + continue; + } + let foundMatchingMapping = false; + if (!tagRendering.multiAnswer) { + for (let i = 0; i < mappings.length; i++) { + const mapping = mappings[i]; + if (mapping.if.matchesProperties(props)) { + categoryCounts[i].push(feature) + foundMatchingMapping = true + break; + } + } + } else { + for (let i = 0; i < mappings.length; i++) { + const mapping = mappings[i]; + if (TagUtils.MatchesMultiAnswer(mapping.if, props)) { + categoryCounts[i].push(feature) + foundMatchingMapping = true + } + } + } + if (!foundMatchingMapping) { + if (tagRendering.freeform?.key !== undefined && props[tagRendering.freeform.key] !== undefined) { + const otherValue = props[tagRendering.freeform.key] + otherCounts[otherValue] = (otherCounts[otherValue] ?? []) + otherCounts[otherValue] .push(feature) + } else { + unknownCount.push(feature) + } + } + } + + if (unknownCount.length + notApplicable.length === features.length) { + console.log("Returning no label nor data: all features are unkown or notApplicable") + return {labels: undefined, data: undefined} + } + + let otherGrouped : T[] = []; + const otherLabels: string[] = [] + const otherData: T[][] = [] + const sortedOtherCounts: [string, T[]][] = [] + for (const v in otherCounts) { + sortedOtherCounts.push([v, otherCounts[v]]); + } + if (options?.sort) { + sortedOtherCounts.sort((a, b) => b[1].length - a[1].length) + } + for (const [v, count] of sortedOtherCounts) { + if (count.length >= (options.groupToOtherCutoff ?? 3)) { + otherLabels.push(v) + otherData.push(otherCounts[v]) + } else { + otherGrouped.push(...count); + } + } + + + const labels = ["Unknown", "Other", "Not applicable", ...mappings?.map(m => m.then.txt) ?? [], ...otherLabels] + const data : T[][] = [unknownCount, otherGrouped, notApplicable, ...categoryCounts, ...otherData] + + return {labels, data} + } + } \ No newline at end of file diff --git a/UI/StatisticsGUI.ts b/UI/StatisticsGUI.ts index fa454d50d2..36e22e1b55 100644 --- a/UI/StatisticsGUI.ts +++ b/UI/StatisticsGUI.ts @@ -7,61 +7,233 @@ import ChartJs from "./Base/ChartJs"; import Loading from "./Base/Loading"; import {Utils} from "../Utils"; import Combine from "./Base/Combine"; +import BaseUIElement from "./BaseUIElement"; +import TagRenderingChart from "./BigComponents/TagRenderingChart"; +import TagRenderingConfig from "../Models/ThemeConfig/TagRenderingConfig"; +import {ChartConfiguration} from "chart.js"; +import {FixedUiElement} from "./Base/FixedUiElement"; export default class StatisticsGUI { - - public static setup(): void{ + + private static readonly homeUrl = "https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/Tools/stats/" + private static readonly stats_files = "file-overview.json" + private readonly index = UIEventSource.FromPromise(Utils.downloadJson(StatisticsGUI.homeUrl + StatisticsGUI.stats_files)) - new VariableUiElement(index.map(paths => { + public setup(): void { + + + new VariableUiElement(this.index.map(paths => { if (paths === undefined) { return new Loading("Loading overview...") } const downloaded = new UIEventSource<{ features: ChangeSetData[] }[]>([]) for (const filepath of paths) { - Utils.downloadJson(homeUrl + filepath).then(data => { + Utils.downloadJson(StatisticsGUI.homeUrl + filepath).then(data => { + data.features.forEach(item => { + item.properties = {...item.properties, ...item.properties.metadata} + delete item.properties.metadata + }) downloaded.data.push(data) downloaded.ping() }) } - return new VariableUiElement(downloaded.map(downloaded => { - const themeBreakdown = new Map() - for (const feats of downloaded) { - console.log("Feats:", feats) - for (const feat of feats.features) { - const key = feat.properties.metadata.theme - const count = themeBreakdown.get(key) ?? 0 - themeBreakdown.set(key, count + 1) - } - } + return new Combine([ + new VariableUiElement(downloaded.map(dl => "Downloaded " + dl.length + " items")), + new VariableUiElement(downloaded.map(l => [...l]).stabilized(250).map(downloaded => { + const overview = ChangesetsOverview.fromDirtyData([].concat(...downloaded.map(d => d.features))) + .filter(cs => new Date(cs.properties.date) > new Date(2022,6,1)) + + // return overview.breakdownPerDay(overview.themeBreakdown) + return overview.breakdownPer(overview.themeBreakdown, "month") + })).SetClass("block w-full h-full") + ]).SetClass("block w-full h-full") - const keys = Array.from(themeBreakdown.keys()) - const values = keys.map( k => themeBreakdown.get(k)) - console.log(keys, values) - return new Combine([ - "Got " + downloaded.length + " files out of " + paths.length, - new ChartJs({ - type: "pie", - data: { - datasets: [{data: values}], - labels: keys - } - }).SetClass("w-1/3 h-full") - ]).SetClass("block w-full h-full") - })).SetClass("block w-full h-full") })).SetClass("block w-full h-full").AttachTo("maindiv") } - } -const homeUrl = "https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/Tools/stats/" -const stats_files = "file-overview.json" -const index = UIEventSource.FromPromise(Utils.downloadJson(homeUrl + stats_files)) +class ChangesetsOverview { + private static readonly theme_remappings = { + "metamap": "maps", + "groen": "buurtnatuur", + "updaten van metadata met mapcomplete": "buurtnatuur", + "Toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur", + "wiki:mapcomplete/fritures": "fritures", + "wiki:MapComplete/Fritures": "fritures", + "lits": "lit", + "pomp": "cyclofix", + "wiki:user:joost_schouppe/campersite": "campersite", + "wiki-user-joost_schouppe-geveltuintjes": "geveltuintjes", + "wiki-user-joost_schouppe-campersite": "campersite", + "wiki-User-joost_schouppe-campersite": "campersite", + "wiki-User-joost_schouppe-geveltuintjes": "geveltuintjes", + "wiki:User:joost_schouppe/campersite": "campersite", + "arbres": "arbres_llefia", + "aed_brugge": "aed", + "https://llefia.org/arbres/mapcomplete.json": "arbres_llefia", + "https://llefia.org/arbres/mapcomplete1.json": "arbres_llefia", + "toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur", + "testing mapcomplete 0.0.0": "buurtnatuur", + "entrances": "indoor", + "https://raw.githubusercontent.com/osmbe/play/master/mapcomplete/geveltuinen/geveltuinen.json": "geveltuintjes" + } + private readonly _meta: ChangeSetData[]; + + public static fromDirtyData(meta: ChangeSetData[]){ + return new ChangesetsOverview(meta.map(cs => ChangesetsOverview.cleanChangesetData(cs))) + } + + private constructor(meta: ChangeSetData[]) { + this._meta = meta; + } + + public filter(predicate: (cs: ChangeSetData) => boolean) { + return new ChangesetsOverview(this._meta.filter(predicate)) + } + + private static cleanChangesetData(cs: ChangeSetData): ChangeSetData { + if (cs.properties.theme === undefined) { + cs.properties.theme = cs.properties.comment.substr(cs.properties.comment.lastIndexOf("#") + 1) + } + cs.properties.theme = cs.properties.theme.toLowerCase() + const remapped = ChangesetsOverview.theme_remappings[cs.properties.theme] + cs.properties.theme = remapped ?? cs.properties.theme + if (cs.properties.theme.startsWith("https://raw.githubusercontent.com/")) { + cs.properties.theme = "gh://" + cs.properties.theme.substr("https://raw.githubusercontent.com/".length) + } + if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) { + cs.properties.theme = "EMPTY CS" + } + try { + cs.properties.host = new URL(cs.properties.host).host + } catch (e) { + + } + return cs + } + + public themeBreakdown = new TagRenderingConfig({ + id: "theme-breakdown", + question: "What theme was used?", + freeform: { + key: "theme" + }, + render: "{theme}" + }, "statistics.themes") + + public ThemeBreakdown(): BaseUIElement { + return new TagRenderingChart( + this._meta, + this.themeBreakdown, + { + chartType: "doughnut", + sort: true, + groupToOtherCutoff: 25 + } + ) + } + + public getAllDays(perMonth = false): string[] { + let earliest: Date = undefined + let latest: Date = undefined; + let allDates = new Set(); + this._meta.forEach((value, key) => { + const d = new Date(value.properties.date); + Utils.SetMidnight(d) + if(perMonth){ + d.setUTCDate(1) + } + if (earliest === undefined) { + earliest = d + } else if (d < earliest) { + earliest = d + } + if (latest === undefined) { + latest = d + } else if (d > latest) { + latest = d + } + allDates.add(d.toISOString()) + }) + + while (earliest < latest) { + earliest.setDate(earliest.getDate() + 1) + allDates.add(earliest.toISOString()) + } + const days = Array.from(allDates) + days.sort() + return days + } + + public breakdownPer(tr: TagRenderingConfig, period: "day" | "month" = "day" ): BaseUIElement { + const {labels, data} = TagRenderingChart.extractDataAndLabels(tr, this._meta, { + sort: true + }) + if (labels === undefined || data === undefined) { + return new FixedUiElement("No labels or data given...") + } + // labels: ["cyclofix", "buurtnatuur", ...]; data : [ ["cyclofix-changeset", "cyclofix-changeset", ...], ["buurtnatuur-cs", "buurtnatuur-cs"], ... ] + + const datasets: { label: string /*themename*/, data: number[]/*counts per day*/, backgroundColor: string }[] = [] + + const allDays = this.getAllDays() + for (let i = 0; i < labels.length; i++) { + const label = labels[i]; + const changesetsForTheme = data[i] + const perDay: ChangeSetData[][] = [] + for (const day of allDays) { + const today: ChangeSetData[] = [] + for (const changeset of changesetsForTheme) { + const csDate = new Date(changeset.properties.date) + Utils.SetMidnight(csDate) + if(period === "month"){ + csDate.setUTCDate(1) + } + if (csDate.toISOString() !== day) { + continue + } + today.push(changeset) + } + perDay.push(today) + } + datasets.push({ + data: perDay.map(cs => cs.length), + backgroundColor: TagRenderingChart.backgroundColors[i % TagRenderingChart.backgroundColors.length], + label + }) + } + + const perDayData = { + labels: allDays.map(d => d.substr(0, d.indexOf("T"))), + datasets + } + + const config = { + type: 'bar', + data: perDayData, + options: { + responsive: true, + scales: { + x: { + stacked: true, + }, + y: { + stacked: true + } + } + } + } + + return new ChartJs(config) + } + +} interface ChangeSetData { "id": number, @@ -92,11 +264,9 @@ interface ChangeSetData { "harmful": any, "checked": boolean, "check_date": any, - "metadata": { - "host": string, - "theme": string, - "imagery": string, - "language": string - } + "host": string, + "theme": string, + "imagery": string, + "language": string } } diff --git a/Utils.ts b/Utils.ts index f727cbad97..2f4b10eb59 100644 --- a/Utils.ts +++ b/Utils.ts @@ -1039,5 +1039,12 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be } return result } + + public static SetMidnight(d : Date): void{ + d.setUTCHours(0) + d.setUTCSeconds(0) + d.setUTCMilliseconds(0) + d.setUTCMinutes(0) + } } diff --git a/test.ts b/test.ts index e69de29bb2..1336dfbca8 100644 --- a/test.ts +++ b/test.ts @@ -0,0 +1,3 @@ +import StatisticsGUI from "./UI/StatisticsGUI"; + +new StatisticsGUI().setup() \ No newline at end of file