Robin-van-der-Linde/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

More work on statistics

Browse source
This commit is contained in:
pietervdvn 2022-08-18 23:37:44 +02:00
parent 56b1337743
commit 5ef9a57bb0
5 changed files with 325 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
Docs/Tools/GenerateSeries.ts
View file

@ -154,57 +154,7 @@ interface ChangeSetData {
}
}
const theme_remappings = {
"metamap": "maps",
"groen": "buurtnatuur",
"updaten van metadata met mapcomplete": "buurtnatuur",
"Toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur",
"wiki:mapcomplete/fritures": "fritures",
"wiki:MapComplete/Fritures": "fritures",
"lits": "lit",
"pomp": "cyclofix",
"wiki:user:joost_schouppe/campersite": "campersite",
"wiki-user-joost_schouppe-geveltuintjes": "geveltuintjes",
"wiki-user-joost_schouppe-campersite": "campersite",
"wiki-User-joost_schouppe-campersite": "campersite",
"wiki-User-joost_schouppe-geveltuintjes": "geveltuintjes",
"wiki:User:joost_schouppe/campersite": "campersite",
"arbres": "arbres_llefia",
"aed_brugge": "aed",
"https://llefia.org/arbres/mapcomplete.json": "arbres_llefia",
"https://llefia.org/arbres/mapcomplete1.json": "arbres_llefia",
"toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur",
"testing mapcomplete 0.0.0": "buurtnatuur",
"https://raw.githubusercontent.com/osmbe/play/master/mapcomplete/geveltuinen/geveltuinen.json": "geveltuintjes"
}
class ChangesetDataTools {
public static cleanChangesetData(cs: ChangeSetData): ChangeSetData {
if (cs.properties.metadata.theme === undefined) {
cs.properties.metadata.theme = cs.properties.comment.substr(cs.properties.comment.lastIndexOf("#") + 1)
}
cs.properties.metadata.theme = cs.properties.metadata.theme.toLowerCase()
const remapped = theme_remappings[cs.properties.metadata.theme]
cs.properties.metadata.theme = remapped ?? cs.properties.metadata.theme
if (cs.properties.metadata.theme.startsWith("https://raw.githubusercontent.com/")) {
cs.properties.metadata.theme = "gh://" + cs.properties.metadata.theme.substr("https://raw.githubusercontent.com/".length)
}
if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) {
cs.properties.metadata.theme = "EMPTY CS"
}
try {
cs.properties.metadata.host = new URL(cs.properties.metadata.host).host
} catch (e) {
}
if (cs.properties.metadata["answer"] > 100) {
console.log("Lots of answers for https://osm.org/changeset/" + cs.id)
}
return cs
}
}
interface PlotSpec {
name: string,
@ -827,8 +777,7 @@ async function main(): Promise<void> {
const allPaths = readdirSync(targetDir)
.filter(p => p.startsWith("stats.") && p.endsWith(".json"));
let allFeatures: ChangeSetData[] = [].concat(...allPaths
.map(path => JSON.parse(readFileSync("Docs/Tools/stats/" + path, "utf-8")).features
.map(cs => ChangesetDataTools.cleanChangesetData(cs))));
.map(path => JSON.parse(readFileSync("Docs/Tools/stats/" + path, "utf-8")).features));
allFeatures = allFeatures.filter(f => f.properties.editor === null || f.properties.editor.toLowerCase().startsWith("mapcomplete"))
const emptyCS = allFeatures.filter(f => f.properties.metadata.theme === "EMPTY CS")
@ -843,18 +792,16 @@ async function main(): Promise<void> {
const allFiles = readdirSync("Docs/Tools/stats").filter(p => p.endsWith(".json"))
writeFileSync("Docs/Tools/stats/file-overview.json", JSON.stringify(allFiles))
/*
await createMiscGraphs(allFeatures, emptyCS)
const grbOnly = allFeatures.filter(f => f.properties.metadata.theme === "grb")
allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "grb")
await createGraphs(allFeatures, "")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020")
await createGraphs(allFeatures, "")
/*await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022")
await createGraphs(allFeatures.filter(f => f.properties.metadata.theme === "toerisme_vlaanderen"), " met pin je punt", 0)
await createGraphs(grbOnly, " with the GRB import tool", 0)
*/
await createGraphs(grbOnly, " with the GRB import tool", 0)*/
}
main().then(_ => console.log("All done!"))

181
UI/BigComponents/TagRenderingChart.ts
View file

@ -1,10 +1,15 @@
import ChartJs from "../Base/ChartJs";
import {OsmFeature} from "../../Models/OsmFeature";
import TagRenderingConfig from "../../Models/ThemeConfig/TagRenderingConfig";
import {ChartConfiguration} from 'chart.js';
import Combine from "../Base/Combine";
import {TagUtils} from "../../Logic/Tags/TagUtils";
export interface TagRenderingChartOptions {
groupToOtherCutoff?: 3 | number,
sort?: boolean
}
export default class TagRenderingChart extends Combine {
private static readonly unkownColor = 'rgba(128, 128, 128, 0.2)'
@ -16,7 +21,7 @@ export default class TagRenderingChart extends Combine {
private static readonly notApplicableBorderColor = 'rgba(255, 0, 0)'
private static readonly backgroundColors = [
public static readonly backgroundColors = [
'rgba(255, 99, 132, 0.2)',
'rgba(54, 162, 235, 0.2)',
'rgba(255, 206, 86, 0.2)',
@ -37,90 +42,27 @@ export default class TagRenderingChart extends Combine {
/**
* Creates a chart about this tagRendering for the given data
*/
constructor(features: OsmFeature[], tagRendering: TagRenderingConfig, options?: {
chartclasses?: string,
constructor(features: { properties: Record<string, string> }[], tagRendering: TagRenderingConfig, options?: TagRenderingChartOptions & { chartclasses?: string,
chartstyle?: string,
includeTitle?: boolean,
groupToOtherCutoff?: 3 | number
}) {
const mappings = tagRendering.mappings ?? []
if (mappings.length === 0 && tagRendering.freeform?.key === undefined) {
chartType?: "pie" | "bar" | "doughnut" }) {
if (tagRendering.mappings?.length === 0 && tagRendering.freeform?.key === undefined) {
super([])
this.SetClass("hidden")
return;
}
let unknownCount = 0;
const categoryCounts = mappings.map(_ => 0)
const otherCounts: Record<string, number> = {}
let notApplicable = 0;
let barchartMode = tagRendering.multiAnswer;
for (const feature of features) {
const props = feature.properties
if (tagRendering.condition !== undefined && !tagRendering.condition.matchesProperties(props)) {
notApplicable++;
continue;
}
if (!tagRendering.IsKnown(props)) {
unknownCount++;
continue;
}
let foundMatchingMapping = false;
if (!tagRendering.multiAnswer) {
for (let i = 0; i < mappings.length; i++) {
const mapping = mappings[i];
if (mapping.if.matchesProperties(props)) {
categoryCounts[i]++
foundMatchingMapping = true
break;
}
}
} else {
for (let i = 0; i < mappings.length; i++) {
const mapping = mappings[i];
if (TagUtils.MatchesMultiAnswer( mapping.if, props)) {
categoryCounts[i]++
foundMatchingMapping = true
}
}
}
if (!foundMatchingMapping) {
if (tagRendering.freeform?.key !== undefined && props[tagRendering.freeform.key] !== undefined) {
const otherValue = props[tagRendering.freeform.key]
otherCounts[otherValue] = (otherCounts[otherValue] ?? 0) + 1
} else {
unknownCount++
}
}
}
if (unknownCount + notApplicable === features.length) {
const {labels, data} = TagRenderingChart.extractDataAndLabels(tagRendering, features, options)
if (labels === undefined || data === undefined) {
super([])
this.SetClass("hidden")
return
}
let otherGrouped = 0;
const otherLabels: string[] = []
const otherData : number[] = []
for (const v in otherCounts) {
const count = otherCounts[v]
if(count >= (options.groupToOtherCutoff ?? 3)){
otherLabels.push(v)
otherData.push(otherCounts[v])
}else{
otherGrouped++;
}
return
}
const labels = ["Unknown", "Other", "Not applicable", ...mappings?.map(m => m.then.txt) ?? [], ...otherLabels]
const data = [unknownCount, otherGrouped, notApplicable, ...categoryCounts, ... otherData]
const borderColor = [TagRenderingChart.unkownBorderColor, TagRenderingChart.otherBorderColor, TagRenderingChart.notApplicableBorderColor]
const backgroundColor = [TagRenderingChart.unkownColor, TagRenderingChart.otherColor, TagRenderingChart.notApplicableColor]
while (borderColor.length < data.length) {
borderColor.push(...TagRenderingChart.borderColors)
@ -128,24 +70,27 @@ export default class TagRenderingChart extends Combine {
}
for (let i = data.length; i >= 0; i--) {
if (data[i] === 0) {
if (data[i]?.length === 0) {
labels.splice(i, 1)
data.splice(i, 1)
borderColor.splice(i, 1)
backgroundColor.splice(i, 1)
}
}
if(labels.length > 9){
let barchartMode = tagRendering.multiAnswer;
if (labels.length > 9) {
barchartMode = true;
}
const config = <ChartConfiguration>{
type: barchartMode ? 'bar' : 'doughnut',
type: options.chartType ?? (barchartMode ? 'bar' : 'doughnut'),
data: {
labels,
datasets: [{
data,
data: data.map(l => l.length),
backgroundColor,
borderColor,
borderWidth: 1,
@ -169,11 +114,91 @@ export default class TagRenderingChart extends Combine {
super([
options?.includeTitle ? (tagRendering.question.Clone() ?? tagRendering.id) : undefined,
options?.includeTitle ? (tagRendering.question.Clone() ?? tagRendering.id) : undefined,
chart])
this.SetClass("block")
}
public static extractDataAndLabels<T extends {properties: Record<string, string>}>(tagRendering: TagRenderingConfig, features: T[], options?:TagRenderingChartOptions): {labels: string[], data: T[][]} {
const mappings = tagRendering.mappings ?? []
options = options ?? {}
let unknownCount : T[] = [];
const categoryCounts : T[][]= mappings.map(_ => [])
const otherCounts: Record<string, T[]> = {}
let notApplicable : T[] = [];
for (const feature of features) {
const props = feature.properties
if (tagRendering.condition !== undefined && !tagRendering.condition.matchesProperties(props)) {
notApplicable.push(feature);
continue;
}
if (!tagRendering.IsKnown(props)) {
unknownCount.push(feature);
continue;
}
let foundMatchingMapping = false;
if (!tagRendering.multiAnswer) {
for (let i = 0; i < mappings.length; i++) {
const mapping = mappings[i];
if (mapping.if.matchesProperties(props)) {
categoryCounts[i].push(feature)
foundMatchingMapping = true
break;
}
}
} else {
for (let i = 0; i < mappings.length; i++) {
const mapping = mappings[i];
if (TagUtils.MatchesMultiAnswer(mapping.if, props)) {
categoryCounts[i].push(feature)
foundMatchingMapping = true
}
}
}
if (!foundMatchingMapping) {
if (tagRendering.freeform?.key !== undefined && props[tagRendering.freeform.key] !== undefined) {
const otherValue = props[tagRendering.freeform.key]
otherCounts[otherValue] = (otherCounts[otherValue] ?? [])
otherCounts[otherValue] .push(feature)
} else {
unknownCount.push(feature)
}
}
}
if (unknownCount.length + notApplicable.length === features.length) {
console.log("Returning no label nor data: all features are unkown or notApplicable")
return {labels: undefined, data: undefined}
}
let otherGrouped : T[] = [];
const otherLabels: string[] = []
const otherData: T[][] = []
const sortedOtherCounts: [string, T[]][] = []
for (const v in otherCounts) {
sortedOtherCounts.push([v, otherCounts[v]]);
}
if (options?.sort) {
sortedOtherCounts.sort((a, b) => b[1].length - a[1].length)
}
for (const [v, count] of sortedOtherCounts) {
if (count.length >= (options.groupToOtherCutoff ?? 3)) {
otherLabels.push(v)
otherData.push(otherCounts[v])
} else {
otherGrouped.push(...count);
}
}
const labels = ["Unknown", "Other", "Not applicable", ...mappings?.map(m => m.then.txt) ?? [], ...otherLabels]
const data : T[][] = [unknownCount, otherGrouped, notApplicable, ...categoryCounts, ...otherData]
return {labels, data}
}
}

246
UI/StatisticsGUI.ts
View file

@ -7,61 +7,233 @@ import ChartJs from "./Base/ChartJs";
import Loading from "./Base/Loading";
import {Utils} from "../Utils";
import Combine from "./Base/Combine";
import BaseUIElement from "./BaseUIElement";
import TagRenderingChart from "./BigComponents/TagRenderingChart";
import TagRenderingConfig from "../Models/ThemeConfig/TagRenderingConfig";
import {ChartConfiguration} from "chart.js";
import {FixedUiElement} from "./Base/FixedUiElement";
export default class StatisticsGUI {
public static setup(): void{
private static readonly homeUrl = "https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/Tools/stats/"
private static readonly stats_files = "file-overview.json"
private readonly index = UIEventSource.FromPromise(Utils.downloadJson(StatisticsGUI.homeUrl + StatisticsGUI.stats_files))
new VariableUiElement(index.map(paths => {
public setup(): void {
new VariableUiElement(this.index.map(paths => {
if (paths === undefined) {
return new Loading("Loading overview...")
}
const downloaded = new UIEventSource<{ features: ChangeSetData[] }[]>([])
for (const filepath of paths) {
Utils.downloadJson(homeUrl + filepath).then(data => {
Utils.downloadJson(StatisticsGUI.homeUrl + filepath).then(data => {
data.features.forEach(item => {
item.properties = {...item.properties, ...item.properties.metadata}
delete item.properties.metadata
})
downloaded.data.push(data)
downloaded.ping()
})
}
return new VariableUiElement(downloaded.map(downloaded => {
const themeBreakdown = new Map<string, number>()
for (const feats of downloaded) {
console.log("Feats:", feats)
for (const feat of feats.features) {
const key = feat.properties.metadata.theme
const count = themeBreakdown.get(key) ?? 0
themeBreakdown.set(key, count + 1)
}
}
return new Combine([
new VariableUiElement(downloaded.map(dl => "Downloaded " + dl.length + " items")),
new VariableUiElement(downloaded.map(l => [...l]).stabilized(250).map(downloaded => {
const overview = ChangesetsOverview.fromDirtyData([].concat(...downloaded.map(d => d.features)))
.filter(cs => new Date(cs.properties.date) > new Date(2022,6,1))
// return overview.breakdownPerDay(overview.themeBreakdown)
return overview.breakdownPer(overview.themeBreakdown, "month")
})).SetClass("block w-full h-full")
]).SetClass("block w-full h-full")
const keys = Array.from(themeBreakdown.keys())
const values = keys.map( k => themeBreakdown.get(k))
console.log(keys, values)
return new Combine([
"Got " + downloaded.length + " files out of " + paths.length,
new ChartJs({
type: "pie",
data: {
datasets: [{data: values}],
labels: keys
}
}).SetClass("w-1/3 h-full")
]).SetClass("block w-full h-full")
})).SetClass("block w-full h-full")
})).SetClass("block w-full h-full").AttachTo("maindiv")
}
}
const homeUrl = "https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/Tools/stats/"
const stats_files = "file-overview.json"
const index = UIEventSource.FromPromise(Utils.downloadJson(homeUrl + stats_files))
class ChangesetsOverview {
private static readonly theme_remappings = {
"metamap": "maps",
"groen": "buurtnatuur",
"updaten van metadata met mapcomplete": "buurtnatuur",
"Toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur",
"wiki:mapcomplete/fritures": "fritures",
"wiki:MapComplete/Fritures": "fritures",
"lits": "lit",
"pomp": "cyclofix",
"wiki:user:joost_schouppe/campersite": "campersite",
"wiki-user-joost_schouppe-geveltuintjes": "geveltuintjes",
"wiki-user-joost_schouppe-campersite": "campersite",
"wiki-User-joost_schouppe-campersite": "campersite",
"wiki-User-joost_schouppe-geveltuintjes": "geveltuintjes",
"wiki:User:joost_schouppe/campersite": "campersite",
"arbres": "arbres_llefia",
"aed_brugge": "aed",
"https://llefia.org/arbres/mapcomplete.json": "arbres_llefia",
"https://llefia.org/arbres/mapcomplete1.json": "arbres_llefia",
"toevoegen of dit natuurreservaat toegangkelijk is": "buurtnatuur",
"testing mapcomplete 0.0.0": "buurtnatuur",
"entrances": "indoor",
"https://raw.githubusercontent.com/osmbe/play/master/mapcomplete/geveltuinen/geveltuinen.json": "geveltuintjes"
}
private readonly _meta: ChangeSetData[];
public static fromDirtyData(meta: ChangeSetData[]){
return new ChangesetsOverview(meta.map(cs => ChangesetsOverview.cleanChangesetData(cs)))
}
private constructor(meta: ChangeSetData[]) {
this._meta = meta;
}
public filter(predicate: (cs: ChangeSetData) => boolean) {
return new ChangesetsOverview(this._meta.filter(predicate))
}
private static cleanChangesetData(cs: ChangeSetData): ChangeSetData {
if (cs.properties.theme === undefined) {
cs.properties.theme = cs.properties.comment.substr(cs.properties.comment.lastIndexOf("#") + 1)
}
cs.properties.theme = cs.properties.theme.toLowerCase()
const remapped = ChangesetsOverview.theme_remappings[cs.properties.theme]
cs.properties.theme = remapped ?? cs.properties.theme
if (cs.properties.theme.startsWith("https://raw.githubusercontent.com/")) {
cs.properties.theme = "gh://" + cs.properties.theme.substr("https://raw.githubusercontent.com/".length)
}
if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) {
cs.properties.theme = "EMPTY CS"
}
try {
cs.properties.host = new URL(cs.properties.host).host
} catch (e) {
}
return cs
}
public themeBreakdown = new TagRenderingConfig({
id: "theme-breakdown",
question: "What theme was used?",
freeform: {
key: "theme"
},
render: "{theme}"
}, "statistics.themes")
public ThemeBreakdown(): BaseUIElement {
return new TagRenderingChart(
<any>this._meta,
this.themeBreakdown,
{
chartType: "doughnut",
sort: true,
groupToOtherCutoff: 25
}
)
}
public getAllDays(perMonth = false): string[] {
let earliest: Date = undefined
let latest: Date = undefined;
let allDates = new Set<string>();
this._meta.forEach((value, key) => {
const d = new Date(value.properties.date);
Utils.SetMidnight(d)
if(perMonth){
d.setUTCDate(1)
}
if (earliest === undefined) {
earliest = d
} else if (d < earliest) {
earliest = d
}
if (latest === undefined) {
latest = d
} else if (d > latest) {
latest = d
}
allDates.add(d.toISOString())
})
while (earliest < latest) {
earliest.setDate(earliest.getDate() + 1)
allDates.add(earliest.toISOString())
}
const days = Array.from(allDates)
days.sort()
return days
}
public breakdownPer(tr: TagRenderingConfig, period: "day" | "month" = "day" ): BaseUIElement {
const {labels, data} = TagRenderingChart.extractDataAndLabels(tr, <any>this._meta, {
sort: true
})
if (labels === undefined || data === undefined) {
return new FixedUiElement("No labels or data given...")
}
// labels: ["cyclofix", "buurtnatuur", ...]; data : [ ["cyclofix-changeset", "cyclofix-changeset", ...], ["buurtnatuur-cs", "buurtnatuur-cs"], ... ]
const datasets: { label: string /*themename*/, data: number[]/*counts per day*/, backgroundColor: string }[] = []
const allDays = this.getAllDays()
for (let i = 0; i < labels.length; i++) {
const label = labels[i];
const changesetsForTheme = <ChangeSetData[]><any>data[i]
const perDay: ChangeSetData[][] = []
for (const day of allDays) {
const today: ChangeSetData[] = []
for (const changeset of changesetsForTheme) {
const csDate = new Date(changeset.properties.date)
Utils.SetMidnight(csDate)
if(period === "month"){
csDate.setUTCDate(1)
}
if (csDate.toISOString() !== day) {
continue
}
today.push(changeset)
}
perDay.push(today)
}
datasets.push({
data: perDay.map(cs => cs.length),
backgroundColor: TagRenderingChart.backgroundColors[i % TagRenderingChart.backgroundColors.length],
label
})
}
const perDayData = {
labels: allDays.map(d => d.substr(0, d.indexOf("T"))),
datasets
}
const config = <ChartConfiguration>{
type: 'bar',
data: perDayData,
options: {
responsive: true,
scales: {
x: {
stacked: true,
},
y: {
stacked: true
}
}
}
}
return new ChartJs(config)
}
}
interface ChangeSetData {
"id": number,
@ -92,11 +264,9 @@ interface ChangeSetData {
"harmful": any,
"checked": boolean,
"check_date": any,
"metadata": {
"host": string,
"theme": string,
"imagery": string,
"language": string
}
"host": string,
"theme": string,
"imagery": string,
"language": string
}
}

7
Utils.ts
View file

@ -1039,5 +1039,12 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
}
return result
}
public static SetMidnight(d : Date): void{
d.setUTCHours(0)
d.setUTCSeconds(0)
d.setUTCMilliseconds(0)
d.setUTCMinutes(0)
}
}

3
test.ts
View file

@ -0,0 +1,3 @@
import StatisticsGUI from "./UI/StatisticsGUI";
new StatisticsGUI().setup()