Graph generation merges changesets which are close with each other, in order to mimic the modern behaviour

Docs/Tools/csvGrapher.py

@@ -1,5 +1,4 @@
 import csv
-import string
 from datetime import datetime
 
 from matplotlib import pyplot
@@ -7,6 +6,7 @@ import re
 
 useLegend = True
 
+
 def counts(lst):
     counts = {}
     for v in lst:
@@ -167,8 +167,9 @@ def create_contributors_per_total_cs(contents, extra_text = "", cutoff=25, per_d
     pyplot.title("Contributors per total number of changesets" + extra_text)
     pyplot.ylabel("Number of contributors")
     pyplot.xlabel("Mapping days with MapComplete" if per_day else "Number of changesets with MapComplete")
-    pyplot.savefig("Contributors per total number of "+("mapping days" if per_day else "changesets")+extra_text+".png", dpi=400)
-
+    pyplot.savefig(
+        "Contributors per total number of " + ("mapping days" if per_day else "changesets") + extra_text + ".png",
+        dpi=400)
 
 
 def create_theme_breakdown(stats, fileExtra="", cutoff=15):
@@ -203,6 +204,7 @@ def create_theme_breakdown(stats, fileExtra="", cutoff=15):
                    bbox_inches='tight')
     return themes
 
+
 def summed_changes_per(contents, extraText, sum_column=5):
     newPerDay = build_hist(contents, 0, 5)
     kv = newPerDay.flatten(sum)
@@ -226,6 +228,7 @@ def summed_changes_per(contents, extraText, sum_column=5):
         pyplot.legend()
     pyplot.savefig(text)
 
+
 def cumulative_changes_per(contents, index, subject, filenameextra="", cutoff=5, cumulative=True, sort=True):
     print("Creating graph about " + subject + filenameextra)
     themes = Hist("date")
@@ -315,7 +318,7 @@ def sortable_user_number(kv):
 
 
 def create_graphs(contents):
-    summed_changes_per(contents, "")
+    # summed_changes_per(contents, "")
     create_contributors_per_total_cs(contents)
     create_contributors_per_total_cs(contents, per_day=True)
 
@@ -345,8 +348,7 @@ def create_graphs(contents):
                                sort=sortable_user_number)
         cumulative_changes_per(contents_filtered, 4, "version number", extratext, cutoff=1, sort=sortable_user_number)
         cumulative_changes_per(contents_filtered, 8, "host", extratext, cutoff=1)
-        summed_changes_per(contents_filtered, "for year "+str(year))
-
+        # summed_changes_per(contents_filtered, "for year " + str(year))
 
 
 def create_per_theme_graphs(contents, cutoff=10):
@@ -359,10 +361,8 @@ def create_per_theme_graphs(contents, cutoff=10):
         contributors = set(map(lambda row: row[1], filtered))
         if len(contributors) >= 2:
             cumulative_changes_per(filtered, 1, "contributor", " for theme " + theme, cutoff=1)
-        if len(filtered) > 25:
-            summed_changes_per(filtered, "for theme "+theme)
-
-
+        # if len(filtered) > 25:
+        #    summed_changes_per(filtered, "for theme " + theme)
 
 
 def create_per_contributor_graphs(contents, least_needed_changesets):
@@ -375,8 +375,8 @@ def create_per_contributor_graphs(contents, least_needed_changesets):
         themes = set(map(lambda row: row[3], filtered))
         if len(themes) >= 2:
             cumulative_changes_per(filtered, 3, "theme", " for contributor " + contrib, cutoff=1)
-        if len(filtered) > 25:
-            summed_changes_per(filtered, "for contributor "+contrib)
+        # if len(filtered) > 25:
+        #     summed_changes_per(filtered, "for contributor " + contrib)
 
 
 theme_remappings = {
@@ -424,6 +424,32 @@ def clean_input(contents):
         yield row
 
 
+# Merges changesets of the same theme and the samecontributos within the same hour, so that the stats are comparable
+def mergeChangesets(contents):
+    open_changesets = dict()  # {contributor --> {theme --> hour of last change}}
+    for row in contents:
+        theme = row[3]
+        contributor = row[1]
+        date = datetime.strptime(row[0], "%Y-%m-%dT%H:%M:%SZ")
+        if (contributor not in open_changesets):
+            open_changesets[contributor] = dict()
+        perTheme = open_changesets[contributor]
+        if (theme in perTheme):
+            lastChange = perTheme[theme]
+            diff = (date - lastChange).total_seconds()
+            if(diff > 60*60):
+                yield row
+        else:
+            yield row
+        perTheme[theme] = date
+        
+
+# Removes the time from the date component    
+def datesOnly(contents):
+    for row in contents:
+        row[0] = row[0].split("T")[0]
+
+
 def contributor_count(stats, index=1, item="contributor"):
     seen_contributors = set()
     for line in stats:
@@ -434,10 +460,13 @@ def contributor_count(stats, index=1, item = "contributor"):
         seen_contributors.add(contributor)
         print(line)
 
+
 def main():
     print("Creating graphs...")
     with open('stats.csv', newline='') as csvfile:
         stats = list(clean_input(csv.reader(csvfile, delimiter=',', quotechar='"')))
+        stats = list(mergeChangesets(stats))
+        datesOnly(stats)
         print("Found " + str(len(stats)) + " changesets")
 
         #  contributor_count(stats, 3, "theme")