more useful log, closer to intended deployable code

view_data_across_atp_datasets.py

@@ -5,6 +5,7 @@ import os
 import json
 import compute_local_time
 import datetime
+from collections import defaultdict
 
 config = __import__("0_config")
 obtain_atp_data = __import__("2_obtain_atp_data")
@@ -13,7 +14,6 @@ obtain_atp_data = __import__("2_obtain_atp_data")
 def download():
     for entry in obtain_atp_data.get_atp_history()[::-1]:
         run_id = entry['run_id']
-        print(run_id)
         obtain_atp_data.download_specific_atp_run(run_id)
 
 
@@ -35,6 +35,7 @@ def view():
     # for zabka see
     # https://github.com/alltheplaces/alltheplaces/issues/5888 https://github.com/alltheplaces/alltheplaces/pull/5586
     atp_code = "zabka_pl"
+    collected_for_given_weekday = defaultdict(list)
     for entry in obtain_atp_data.get_atp_history()[::-1]:
         run_id = entry['run_id']
         run_metadata = obtain_atp_data.get_atp_specific_spider_run_metadata(run_id, atp_code)
@@ -42,26 +43,19 @@ def view():
         print(run_id)
         folder = obtain_atp_data.atp_unpacked_folder(run_id)
         source_atp_filepath = folder + atp_code + ".geojson"
+        if os.path.isfile(source_atp_filepath) == False:
+            print("file", source_atp_filepath, "is missing")
+            continue
         with open(source_atp_filepath) as file:
             try:
                 data = json.load(file)
                 rich.print("spider:collection_time", data['dataset_attributes'].get("spider:collection_time"))
                 for feature in data['features']:
                     if relevant_one(feature):
-                        # rich.print(feature['properties']['addr:city'])
+                        print()
                         opening_hours = feature['properties']['opening_hours']
-                        rich.print(opening_hours)
-                        oh = parser.OpeningHours(opening_hours)
-                        oh.display()
-                        for day in parser.OpeningHours.DAYS_OF_WEEK:
-                            stringified = []
-                            for time_selector in oh.parsed[day]:
-                                stringified.append(str(time_selector)) # .from_hours, .from_minutes, .to_hours, .to_minutes
-                            reduced_to_canonical_string = ",".join(stringified)
-                            print(day, reduced_to_canonical_string)
+                        print(opening_hours)
 
-                        # rich.print(feature)
-                        rich.print(feature['geometry']['coordinates'])
                         longitude = feature['geometry']['coordinates'][0]
                         latitude = feature['geometry']['coordinates'][1]
 
@@ -71,10 +65,23 @@ def view():
                             print('in local t', start_time)
                             print('in local t', start_time.date())
                             weekday_index = start_time.date().weekday()
+
+                            oh = parser.OpeningHours(opening_hours)
+                            oh.display()
+
                             for offset in range(7):
                                 date = start_time.date() + datetime.timedelta(days=offset)
                                 day_code = parser.OpeningHours.day_of_week_code(weekday_index + offset)
-                                print(offset, date, day_code)
+
+                                stringified = []
+                                for time_selector in oh.parsed[day_code]:
+                                    stringified.append(str(time_selector)) # .from_hours, .from_minutes, .to_hours, .to_minutes
+                                reduced_to_canonical_string = ",".join(stringified)
+
+                                print(offset, date, day_code, reduced_to_canonical_string)
+
+                                collected_for_given_weekday[day_code].append({"opening_hours": reduced_to_canonical_string, "date": date})
+
                             print('finish_time', run_metadata['finish_time'].replace("T", " "))
                             finish_time = compute_local_time.convert_utc_timestamp_string_to_local(run_metadata['finish_time'], latitude, longitude)
                             print('in local ti', finish_time)
@@ -86,7 +93,7 @@ def view():
 
             except json.decoder.JSONDecodeError as e:
                 print(e)
-
+    rich.print(collected_for_given_weekday)
 # {"type": "Feature", "id": "y0P1sd7RSJB-6RBe-vTNFRUDZEg=", "properties": {"ref": "Z2220", "@source_uri": "https://www.zabka.pl/app/uploads/locator-store-data.json", "@spider": "zabka_pl", "name:pl": "\u017babka", "name:uk": "\u0416\u0430\u0431\u043a\u0430", "shop": "convenience", "addr:street_address": "ul. Miko\u0142ajczyka/os. Kalinowe 12c", "addr:city": "Krak\u00f3w", "addr:country": "PL", "name": "\u017babka", "website": "https://www.zabka.pl/znajdz-sklep/ID02220-krakow-ul-mikolajczyka-os-kalinowe-12c/", "opening_hours": "Mo-Sa 06:00-23:00", "brand": "\u017babka", "brand:wikidata": "Q2589061", "nsi_id": "zabka-f55ead"}, "geometry": {"type": "Point", "coordinates": [20.014811, 50.092896]}},