From 75f62c66e33a92f2c51b68ea0a21358387f08441 Mon Sep 17 00:00:00 2001
From: Mateusz Konieczny <matkoniecz@gmail.com>
Date: Sat, 22 Mar 2025 08:53:19 +0100
Subject: [PATCH] separate pile of success markers into own folder
---
5_generate_graticule_reports.py | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/5_generate_graticule_reports.py b/5_generate_graticule_reports.py
index f435a9f..aa57bbb 100644
--- a/5_generate_graticule_reports.py
+++ b/5_generate_graticule_reports.py
@@ -75,6 +75,8 @@ def run_graticule_cover(area):
os.makedirs(graticule_cache(area))
if os.path.isdir(config.output_folder()) == False:
os.makedirs(config.output_folder())
+ if os.path.isdir(cache_for_specific_spider_part_success_markers(area)) == False:
+ os.makedirs(cache_for_specific_spider_part_success_markers(area))
for lat_anchor in range(area['min_lat'], area['max_lat']):
for lon_anchor in range(area['min_lon'], area['max_lon']):
if os.path.isdir(specific_graticule_cache_for_atp_osm_input(area, lat=lat_anchor, lon=lon_anchor)) == False:
@@ -202,8 +204,11 @@ def split_osm_data_across_graticules(graticule_coverage):
offset_lon = 179
add_entry_to_graticule_file(entry, 'osm', offset_lat, offset_lon, graticule_coverage)
-def graticule_data_specific_spider_subpart_success_marker_filepath(area, atp_code):
- return graticule_cache(area) + "graticule_data_built_subpart_atp_spider_" + atp_code + ".success"
+def cache_for_specific_spider_part_success_markers(area):
+ return graticule_cache(area) + "graticule_data_built_subpart_atp_spiders" + "/"
+
+def spider_subpart_success_marker_filepath(area, atp_code):
+ return cache_for_specific_spider_part_success_markers(area) + atp_code + ".success"
def delete_merged_graticule_files_for_atp_data_if_any(area):
# removes debris, if merging was interrupted
@@ -228,7 +233,7 @@ def graticule_data_subpart_atp_data_merge_success_marker_filepath(area):
def prepare_atp_graticule_files(graticule_coverage):
atp_codes = list(data_iterator.all_spider_codes_iterator())
for index, atp_code in enumerate(atp_codes):
- spider_data_split_success_marker_filepath = graticule_data_specific_spider_subpart_success_marker_filepath(graticule_coverage, atp_code)
+ spider_data_split_success_marker_filepath = spider_subpart_success_marker_filepath(graticule_coverage, atp_code)
if os.path.isfile(spider_data_split_success_marker_filepath):
print(atp_code, "is done already, no need to do anything here")
continue