1
0
Fork 0

separate pile of success markers into own folder

This commit is contained in:
Mateusz Konieczny 2025-03-22 08:53:19 +01:00
parent 3dc274ce1e
commit 75f62c66e3

View file

@ -75,6 +75,8 @@ def run_graticule_cover(area):
os.makedirs(graticule_cache(area))
if os.path.isdir(config.output_folder()) == False:
os.makedirs(config.output_folder())
if os.path.isdir(cache_for_specific_spider_part_success_markers(area)) == False:
os.makedirs(cache_for_specific_spider_part_success_markers(area))
for lat_anchor in range(area['min_lat'], area['max_lat']):
for lon_anchor in range(area['min_lon'], area['max_lon']):
if os.path.isdir(specific_graticule_cache_for_atp_osm_input(area, lat=lat_anchor, lon=lon_anchor)) == False:
@ -202,8 +204,11 @@ def split_osm_data_across_graticules(graticule_coverage):
offset_lon = 179
add_entry_to_graticule_file(entry, 'osm', offset_lat, offset_lon, graticule_coverage)
def graticule_data_specific_spider_subpart_success_marker_filepath(area, atp_code):
return graticule_cache(area) + "graticule_data_built_subpart_atp_spider_" + atp_code + ".success"
def cache_for_specific_spider_part_success_markers(area):
return graticule_cache(area) + "graticule_data_built_subpart_atp_spiders" + "/"
def spider_subpart_success_marker_filepath(area, atp_code):
return cache_for_specific_spider_part_success_markers(area) + atp_code + ".success"
def delete_merged_graticule_files_for_atp_data_if_any(area):
# removes debris, if merging was interrupted
@ -228,7 +233,7 @@ def graticule_data_subpart_atp_data_merge_success_marker_filepath(area):
def prepare_atp_graticule_files(graticule_coverage):
atp_codes = list(data_iterator.all_spider_codes_iterator())
for index, atp_code in enumerate(atp_codes):
spider_data_split_success_marker_filepath = graticule_data_specific_spider_subpart_success_marker_filepath(graticule_coverage, atp_code)
spider_data_split_success_marker_filepath = spider_subpart_success_marker_filepath(graticule_coverage, atp_code)
if os.path.isfile(spider_data_split_success_marker_filepath):
print(atp_code, "is done already, no need to do anything here")
continue