Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Small improvements to caching script

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-04-22 16:01:43 +02:00
parent 0dec1d0f75
commit b8f46f6b54
1 changed files with 32 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

39
scripts/generateCache.ts
View file

@ -16,6 +16,8 @@ import * as OsmToGeoJson from "osmtogeojson";
import {Script} from "vm"; import {Script} from "vm";
import MetaTagging from "../Logic/MetaTagging"; import MetaTagging from "../Logic/MetaTagging";
import State from "../State"; import State from "../State";
import {createEvalAwarePartialHost} from "ts-node/dist/repl";
import {fail} from "assert";
function createOverpassObject(theme: LayoutConfig) { function createOverpassObject(theme: LayoutConfig) {
let filters: TagsFilter[] = []; let filters: TagsFilter[] = [];
@ -48,14 +50,15 @@ function createOverpassObject(theme: LayoutConfig) {
return new Overpass(new Or(filters), extraScripts); return new Overpass(new Or(filters), extraScripts);
} }
function saveResponse(chunks: string[], targetDir: string) { function saveResponse(chunks: string[], targetDir: string): boolean {
const contents = chunks.join("") const contents = chunks.join("")
if (contents.startsWith("<?xml")) { if (contents.startsWith("<?xml")) {
// THis is an error message // THis is an error message
console.error("Failed to create ", targetDir, "probably over quota") console.error("Failed to create ", targetDir, "probably over quota: ", contents)
return; return false;
} }
writeFileSync(targetDir, contents) writeFileSync(targetDir, contents)
return true
} }
function rawJsonName(targetDir: string, x: number, y: number, z: number): string { function rawJsonName(targetDir: string, x: number, y: number, z: number): string {
@ -70,17 +73,20 @@ function metaJsonName(targetDir: string, x: number, y: number, z: number): strin
return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json" return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json"
} }
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) { async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/* : {failed: number, skipped :number} */ {
let downloaded = 0 let downloaded = 0
let failed = 0
let skipped = 0
for (let x = r.xstart; x <= r.xend; x++) { for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) { for (let y = r.ystart; y <= r.yend; y++) {
downloaded++; downloaded++;
const filename = rawJsonName(targetdir, x, y, r.zoomlevel) const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
if (existsSync(filename)) { if (existsSync(filename)) {
console.log("Already exists: ", filename) console.log("Already exists: ", filename)
skipped++
continue; continue;
} }
console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total) console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total, "failed: ", failed, "skipped: ", skipped)
const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y) const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y)
const bounds = { const bounds = {
@ -92,10 +98,11 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]") const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
let gotResponse = false let gotResponse = false
let success = false;
ScriptUtils.DownloadJSON(url, ScriptUtils.DownloadJSON(url,
chunks => { chunks => {
gotResponse = true; gotResponse = true;
saveResponse(chunks, filename) success = saveResponse(chunks, filename)
}) })
while (!gotResponse) { while (!gotResponse) {
@ -106,9 +113,20 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)
} }
} }
if(!success){
failed++;
console.log("Hit the rate limit - waiting 90s")
for (let i = 0; i < 90; i++) {
console.log(90 - i)
await ScriptUtils.sleep(1000)
}
}
} }
} }
return {failed: failed, skipped: skipped}
} }
async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) { async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) {
@ -189,8 +207,15 @@ async function main(args: string[]) {
const overpass = createOverpassObject(theme) const overpass = createOverpassObject(theme)
let failed = 0;
do {
const cachingResult = await downloadRaw(targetdir, tileRange, overpass)
failed = cachingResult.failed
if (failed > 0) {
ScriptUtils.sleep(30000)
}
} while (failed > 0)
await downloadRaw(targetdir, tileRange, overpass)
await postProcess(targetdir, tileRange, theme) await postProcess(targetdir, tileRange, theme)
} }