contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

FIx caching script, some cleanup

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-12-13 13:22:23 +01:00
parent e922768f99
commit 5b513b1248
3 changed files with 141 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

46
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -8,6 +8,9 @@ import GeoJsonSource from "../Sources/GeoJsonSource";
import {BBox} from "../../BBox";
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
private static whitelistCache = new Map<string, any>()
constructor(layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer & Tiled) => void,
state: {
@ -30,18 +33,26 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}", layer.layerDef.id)
Utils.downloadJsonCached(whitelistUrl, 1000*60*60).then(
json => {
const data = new Map<number, Set<number>>();
for (const x in json) {
data.set(Number(x), new Set(json[x]))
if (DynamicGeoJsonTileSource.whitelistCache.has(whitelistUrl)) {
whitelist = DynamicGeoJsonTileSource.whitelistCache.get(whitelistUrl)
} else {
Utils.downloadJsonCached(whitelistUrl, 1000 * 60 * 60).then(
json => {
const data = new Map<number, Set<number>>();
for (const x in json) {
if(x === "zoom"){
continue
}
data.set(Number(x), new Set(json[x]))
}
console.log("The whitelist is", data, "based on ", json, "from", whitelistUrl)
whitelist = data
DynamicGeoJsonTileSource.whitelistCache.set(whitelistUrl, whitelist)
}
console.log("The whitelist is", data, "based on ", json, "from", whitelistUrl)
whitelist = data
}
).catch(err => {
console.warn("No whitelist found for ", layer.layerDef.id, err)
})
).catch(err => {
console.warn("No whitelist found for ", layer.layerDef.id, err)
})
}
}
const seenIds = new Set<string>();
@ -53,7 +64,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
if (whitelist !== undefined) {
const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2])
if (!isWhiteListed) {
console.log("Not downloading tile", ...zxy, "as it is not on the whitelist")
console.debug("Not downloading tile", ...zxy, "as it is not on the whitelist")
return undefined;
}
}
@ -77,4 +88,15 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
}
public static RegisterWhitelist(url: string, json: any) {
const data = new Map<number, Set<number>>();
for (const x in json) {
if(x === "zoom"){
continue
}
data.set(Number(x), new Set(json[x]))
}
DynamicGeoJsonTileSource.whitelistCache.set(url, data)
}
}

196
UI/AutomatonGui.ts
View file

@ -26,40 +26,81 @@ import List from "./Base/List";
import {QueryParameters} from "../Logic/Web/QueryParameters";
import {SubstitutedTranslation} from "./SubstitutedTranslation";
import {AutoAction} from "./Popup/AutoApplyButton";
import DynamicGeoJsonTileSource from "../Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource";
class AutomatonGui extends Combine {
constructor() {
class AutomationPanel extends Combine{
const osmConnection = new OsmConnection({
allElements: undefined,
changes: undefined,
layoutName: "automaton",
singlePage: false,
oauth_token: QueryParameters.GetQueryParameter("oauth_token", "OAuth token")
});
constructor(layoutToUse: LayoutConfig, indices: number[], extraCommentText: UIEventSource<string>, tagRenderingToAutomate: { layer: LayerConfig, tagRendering: TagRenderingConfig }) {
const layerId = tagRenderingToAutomate.layer.id
const trId = tagRenderingToAutomate.tagRendering.id
const tileState = LocalStorageSource.GetParsed("automation-tile_state-" + layerId + "-" + trId, {})
const logMessages = new UIEventSource<string[]>([])
if (indices === undefined) {
throw ("No tiles loaded - can not automate")
}
super([
new Combine([Svg.robot_svg().SetClass("w-24 h-24 p-4 rounded-full subtle-background"),
new Combine([new Title("MapComplete Automaton", 1),
"This page helps to automate certain tasks for a theme. Expert use only."
]).SetClass("flex flex-col m-4")
]).SetClass("flex"),
new Toggle(
AutomatonGui.GenerateMainPanel(),
new SubtleButton(Svg.osm_logo_svg(), "Login to get started").onClick(() => osmConnection.AttemptLogin()),
osmConnection.isLoggedIn
)])
const nextTileToHandle = tileState.map(handledTiles => {
for (const index of indices) {
if (handledTiles[index] !== undefined) {
// Already handled
continue
}
return index
}
return undefined
})
nextTileToHandle.addCallback(t => console.warn("Next tile to handle is", t))
const neededTimes = new UIEventSource<number[]>([])
const automaton = new VariableUiElement(nextTileToHandle.map(tileIndex => {
if (tileIndex === undefined) {
return new FixedUiElement("All done!").SetClass("thanks")
}
console.warn("Triggered map on nextTileToHandle",tileIndex)
const start = new Date()
return AutomationPanel.TileHandler(layoutToUse, tileIndex, layerId, tagRenderingToAutomate.tagRendering, extraCommentText,(result, logMessage) => {
const end = new Date()
const timeNeeded = (end.getTime() - start.getTime()) / 1000;
neededTimes.data.push(timeNeeded)
neededTimes.ping()
tileState.data[tileIndex] = result
tileState.ping();
if(logMessage !== undefined){
logMessages.data.push(logMessage)
logMessages.ping();
}
});
}))
const statistics = new VariableUiElement(tileState.map(states => {
let total = 0
const perResult = new Map<string, number>()
for (const key in states) {
total++
const result = states[key]
perResult.set(result, (perResult.get(result) ?? 0) + 1)
}
let sum = 0
neededTimes.data.forEach(v => {
sum = sum + v
})
let timePerTile = sum / neededTimes.data.length
return new Combine(["Handled " + total + "/" + indices.length + " tiles: ",
new List(Array.from(perResult.keys()).map(key => key + ": " + perResult.get(key))),
"Handling one tile needs " + (Math.floor(timePerTile * 100) / 100) + "s on average. Estimated time left: " + Math.floor((indices.length - total) * timePerTile) + "s"
]).SetClass("flex flex-col")
}))
super([statistics, automaton, new VariableUiElement(logMessages.map(logMessages => new List(logMessages)))])
this.SetClass("flex flex-col")
}
private static startedTiles = new Set<number>()
private static TileHandler(layoutToUse: LayoutConfig, tileIndex: number, targetLayer: string, targetAction: TagRenderingConfig, extraCommentText: UIEventSource<string>, whenDone: ((result: string) => void)): BaseUIElement {
if (AutomatonGui.startedTiles.has(tileIndex)) {
throw "Already started tile " + tileIndex
}
AutomatonGui.startedTiles.add(tileIndex)
private static TileHandler(layoutToUse: LayoutConfig, tileIndex: number, targetLayer: string, targetAction: TagRenderingConfig, extraCommentText: UIEventSource<string>, whenDone: ((result: string, logMessage?: string) => void)): BaseUIElement {
const state = new MapState(layoutToUse, {attemptLogin: false})
extraCommentText.syncWith( state.changes.extraComment)
@ -107,6 +148,7 @@ class AutomatonGui extends Combine {
let handled = 0
let inspected = 0
let log = []
for (const targetTile of targetTiles.data) {
for (const ffs of targetTile.features.data) {
@ -115,7 +157,9 @@ class AutomatonGui extends Combine {
stateToShow.setData("Inspected " + inspected + " features, updated " + handled + " features")
}
const feature = ffs.feature
const rendering = targetAction.GetRenderValue(feature.properties).txt
const renderingTr = targetAction.GetRenderValue(feature.properties)
const rendering = renderingTr.txt
log.push("<a href='https://openstreetmap.org/"+feature.properties.id+"' target='_blank'>"+feature.properties.id+"</a>: "+new SubstitutedTranslation(renderingTr, new UIEventSource<any>(feature.properties)).ConstructElement().innerText)
const actions = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering)
.map(obj => obj.special))
for (const action of actions) {
@ -136,15 +180,16 @@ class AutomatonGui extends Combine {
if (inspected === 0) {
whenDone("empty")
return;
return true;
}
if (handled === 0) {
window.setTimeout(() => whenDone("no-action"), 1000)
whenDone("no-action","Inspected "+inspected+" elements: "+log.join("; "))
}else{
state.changes.flushChanges("handled tile automatically, time to flush!")
whenDone("fixed")
whenDone("fixed", "Updated " + handled+" elements, inspected "+inspected+": "+log.join("; "))
}
return true;
}, [targetTiles])
@ -153,70 +198,39 @@ class AutomatonGui extends Combine {
new VariableUiElement(stateToShow)]).SetClass("flex flex-col")
}
private static AutomationPanel(layoutToUse: LayoutConfig, indices: number[], extraCommentText: UIEventSource<string>, tagRenderingToAutomate: { layer: LayerConfig, tagRendering: TagRenderingConfig }): BaseUIElement {
const layerId = tagRenderingToAutomate.layer.id
const trId = tagRenderingToAutomate.tagRendering.id
const tileState = LocalStorageSource.GetParsed("automation-tile_state-" + layerId + "-" + trId, {})
if (indices === undefined) {
return new FixedUiElement("No tiles loaded - can not automate")
}
const nextTileToHandle = tileState.map(handledTiles => {
for (const index of indices) {
if (handledTiles[index] !== undefined) {
// Already handled
continue
}
return index
}
return undefined
})
nextTileToHandle.addCallback(t => console.warn("Next tile to handle is", t))
const neededTimes = new UIEventSource<number[]>([])
const automaton = new VariableUiElement(nextTileToHandle.map(tileIndex => {
if (tileIndex === undefined) {
return new FixedUiElement("All done!").SetClass("thanks")
}
console.warn("Triggered map on nextTileToHandle",tileIndex)
const start = new Date()
return AutomatonGui.TileHandler(layoutToUse, tileIndex, layerId, tagRenderingToAutomate.tagRendering, extraCommentText,(result) => {
const end = new Date()
const timeNeeded = (end.getTime() - start.getTime()) / 1000;
neededTimes.data.push(timeNeeded)
neededTimes.ping()
tileState.data[tileIndex] = result
tileState.ping();
});
}))
}
const statistics = new VariableUiElement(tileState.map(states => {
let total = 0
const perResult = new Map<string, number>()
for (const key in states) {
total++
const result = states[key]
perResult.set(result, (perResult.get(result) ?? 0) + 1)
}
let sum = 0
neededTimes.data.forEach(v => {
sum = sum + v
})
let timePerTile = sum / neededTimes.data.length
class AutomatonGui {
return new Combine(["Handled " + total + "/" + indices.length + " tiles: ",
new List(Array.from(perResult.keys()).map(key => key + ": " + perResult.get(key))),
"Handling one tile needs " + (Math.floor(timePerTile * 100) / 100) + "s on average. Estimated time left: " + Math.floor((indices.length - total) * timePerTile) + "s"
]).SetClass("flex flex-col")
}))
constructor() {
return new Combine([statistics, automaton]).SetClass("flex flex-col")
const osmConnection = new OsmConnection({
allElements: undefined,
changes: undefined,
layoutName: "automaton",
singlePage: false,
oauth_token: QueryParameters.GetQueryParameter("oauth_token", "OAuth token")
});
new Combine([
new Combine([Svg.robot_svg().SetClass("w-24 h-24 p-4 rounded-full subtle-background"),
new Combine([new Title("MapComplete Automaton", 1),
"This page helps to automate certain tasks for a theme. Expert use only."
]).SetClass("flex flex-col m-4")
]).SetClass("flex"),
new Toggle(
AutomatonGui.GenerateMainPanel(),
new SubtleButton(Svg.osm_logo_svg(), "Login to get started").onClick(() => osmConnection.AttemptLogin()),
osmConnection.isLoggedIn
)]) .SetClass("block p-4")
.AttachTo("main")
}
private static GenerateMainPanel(): BaseUIElement {
const themeSelect = new DropDown<string>("Select a theme",
@ -249,6 +263,7 @@ class AutomatonGui extends Combine {
}
let indexes : number[] = [];
const tilesS = tiles["success"]
DynamicGeoJsonTileSource.RegisterWhitelist(tilepath.GetValue().data , tilesS)
const z = Number(tilesS["zoom"])
for (const key in tilesS) {
if (key === "zoom") {
@ -282,7 +297,9 @@ class AutomatonGui extends Combine {
themeSelect,
"Specify the path to a tile overview. This is a hosted .json of the format {x : [y0, y1, y2], x1: [y0, ...]} where x is a string and y are numbers",
tilepath,
"Add an extra comment:",
extraComment,
new VariableUiElement(extraComment.GetValue().map(c => "Your comment is "+c.length+"/200 characters long")).SetClass("subtle"),
new VariableUiElement(tilesToRunOver.map(t => {
if (t === undefined) {
return "No path given or still loading..."
@ -331,7 +348,7 @@ class AutomatonGui extends Combine {
return new Combine([
pickAuto,
new VariableUiElement(pickAuto.GetValue().map(auto => auto === undefined ? undefined : AutomatonGui.AutomationPanel(layoutToUse, tilesPerIndex.data, extraComment.GetValue(), auto)))])
new VariableUiElement(pickAuto.GetValue().map(auto => auto === undefined ? undefined : new AutomationPanel(layoutToUse, tilesPerIndex.data, extraComment.GetValue(), auto)))])
}, [tilesPerIndex])).SetClass("flex flex-col")
@ -347,5 +364,4 @@ class AutomatonGui extends Combine {
MinimapImplementation.initialize()
new AutomatonGui()
.SetClass("block p-4")
.AttachTo("main")

1
scripts/generateCache.ts
View file

@ -219,6 +219,7 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
getFeatureById: getFeatureById
},
layer,
{},
{
includeDates: false,
includeNonDates: true