Lots of refactoring, first version of the import helper

This commit is contained in:
Pieter Vander Vennet 2022-01-19 20:34:04 +01:00
parent 612b8136ad
commit 3402ac0954
54 changed files with 1104 additions and 315 deletions

View file

@ -0,0 +1,242 @@
import {BBox} from "../../Logic/BBox";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
import Combine from "../Base/Combine";
import Title from "../Base/Title";
import {Overpass} from "../../Logic/Osm/Overpass";
import {UIEventSource} from "../../Logic/UIEventSource";
import Constants from "../../Models/Constants";
import RelationsTracker from "../../Logic/Osm/RelationsTracker";
import {VariableUiElement} from "../Base/VariableUIElement";
import {FixedUiElement} from "../Base/FixedUiElement";
import {FlowStep} from "./FlowStep";
import Loading from "../Base/Loading";
import {SubtleButton} from "../Base/SubtleButton";
import Svg from "../../Svg";
import {Utils} from "../../Utils";
import {IdbLocalStorage} from "../../Logic/Web/IdbLocalStorage";
import Minimap from "../Base/Minimap";
import BaseLayer from "../../Models/BaseLayer";
import AvailableBaseLayers from "../../Logic/Actors/AvailableBaseLayers";
import Loc from "../../Models/Loc";
import Attribution from "../BigComponents/Attribution";
import ShowDataLayer from "../ShowDataLayer/ShowDataLayer";
import StaticFeatureSource from "../../Logic/FeatureSource/Sources/StaticFeatureSource";
import ValidatedTextField from "../Input/ValidatedTextField";
import {LocalStorageSource} from "../../Logic/Web/LocalStorageSource";
import * as currentview from "../../assets/layers/current_view/current_view.json"
import * as import_candidate from "../../assets/layers/import_candidate/import_candidate.json"
import {GeoOperations} from "../../Logic/GeoOperations";
import FeatureInfoBox from "../Popup/FeatureInfoBox";
/**
* Given the data to import, the bbox and the layer, will query overpass for similar items
*/
export default class ConflationChecker extends Combine implements FlowStep<any> {
public readonly IsValid
public readonly Value
constructor(
state,
params: { bbox: BBox, layer: LayerConfig, geojson: any }) {
const bbox = params.bbox.padAbsolute(0.0001)
const layer = params.layer;
const toImport = params.geojson;
let overpassStatus = new UIEventSource<{ error: string } | "running" | "success" | "idle" | "cached" >("idle")
const fromLocalStorage = IdbLocalStorage.Get<[any, Date]>("importer-overpass-cache-" + layer.id, {
whenLoaded: (v) => {
if (v !== undefined) {
console.log("Loaded from local storage:", v)
const [geojson, date] = v;
const timeDiff = (new Date().getTime() - date.getTime()) / 1000;
console.log("The cache is ", timeDiff, "seconds old")
if (timeDiff < 24 * 60 * 60) {
// Recently cached!
overpassStatus.setData("cached")
return;
}
}
// Load the data!
const url = Constants.defaultOverpassUrls[1]
const relationTracker = new RelationsTracker()
const overpass = new Overpass(params.layer.source.osmTags, [], url, new UIEventSource<number>(180), relationTracker, true)
console.log("Loading from overpass!")
overpassStatus.setData("running")
overpass.queryGeoJson(bbox).then(
([data, date] ) => {
console.log("Received overpass-data: ", data.features.length,"features are loaded at ", date);
overpassStatus.setData("success")
fromLocalStorage.setData([data, date])
},
(error) => {overpassStatus.setData({error})})
}
});
const geojson : UIEventSource<any> = fromLocalStorage.map(d => {
if (d === undefined) {
return undefined
}
return d[0]
})
const background = new UIEventSource<BaseLayer>(AvailableBaseLayers.osmCarto)
const location = new UIEventSource<Loc>({lat: 0, lon: 0, zoom: 1})
const currentBounds = new UIEventSource<BBox>(undefined)
const zoomLevel = ValidatedTextField.InputForType("pnat")
zoomLevel.SetClass("ml-1 border border-black")
zoomLevel.GetValue().syncWith(LocalStorageSource.Get("importer-zoom-level", "14"), true)
const osmLiveData = Minimap.createMiniMap({
allowMoving: true,
location,
background,
bounds: currentBounds,
attribution: new Attribution(location, state.osmConnection.userDetails, undefined, currentBounds)
})
osmLiveData.SetClass("w-full").SetStyle("height: 500px")
const preview = new StaticFeatureSource(geojson.map(geojson => {
if(geojson?.features === undefined){
return []
}
const zoomedEnough: boolean = osmLiveData.location.data.zoom >= Number(zoomLevel.GetValue().data)
if(!zoomedEnough){
return []
}
const bounds = osmLiveData.bounds.data
return geojson.features.filter(f => BBox.get(f).overlapsWith(bounds))
}, [osmLiveData.bounds, zoomLevel.GetValue()]), false);
new ShowDataLayer({
layerToShow:new LayerConfig(currentview),
state,
leafletMap: osmLiveData.leafletMap,
enablePopups: undefined,
zoomToFeatures: true,
features: new StaticFeatureSource([
bbox.asGeoJson({})
], false)
})
new ShowDataLayer({
layerToShow:layer,
state,
leafletMap: osmLiveData.leafletMap,
popup: (tags, layer) => new FeatureInfoBox(tags, layer, state),
zoomToFeatures: false,
features: preview
})
new ShowDataLayer({
layerToShow:new LayerConfig(import_candidate),
state,
leafletMap: osmLiveData.leafletMap,
popup: (tags, layer) => new FeatureInfoBox(tags, layer, state),
zoomToFeatures: false,
features: new StaticFeatureSource(toImport.features, false)
})
const nearbyCutoff = ValidatedTextField.InputForType("pnat")
nearbyCutoff.SetClass("ml-1 border border-black")
nearbyCutoff.GetValue().syncWith(LocalStorageSource.Get("importer-cutoff", "25"), true)
const matchedFeaturesMap = Minimap.createMiniMap({
allowMoving: true,
background
})
matchedFeaturesMap.SetClass("w-full").SetStyle("height: 500px")
// Featuresource showing OSM-features which are nearby a toImport-feature
const nearbyFeatures = new StaticFeatureSource(geojson.map(osmData => {
if(osmData?.features === undefined){
return []
}
const maxDist = Number(nearbyCutoff.GetValue().data)
return osmData.features.filter(f =>
toImport.features.some(imp =>
maxDist >= GeoOperations.distanceBetween(imp.geometry.coordinates, GeoOperations.centerpointCoordinates(f))) )
}, [nearbyCutoff.GetValue()]), false);
// Featuresource showing OSM-features which are nearby a toImport-feature
const toImportWithNearby = new StaticFeatureSource(geojson.map(osmData => {
if(osmData?.features === undefined){
return []
}
const maxDist = Number(nearbyCutoff.GetValue().data)
return toImport.features.filter(imp =>
osmData.features.some(f =>
maxDist >= GeoOperations.distanceBetween(imp.geometry.coordinates, GeoOperations.centerpointCoordinates(f))) )
}, [nearbyCutoff.GetValue()]), false);
new ShowDataLayer({
layerToShow:layer,
state,
leafletMap: matchedFeaturesMap.leafletMap,
popup: (tags, layer) => new FeatureInfoBox(tags, layer, state),
zoomToFeatures: true,
features: nearbyFeatures
})
new ShowDataLayer({
layerToShow:new LayerConfig(import_candidate),
state,
leafletMap: matchedFeaturesMap.leafletMap,
popup: (tags, layer) => new FeatureInfoBox(tags, layer, state),
zoomToFeatures: false,
features: toImportWithNearby
})
super([
new Title("Comparison with existing data"),
new VariableUiElement(overpassStatus.map(d => {
if (d === "idle") {
return new Loading("Checking local storage...")
}
if (d["error"] !== undefined) {
return new FixedUiElement("Could not load latest data from overpass: " + d["error"]).SetClass("alert")
}
if(d === "running"){
return new Loading("Querying overpass...")
}
if(d === "cached"){
return new FixedUiElement("Fetched data from local storage")
}
if(d === "success"){
return new FixedUiElement("Data loaded")
}
return new FixedUiElement("Unexpected state "+d).SetClass("alert")
})),
new VariableUiElement(
geojson.map(geojson => {
if (geojson === undefined) {
return undefined;
}
return new SubtleButton(Svg.download_svg(), "Download the loaded geojson from overpass").onClick(() => {
Utils.offerContentsAsDownloadableFile(JSON.stringify(geojson, null, " "), "mapcomplete-" + layer.id + ".geojson", {
mimetype: "application/json+geo"
})
});
})),
new Title("Live data on OSM"),
osmLiveData,
new Combine(["The live data is shown if the zoomlevel is at least ", zoomLevel, ". The current zoom level is ", new VariableUiElement(osmLiveData.location.map(l => ""+l.zoom))]).SetClass("flex"),
new Title("Nearby features"),
new Combine([ "The following map shows features to import which have an OSM-feature within ", nearbyCutoff, "meter"]).SetClass("flex"),
new FixedUiElement("The red elements on the following map will <b>not</b> be imported!").SetClass("alert"),
"Set the range to 0 or 1 if you want to import them all",
matchedFeaturesMap
])
this.IsValid = new UIEventSource(false)
this.Value = new UIEventSource(undefined)
}
}

156
UI/ImportFlow/DataPanel.ts Normal file
View file

@ -0,0 +1,156 @@
import Combine from "../Base/Combine";
import {UIEventSource} from "../../Logic/UIEventSource";
import {BBox} from "../../Logic/BBox";
import UserRelatedState from "../../Logic/State/UserRelatedState";
import Translations from "../i18n/Translations";
import {AllKnownLayouts} from "../../Customizations/AllKnownLayouts";
import Constants from "../../Models/Constants";
import {DropDown} from "../Input/DropDown";
import {Utils} from "../../Utils";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
import BaseLayer from "../../Models/BaseLayer";
import AvailableBaseLayers from "../../Logic/Actors/AvailableBaseLayers";
import Loc from "../../Models/Loc";
import Minimap from "../Base/Minimap";
import Attribution from "../BigComponents/Attribution";
import ShowDataMultiLayer from "../ShowDataLayer/ShowDataMultiLayer";
import FilteredLayer, {FilterState} from "../../Models/FilteredLayer";
import StaticFeatureSource from "../../Logic/FeatureSource/Sources/StaticFeatureSource";
import Toggle from "../Input/Toggle";
import Table from "../Base/Table";
import {VariableUiElement} from "../Base/VariableUIElement";
import {FixedUiElement} from "../Base/FixedUiElement";
import {FlowStep} from "./FlowStep";
import {Layer} from "leaflet";
/**
* Shows the data to import on a map, asks for the correct layer to be selected
*/
export class DataPanel extends Combine implements FlowStep<{ bbox: BBox, layer: LayerConfig, geojson: any }>{
public readonly IsValid: UIEventSource<boolean>;
public readonly Value: UIEventSource<{ bbox: BBox, layer: LayerConfig, geojson: any }>
constructor(
state: UserRelatedState,
geojson: { features: { properties: any, geometry: { coordinates: [number, number] } }[] }) {
const t = Translations.t.importHelper;
const propertyKeys = new Set<string>()
console.log("Datapanel input got ", geojson)
for (const f of geojson.features) {
Object.keys(f.properties).forEach(key => propertyKeys.add(key))
}
const availableLayers = AllKnownLayouts.AllPublicLayers().filter(l => l.name !== undefined && Constants.priviliged_layers.indexOf(l.id) < 0)
const layerPicker = new DropDown("Which layer does this import match with?",
[{shown: t.selectLayer, value: undefined}].concat(availableLayers.map(l => ({
shown: l.name,
value: l
})))
)
let autodetected = new UIEventSource(false)
for (const layer of availableLayers) {
const mismatched = geojson.features.some(f =>
!layer.source.osmTags.matchesProperties(f.properties)
)
if (!mismatched) {
layerPicker.GetValue().setData(layer);
layerPicker.GetValue().addCallback(_ => autodetected.setData(false))
autodetected.setData(true)
break;
}
}
const withId = geojson.features.map((f, i) => {
const copy = Utils.Clone(f)
copy.properties.id = "to-import/" + i
return copy
})
const matching: UIEventSource<{ properties: any, geometry: { coordinates: [number, number] } }[]> = layerPicker.GetValue().map((layer: LayerConfig) => {
if (layer === undefined) {
return undefined;
}
const matching: { properties: any, geometry: { coordinates: [number, number] } }[] = []
for (const feature of withId) {
if (layer.source.osmTags.matchesProperties(feature.properties)) {
matching.push(feature)
}
}
return matching
})
const background = new UIEventSource<BaseLayer>(AvailableBaseLayers.osmCarto)
const location = new UIEventSource<Loc>({lat: 0, lon: 0, zoom: 1})
const currentBounds = new UIEventSource<BBox>(undefined)
const map = Minimap.createMiniMap({
allowMoving: true,
location,
background,
bounds: currentBounds,
attribution: new Attribution(location, state.osmConnection.userDetails, undefined, currentBounds)
})
map.SetClass("w-full").SetStyle("height: 500px")
new ShowDataMultiLayer({
layers: new UIEventSource<FilteredLayer[]>(AllKnownLayouts.AllPublicLayers().map(l => ({
layerDef: l,
isDisplayed: new UIEventSource<boolean>(true),
appliedFilters: new UIEventSource<Map<string, FilterState>>(undefined)
}))),
zoomToFeatures: true,
features: new StaticFeatureSource(matching, false),
state: {
...state,
filteredLayers: new UIEventSource<FilteredLayer[]>(undefined),
backgroundLayer: background
},
leafletMap: map.leafletMap,
})
var bbox = matching.map(feats => BBox.bboxAroundAll(feats.map(f => new BBox([f.geometry.coordinates]))))
super([
"Has " + geojson.features.length + " features",
layerPicker,
new Toggle("Automatically detected layer", undefined, autodetected),
new Table(["", "Key", "Values", "Unique values seen"],
Array.from(propertyKeys).map(key => {
const uniqueValues = Utils.Dedup(Utils.NoNull(geojson.features.map(f => f.properties[key])))
uniqueValues.sort()
return [geojson.features.filter(f => f.properties[key] !== undefined).length + "", key, uniqueValues.join(", "), "" + uniqueValues.length]
})
).SetClass("zebra-table table-auto"),
new VariableUiElement(matching.map(matching => {
if (matching === undefined) {
return undefined
}
const diff = geojson.features.length - matching.length;
if (diff === 0) {
return undefined
}
const obligatory = layerPicker.GetValue().data?.source?.osmTags?.asHumanString(false, false, {});
return new FixedUiElement(`${diff} features will _not_ match this layer. Make sure that all obligatory objects are present: ${obligatory}`).SetClass("alert");
})),
map
]);
this.Value = bbox.map(bbox =>
({
bbox,
geojson,
layer: layerPicker.GetValue().data
}), [layerPicker.GetValue()])
this.IsValid = matching.map(matching => {
if (matching === undefined) {
return false
}
const diff = geojson.features.length - matching.length;
return diff === 0;
})
}
}

105
UI/ImportFlow/FlowStep.ts Normal file
View file

@ -0,0 +1,105 @@
import {UIEventSource} from "../../Logic/UIEventSource";
import Combine from "../Base/Combine";
import BaseUIElement from "../BaseUIElement";
import {SubtleButton} from "../Base/SubtleButton";
import Svg from "../../Svg";
import Translations from "../i18n/Translations";
import {VariableUiElement} from "../Base/VariableUIElement";
import Toggle from "../Input/Toggle";
import {UIElement} from "../UIElement";
export interface FlowStep<T> extends BaseUIElement{
readonly IsValid: UIEventSource<boolean>
readonly Value: UIEventSource<T>
}
export class FlowPanelFactory<T> {
private _initial: FlowStep<any>;
private _steps: ((x: any) => FlowStep<any>)[];
private _stepNames: string[];
private constructor(initial: FlowStep<any>, steps: ((x:any) => FlowStep<any>)[], stepNames: string[]) {
this._initial = initial;
this._steps = steps;
this._stepNames = stepNames;
}
public static start<TOut> (step: FlowStep<TOut>): FlowPanelFactory<TOut>{
return new FlowPanelFactory(step, [], [])
}
public then<TOut>(name: string, construct: ((t:T) => FlowStep<TOut>)): FlowPanelFactory<TOut>{
return new FlowPanelFactory<TOut>(
this._initial,
this._steps.concat([construct]),
this._stepNames.concat([name])
)
}
public finish(construct: ((t: T, backButton?: BaseUIElement) => BaseUIElement)) : BaseUIElement {
// Construct all the flowpanels step by step (in reverse order)
const nextConstr : ((t:any, back?: UIElement) => BaseUIElement)[] = this._steps.map(_ => undefined)
nextConstr.push(construct)
for (let i = this._steps.length - 1; i >= 0; i--){
const createFlowStep : (value) => FlowStep<any> = this._steps[i];
nextConstr[i] = (value, backButton) => {
console.log("Creating flowSTep ", this._stepNames[i])
const flowStep = createFlowStep(value)
return new FlowPanel(flowStep, nextConstr[i + 1], backButton);
}
}
return new FlowPanel(this._initial, nextConstr[0],undefined)
}
}
export class FlowPanel<T> extends Toggle {
constructor(
initial: (FlowStep<T>),
constructNextstep: ((input: T, backButton: BaseUIElement) => BaseUIElement),
backbutton?: BaseUIElement
) {
const t = Translations.t.general;
const currentStepActive = new UIEventSource(true);
let nextStep: UIEventSource<BaseUIElement>= new UIEventSource<BaseUIElement>(undefined)
const backButtonForNextStep = new SubtleButton(Svg.back_svg(), t.back).onClick(() => {
currentStepActive.setData(true)
})
let elements : (BaseUIElement | string)[] = []
if(initial !== undefined){
// Startup the flow
elements = [
initial,
new Combine([
backbutton,
new Toggle(
new SubtleButton(Svg.back_svg().SetStyle("transform: rotate(180deg);"), t.next).onClick(() => {
const v = initial.Value.data;
nextStep.setData(constructNextstep(v, backButtonForNextStep))
currentStepActive.setData(false)
}),
"Select a valid value to continue",
initial.IsValid
)
]).SetClass("flex w-full justify-end space-x-2")
]
}
super(
new Combine(elements).SetClass("h-full flex flex-col justify-between"),
new VariableUiElement(nextStep),
currentStepActive
);
}
}

View file

@ -0,0 +1,67 @@
import Combine from "../Base/Combine";
import {LoginToggle} from "../Popup/LoginButton";
import Toggle from "../Input/Toggle";
import LanguagePicker from "../LanguagePicker";
import BackToIndex from "../BigComponents/BackToIndex";
import UserRelatedState from "../../Logic/State/UserRelatedState";
import BaseUIElement from "../BaseUIElement";
import MoreScreen from "../BigComponents/MoreScreen";
import MinimapImplementation from "../Base/MinimapImplementation";
import Translations from "../i18n/Translations";
import Constants from "../../Models/Constants";
import {FlowPanel, FlowPanelFactory} from "./FlowStep";
import {RequestFile} from "./RequestFile";
import {DataPanel} from "./DataPanel";
import {FixedUiElement} from "../Base/FixedUiElement";
import ConflationChecker from "./ConflationChecker";
export default class ImportHelperGui extends LoginToggle {
constructor() {
const t = Translations.t.importHelper;
const state = new UserRelatedState(undefined)
// We disable the userbadge, as various 'showData'-layers will give a read-only view in this case
state.featureSwitchUserbadge.setData(false)
const leftContents: BaseUIElement[] = [
new BackToIndex().SetClass("block pl-4"),
LanguagePicker.CreateLanguagePicker(Translations.t.importHelper.title.SupportedLanguages())?.SetClass("mt-4 self-end flex-col"),
].map(el => el?.SetClass("pl-4"))
const leftBar = new Combine([
new Combine(leftContents).SetClass("sticky top-4 m-4")
]).SetClass("block w-full md:w-2/6 lg:w-1/6")
const mainPanel =
FlowPanelFactory
.start(new RequestFile())
.then("datapanel", geojson => new DataPanel(state, geojson))
.then("conflation", v => new ConflationChecker(state, v))
.finish(_ => new FixedUiElement("All done!"))
super(
new Toggle(
new Combine([
leftBar,
mainPanel.SetClass("m-8 w-full mb-24")
]).SetClass("h-full block md:flex")
,
new Combine([
t.lockNotice.Subs(Constants.userJourney),
MoreScreen.CreateProffessionalSerivesButton()
])
,
state.osmConnection.userDetails.map(ud => ud.csCount >= Constants.userJourney.importHelperUnlock)),
"Login needed...",
state)
}
}
MinimapImplementation.initialize()
new ImportHelperGui().AttachTo("main")

View file

@ -0,0 +1,145 @@
import Combine from "../Base/Combine";
import {UIEventSource} from "../../Logic/UIEventSource";
import Translations from "../i18n/Translations";
import {SubtleButton} from "../Base/SubtleButton";
import {VariableUiElement} from "../Base/VariableUIElement";
import {FixedUiElement} from "../Base/FixedUiElement";
import Title from "../Base/Title";
import InputElementMap from "../Input/InputElementMap";
import BaseUIElement from "../BaseUIElement";
import FileSelectorButton from "../Input/FileSelectorButton";
import {FlowStep} from "./FlowStep";
class FileSelector extends InputElementMap<FileList, { name: string, contents: Promise<string> }> {
constructor(label: BaseUIElement) {
super(
new FileSelectorButton(label, {allowMultiple: false, acceptType: "*"}),
(x0, x1) => {
// Total hack: x1 is undefined is the backvalue - we effectively make this a one-way-story
return x1 === undefined || x0 === x1;
},
filelist => {
if (filelist === undefined) {
return undefined
}
const file = filelist.item(0)
return {name: file.name, contents: file.text()}
},
_ => undefined
)
}
}
/**
* The first step in the import flow: load a file and validate that it is a correct geojson or CSV file
*/
export class RequestFile extends Combine implements FlowStep<any> {
public readonly IsValid: UIEventSource<boolean>
/**
* The loaded GeoJSON
*/
public readonly Value: UIEventSource<any>
constructor() {
const t = Translations.t.importHelper;
const csvSelector = new FileSelector(new SubtleButton(undefined, t.selectFile))
const loadedFiles = new VariableUiElement(csvSelector.GetValue().map(file => {
if (file === undefined) {
return t.noFilesLoaded.SetClass("alert")
}
return t.loadedFilesAre.Subs({file: file.name}).SetClass("thanks")
}))
const text = UIEventSource.flatten(
csvSelector.GetValue().map(v => {
if (v === undefined) {
return undefined
}
return UIEventSource.FromPromise(v.contents)
}))
const asGeoJson: UIEventSource<any | { error: string }> = text.map(src => {
if (src === undefined) {
return undefined
}
try {
const parsed = JSON.parse(src)
if (parsed["type"] !== "FeatureCollection") {
return {error: "The loaded JSON-file is not a geojson-featurecollection"}
}
if (parsed.features.some(f => f.geometry.type != "Point")) {
return {error: "The loaded JSON-file should only contain points"}
}
return parsed;
} catch (e) {
// Loading as CSV
const lines = src.split("\n")
const header = lines[0].split(",")
lines.splice(0, 1)
if (header.indexOf("lat") < 0 || header.indexOf("lon") < 0) {
return {error: "The header does not contain `lat` or `lon`"}
}
const features = []
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line.trim() === "") {
continue
}
const attrs = line.split(",")
const properties = {}
for (let i = 0; i < header.length; i++) {
properties[header[i]] = attrs[i];
}
const coordinates = [Number(properties["lon"]), Number(properties["lat"])]
delete properties["lat"]
delete properties["lon"]
if (coordinates.some(isNaN)) {
return {error: "A coordinate could not be parsed for line " + (i + 2)}
}
const f = {
type: "Feature",
properties,
geometry: {
type: "Point",
coordinates
}
};
features.push(f)
}
return {
type: "FeatureCollection",
features
}
}
})
const errorIndicator = new VariableUiElement(asGeoJson.map(v => {
if (v === undefined) {
return undefined;
}
if (v?.error === undefined) {
return undefined;
}
return new FixedUiElement(v?.error).SetClass("alert");
}))
super([
new Title(t.title, 1),
t.description,
csvSelector,
loadedFiles,
errorIndicator
]);
this.IsValid = asGeoJson.map(geojson => geojson !== undefined && geojson["error"] === undefined)
this.Value = asGeoJson
}
}