Bastian-Greshake-Tzovaras/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Some more experimentation

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-02-29 14:54:14 +01:00
parent 42e99f3aca
commit aa88a3f964
3 changed files with 13 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

4
package.json
View file

@ -1,6 +1,6 @@
{ {
"name": "mapcomplete", "name": "mapcomplete",
"version": "0.40.0", "version": "0.40.1",
"repository": "https://github.com/pietervdvn/MapComplete", "repository": "https://github.com/pietervdvn/MapComplete",
"description": "A small website to edit OSM easily", "description": "A small website to edit OSM easily",
"bugs": "https://github.com/pietervdvn/MapComplete/issues", "bugs": "https://github.com/pietervdvn/MapComplete/issues",
@ -41,7 +41,7 @@
], ],
"country_coder_host": "https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/latlon2country", "country_coder_host": "https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/latlon2country",
"nominatimEndpoint": "https://geocoding.geofabrik.de/b75350b1cfc34962ac49824fe5b582dc/", "nominatimEndpoint": "https://geocoding.geofabrik.de/b75350b1cfc34962ac49824fe5b582dc/",
"jsonld-proxy": "http://127.0.0.1:2346/extractgraph?url={url}" "jsonld-proxy": "http://cache.mapcomplete.org/extractgraph?url={url}"
}, },
"scripts": { "scripts": {
"start": "npm run generate:layeroverview && npm run strt", "start": "npm run generate:layeroverview && npm run strt",

19
scripts/importscripts/compareWebsiteData.ts
View file

@ -1,10 +1,9 @@
import fs from "fs" import fs from "fs"
// import readline from "readline" import readline from "readline"
import Script from "../Script" import Script from "../Script"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader" import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator" import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator"
// vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/ // vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/
/*
class CompareWebsiteData extends Script { class CompareWebsiteData extends Script {
constructor() { constructor() {
super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir") super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir")
@ -28,19 +27,20 @@ class CompareWebsiteData extends Script {
tags = tags.replace(/""/g, "\"") tags = tags.replace(/""/g, "\"")
const data = JSON.parse(tags) const data = JSON.parse(tags)
const website = data.website //this.urlFormatter.reformat(data.website) try{
if(!website.startsWith("https://stores.delhaize.be")){
return false const website = this.urlFormatter.reformat(data.website)
}
console.log(website) console.log(website)
const jsonld = await this.getWithCache(cachedir, website) const jsonld = await this.getWithCache(cachedir, website)
console.log(jsonld)
if(Object.keys(jsonld).length === 0){ if(Object.keys(jsonld).length === 0){
return false return false
} }
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data) const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n") fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n\n")
return true return true
}catch (e) {
console.error("Could not download ", data.website)
}
} }
async main(args: string[]): Promise<void> { async main(args: string[]): Promise<void> {
@ -66,7 +66,7 @@ class CompareWebsiteData extends Script {
handled ++ handled ++
diffed = diffed + (madeComparison ? 1 : 0) diffed = diffed + (madeComparison ? 1 : 0)
if(handled % 1000 == 0){ if(handled % 1000 == 0){
// console.log("Handled ",handled," got ",diffed,"diff results") console.log("Handled ",handled," got ",diffed,"diff results")
} }
} catch (e) { } catch (e) {
// console.error(e) // console.error(e)
@ -77,4 +77,3 @@ class CompareWebsiteData extends Script {
} }
new CompareWebsiteData().run() new CompareWebsiteData().run()
*/

3
src/Logic/Web/LinkedDataLoader.ts
View file

@ -48,7 +48,8 @@ export default class LinkedDataLoader {
] ]
private static ignoreTypes = [ private static ignoreTypes = [
"Breadcrumblist" "Breadcrumblist",
"http://schema.org/SearchAction"
] ]
static async geoToGeometry(geo): Promise<Geometry> { static async geoToGeometry(geo): Promise<Geometry> {