ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Some more experimentation

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-02-29 14:54:14 +01:00
parent 42e99f3aca
commit aa88a3f964
3 changed files with 13 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

4
package.json
View file

@ -1,6 +1,6 @@
{
"name": "mapcomplete",
"version": "0.40.0",
"version": "0.40.1",
"repository": "https://github.com/pietervdvn/MapComplete",
"description": "A small website to edit OSM easily",
"bugs": "https://github.com/pietervdvn/MapComplete/issues",
@ -41,7 +41,7 @@
],
"country_coder_host": "https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/latlon2country",
"nominatimEndpoint": "https://geocoding.geofabrik.de/b75350b1cfc34962ac49824fe5b582dc/",
"jsonld-proxy": "http://127.0.0.1:2346/extractgraph?url={url}"
"jsonld-proxy": "http://cache.mapcomplete.org/extractgraph?url={url}"
},
"scripts": {
"start": "npm run generate:layeroverview && npm run strt",

19
scripts/importscripts/compareWebsiteData.ts
View file

@ -1,10 +1,9 @@
import fs from "fs"
// import readline from "readline"
import readline from "readline"
import Script from "../Script"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator"
// vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/
/*
class CompareWebsiteData extends Script {
constructor() {
super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir")
@ -28,19 +27,20 @@ class CompareWebsiteData extends Script {
tags = tags.replace(/""/g, "\"")
const data = JSON.parse(tags)
const website = data.website //this.urlFormatter.reformat(data.website)
if(!website.startsWith("https://stores.delhaize.be")){
return false
}
try{
const website = this.urlFormatter.reformat(data.website)
console.log(website)
const jsonld = await this.getWithCache(cachedir, website)
console.log(jsonld)
if(Object.keys(jsonld).length === 0){
return false
}
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n")
fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n\n")
return true
}catch (e) {
console.error("Could not download ", data.website)
}
}
async main(args: string[]): Promise<void> {
@ -66,7 +66,7 @@ class CompareWebsiteData extends Script {
handled ++
diffed = diffed + (madeComparison ? 1 : 0)
if(handled % 1000 == 0){
// console.log("Handled ",handled," got ",diffed,"diff results")
console.log("Handled ",handled," got ",diffed,"diff results")
}
} catch (e) {
// console.error(e)
@ -77,4 +77,3 @@ class CompareWebsiteData extends Script {
}
new CompareWebsiteData().run()
*/

3
src/Logic/Web/LinkedDataLoader.ts
View file

@ -48,7 +48,8 @@ export default class LinkedDataLoader {
]
private static ignoreTypes = [
"Breadcrumblist"
"Breadcrumblist",
"http://schema.org/SearchAction"
]
static async geoToGeometry(geo): Promise<Geometry> {