From aa88a3f96469343da9f5bb121b191ac76b0164d5 Mon Sep 17 00:00:00 2001
From: Pieter Vander Vennet <pietervdvn@posteo.net>
Date: Thu, 29 Feb 2024 14:54:14 +0100
Subject: [PATCH] Some more experimentation

---
 package.json                                |  4 ++--
 scripts/importscripts/compareWebsiteData.ts | 19 +++++++++----------
 src/Logic/Web/LinkedDataLoader.ts           |  3 ++-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/package.json b/package.json
index cccf4e0868..606a312315 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mapcomplete",
-  "version": "0.40.0",
+  "version": "0.40.1",
   "repository": "https://github.com/pietervdvn/MapComplete",
   "description": "A small website to edit OSM easily",
   "bugs": "https://github.com/pietervdvn/MapComplete/issues",
@@ -41,7 +41,7 @@
     ],
     "country_coder_host": "https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/latlon2country",
     "nominatimEndpoint": "https://geocoding.geofabrik.de/b75350b1cfc34962ac49824fe5b582dc/",
-    "jsonld-proxy": "http://127.0.0.1:2346/extractgraph?url={url}"
+    "jsonld-proxy": "http://cache.mapcomplete.org/extractgraph?url={url}"
   },
   "scripts": {
     "start": "npm run generate:layeroverview && npm run strt",
diff --git a/scripts/importscripts/compareWebsiteData.ts b/scripts/importscripts/compareWebsiteData.ts
index 87233d34d6..24ff5def51 100644
--- a/scripts/importscripts/compareWebsiteData.ts
+++ b/scripts/importscripts/compareWebsiteData.ts
@@ -1,10 +1,9 @@
 import fs from "fs"
-// import readline from "readline"
+import readline from "readline"
 import Script from "../Script"
 import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
 import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator"
 // vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/
-/*
 class CompareWebsiteData extends Script {
     constructor() {
         super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir")
@@ -28,19 +27,20 @@ class CompareWebsiteData extends Script {
         tags = tags.replace(/""/g, "\"")
         const data = JSON.parse(tags)
 
-        const website = data.website //this.urlFormatter.reformat(data.website)
-        if(!website.startsWith("https://stores.delhaize.be")){
-            return false
-        }
+        try{
+
+        const website = this.urlFormatter.reformat(data.website)
         console.log(website)
         const jsonld = await this.getWithCache(cachedir, website)
-        console.log(jsonld)
         if(Object.keys(jsonld).length === 0){
             return false
         }
         const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
-        fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n")
+        fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n\n")
         return true
+        }catch (e) {
+            console.error("Could not download ", data.website)
+        }
     }
 
     async main(args: string[]): Promise<void> {
@@ -66,7 +66,7 @@ class CompareWebsiteData extends Script {
                 handled ++
                 diffed = diffed + (madeComparison ? 1 : 0)
                 if(handled % 1000 == 0){
-                 //   console.log("Handled ",handled," got ",diffed,"diff results")
+                    console.log("Handled ",handled," got ",diffed,"diff results")
                 }
             } catch (e) {
                // console.error(e)
@@ -77,4 +77,3 @@ class CompareWebsiteData extends Script {
 }
 
 new CompareWebsiteData().run()
-*/
diff --git a/src/Logic/Web/LinkedDataLoader.ts b/src/Logic/Web/LinkedDataLoader.ts
index 5c317bccdf..3ff73c83d0 100644
--- a/src/Logic/Web/LinkedDataLoader.ts
+++ b/src/Logic/Web/LinkedDataLoader.ts
@@ -48,7 +48,8 @@ export default class LinkedDataLoader {
     ]
 
     private static ignoreTypes = [
-        "Breadcrumblist"
+        "Breadcrumblist",
+       "http://schema.org/SearchAction"
     ]
 
     static async geoToGeometry(geo): Promise<Geometry> {