midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Chore: linting

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-04-13 02:40:21 +02:00
parent 4625ad9a5c
commit 097141f944
307 changed files with 5346 additions and 2147 deletions
Download patch file Download diff file Expand all files Collapse all files

49
scripts/importscripts/compareWebsiteData.ts
View file

@ -6,13 +6,15 @@ import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator"
// vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/
class CompareWebsiteData extends Script {
constructor() {
super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir")
super(
"Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir"
)
}
private readonly urlFormatter = new UrlValidator()
async getWithCache(cachedir : string, url: string): Promise<any>{
const filename= cachedir+"/"+encodeURIComponent(url)
if(fs.existsSync(filename)){
async getWithCache(cachedir: string, url: string): Promise<any> {
const filename = cachedir + "/" + encodeURIComponent(url)
if (fs.existsSync(filename)) {
return JSON.parse(fs.readFileSync(filename, "utf-8"))
}
const jsonLd = await LinkedDataLoader.fetchJsonLd(url, undefined, true)
@ -20,25 +22,24 @@ class CompareWebsiteData extends Script {
fs.writeFileSync(filename, JSON.stringify(jsonLd))
return jsonLd
}
async handleEntry(line: string, cachedir: string, targetfile: string) : Promise<boolean>{
async handleEntry(line: string, cachedir: string, targetfile: string): Promise<boolean> {
const id = JSON.parse(line.split(",")[0])
let tags = line.substring(line.indexOf("{") - 1)
tags = tags.substring(1, tags.length - 1)
tags = tags.replace(/""/g, "\"")
tags = tags.replace(/""/g, '"')
const data = JSON.parse(tags)
try{
const website = this.urlFormatter.reformat(data.website)
console.log(website)
const jsonld = await this.getWithCache(cachedir, website)
if(Object.keys(jsonld).length === 0){
return false
}
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n\n")
return true
}catch (e) {
try {
const website = this.urlFormatter.reformat(data.website)
console.log(website)
const jsonld = await this.getWithCache(cachedir, website)
if (Object.keys(jsonld).length === 0) {
return false
}
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
fs.appendFileSync(targetfile, id + ", " + JSON.stringify(diff) + "\n\n")
return true
} catch (e) {
console.error("Could not download ", data.website)
}
}
@ -48,7 +49,6 @@ class CompareWebsiteData extends Script {
throw "Not enough arguments"
}
const readInterface = readline.createInterface({
input: fs.createReadStream(args[0]),
})
@ -59,20 +59,19 @@ class CompareWebsiteData extends Script {
fs.writeFileSync(targetfile, "id, diff-json\n")
for await (const line of readInterface) {
try {
if(line.startsWith("\"id\"")){
if (line.startsWith('"id"')) {
continue
}
const madeComparison = await this.handleEntry(line, args[1], targetfile)
handled ++
handled++
diffed = diffed + (madeComparison ? 1 : 0)
if(handled % 1000 == 0){
console.log("Handled ",handled," got ",diffed,"diff results")
if (handled % 1000 == 0) {
console.log("Handled ", handled, " got ", diffed, "diff results")
}
} catch (e) {
// console.error(e)
// console.error(e)
}
}
}
}