diff --git a/scripts/generateReviewsAnalysis.ts b/scripts/generateReviewsAnalysis.ts new file mode 100644 index 000000000..cff0a9f03 --- /dev/null +++ b/scripts/generateReviewsAnalysis.ts @@ -0,0 +1,159 @@ +import Script from "./Script" +import * as fs from "fs" +import { Review } from "mangrove-reviews-typescript" +import { parse } from "csv-parse" +import { Feature, FeatureCollection, Point } from "geojson" + +export default class GenerateReviewsAnalysis extends Script { + constructor() { + super("Analyses a CSV-file with Mangrove reviews") + } + + async analyze(datapath: string) { + const reviews = await this.parseCsv(datapath) + + const clientWebsites: Record = {} + const themeHist: Record = {} + const languageHist: Record = {} + + const geojsonFeatures: Feature>[] = [] + + for (const review of reviews) { + try { + const client = new URL(review.metadata.client_id) + clientWebsites[client.host] = 1 + (clientWebsites[client.host] ?? 0) + if ( + client.host.indexOf("mapcomplete") >= 0 || + client.host.indexOf("pietervdvn") >= 0 + ) { + let theme = client.pathname.substring(client.pathname.lastIndexOf("/") + 1) + if (theme.endsWith(".html")) { + theme = theme.substring(0, theme.length - 5) + } + if (theme === "theme") { + // THis is a custom layout + theme = + client.searchParams.get("layout") ?? + client.searchParams.get("userlayout") + } + theme = "https://mapcomplete.osm.be/" + theme + themeHist[theme] = (themeHist[theme] ?? 0) + 1 + + const language = client.searchParams.get("language") + languageHist[language] = (languageHist[language] ?? 0) + 1 + } + } catch (e) { + console.error("Not a url:", review.metadata.client_id) + } + + try { + const geo = new URL(review.sub) + if (geo.protocol !== "geo:") { + continue + } + const [lat, lon] = geo.pathname.split(",").map(Number) + console.log(lat, lon) + geojsonFeatures.push({ + geometry: { + type: "Point", + coordinates: [lon, lat], + }, + type: "Feature", + properties: { + name: geo.searchParams.get("q"), + rating: "" + review.rating, + opinion: review.opinion, + client: review.metadata.client_id, + nickname: review.metadata.nickname, + }, + }) + } catch (e) { + console.error(e) + } + } + this.print("Website", clientWebsites) + this.print("Theme", themeHist) + this.print("language", languageHist) + const fc: FeatureCollection = { + type: "FeatureCollection", + features: geojsonFeatures, + } + fs.writeFileSync( + "../MapComplete-data/reviews.geojson", + + JSON.stringify(fc), + { encoding: "utf-8" } + ) + } + + async main(args: string[]): Promise { + const datapath = args[0] ?? "../MapComplete-data/mangrove.reviews_1674234503.csv" + await this.analyze(datapath) + } + + private sort(record: Record): Record { + record = { ...record } + const result: Record = {} + do { + let maxKey: string = undefined + let maxCount: number = -999 + + for (const key in record) { + const c = record[key] + if (c > maxCount) { + maxCount = c + maxKey = key + } + } + result[maxKey] = maxCount + delete record[maxKey] + } while (Object.keys(record).length > 0) + + return result + } + + private print(type: string, histogram: Record) { + console.log(type, this.sort(histogram)) + } + + private parseCsv(datapath: string): Promise { + const header: string[] = [ + "signature", + "pem", + "iat", + "sub", + "rating", + "opinion", + "images", + "metadata", + ] + return new Promise((resolve) => { + const parser = parse({ delimiter: "," }, function (err, data) { + const asJson: Review[] = [] + for (let i = 1; i < data.length; i++) { + const line = data[i] + const entry: Review = { sub: undefined } + for (let c = 0; c < line.length; c++) { + const key: string = header[c] + let value = line[c] + if (value === "none") { + value = null + } else if (key === "images" || key === "metadata") { + try { + value = JSON.parse(value) + } catch (e) { + console.log("Could not parse", value, "\n", line) + } + } + entry[key] = value + } + asJson.push(entry) + } + resolve(asJson) + }) + fs.createReadStream(datapath).pipe(parser) + }) + } +} + +new GenerateReviewsAnalysis().run()