import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import SmallLicense from "../src/Models/smallLicense"
import ScriptUtils from "./ScriptUtils"
import Script from "./Script"
import { Utils } from "../src/Utils"
const prompt = require("prompt-sync")()
export class GenerateLicenseInfo extends Script {
private static readonly needsLicenseRef = new Set(
ScriptUtils.readDirRecSync("./LICENSES")
.map((p) => p.substring(p.lastIndexOf("/") + 1))
.filter((p) => p.startsWith("LicenseRef-"))
.map((p) => p.substring("LicenseRef-".length))
.map((p) => p.substring(0, p.lastIndexOf(".")))
)
constructor() {
super("Validates the licenses and compiles them into one single asset file")
}
static defaultLicenses() {
const knownLicenses = new Map<string, SmallLicense>()
knownLicenses.set("me", {
authors: ["Pieter Vander Vennet"],
path: undefined,
license: "CC0",
sources: [],
})
knownLicenses.set("streetcomplete", {
authors: ["Tobias Zwick (westnordost)"],
path: undefined,
license: "CC0",
sources: [
"https://github.com/streetcomplete/StreetComplete/tree/master/res/graphics",
"https://f-droid.org/packages/de.westnordost.streetcomplete/",
],
})
knownLicenses.set("temaki", {
authors: ["Temaki"],
path: undefined,
license: "CC0",
sources: [
"https://github.com/ideditor/temaki",
"https://ideditor.github.io/temaki/docs/",
],
})
knownLicenses.set("maki", {
authors: ["Maki"],
path: undefined,
license: "CC0",
sources: ["https://labs.mapbox.com/maki-icons/"],
})
knownLicenses.set("t", {
authors: [],
path: undefined,
license: "CC0; trivial",
sources: [],
})
knownLicenses.set("na", {
authors: [],
path: undefined,
license: "CC0",
sources: [],
})
knownLicenses.set("carto", {
authors: ["OSM-Carto"],
path: undefined,
license: "CC0",
sources: [""],
})
knownLicenses.set("tv", {
authors: ["Toerisme Vlaanderen"],
path: undefined,
license: "CC0",
sources: [
"https://toerismevlaanderen.be/pinjepunt",
"https://mapcomplete.org/toerisme_vlaanderenn",
],
})
knownLicenses.set("tvf", {
authors: ["Jo De Baerdemaeker "],
path: undefined,
license: "All rights reserved",
sources: ["https://www.studiotype.be/fonts/flandersart"],
})
knownLicenses.set("twemoji", {
authors: ["Twemoji"],
path: undefined,
license: "CC-BY 4.0",
sources: ["https://github.com/twitter/twemoji"],
})
return knownLicenses
}
validateLicenseInfo(l: SmallLicense) {
l.sources.map((s) => {
try {
return new URL(s)
} catch (e) {
throw "Could not parse URL " + s + " for a license for " + l.path + " due to " + e
}
})
}
/**
* Sweeps the entire 'assets/' (except assets/generated) directory for image files and any 'license_info.json'-file.
* Checks that the license info is included for each of them and generates a compiles license_info.json for those
*/
generateLicenseInfos(paths: string[]): SmallLicense[] {
const licenses = []
for (const path of paths) {
try {
const parsed = JSON.parse(readFileSync(path, { encoding: "utf8" }))
if (Array.isArray(parsed)) {
const l: SmallLicense[] = parsed
for (const smallLicens of l) {
smallLicens.path =
path.substring(0, path.length - "license_info.json".length) +
smallLicens.path
}
licenses.push(...l)
} else {
const smallLicens: SmallLicense = parsed
smallLicens.path =
path.substring(0, 1 + path.lastIndexOf("/")) + smallLicens.path
licenses.push(smallLicens)
}
} catch (e) {
console.error("Error: ", e, "while handling", path)
}
}
return licenses
}
async mostlyWhite(allIcons: string[]) {
const whitePaths = new Set<string>()
for (const icon of allIcons) {
if (!icon.endsWith(".svg")) {
continue
}
const svg = await ScriptUtils.ReadSvg(icon)
const colours = new Set<string>()
Utils.WalkObject(
svg,
(leaf) => {
const style = leaf["style"].split(";")
for (const styleElement of style) {
const [key, value] = styleElement.split(":").map((x) => x.trim())
if (value === "none") {
continue
}
if (key === "fill" || key === "stroke") {
colours.add(value)
}
return colours
}
},
(leaf) => typeof leaf["style"] === "string"
)
if (colours.size === 0) {
continue
}
const whiteColours = Array.from(colours).map((c) => {
const rgb = Utils.color(c)
if (!rgb) {
console.log("Could not parse ", c)
return false
}
const { r, g, b } = rgb
return r > 245 && g > 245 && b > 245
})
const hasDark = whiteColours.some((isWhite) => !isWhite)
if (!hasDark) {
whitePaths.add(icon)
}
}
return whitePaths
}
missingLicenseInfos(licenseInfos: SmallLicense[], allIcons: string[]) {
const missing = []
const knownPaths = new Set<string>()
for (const licenseInfo of licenseInfos) {
knownPaths.add(licenseInfo.path)
}
for (const iconPath of allIcons) {
if (iconPath.indexOf("license_info.json") >= 0) {
continue
}
if (knownPaths.has(iconPath)) {
continue
}
missing.push(iconPath)
}
return missing
}
promptLicenseFor(path): SmallLicense {
const knownLicenses = GenerateLicenseInfo.defaultLicenses()
console.log("License abbreviations:")
knownLicenses.forEach((value, key) => {
console.log(key, " => ", value)
})
const author = prompt(
"What is the author for artwork " + path + "? (or: [Q]uit, [S]kip) > "
)
path = path.substring(path.lastIndexOf("/") + 1)
if (knownLicenses.has(author)) {
const license = knownLicenses.get(author)
license.path = path
return license
}
if (author == "s") {
return null
}
if (author == "Q" || author == "q" || author == "") {
throw "Quitting now!"
}
return {
authors: author.split(";"),
path: path,
license: prompt("What is the license for artwork " + path + "? > "),
sources: prompt("Where was this artwork found? > ").split(";"),
}
}
createLicenseInfoFor(path): void {
const li = this.promptLicenseFor(path)
if (li == null) {
return
}
writeFileSync(path + ".license_info.json", JSON.stringify(li, null, " "))
}
/**
* Rewrites a license into a SPDX-valid-ID.
* Might involve some guesswork (e.g. 'CC-BY-SA' --> 'CC-BY-SA 4.0"
* @param licenseId
*/
toSPDXCompliantLicense(licenseId: string): string {
licenseId = licenseId.trim()
// https://spdx.org/licenses/
const mappings: Record<string, string> = {
"CC-0": "CC0-1.0",
CC0: "CC0-1.0",
"CC-BY-4.0-INTERNATIONAL": "CC-BY-4.0",
"CC-4.0": "CC-BY-4.0",
"CC-BY": "CC-BY-4.0",
"CC-BY-SA-4.0-INTERNATIONAL": "CC-BY-SA-4.0",
"CC-BY-SA": "CC-BY-SA-4.0",
"CREATIVE-COMMONS-4.0-BY-NC": "CC-BY-NC-4.0",
"CC-BY-SA-3.0-UNPORTED": "CC-BY-SA-3.0",
"ISC-LICENSE": "ISC",
"LOGO-BY-THE-GOVERNMENT": "LOGO",
PD: "PUBLIC-DOMAIN",
"LOGO-(ALL-RIGHTS-RESERVED)": "LOGO",
/* ALL-RIGHTS-RESERVED:
PD:
PUBLIC-DOMAIN:
TRIVIAL: //*/
}
return mappings[licenseId] ?? licenseId
}
cleanLicenseInfo(allPaths: string[], allLicenseInfos: SmallLicense[]) {
// Read the license info file from the generated assets, creates a compiled license info in every directory
// Note: this removes all the old license infos
for (const licensePath of allPaths) {
unlinkSync(licensePath)
}
const perDirectory = new Map<string, SmallLicense[]>()
for (const license of allLicenseInfos) {
const p = license.path
const dir = p.substring(0, p.lastIndexOf("/"))
license.path = p.substring(dir.length + 1)
if (!perDirectory.has(dir)) {
perDirectory.set(dir, [])
}
const cloned: SmallLicense = {
// We make a clone to force the order of the keys
path: license.path,
license: license.license,
authors: license.authors,
sources: license.sources,
}
cloned.license = Utils.Dedup(
cloned.license.split(";").map((l) => this.toSPDXCompliantLicense(l))
).join("; ")
if (cloned.license === "CC0-1.0; TRIVIAL") {
cloned.license = "TRIVIAL"
}
if (cloned.license === "LOGO; ALL-RIGHTS-RESERVED") {
cloned.license = "LOGO"
}
cloned.license = cloned.license.split("; ").join(" AND ")
perDirectory.get(dir).push(cloned)
}
perDirectory.forEach((licenses, dir) => {
for (let i = licenses.length - 1; i >= 0; i--) {
const license = licenses[i]
const path = dir + "/" + license.path
if (!existsSync(path)) {
console.log(
"Found license for now missing file: ",
path,
" - removing this license"
)
licenses.splice(i, 1)
}
}
licenses.sort((a, b) => (a.path < b.path ? -1 : 1))
const path = dir + "/license_info.json"
if (licenses.length === 0) {
console.log("Removing", path, "as it is empty")
// No need to _actually_ unlik, this is done above
} else {
writeFileSync(path, JSON.stringify(licenses, null, 2))
}
})
}
queryMissingLicenses(missingLicenses: string[]) {
process.on("SIGINT", function () {
console.log("Aborting... Bye!")
process.exit()
})
let i = 1
for (const missingLicens of missingLicenses) {
console.log(i + " / " + missingLicenses.length)
i++
if (i < missingLicenses.length - 5) {
// continue
}
this.createLicenseInfoFor(missingLicens)
}
console.log("You're through!")
}
/**
* Creates the humongous license_info in the generated assets, containing all licenses with a path relative to the root
* @param licensePaths
*/
createFullLicenseOverview(licensePaths: string[], mostlyWhite: string[]) {
const allLicenses: SmallLicense[] = []
for (const licensePath of licensePaths) {
if (!existsSync(licensePath)) {
continue
}
const licenses = <SmallLicense[]>(
JSON.parse(readFileSync(licensePath, { encoding: "utf8" }))
)
for (const license of licenses) {
this.validateLicenseInfo(license)
const dir = licensePath.substring(
0,
licensePath.length - "license_info.json".length
)
license.path = dir + license.path
if (mostlyWhite.some((l) => license.path === l)) {
license["mostly_white"] = true
}
allLicenses.push(license)
}
}
writeFileSync(
"./src/assets/generated/license_info.json",
JSON.stringify(allLicenses, null, " ")
)
}
async main(args: string[]) {
console.log("Checking and compiling license info")
if (!existsSync("./src/assets/generated")) {
mkdirSync("./src/assets/generated")
}
const contents = ScriptUtils.readDirRecSync("./assets").filter(
(entry) => entry.indexOf("./assets/generated") != 0
)
const licensePaths = contents.filter((entry) => entry.indexOf("license_info.json") >= 0)
const licenseInfos = this.generateLicenseInfos(licensePaths)
const artwork = contents.filter(
(pth) => pth.match(/(.svg|.png|.jpg|.ttf|.otf|.woff|.jpeg)$/i) != null
)
const missingLicenses = this.missingLicenseInfos(licenseInfos, artwork)
const mostlyWhite: Set<string> = await this.mostlyWhite(artwork)
if (args.indexOf("--prompt") >= 0 || args.indexOf("--query") >= 0) {
this.queryMissingLicenses(missingLicenses)
return this.main([])
}
const invalidLicenses = licenseInfos
.filter((l) => (l.license ?? "") === "")
.map((l) => `License for artwork ${l.path} is empty string or undefined`)
for (const licenseInfo of licenseInfos) {
const isTrivial = licenseInfo.license
.split(";")
.map((l) => l.trim().toLowerCase())
.some((s) => s.endsWith("trivial"))
if (licenseInfo.sources.length + licenseInfo.authors.length == 0 && !isTrivial) {
invalidLicenses.push(
"Invalid license: No sources nor authors given in the license for " +
JSON.stringify(licenseInfo)
)
continue
}
for (const source of licenseInfo.sources) {
if (source == "") {
invalidLicenses.push(
"Invalid license: empty string in " + JSON.stringify(licenseInfo)
)
}
try {
new URL(source)
} catch {
invalidLicenses.push("Not a valid URL: " + source)
}
}
const spdxPath = licenseInfo.path + ".license"
const spdxContent = [
"SPDX-FileCopyrightText: " + licenseInfo.authors.join("; "),
"SPDX-License-Identifier: " +
licenseInfo.license
.split(" AND ")
.map((s) => this.addLicenseRef(s))
.join(" AND "),
]
writeFileSync(spdxPath, spdxContent.join("\n"))
}
if (missingLicenses.length > 0 || invalidLicenses.length) {
const msg = `There are ${missingLicenses.length} licenses missing and ${invalidLicenses.length} invalid licenses.`
console.log(missingLicenses.concat(invalidLicenses).join("\n"))
console.error(msg)
if (args.indexOf("--no-fail") < 0) {
throw msg
}
}
this.cleanLicenseInfo(licensePaths, licenseInfos)
this.createFullLicenseOverview(licensePaths, Array.from(mostlyWhite))
}
/**
* Some licenses need "LicenseRef-" to be added to make reuse lint work
* @param s
* @private
*/
private addLicenseRef(s: string): string {
if (GenerateLicenseInfo.needsLicenseRef.has(s)) {
return "LicenseRef-" + s
}
return s
}
}
new GenerateLicenseInfo().run()