Add linked data module which scrapes websites

This commit is contained in:
Pieter Vander Vennet 2024-02-22 18:58:34 +01:00
parent 2af6af7630
commit 35c31f9861
15 changed files with 870 additions and 130 deletions

View file

@ -1,13 +1,15 @@
import { Validator } from "../Validator"
export default class UrlValidator extends Validator {
constructor(name?: string, explanation?: string) {
private readonly _forceHttps: boolean
constructor(name?: string, explanation?: string, forceHttps?: boolean) {
super(
name ?? "url",
explanation ??
"The validatedTextField will format URLs to always be valid and have a https://-header (even though the 'https'-part will be hidden from the user. Furthermore, some tracking parameters will be removed",
"url"
)
this._forceHttps = forceHttps ?? false
}
reformat(str: string): string {
try {
@ -22,6 +24,9 @@ export default class UrlValidator extends Validator {
} else {
url = new URL(str)
}
if (this._forceHttps) {
url.protocol = "https:"
}
const blacklistedTrackingParams = [
"fbclid", // Oh god, how I hate the fbclid. Let it burn, burn in hell!
"gclid",