diff --git a/Logic/Web/Wikidata.ts b/Logic/Web/Wikidata.ts index 9364750012..d4049a24e6 100644 --- a/Logic/Web/Wikidata.ts +++ b/Logic/Web/Wikidata.ts @@ -1,6 +1,6 @@ import {Utils} from "../../Utils"; import {UIEventSource} from "../UIEventSource"; -import * as wds from "wikibase-sdk" +import * as wds from "wikidata-sdk" export class WikidataResponse { public readonly id: string @@ -126,13 +126,22 @@ export interface WikidataSearchoptions { maxCount?: 20 | number } +export interface WikidataAdvancedSearchoptions extends WikidataSearchoptions { + instanceOf?: number[]; + notInstanceOf?: number[] +} + + /** * Utility functions around wikidata */ export default class Wikidata { private static readonly _identifierPrefixes = ["Q", "L"].map(str => str.toLowerCase()) - private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:", "https://www.wikidata.org/wiki/", "Lexeme:"].map(str => str.toLowerCase()) + private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:", + "https://www.wikidata.org/wiki/", + "http://www.wikidata.org/entity/", + "Lexeme:"].map(str => str.toLowerCase()) private static readonly _cache = new Map>() @@ -147,25 +156,51 @@ export default class Wikidata { Wikidata._cache.set(key, src) return src; } - - public static async searchAdvanced(text: string, options: WikidataSearchoptions & { - instanceOf: number}){ + + /** + * Given a search text, searches for the relevant wikidata entries, excluding pages "outside of the main tree", e.g. disambiguation pages. + * Optionally, an 'instance of' can be given to limit the scope, e.g. instanceOf:5 (humans) will only search for humans + */ + public static async searchAdvanced(text: string, options: WikidataAdvancedSearchoptions): Promise<{ + id: string, + relevance?: number, + label: string, + description?: string + }[]> { + let instanceOf = "" + if (options?.instanceOf !== undefined && options.instanceOf.length > 0) { + const phrases = options.instanceOf.map(q => `{ ?item wdt:P31/wdt:P279* wd:Q${q}. }`) + instanceOf = "{"+ phrases.join(" UNION ") + "}" + } + const forbidden = (options?.notInstanceOf ?? []) + .concat([17379835]) // blacklist 'wikimedia pages outside of the main knowledge tree', e.g. disambiguation pages + const minusPhrases = forbidden.map(q => `MINUS {?item wdt:P31/wdt:P279* wd:Q${q} .}`) const sparql = `SELECT * WHERE { SERVICE wikibase:mwapi { bd:serviceParam wikibase:api "EntitySearch" . - bd:serviceParam wikibase:endpoint "www.wikidata.org" . - bd:serviceParam mwapi:search "${text}" . - bd:serviceParam mwapi:language "${options.lang}" . - ?item wikibase:apiOutputItem mwapi:item . - ?num wikibase:apiOrdinal true . - } - ?item (wdt:P279|wdt:P31) wd:Q${options.instanceOf} - } ORDER BY ASC(?num) LIMIT ${options.maxCount}` + bd:serviceParam wikibase:endpoint "www.wikidata.org" . + bd:serviceParam mwapi:search "${text}" . + bd:serviceParam mwapi:language "${options.lang}" . + ?item wikibase:apiOutputItem mwapi:item . + ?num wikibase:apiOrdinal true . + bd:serviceParam wikibase:limit ${Math.round((options.maxCount ?? 20) * 1.5) /*Some padding for disambiguation pages */} . + ?label wikibase:apiOutput mwapi:label . + ?description wikibase:apiOutput "@description" . + } + ${instanceOf} + ${minusPhrases.join("\n ")} + } ORDER BY ASC(?num) LIMIT ${options.maxCount ?? 20}` const url = wds.sparqlQuery(sparql) - const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"}) - return result.results.bindings - + const result = await Utils.downloadJson(url) + /*The full uri of the wikidata-item*/ + + return result.results.bindings.map(({item, label, description, num}) => ({ + relevance: num?.value, + id: item?.value, + label: label?.value, + description: description?.value + })) } public static async search( @@ -215,39 +250,28 @@ export default class Wikidata { public static async searchAndFetch( search: string, - options?: WikidataSearchoptions + options?: WikidataAdvancedSearchoptions ): Promise { - const maxCount = options.maxCount // We provide some padding to filter away invalid values - options.maxCount = Math.ceil((options.maxCount ?? 20) * 1.5) - const searchResults = await Wikidata.search(search, options) - const maybeResponses = await Promise.all(searchResults.map(async r => { - try { - return await Wikidata.LoadWikidataEntry(r.id).AsPromise() - } catch (e) { - console.error(e) - return undefined; - } - })) - const responses = maybeResponses - .map(r => r["success"]) - .filter(wd => { - if (wd === undefined) { - return false; + const searchResults = await Wikidata.searchAdvanced(search, options) + const maybeResponses = await Promise.all( + searchResults.map(async r => { + try { + console.log("Loading ", r.id) + return await Wikidata.LoadWikidataEntry(r.id).AsPromise() + } catch (e) { + console.error(e) + return undefined; } - if (wd.claims.get("P31" /*Instance of*/)?.has("Q4167410"/* Wikimedia Disambiguation page*/)) { - return false; - } - return true; - }) - responses.splice(maxCount, responses.length - maxCount) - return responses + })) + return Utils.NoNull(maybeResponses.map(r => r["success"])) } /** * Gets the 'key' segment from a URL - * + * * Wikidata.ExtractKey("https://www.wikidata.org/wiki/Lexeme:L614072") // => "L614072" + * Wikidata.ExtractKey("http://www.wikidata.org/entity/Q55008046") // => "QQ55008046" */ public static ExtractKey(value: string | number): string { if (typeof value === "number") { @@ -291,6 +315,35 @@ export default class Wikidata { return undefined; } + /** + * Converts 'Q123' into 123, returns undefined if invalid + * + * Wikidata.QIdToNumber("Q123") // => 123 + * Wikidata.QIdToNumber(" Q123 ") // => 123 + * Wikidata.QIdToNumber(" X123 ") // => undefined + * Wikidata.QIdToNumber(" Q123X ") // => undefined + * Wikidata.QIdToNumber(undefined) // => undefined + * Wikidata.QIdToNumber(123) // => 123 + */ + public static QIdToNumber(q: string | number): number | undefined { + if(q === undefined || q === null){ + return + } + if(typeof q === "number"){ + return q + } + q = q.trim() + if (!q.startsWith("Q")) { + return + } + q = q.substr(1) + const n = Number(q) + if (isNaN(n)) { + return + } + return n + } + public static IdToArticle(id: string) { if (id.startsWith("Q")) { return "https://wikidata.org/wiki/" + id diff --git a/UI/Input/ValidatedTextField.ts b/UI/Input/ValidatedTextField.ts index 5e8f21fbab..15fe25b7eb 100644 --- a/UI/Input/ValidatedTextField.ts +++ b/UI/Input/ValidatedTextField.ts @@ -250,13 +250,15 @@ class WikidataTextField extends TextFieldDef { ["subarg", "doc"], [["removePrefixes", "remove these snippets of text from the start of the passed string to search"], ["removePostfixes", "remove these snippets of text from the end of the passed string to search"], + ["instanceOf","A list of Q-identifier which indicates that the search results _must_ be an entity of this type, e.g. [`Q5`](https://www.wikidata.org/wiki/Q5) for humans"], + ["notInstanceof","A list of Q-identifiers which indicates that the search results _must not_ be an entity of this type, e.g. [`Q79007`](https://www.wikidata.org/wiki/Q79007) to filter away all streets from the search results"] ] )]) ]]), new Title("Example usage"), `The following is the 'freeform'-part of a layer config which will trigger a search for the wikidata item corresponding with the name of the selected feature. It will also remove '-street', '-square', ... if found at the end of the name -\`\`\` +\`\`\`json "freeform": { "key": "name:etymology:wikidata", "type": "wikidata", @@ -269,11 +271,29 @@ class WikidataTextField extends TextFieldDef { "path", "square", "plaza", - ] + ], + "#": "Remove streets and parks from the search results:" + "notInstanceOf": ["Q79007","Q22698"] } + ] } -\`\`\`` +\`\`\` + +Another example is to search for species and trees: + +\`\`\`json + "freeform": { + "key": "species:wikidata", + "type": "wikidata", + "helperArgs": [ + "species", + { + "instanceOf": [10884, 16521] + }] + } +\`\`\` +` ])); } @@ -304,9 +324,9 @@ class WikidataTextField extends TextFieldDef { const args = inputHelperOptions.args ?? [] const searchKey = args[0] ?? "name" - let searchFor = inputHelperOptions.feature?.properties[searchKey]?.toLowerCase() + let searchFor = (inputHelperOptions.feature?.properties[searchKey]?.toLowerCase() ?? "") - const options = args[1] + const options: any = args[1] if (searchFor !== undefined && options !== undefined) { const prefixes = options["removePrefixes"] const postfixes = options["removePostfixes"] @@ -325,10 +345,18 @@ class WikidataTextField extends TextFieldDef { } } + + let instanceOf : number[] = Utils.NoNull((options?.instanceOf ?? []).map(i => Wikidata.QIdToNumber(i))) + let notInstanceOf : number[] = Utils.NoNull((options?.notInstanceOf ?? []).map(i => Wikidata.QIdToNumber(i))) + console.log("Instance of", instanceOf) + + return new WikidataSearchBox({ value: currentValue, - searchText: new UIEventSource(searchFor) + searchText: new UIEventSource(searchFor), + instanceOf, + notInstanceOf }) } } diff --git a/UI/SpecialVisualizations.ts b/UI/SpecialVisualizations.ts index 046bd28f60..3203553932 100644 --- a/UI/SpecialVisualizations.ts +++ b/UI/SpecialVisualizations.ts @@ -46,6 +46,8 @@ import {LoginToggle} from "./Popup/LoginButton"; import {start} from "repl"; import {SubstitutedTranslation} from "./SubstitutedTranslation"; import {TextField} from "./Input/TextField"; +import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata"; +import {Translation} from "./i18n/Translation"; export interface SpecialVisualization { funcName: string, @@ -159,19 +161,19 @@ class CloseNoteButton implements SpecialVisualization { tags.ping() }) }) - - if((params.minZoom??"") !== "" && !isNaN(Number(params.minZoom))){ - closeButton = new Toggle( + + if ((params.minZoom ?? "") !== "" && !isNaN(Number(params.minZoom))) { + closeButton = new Toggle( closeButton, params.zoomButton ?? "", - state. locationControl.map(l => l.zoom >= Number(params.minZoom)) + state.locationControl.map(l => l.zoom >= Number(params.minZoom)) ) } - + return new LoginToggle(new Toggle( t.isClosed.SetClass("thanks"), closeButton, - + isClosed ), t.loginToClose, state) } @@ -180,7 +182,7 @@ class CloseNoteButton implements SpecialVisualization { export default class SpecialVisualizations { - public static specialVisualizations : SpecialVisualization[] = SpecialVisualizations.init() + public static specialVisualizations: SpecialVisualization[] = SpecialVisualizations.init() public static HelpMessage() { @@ -207,28 +209,28 @@ export default class SpecialVisualizations { )); return new Combine([ - new Combine([ - - new Title("Special tag renderings", 1), - - "In a tagrendering, some special values are substituted by an advanced UI-element. This allows advanced features and visualizations to be reused by custom themes or even to query third-party API's.", - "General usage is `{func_name()}`, `{func_name(arg, someotherarg)}` or `{func_name(args):cssStyle}`. Note that you _do not_ need to use quotes around your arguments, the comma is enough to separate them. This also implies you cannot use a comma in your args", - new Title("Using expanded syntax",4), - `Instead of using \`{"render": {"en": "{some_special_visualisation(some_arg, some other really long message, more args)} , "nl": "{some_special_visualisation(some_arg, een boodschap in een andere taal, more args)}}, one can also write`, - new FixedUiElement(JSON.stringify({ - render: { - special:{ - type: "some_special_visualisation", - "argname": "some_arg", - "message":{ - en:"some other really long message", - nl: "een boodschap in een andere taal" - }, - "other_arg_name":"more args" + new Combine([ + + new Title("Special tag renderings", 1), + + "In a tagrendering, some special values are substituted by an advanced UI-element. This allows advanced features and visualizations to be reused by custom themes or even to query third-party API's.", + "General usage is `{func_name()}`, `{func_name(arg, someotherarg)}` or `{func_name(args):cssStyle}`. Note that you _do not_ need to use quotes around your arguments, the comma is enough to separate them. This also implies you cannot use a comma in your args", + new Title("Using expanded syntax", 4), + `Instead of using \`{"render": {"en": "{some_special_visualisation(some_arg, some other really long message, more args)} , "nl": "{some_special_visualisation(some_arg, een boodschap in een andere taal, more args)}}, one can also write`, + new FixedUiElement(JSON.stringify({ + render: { + special: { + type: "some_special_visualisation", + "argname": "some_arg", + "message": { + en: "some other really long message", + nl: "een boodschap in een andere taal" + }, + "other_arg_name": "more args" + } } - } - })).SetClass("code") - ]).SetClass("flex flex-col"), + })).SetClass("code") + ]).SetClass("flex flex-col"), ...helpTexts ] ).SetClass("flex flex-col"); @@ -297,6 +299,32 @@ export default class SpecialVisualizations { ) }, + { + funcName: "wikidata_label", + docs: "Shows the label of the corresponding wikidata-item", + args: [ + { + name: "keyToShowWikidataFor", + doc: "Use the wikidata entry from this key to show the label", + defaultValue: "wikidata" + } + ], + example: "`{wikidata_label()}` is a basic example, `{wikipedia(name:etymology:wikidata)}` to show the label itself", + constr: (_, tagsSource, args) => + new VariableUiElement( + tagsSource.map(tags => tags[args[0]]) + .map(wikidata => { + wikidata = Utils.NoEmpty(wikidata?.split(";")?.map(wd => wd.trim()) ?? [])[0] + const entry = Wikidata.LoadWikidataEntry(wikidata) + return new VariableUiElement(entry.map(e => { + if (e === undefined || e["success"] === undefined) { + return wikidata + } + const response = e["success"] + return Translation.fromMap(response.labels) + })) + })) + }, { funcName: "minimap", docs: "A small map showing the selected feature.", @@ -482,7 +510,7 @@ export default class SpecialVisualizations { docs: "Downloads a JSON from the given URL, e.g. '{live(example.org/data.json, shorthand:x.y.z, other:a.b.c, shorthand)}' will download the given file, will create an object {shorthand: json[x][y][z], other: json[a][b][c] out of it and will return 'other' or 'json[a][b][c]. This is made to use in combination with tags, e.g. {live({url}, {url:format}, needed_value)}", example: "{live({url},{url:format},hour)} {live(https://data.mobility.brussels/bike/api/counts/?request=live&featureID=CB2105,hour:data.hour_cnt;day:data.day_cnt;year:data.year_cnt,hour)}", args: [{ - name: "Url", + name: "Url", doc: "The URL to load", required: true }, { @@ -783,7 +811,7 @@ export default class SpecialVisualizations { const textField = new TextField( { placeholder: t.addCommentPlaceholder, - inputStyle: "width: 100%; height: 6rem;", + inputStyle: "width: 100%; height: 6rem;", textAreaRows: 3, htmlType: "area" } @@ -846,7 +874,7 @@ export default class SpecialVisualizations { textField, new Combine([ stateButtons.SetClass("sm:mr-2"), - new Toggle(addCommentButton, + new Toggle(addCommentButton, new Combine([t.typeText]).SetClass("flex items-center h-full subtle"), textField.GetValue().map(t => t !== undefined && t.length >= 1)).SetClass("sm:mr-2") ]).SetClass("sm:flex sm:justify-between sm:items-stretch") @@ -947,7 +975,7 @@ export default class SpecialVisualizations { ] specialVisualizations.push(new AutoApplyButton(specialVisualizations)) - + return specialVisualizations; } diff --git a/UI/Wikipedia/WikidataSearchBox.ts b/UI/Wikipedia/WikidataSearchBox.ts index c3df838cdd..87afd99054 100644 --- a/UI/Wikipedia/WikidataSearchBox.ts +++ b/UI/Wikipedia/WikidataSearchBox.ts @@ -17,14 +17,20 @@ export default class WikidataSearchBox extends InputElement { IsSelected: UIEventSource = new UIEventSource(false); private readonly wikidataId: UIEventSource private readonly searchText: UIEventSource + private readonly instanceOf?: number[]; + private readonly notInstanceOf?: number[]; constructor(options?: { searchText?: UIEventSource, - value?: UIEventSource + value?: UIEventSource, + notInstanceOf?: number[], + instanceOf?: number[] }) { super(); this.searchText = options?.searchText this.wikidataId = options?.value ?? new UIEventSource(undefined); + this.instanceOf = options?.instanceOf + this.notInstanceOf = options?.notInstanceOf } GetValue(): UIEventSource { @@ -59,7 +65,9 @@ export default class WikidataSearchBox extends InputElement { if (promise === undefined) { promise = Wikidata.searchAndFetch(searchText, { lang, - maxCount: 5 + maxCount: 5, + notInstanceOf: this.notInstanceOf, + instanceOf: this.instanceOf } ) WikidataSearchBox._searchCache.set(key, promise) @@ -75,13 +83,15 @@ export default class WikidataSearchBox extends InputElement { return new Combine([Translations.t.general.wikipedia.failed.Clone().SetClass("alert"), searchFailMessage.data]) } + if (searchField.GetValue().data.length === 0) { + return Translations.t.general.wikipedia.doSearch + } + if (searchResults.length === 0) { return Translations.t.general.wikipedia.noResults.Subs({search: searchField.GetValue().data ?? ""}) } - if (searchResults.length === 0) { - return Translations.t.general.wikipedia.doSearch - } + return new Combine(searchResults.map(wikidataresponse => { const el = WikidataPreviewBox.WikidataResponsePreview(wikidataresponse).SetClass("rounded-xl p-1 sm:p-2 md:p-3 m-px border-2 sm:border-4 transition-colors") diff --git a/assets/layers/etymology/etymology.json b/assets/layers/etymology/etymology.json index cf45d7a5d6..1447c7d812 100644 --- a/assets/layers/etymology/etymology.json +++ b/assets/layers/etymology/etymology.json @@ -52,6 +52,7 @@ "helperArgs": [ "name", { + "notInstanceOf": ["Q79007","Q22698"], "removePostfixes": [ "steenweg", "heirbaan", @@ -70,7 +71,9 @@ "wegel", "kerk", "church", - "kaai" + "kaai", + "park", + "parque" ] } ] diff --git a/assets/layers/tree_node/tree_node.json b/assets/layers/tree_node/tree_node.json index 821d2650b1..3035cc7ef0 100644 --- a/assets/layers/tree_node/tree_node.json +++ b/assets/layers/tree_node/tree_node.json @@ -314,6 +314,34 @@ ] } }, + { + "id": "tree-species-wikidata", + "question": { + "en": "What species is this tree?" + }, + "render": { + "*":"{wikipedia(species:wikidata):max-height: 25rem}" + }, + "freeform": { + "key": "species:wikidata", + "type": "wikidata", + "helperArgs": [ + "species", + { + "instanceOf": [10884, 16521] + }] + } + }, + { + "id": "tree-wikipedia", + "#": "If this tree has a wikipedia article, show it. People can _only_ set the species though!", + "render": { + "*":"{wikipedia()}" + }, + "condition": { + "or": ["wikipedia~*","wikidata~*"] + } + }, { "render": { "nl": "Naam: {name}", diff --git a/test.ts b/test.ts index 9a6b02f972..4640df0ca4 100644 --- a/test.ts +++ b/test.ts @@ -1,31 +1,17 @@ -import Combine from "./UI/Base/Combine"; -import ValidatedTextField from "./UI/Input/ValidatedTextField"; -import Title from "./UI/Base/Title"; -import {FixedUiElement} from "./UI/Base/FixedUiElement"; import {VariableUiElement} from "./UI/Base/VariableUIElement"; import {UIEventSource} from "./Logic/UIEventSource"; -import {Translation} from "./UI/i18n/Translation"; +import Wikidata from "./Logic/Web/Wikidata"; +import Combine from "./UI/Base/Combine"; +import {FixedUiElement} from "./UI/Base/FixedUiElement"; -new Combine( - ValidatedTextField.AvailableTypes().map(key => { - let inp; - const feedback = new UIEventSource(undefined) - try { - inp = ValidatedTextField.ForType(key).ConstructInputElement({ - feedback, - country: () => "be" - }); - } catch (e) { - console.error(e) - inp = new FixedUiElement(e).SetClass("alert") - } - - return new Combine([ - new Title(key), - inp, - new VariableUiElement(inp.GetValue()), - new VariableUiElement(feedback.map(v => v?.SetClass("alert"))) - ]); - } - ) -).AttachTo("maindiv") \ No newline at end of file +const result = UIEventSource.FromPromise( + Wikidata.searchAdvanced("WOlf", { + lang: "nl", + maxCount: 100, + instanceOf: 5 + }) +) +result.addCallbackAndRunD(r => console.log(r)) +new VariableUiElement(result.map(items =>new Combine( (items??[])?.map(i => + new FixedUiElement(JSON.stringify(i, null, " ")).SetClass("p-4 block") +)) )).SetClass("flex flex-col").AttachTo("maindiv") \ No newline at end of file