Improve wikidata search with filtering, add search box for species to trees

Logic/Web/Wikidata.ts

@@ -1,6 +1,6 @@
 import {Utils} from "../../Utils";
 import {UIEventSource} from "../UIEventSource";
-import * as wds from "wikibase-sdk"
+import * as wds from "wikidata-sdk"
 
 export class WikidataResponse {
     public readonly id: string
@@ -126,13 +126,22 @@ export interface WikidataSearchoptions {
     maxCount?: 20 | number
 }
 
+export interface WikidataAdvancedSearchoptions extends WikidataSearchoptions {
+    instanceOf?: number[];
+    notInstanceOf?: number[]
+}
+
+
 /**
  * Utility functions around wikidata
  */
 export default class Wikidata {
 
     private static readonly _identifierPrefixes = ["Q", "L"].map(str => str.toLowerCase())
-    private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:", "https://www.wikidata.org/wiki/", "Lexeme:"].map(str => str.toLowerCase())
+    private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:", 
+        "https://www.wikidata.org/wiki/",
+        "http://www.wikidata.org/entity/",
+        "Lexeme:"].map(str => str.toLowerCase())
 
 
     private static readonly _cache = new Map<string, UIEventSource<{ success: WikidataResponse } | { error: any }>>()
@@ -147,25 +156,51 @@ export default class Wikidata {
         Wikidata._cache.set(key, src)
         return src;
     }
-    
+
-    public static async searchAdvanced(text: string, options: WikidataSearchoptions & {
+    /**
-        instanceOf: number}){
+     * Given a search text, searches for the relevant wikidata entries, excluding pages "outside of the main tree", e.g. disambiguation pages.
+     * Optionally, an 'instance of' can be given to limit the scope, e.g. instanceOf:5 (humans) will only search for humans
+     */
+    public static async searchAdvanced(text: string, options: WikidataAdvancedSearchoptions): Promise<{
+        id: string,
+        relevance?: number,
+        label: string,
+        description?: string
+    }[]> {
+        let instanceOf = ""
+        if (options?.instanceOf !== undefined && options.instanceOf.length > 0) {
+           const phrases = options.instanceOf.map(q => `{ ?item wdt:P31/wdt:P279* wd:Q${q}. }`)
+            instanceOf = "{"+ phrases.join(" UNION ") + "}"
+        }
+        const forbidden = (options?.notInstanceOf ?? [])
+            .concat([17379835]) // blacklist 'wikimedia pages outside of the main knowledge tree', e.g. disambiguation pages
+        const minusPhrases = forbidden.map(q => `MINUS {?item wdt:P31/wdt:P279* wd:Q${q} .}`)
         const sparql = `SELECT * WHERE {
             SERVICE wikibase:mwapi {
                 bd:serviceParam wikibase:api "EntitySearch" .
-                    bd:serviceParam wikibase:endpoint "www.wikidata.org" .
+                bd:serviceParam wikibase:endpoint "www.wikidata.org" .
-                    bd:serviceParam mwapi:search "${text}" .
+                bd:serviceParam mwapi:search "${text}" .
-                    bd:serviceParam mwapi:language "${options.lang}" .
+                bd:serviceParam mwapi:language "${options.lang}" .
-                    ?item wikibase:apiOutputItem mwapi:item .
+                ?item wikibase:apiOutputItem mwapi:item .
-                    ?num wikibase:apiOrdinal true .
+                ?num wikibase:apiOrdinal true .
-            }
+                bd:serviceParam wikibase:limit ${Math.round((options.maxCount ?? 20) * 1.5) /*Some padding for disambiguation pages */} .
-                ?item (wdt:P279|wdt:P31) wd:Q${options.instanceOf}
+                ?label wikibase:apiOutput mwapi:label .
-        } ORDER BY ASC(?num) LIMIT ${options.maxCount}`
+                ?description wikibase:apiOutput "@description" .
+            } 
+            ${instanceOf}
+            ${minusPhrases.join("\n    ")}
+        } ORDER BY ASC(?num) LIMIT ${options.maxCount ?? 20}`
         const url = wds.sparqlQuery(sparql)
 
-        const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
+        const result = await Utils.downloadJson(url)
-        return result.results.bindings
+        /*The full uri of the wikidata-item*/
-        
+
+        return result.results.bindings.map(({item, label, description, num}) => ({
+            relevance: num?.value,
+            id: item?.value,
+            label: label?.value,
+            description: description?.value
+        }))
     }
 
     public static async search(
@@ -215,39 +250,28 @@ export default class Wikidata {
 
     public static async searchAndFetch(
         search: string,
-        options?: WikidataSearchoptions
+        options?: WikidataAdvancedSearchoptions
     ): Promise<WikidataResponse[]> {
-        const maxCount = options.maxCount
         // We provide some padding to filter away invalid values
-        options.maxCount = Math.ceil((options.maxCount ?? 20) * 1.5)
+        const searchResults = await Wikidata.searchAdvanced(search, options)
-        const searchResults = await Wikidata.search(search, options)
+        const maybeResponses = await Promise.all(
-        const maybeResponses = await Promise.all(searchResults.map(async r => {
+            searchResults.map(async r => {
-            try {
+                try {
-                return await Wikidata.LoadWikidataEntry(r.id).AsPromise()
+                    console.log("Loading ", r.id)
-            } catch (e) {
+                    return await Wikidata.LoadWikidataEntry(r.id).AsPromise()
-                console.error(e)
+                } catch (e) {
-                return undefined;
+                    console.error(e)
-            }
+                    return undefined;
-        }))
-        const responses = maybeResponses
-            .map(r => <WikidataResponse>r["success"])
-            .filter(wd => {
-                if (wd === undefined) {
-                    return false;
                 }
-                if (wd.claims.get("P31" /*Instance of*/)?.has("Q4167410"/* Wikimedia Disambiguation page*/)) {
+            }))
-                    return false;
+        return Utils.NoNull(maybeResponses.map(r => <WikidataResponse>r["success"]))
-                }
-                return true;
-            })
-        responses.splice(maxCount, responses.length - maxCount)
-        return responses
     }
 
     /**
      * Gets the 'key' segment from a URL
-     * 
+     *
      * Wikidata.ExtractKey("https://www.wikidata.org/wiki/Lexeme:L614072") // => "L614072"
+     * Wikidata.ExtractKey("http://www.wikidata.org/entity/Q55008046") // => "QQ55008046"
      */
     public static ExtractKey(value: string | number): string {
         if (typeof value === "number") {
@@ -291,6 +315,35 @@ export default class Wikidata {
         return undefined;
     }
 
+    /**
+     * Converts 'Q123' into 123, returns undefined if invalid
+     *
+     * Wikidata.QIdToNumber("Q123") // => 123
+     * Wikidata.QIdToNumber("  Q123  ") // => 123
+     *  Wikidata.QIdToNumber("  X123  ") // => undefined
+     * Wikidata.QIdToNumber("  Q123X  ") // => undefined
+     * Wikidata.QIdToNumber(undefined) // => undefined
+     * Wikidata.QIdToNumber(123) // => 123
+     */
+    public static QIdToNumber(q: string | number): number | undefined {
+        if(q === undefined || q === null){
+            return
+        }
+        if(typeof q === "number"){
+            return q
+        }
+        q = q.trim()
+        if (!q.startsWith("Q")) {
+            return
+        }
+        q = q.substr(1)
+        const n = Number(q)
+        if (isNaN(n)) {
+            return
+        }
+        return n
+    }
+
     public static IdToArticle(id: string) {
         if (id.startsWith("Q")) {
             return "https://wikidata.org/wiki/" + id

UI/Input/ValidatedTextField.ts

@@ -250,13 +250,15 @@ class WikidataTextField extends TextFieldDef {
                                 ["subarg", "doc"],
                                 [["removePrefixes", "remove these snippets of text from the start of the passed string to search"],
                                     ["removePostfixes", "remove these snippets of text from the end of the passed string to search"],
+                                    ["instanceOf","A list of Q-identifier which indicates that the search results _must_ be an entity of this type, e.g. [`Q5`](https://www.wikidata.org/wiki/Q5) for humans"],
+                                        ["notInstanceof","A list of Q-identifiers which indicates that the search results _must not_ be an entity of this type, e.g. [`Q79007`](https://www.wikidata.org/wiki/Q79007) to filter away all streets from the search results"]
                                 ]
                             )])
                         ]]),
                 new Title("Example usage"),
                 `The following is the 'freeform'-part of a layer config which will trigger a search for the wikidata item corresponding with the name of the selected feature. It will also remove '-street', '-square', ... if found at the end of the name
 
-\`\`\`
+\`\`\`json
 "freeform": {
     "key": "name:etymology:wikidata",
     "type": "wikidata",
@@ -269,11 +271,29 @@ class WikidataTextField extends TextFieldDef {
                 "path",
                 "square",
                 "plaza",
-            ]
+            ],
+            "#": "Remove streets and parks from the search results:"
+             "notInstanceOf": ["Q79007","Q22698"] 
         }
+        
     ]
 }
-\`\`\``
+\`\`\`
+
+Another example is to search for species and trees:
+
+\`\`\`json
+ "freeform": {
+        "key": "species:wikidata",
+        "type": "wikidata",
+        "helperArgs": [
+          "species",
+          {
+          "instanceOf": [10884, 16521]
+        }]
+      }
+\`\`\`
+`
             ]));
     }
 
@@ -304,9 +324,9 @@ class WikidataTextField extends TextFieldDef {
         const args = inputHelperOptions.args ?? []
         const searchKey = args[0] ?? "name"
 
-        let searchFor = <string>inputHelperOptions.feature?.properties[searchKey]?.toLowerCase()
+        let searchFor = <string>(inputHelperOptions.feature?.properties[searchKey]?.toLowerCase() ?? "")
 
-        const options = args[1]
+        const options: any = args[1]
         if (searchFor !== undefined && options !== undefined) {
             const prefixes = <string[]>options["removePrefixes"]
             const postfixes = <string[]>options["removePostfixes"]
@@ -325,10 +345,18 @@ class WikidataTextField extends TextFieldDef {
             }
 
         }
+        
+        let instanceOf : number[] =  Utils.NoNull((options?.instanceOf ?? []).map(i => Wikidata.QIdToNumber(i)))
+        let notInstanceOf : number[] = Utils.NoNull((options?.notInstanceOf ?? []).map(i => Wikidata.QIdToNumber(i)))
 
+        console.log("Instance of", instanceOf)
+        
+        
         return new WikidataSearchBox({
             value: currentValue,
-            searchText: new UIEventSource<string>(searchFor)
+            searchText: new UIEventSource<string>(searchFor),
+            instanceOf,
+            notInstanceOf
         })
     }
 }

UI/SpecialVisualizations.ts

@@ -46,6 +46,8 @@ import {LoginToggle} from "./Popup/LoginButton";
 import {start} from "repl";
 import {SubstitutedTranslation} from "./SubstitutedTranslation";
 import {TextField} from "./Input/TextField";
+import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata";
+import {Translation} from "./i18n/Translation";
 
 export interface SpecialVisualization {
     funcName: string,
@@ -159,19 +161,19 @@ class CloseNoteButton implements SpecialVisualization {
                     tags.ping()
                 })
         })
-        
+
-        if((params.minZoom??"") !== "" && !isNaN(Number(params.minZoom))){
+        if ((params.minZoom ?? "") !== "" && !isNaN(Number(params.minZoom))) {
-          closeButton =  new Toggle(
+            closeButton = new Toggle(
                 closeButton,
                 params.zoomButton ?? "",
-                state.  locationControl.map(l => l.zoom >= Number(params.minZoom))
+                state.locationControl.map(l => l.zoom >= Number(params.minZoom))
             )
         }
-        
+
         return new LoginToggle(new Toggle(
             t.isClosed.SetClass("thanks"),
             closeButton,
-            
+
             isClosed
         ), t.loginToClose, state)
     }
@@ -180,7 +182,7 @@ class CloseNoteButton implements SpecialVisualization {
 
 export default class SpecialVisualizations {
 
-    public static specialVisualizations : SpecialVisualization[] = SpecialVisualizations.init()
+    public static specialVisualizations: SpecialVisualization[] = SpecialVisualizations.init()
 
     public static HelpMessage() {
 
@@ -207,28 +209,28 @@ export default class SpecialVisualizations {
             ));
 
         return new Combine([
-            new Combine([
+                new Combine([
-                
+
-                new Title("Special tag renderings", 1),
+                    new Title("Special tag renderings", 1),
-                
+
-                "In a tagrendering, some special values are substituted by an advanced UI-element. This allows advanced features and visualizations to be reused by custom themes or even to query third-party API's.",
+                    "In a tagrendering, some special values are substituted by an advanced UI-element. This allows advanced features and visualizations to be reused by custom themes or even to query third-party API's.",
-                "General usage is `{func_name()}`, `{func_name(arg, someotherarg)}` or `{func_name(args):cssStyle}`. Note that you _do not_ need to use quotes around your arguments, the comma is enough to separate them. This also implies you cannot use a comma in your args",
+                    "General usage is `{func_name()}`, `{func_name(arg, someotherarg)}` or `{func_name(args):cssStyle}`. Note that you _do not_ need to use quotes around your arguments, the comma is enough to separate them. This also implies you cannot use a comma in your args",
-                new Title("Using expanded syntax",4),
+                    new Title("Using expanded syntax", 4),
-                `Instead of using \`{"render": {"en": "{some_special_visualisation(some_arg, some other really long message, more args)} , "nl": "{some_special_visualisation(some_arg, een boodschap in een andere taal, more args)}}, one can also write`,
+                    `Instead of using \`{"render": {"en": "{some_special_visualisation(some_arg, some other really long message, more args)} , "nl": "{some_special_visualisation(some_arg, een boodschap in een andere taal, more args)}}, one can also write`,
-                new FixedUiElement(JSON.stringify({
+                    new FixedUiElement(JSON.stringify({
-                    render: {
+                        render: {
-                        special:{
+                            special: {
-                            type: "some_special_visualisation",
+                                type: "some_special_visualisation",
-                            "argname": "some_arg",
+                                "argname": "some_arg",
-                            "message":{
+                                "message": {
-                                en:"some other really long message",
+                                    en: "some other really long message",
-                                nl: "een boodschap in een andere taal"
+                                    nl: "een boodschap in een andere taal"
-                            },
+                                },
-                            "other_arg_name":"more args"
+                                "other_arg_name": "more args"
+                            }
                         }
-                    }
+                    })).SetClass("code")
-                })).SetClass("code")
+                ]).SetClass("flex flex-col"),
-            ]).SetClass("flex flex-col"),
                 ...helpTexts
             ]
         ).SetClass("flex flex-col");
@@ -297,6 +299,32 @@ export default class SpecialVisualizations {
                         )
 
                 },
+                {
+                    funcName: "wikidata_label",
+                    docs: "Shows the label of the corresponding wikidata-item",
+                    args: [
+                        {
+                            name: "keyToShowWikidataFor",
+                            doc: "Use the wikidata entry from this key to show the label",
+                            defaultValue: "wikidata"
+                        }
+                    ],
+                    example: "`{wikidata_label()}` is a basic example, `{wikipedia(name:etymology:wikidata)}` to show the label itself",
+                    constr: (_, tagsSource, args) =>
+                        new VariableUiElement(
+                            tagsSource.map(tags => tags[args[0]])
+                                .map(wikidata => {
+                                    wikidata = Utils.NoEmpty(wikidata?.split(";")?.map(wd => wd.trim()) ?? [])[0]
+                                    const entry = Wikidata.LoadWikidataEntry(wikidata)
+                                    return new VariableUiElement(entry.map(e => {
+                                        if (e === undefined || e["success"] === undefined) {
+                                            return wikidata
+                                        }
+                                        const response = <WikidataResponse>e["success"]
+                                        return Translation.fromMap(response.labels)
+                                    }))
+                                }))
+                },
                 {
                     funcName: "minimap",
                     docs: "A small map showing the selected feature.",
@@ -482,7 +510,7 @@ export default class SpecialVisualizations {
                     docs: "Downloads a JSON from the given URL, e.g. '{live(example.org/data.json, shorthand:x.y.z, other:a.b.c, shorthand)}' will download the given file, will create an object {shorthand: json[x][y][z], other: json[a][b][c] out of it and will return 'other' or 'json[a][b][c]. This is made to use in combination with tags, e.g. {live({url}, {url:format}, needed_value)}",
                     example: "{live({url},{url:format},hour)} {live(https://data.mobility.brussels/bike/api/counts/?request=live&featureID=CB2105,hour:data.hour_cnt;day:data.day_cnt;year:data.year_cnt,hour)}",
                     args: [{
-                        name: "Url", 
+                        name: "Url",
                         doc: "The URL to load",
                         required: true
                     }, {
@@ -783,7 +811,7 @@ export default class SpecialVisualizations {
                         const textField = new TextField(
                             {
                                 placeholder: t.addCommentPlaceholder,
-                            inputStyle: "width: 100%; height: 6rem;",
+                                inputStyle: "width: 100%; height: 6rem;",
                                 textAreaRows: 3,
                                 htmlType: "area"
                             }
@@ -846,7 +874,7 @@ export default class SpecialVisualizations {
                                 textField,
                                 new Combine([
                                     stateButtons.SetClass("sm:mr-2"),
-                                    new Toggle(addCommentButton, 
+                                    new Toggle(addCommentButton,
                                         new Combine([t.typeText]).SetClass("flex items-center h-full subtle"),
                                         textField.GetValue().map(t => t !== undefined && t.length >= 1)).SetClass("sm:mr-2")
                                 ]).SetClass("sm:flex sm:justify-between sm:items-stretch")
@@ -947,7 +975,7 @@ export default class SpecialVisualizations {
             ]
 
         specialVisualizations.push(new AutoApplyButton(specialVisualizations))
-        
+
         return specialVisualizations;
     }
 

UI/Wikipedia/WikidataSearchBox.ts

@@ -17,14 +17,20 @@ export default class WikidataSearchBox extends InputElement<string> {
     IsSelected: UIEventSource<boolean> = new UIEventSource<boolean>(false);
     private readonly wikidataId: UIEventSource<string>
     private readonly searchText: UIEventSource<string>
+    private readonly instanceOf?: number[];
+    private readonly notInstanceOf?: number[];
 
     constructor(options?: {
         searchText?: UIEventSource<string>,
-        value?: UIEventSource<string>
+        value?: UIEventSource<string>,
+        notInstanceOf?: number[],
+        instanceOf?: number[]
     }) {
         super();
         this.searchText = options?.searchText
         this.wikidataId = options?.value ?? new UIEventSource<string>(undefined);
+        this.instanceOf = options?.instanceOf
+        this.notInstanceOf = options?.notInstanceOf
     }
 
     GetValue(): UIEventSource<string> {
@@ -59,7 +65,9 @@ export default class WikidataSearchBox extends InputElement<string> {
             if (promise === undefined) {
                 promise = Wikidata.searchAndFetch(searchText, {
                         lang,
-                        maxCount: 5
+                        maxCount: 5,
+                        notInstanceOf: this.notInstanceOf,
+                        instanceOf: this.instanceOf
                     }
                 )
                 WikidataSearchBox._searchCache.set(key, promise)
@@ -75,13 +83,15 @@ export default class WikidataSearchBox extends InputElement<string> {
                 return new Combine([Translations.t.general.wikipedia.failed.Clone().SetClass("alert"), searchFailMessage.data])
             }
 
+            if (searchField.GetValue().data.length === 0) {
+                return Translations.t.general.wikipedia.doSearch
+            }
+
             if (searchResults.length === 0) {
                 return Translations.t.general.wikipedia.noResults.Subs({search: searchField.GetValue().data ?? ""})
             }
 
-            if (searchResults.length === 0) {
+
-                return Translations.t.general.wikipedia.doSearch
-            }
 
             return new Combine(searchResults.map(wikidataresponse => {
                 const el = WikidataPreviewBox.WikidataResponsePreview(wikidataresponse).SetClass("rounded-xl p-1 sm:p-2 md:p-3 m-px border-2 sm:border-4 transition-colors")

assets/layers/etymology/etymology.json

@@ -52,6 +52,7 @@
         "helperArgs": [
           "name",
           {
+            "notInstanceOf": ["Q79007","Q22698"],
             "removePostfixes": [
               "steenweg",
               "heirbaan",
@@ -70,7 +71,9 @@
               "wegel",
               "kerk",
               "church",
-              "kaai"
+              "kaai",
+              "park",
+              "parque"
             ]
           }
         ]

assets/layers/tree_node/tree_node.json

@@ -314,6 +314,34 @@
         ]
       }
     },
+    {
+      "id": "tree-species-wikidata",
+      "question": {
+        "en": "What species is this tree?"
+      },
+      "render": {
+        "*":"{wikipedia(species:wikidata):max-height: 25rem}" 
+      },
+      "freeform": {
+        "key": "species:wikidata",
+        "type": "wikidata",
+        "helperArgs": [
+          "species",
+          {
+          "instanceOf": [10884, 16521]
+        }]
+      }
+    },
+    {
+      "id": "tree-wikipedia",
+      "#": "If this tree has a wikipedia article, show it. People can _only_ set the species though!",
+      "render": {
+        "*":"{wikipedia()}"
+      },
+      "condition": {
+        "or": ["wikipedia~*","wikidata~*"]
+      }
+    },
     {
       "render": {
         "nl": "Naam: {name}",

test.ts

@@ -1,31 +1,17 @@
-import Combine from "./UI/Base/Combine";
-import ValidatedTextField from "./UI/Input/ValidatedTextField";
-import Title from "./UI/Base/Title";
-import {FixedUiElement} from "./UI/Base/FixedUiElement";
 import {VariableUiElement} from "./UI/Base/VariableUIElement";
 import {UIEventSource} from "./Logic/UIEventSource";
-import {Translation} from "./UI/i18n/Translation";
+import Wikidata from "./Logic/Web/Wikidata";
+import Combine from "./UI/Base/Combine";
+import {FixedUiElement} from "./UI/Base/FixedUiElement";
 
-new Combine(
+const result = UIEventSource.FromPromise(
-    ValidatedTextField.AvailableTypes().map(key => {
+    Wikidata.searchAdvanced("WOlf", {
-            let inp;
+        lang: "nl",
-            const feedback = new UIEventSource<Translation>(undefined)
+        maxCount: 100,
-            try {
+        instanceOf: 5
-                inp = ValidatedTextField.ForType(key).ConstructInputElement({
+    })
-                    feedback,
+)
-                    country: () => "be"
+result.addCallbackAndRunD(r => console.log(r))
-                });
+new VariableUiElement(result.map(items =>new Combine( (items??[])?.map(i => 
-            } catch (e) {
+    new FixedUiElement(JSON.stringify(i, null, "  ")).SetClass("p-4 block")
-                console.error(e)
+))  )).SetClass("flex flex-col").AttachTo("maindiv")
-                inp = new FixedUiElement(e).SetClass("alert")
-            }
-
-            return new Combine([
-                new Title(key),
-                inp,
-                new VariableUiElement(inp.GetValue()),
-                new VariableUiElement(feedback.map(v => v?.SetClass("alert")))
-            ]);
-        }
-    )
-).AttachTo("maindiv")