midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Improve wikidata search with filtering, add search box for species to trees

Browse source
This commit is contained in:
pietervdvn 2022-04-22 01:45:54 +02:00
parent 1271f24160
commit 389e3f18a0
7 changed files with 249 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
Logic/Web/Wikidata.ts
View file

@ -1,6 +1,6 @@
import {Utils} from "../../Utils";
import {UIEventSource} from "../UIEventSource";
import * as wds from "wikibase-sdk"
import * as wds from "wikidata-sdk"
export class WikidataResponse {
public readonly id: string
@ -126,13 +126,22 @@ export interface WikidataSearchoptions {
maxCount?: 20 | number
}
export interface WikidataAdvancedSearchoptions extends WikidataSearchoptions {
instanceOf?: number[];
notInstanceOf?: number[]
}
/**
* Utility functions around wikidata
*/
export default class Wikidata {
private static readonly _identifierPrefixes = ["Q", "L"].map(str => str.toLowerCase())
private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:", "https://www.wikidata.org/wiki/", "Lexeme:"].map(str => str.toLowerCase())
private static readonly _prefixesToRemove = ["https://www.wikidata.org/wiki/Lexeme:",
"https://www.wikidata.org/wiki/",
"http://www.wikidata.org/entity/",
"Lexeme:"].map(str => str.toLowerCase())
private static readonly _cache = new Map<string, UIEventSource<{ success: WikidataResponse } | { error: any }>>()
@ -147,25 +156,51 @@ export default class Wikidata {
Wikidata._cache.set(key, src)
return src;
}
public static async searchAdvanced(text: string, options: WikidataSearchoptions & {
instanceOf: number}){
/**
* Given a search text, searches for the relevant wikidata entries, excluding pages "outside of the main tree", e.g. disambiguation pages.
* Optionally, an 'instance of' can be given to limit the scope, e.g. instanceOf:5 (humans) will only search for humans
*/
public static async searchAdvanced(text: string, options: WikidataAdvancedSearchoptions): Promise<{
id: string,
relevance?: number,
label: string,
description?: string
}[]> {
let instanceOf = ""
if (options?.instanceOf !== undefined && options.instanceOf.length > 0) {
const phrases = options.instanceOf.map(q => `{ ?item wdt:P31/wdt:P279* wd:Q${q}. }`)
instanceOf = "{"+ phrases.join(" UNION ") + "}"
}
const forbidden = (options?.notInstanceOf ?? [])
.concat([17379835]) // blacklist 'wikimedia pages outside of the main knowledge tree', e.g. disambiguation pages
const minusPhrases = forbidden.map(q => `MINUS {?item wdt:P31/wdt:P279* wd:Q${q} .}`)
const sparql = `SELECT * WHERE {
SERVICE wikibase:mwapi {
bd:serviceParam wikibase:api "EntitySearch" .
bd:serviceParam wikibase:endpoint "www.wikidata.org" .
bd:serviceParam mwapi:search "${text}" .
bd:serviceParam mwapi:language "${options.lang}" .
?item wikibase:apiOutputItem mwapi:item .
?num wikibase:apiOrdinal true .
}
?item (wdt:P279|wdt:P31) wd:Q${options.instanceOf}
} ORDER BY ASC(?num) LIMIT ${options.maxCount}`
bd:serviceParam wikibase:endpoint "www.wikidata.org" .
bd:serviceParam mwapi:search "${text}" .
bd:serviceParam mwapi:language "${options.lang}" .
?item wikibase:apiOutputItem mwapi:item .
?num wikibase:apiOrdinal true .
bd:serviceParam wikibase:limit ${Math.round((options.maxCount ?? 20) * 1.5) /*Some padding for disambiguation pages */} .
?label wikibase:apiOutput mwapi:label .
?description wikibase:apiOutput "@description" .
}
${instanceOf}
${minusPhrases.join("\n ")}
} ORDER BY ASC(?num) LIMIT ${options.maxCount ?? 20}`
const url = wds.sparqlQuery(sparql)
const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
return result.results.bindings
const result = await Utils.downloadJson(url)
/*The full uri of the wikidata-item*/
return result.results.bindings.map(({item, label, description, num}) => ({
relevance: num?.value,
id: item?.value,
label: label?.value,
description: description?.value
}))
}
public static async search(
@ -215,39 +250,28 @@ export default class Wikidata {
public static async searchAndFetch(
search: string,
options?: WikidataSearchoptions
options?: WikidataAdvancedSearchoptions
): Promise<WikidataResponse[]> {
const maxCount = options.maxCount
// We provide some padding to filter away invalid values
options.maxCount = Math.ceil((options.maxCount ?? 20) * 1.5)
const searchResults = await Wikidata.search(search, options)
const maybeResponses = await Promise.all(searchResults.map(async r => {
try {
return await Wikidata.LoadWikidataEntry(r.id).AsPromise()
} catch (e) {
console.error(e)
return undefined;
}
}))
const responses = maybeResponses
.map(r => <WikidataResponse>r["success"])
.filter(wd => {
if (wd === undefined) {
return false;
const searchResults = await Wikidata.searchAdvanced(search, options)
const maybeResponses = await Promise.all(
searchResults.map(async r => {
try {
console.log("Loading ", r.id)
return await Wikidata.LoadWikidataEntry(r.id).AsPromise()
} catch (e) {
console.error(e)
return undefined;
}
if (wd.claims.get("P31" /*Instance of*/)?.has("Q4167410"/* Wikimedia Disambiguation page*/)) {
return false;
}
return true;
})
responses.splice(maxCount, responses.length - maxCount)
return responses
}))
return Utils.NoNull(maybeResponses.map(r => <WikidataResponse>r["success"]))
}
/**
* Gets the 'key' segment from a URL
*
*
* Wikidata.ExtractKey("https://www.wikidata.org/wiki/Lexeme:L614072") // => "L614072"
* Wikidata.ExtractKey("http://www.wikidata.org/entity/Q55008046") // => "QQ55008046"
*/
public static ExtractKey(value: string | number): string {
if (typeof value === "number") {
@ -291,6 +315,35 @@ export default class Wikidata {
return undefined;
}
/**
* Converts 'Q123' into 123, returns undefined if invalid
*
* Wikidata.QIdToNumber("Q123") // => 123
* Wikidata.QIdToNumber(" Q123 ") // => 123
* Wikidata.QIdToNumber(" X123 ") // => undefined
* Wikidata.QIdToNumber(" Q123X ") // => undefined
* Wikidata.QIdToNumber(undefined) // => undefined
* Wikidata.QIdToNumber(123) // => 123
*/
public static QIdToNumber(q: string | number): number | undefined {
if(q === undefined || q === null){
return
}
if(typeof q === "number"){
return q
}
q = q.trim()
if (!q.startsWith("Q")) {
return
}
q = q.substr(1)
const n = Number(q)
if (isNaN(n)) {
return
}
return n
}
public static IdToArticle(id: string) {
if (id.startsWith("Q")) {
return "https://wikidata.org/wiki/" + id

40
UI/Input/ValidatedTextField.ts
View file

@ -250,13 +250,15 @@ class WikidataTextField extends TextFieldDef {
["subarg", "doc"],
[["removePrefixes", "remove these snippets of text from the start of the passed string to search"],
["removePostfixes", "remove these snippets of text from the end of the passed string to search"],
["instanceOf","A list of Q-identifier which indicates that the search results _must_ be an entity of this type, e.g. [`Q5`](https://www.wikidata.org/wiki/Q5) for humans"],
["notInstanceof","A list of Q-identifiers which indicates that the search results _must not_ be an entity of this type, e.g. [`Q79007`](https://www.wikidata.org/wiki/Q79007) to filter away all streets from the search results"]
]
)])
]]),
new Title("Example usage"),
`The following is the 'freeform'-part of a layer config which will trigger a search for the wikidata item corresponding with the name of the selected feature. It will also remove '-street', '-square', ... if found at the end of the name
\`\`\`
\`\`\`json
"freeform": {
"key": "name:etymology:wikidata",
"type": "wikidata",
@ -269,11 +271,29 @@ class WikidataTextField extends TextFieldDef {
"path",
"square",
"plaza",
]
],
"#": "Remove streets and parks from the search results:"
"notInstanceOf": ["Q79007","Q22698"]
}
]
}
\`\`\``
\`\`\`
Another example is to search for species and trees:
\`\`\`json
"freeform": {
"key": "species:wikidata",
"type": "wikidata",
"helperArgs": [
"species",
{
"instanceOf": [10884, 16521]
}]
}
\`\`\`
`
]));
}
@ -304,9 +324,9 @@ class WikidataTextField extends TextFieldDef {
const args = inputHelperOptions.args ?? []
const searchKey = args[0] ?? "name"
let searchFor = <string>inputHelperOptions.feature?.properties[searchKey]?.toLowerCase()
let searchFor = <string>(inputHelperOptions.feature?.properties[searchKey]?.toLowerCase() ?? "")
const options = args[1]
const options: any = args[1]
if (searchFor !== undefined && options !== undefined) {
const prefixes = <string[]>options["removePrefixes"]
const postfixes = <string[]>options["removePostfixes"]
@ -325,10 +345,18 @@ class WikidataTextField extends TextFieldDef {
}
}
let instanceOf : number[] = Utils.NoNull((options?.instanceOf ?? []).map(i => Wikidata.QIdToNumber(i)))
let notInstanceOf : number[] = Utils.NoNull((options?.notInstanceOf ?? []).map(i => Wikidata.QIdToNumber(i)))
console.log("Instance of", instanceOf)
return new WikidataSearchBox({
value: currentValue,
searchText: new UIEventSource<string>(searchFor)
searchText: new UIEventSource<string>(searchFor),
instanceOf,
notInstanceOf
})
}
}

92
UI/SpecialVisualizations.ts
View file

@ -46,6 +46,8 @@ import {LoginToggle} from "./Popup/LoginButton";
import {start} from "repl";
import {SubstitutedTranslation} from "./SubstitutedTranslation";
import {TextField} from "./Input/TextField";
import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata";
import {Translation} from "./i18n/Translation";
export interface SpecialVisualization {
funcName: string,
@ -159,19 +161,19 @@ class CloseNoteButton implements SpecialVisualization {
tags.ping()
})
})
if((params.minZoom??"") !== "" && !isNaN(Number(params.minZoom))){
closeButton = new Toggle(
if ((params.minZoom ?? "") !== "" && !isNaN(Number(params.minZoom))) {
closeButton = new Toggle(
closeButton,
params.zoomButton ?? "",
state. locationControl.map(l => l.zoom >= Number(params.minZoom))
state.locationControl.map(l => l.zoom >= Number(params.minZoom))
)
}
return new LoginToggle(new Toggle(
t.isClosed.SetClass("thanks"),
closeButton,
isClosed
), t.loginToClose, state)
}
@ -180,7 +182,7 @@ class CloseNoteButton implements SpecialVisualization {
export default class SpecialVisualizations {
public static specialVisualizations : SpecialVisualization[] = SpecialVisualizations.init()
public static specialVisualizations: SpecialVisualization[] = SpecialVisualizations.init()
public static HelpMessage() {
@ -207,28 +209,28 @@ export default class SpecialVisualizations {
));
return new Combine([
new Combine([
new Title("Special tag renderings", 1),
"In a tagrendering, some special values are substituted by an advanced UI-element. This allows advanced features and visualizations to be reused by custom themes or even to query third-party API's.",
"General usage is `{func_name()}`, `{func_name(arg, someotherarg)}` or `{func_name(args):cssStyle}`. Note that you _do not_ need to use quotes around your arguments, the comma is enough to separate them. This also implies you cannot use a comma in your args",
new Title("Using expanded syntax",4),
`Instead of using \`{"render": {"en": "{some_special_visualisation(some_arg, some other really long message, more args)} , "nl": "{some_special_visualisation(some_arg, een boodschap in een andere taal, more args)}}, one can also write`,
new FixedUiElement(JSON.stringify({
render: {
special:{
type: "some_special_visualisation",
"argname": "some_arg",
"message":{
en:"some other really long message",
nl: "een boodschap in een andere taal"
},
"other_arg_name":"more args"
new Combine([
new Title("Special tag renderings", 1),
"In a tagrendering, some special values are substituted by an advanced UI-element. This allows advanced features and visualizations to be reused by custom themes or even to query third-party API's.",
"General usage is `{func_name()}`, `{func_name(arg, someotherarg)}` or `{func_name(args):cssStyle}`. Note that you _do not_ need to use quotes around your arguments, the comma is enough to separate them. This also implies you cannot use a comma in your args",
new Title("Using expanded syntax", 4),
`Instead of using \`{"render": {"en": "{some_special_visualisation(some_arg, some other really long message, more args)} , "nl": "{some_special_visualisation(some_arg, een boodschap in een andere taal, more args)}}, one can also write`,
new FixedUiElement(JSON.stringify({
render: {
special: {
type: "some_special_visualisation",
"argname": "some_arg",
"message": {
en: "some other really long message",
nl: "een boodschap in een andere taal"
},
"other_arg_name": "more args"
}
}
}
})).SetClass("code")
]).SetClass("flex flex-col"),
})).SetClass("code")
]).SetClass("flex flex-col"),
...helpTexts
]
).SetClass("flex flex-col");
@ -297,6 +299,32 @@ export default class SpecialVisualizations {
)
},
{
funcName: "wikidata_label",
docs: "Shows the label of the corresponding wikidata-item",
args: [
{
name: "keyToShowWikidataFor",
doc: "Use the wikidata entry from this key to show the label",
defaultValue: "wikidata"
}
],
example: "`{wikidata_label()}` is a basic example, `{wikipedia(name:etymology:wikidata)}` to show the label itself",
constr: (_, tagsSource, args) =>
new VariableUiElement(
tagsSource.map(tags => tags[args[0]])
.map(wikidata => {
wikidata = Utils.NoEmpty(wikidata?.split(";")?.map(wd => wd.trim()) ?? [])[0]
const entry = Wikidata.LoadWikidataEntry(wikidata)
return new VariableUiElement(entry.map(e => {
if (e === undefined || e["success"] === undefined) {
return wikidata
}
const response = <WikidataResponse>e["success"]
return Translation.fromMap(response.labels)
}))
}))
},
{
funcName: "minimap",
docs: "A small map showing the selected feature.",
@ -482,7 +510,7 @@ export default class SpecialVisualizations {
docs: "Downloads a JSON from the given URL, e.g. '{live(example.org/data.json, shorthand:x.y.z, other:a.b.c, shorthand)}' will download the given file, will create an object {shorthand: json[x][y][z], other: json[a][b][c] out of it and will return 'other' or 'json[a][b][c]. This is made to use in combination with tags, e.g. {live({url}, {url:format}, needed_value)}",
example: "{live({url},{url:format},hour)} {live(https://data.mobility.brussels/bike/api/counts/?request=live&featureID=CB2105,hour:data.hour_cnt;day:data.day_cnt;year:data.year_cnt,hour)}",
args: [{
name: "Url",
name: "Url",
doc: "The URL to load",
required: true
}, {
@ -783,7 +811,7 @@ export default class SpecialVisualizations {
const textField = new TextField(
{
placeholder: t.addCommentPlaceholder,
inputStyle: "width: 100%; height: 6rem;",
inputStyle: "width: 100%; height: 6rem;",
textAreaRows: 3,
htmlType: "area"
}
@ -846,7 +874,7 @@ export default class SpecialVisualizations {
textField,
new Combine([
stateButtons.SetClass("sm:mr-2"),
new Toggle(addCommentButton,
new Toggle(addCommentButton,
new Combine([t.typeText]).SetClass("flex items-center h-full subtle"),
textField.GetValue().map(t => t !== undefined && t.length >= 1)).SetClass("sm:mr-2")
]).SetClass("sm:flex sm:justify-between sm:items-stretch")
@ -947,7 +975,7 @@ export default class SpecialVisualizations {
]
specialVisualizations.push(new AutoApplyButton(specialVisualizations))
return specialVisualizations;
}

20
UI/Wikipedia/WikidataSearchBox.ts
View file

@ -17,14 +17,20 @@ export default class WikidataSearchBox extends InputElement<string> {
IsSelected: UIEventSource<boolean> = new UIEventSource<boolean>(false);
private readonly wikidataId: UIEventSource<string>
private readonly searchText: UIEventSource<string>
private readonly instanceOf?: number[];
private readonly notInstanceOf?: number[];
constructor(options?: {
searchText?: UIEventSource<string>,
value?: UIEventSource<string>
value?: UIEventSource<string>,
notInstanceOf?: number[],
instanceOf?: number[]
}) {
super();
this.searchText = options?.searchText
this.wikidataId = options?.value ?? new UIEventSource<string>(undefined);
this.instanceOf = options?.instanceOf
this.notInstanceOf = options?.notInstanceOf
}
GetValue(): UIEventSource<string> {
@ -59,7 +65,9 @@ export default class WikidataSearchBox extends InputElement<string> {
if (promise === undefined) {
promise = Wikidata.searchAndFetch(searchText, {
lang,
maxCount: 5
maxCount: 5,
notInstanceOf: this.notInstanceOf,
instanceOf: this.instanceOf
}
)
WikidataSearchBox._searchCache.set(key, promise)
@ -75,13 +83,15 @@ export default class WikidataSearchBox extends InputElement<string> {
return new Combine([Translations.t.general.wikipedia.failed.Clone().SetClass("alert"), searchFailMessage.data])
}
if (searchField.GetValue().data.length === 0) {
return Translations.t.general.wikipedia.doSearch
}
if (searchResults.length === 0) {
return Translations.t.general.wikipedia.noResults.Subs({search: searchField.GetValue().data ?? ""})
}
if (searchResults.length === 0) {
return Translations.t.general.wikipedia.doSearch
}
return new Combine(searchResults.map(wikidataresponse => {
const el = WikidataPreviewBox.WikidataResponsePreview(wikidataresponse).SetClass("rounded-xl p-1 sm:p-2 md:p-3 m-px border-2 sm:border-4 transition-colors")

5
assets/layers/etymology/etymology.json
View file

@ -52,6 +52,7 @@
"helperArgs": [
"name",
{
"notInstanceOf": ["Q79007","Q22698"],
"removePostfixes": [
"steenweg",
"heirbaan",
@ -70,7 +71,9 @@
"wegel",
"kerk",
"church",
"kaai"
"kaai",
"park",
"parque"
]
}
]

28
assets/layers/tree_node/tree_node.json
View file

@ -314,6 +314,34 @@
]
}
},
{
"id": "tree-species-wikidata",
"question": {
"en": "What species is this tree?"
},
"render": {
"*":"{wikipedia(species:wikidata):max-height: 25rem}"
},
"freeform": {
"key": "species:wikidata",
"type": "wikidata",
"helperArgs": [
"species",
{
"instanceOf": [10884, 16521]
}]
}
},
{
"id": "tree-wikipedia",
"#": "If this tree has a wikipedia article, show it. People can _only_ set the species though!",
"render": {
"*":"{wikipedia()}"
},
"condition": {
"or": ["wikipedia~*","wikidata~*"]
}
},
{
"render": {
"nl": "Naam: {name}",

42
test.ts
View file

@ -1,31 +1,17 @@
import Combine from "./UI/Base/Combine";
import ValidatedTextField from "./UI/Input/ValidatedTextField";
import Title from "./UI/Base/Title";
import {FixedUiElement} from "./UI/Base/FixedUiElement";
import {VariableUiElement} from "./UI/Base/VariableUIElement";
import {UIEventSource} from "./Logic/UIEventSource";
import {Translation} from "./UI/i18n/Translation";
import Wikidata from "./Logic/Web/Wikidata";
import Combine from "./UI/Base/Combine";
import {FixedUiElement} from "./UI/Base/FixedUiElement";
new Combine(
ValidatedTextField.AvailableTypes().map(key => {
let inp;
const feedback = new UIEventSource<Translation>(undefined)
try {
inp = ValidatedTextField.ForType(key).ConstructInputElement({
feedback,
country: () => "be"
});
} catch (e) {
console.error(e)
inp = new FixedUiElement(e).SetClass("alert")
}
return new Combine([
new Title(key),
inp,
new VariableUiElement(inp.GetValue()),
new VariableUiElement(feedback.map(v => v?.SetClass("alert")))
]);
}
)
).AttachTo("maindiv")
const result = UIEventSource.FromPromise(
Wikidata.searchAdvanced("WOlf", {
lang: "nl",
maxCount: 100,
instanceOf: 5
})
)
result.addCallbackAndRunD(r => console.log(r))
new VariableUiElement(result.map(items =>new Combine( (items??[])?.map(i =>
new FixedUiElement(JSON.stringify(i, null, " ")).SetClass("p-4 block")
)) )).SetClass("flex flex-col").AttachTo("maindiv")