midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Full code cleanup

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-11-07 16:34:51 +01:00
parent 8e6ee8c87f
commit bd21212eba
246 changed files with 19418 additions and 11729 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Logic/ImageProviders/AllImageProviders.ts
View file

@ -19,9 +19,9 @@ export default class AllImageProviders {
new GenericImageProvider(
[].concat(...Imgur.defaultValuePrefix, ...WikimediaImageProvider.commonsPrefixes, ...Mapillary.valuePrefixes)
)
]
public static defaultKeys = [].concat(AllImageProviders.ImageAttributionSource.map(provider => provider.defaultKeyPrefixes))
@ -32,7 +32,7 @@ export default class AllImageProviders {
return undefined;
}
const cacheKey = tags.data.id+tagKey
const cacheKey = tags.data.id + tagKey
const cached = this._cache.get(cacheKey)
if (cached !== undefined) {
return cached
@ -43,22 +43,22 @@ export default class AllImageProviders {
this._cache.set(cacheKey, source)
const allSources = []
for (const imageProvider of AllImageProviders.ImageAttributionSource) {
let prefixes = imageProvider.defaultKeyPrefixes
if(tagKey !== undefined){
if (tagKey !== undefined) {
prefixes = tagKey
}
const singleSource = imageProvider.GetRelevantUrls(tags, {
prefixes: prefixes
})
allSources.push(singleSource)
singleSource.addCallbackAndRunD(_ => {
const all : ProvidedImage[] = [].concat(...allSources.map(source => source.data))
const all: ProvidedImage[] = [].concat(...allSources.map(source => source.data))
const uniq = []
const seen = new Set<string>()
for (const img of all) {
if(seen.has(img.url)){
if (seen.has(img.url)) {
continue
}
seen.add(img.url)

17
Logic/ImageProviders/GenericImageProvider.ts
View file

@ -10,24 +10,19 @@ export default class GenericImageProvider extends ImageProvider {
this._valuePrefixBlacklist = valuePrefixBlacklist;
}
protected DownloadAttribution(url: string) {
return undefined
}
async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
if (this._valuePrefixBlacklist.some(prefix => value.startsWith(prefix))) {
return []
}
try{
try {
new URL(value)
}catch (_){
} catch (_) {
// Not a valid URL
return []
}
return [Promise.resolve({
key: key,
url: value,
@ -39,5 +34,9 @@ export default class GenericImageProvider extends ImageProvider {
return undefined;
}
protected DownloadAttribution(url: string) {
return undefined
}
}

6
Logic/ImageProviders/ImageProvider.ts
View file

@ -14,7 +14,7 @@ export default abstract class ImageProvider {
public abstract readonly defaultKeyPrefixes: string[]
private _cache = new Map<string, UIEventSource<LicenseInfo>>()
GetAttributionFor(url: string): UIEventSource<LicenseInfo> {
const cached = this._cache.get(url);
if (cached !== undefined) {
@ -27,8 +27,6 @@ export default abstract class ImageProvider {
public abstract SourceIcon(backlinkSource?: string): BaseUIElement;
protected abstract DownloadAttribution(url: string): Promise<LicenseInfo>;
/**
* Given a properies object, maps it onto _all_ the available pictures for this imageProvider
*/
@ -77,4 +75,6 @@ export default abstract class ImageProvider {
public abstract ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]>;
protected abstract DownloadAttribution(url: string): Promise<LicenseInfo>;
}

27
Logic/ImageProviders/Imgur.ts
View file

@ -7,10 +7,9 @@ import {LicenseInfo} from "./LicenseInfo";
export class Imgur extends ImageProvider {
public static readonly defaultValuePrefix = ["https://i.imgur.com"]
public readonly defaultKeyPrefixes: string[] = ["image"];
public static readonly defaultValuePrefix = ["https://i.imgur.com"]
public static readonly singleton = new Imgur();
public readonly defaultKeyPrefixes: string[] = ["image"];
private constructor() {
super();
@ -89,6 +88,17 @@ export class Imgur extends ImageProvider {
return undefined;
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
if (Imgur.defaultValuePrefix.some(prefix => value.startsWith(prefix))) {
return [Promise.resolve({
url: value,
key: key,
provider: this
})]
}
return []
}
protected DownloadAttribution: (url: string) => Promise<LicenseInfo> = async (url: string) => {
const hash = url.substr("https://i.imgur.com/".length).split(".jpg")[0];
@ -112,16 +122,5 @@ export class Imgur extends ImageProvider {
return licenseInfo
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
if (Imgur.defaultValuePrefix.some(prefix => value.startsWith(prefix))) {
return [Promise.resolve({
url: value,
key: key,
provider: this
})]
}
return []
}
}

3
Logic/ImageProviders/ImgurUploader.ts
View file

@ -6,9 +6,8 @@ export default class ImgurUploader {
public readonly queue: UIEventSource<string[]> = new UIEventSource<string[]>([]);
public readonly failed: UIEventSource<string[]> = new UIEventSource<string[]>([]);
public readonly success: UIEventSource<string[]> = new UIEventSource<string[]>([]);
private readonly _handleSuccessUrl: (string) => void;
public maxFileSizeInMegabytes = 10;
private readonly _handleSuccessUrl: (string) => void;
constructor(handleSuccessUrl: (string) => void) {
this._handleSuccessUrl = handleSuccessUrl;

67
Logic/ImageProviders/Mapillary.ts
View file

@ -7,17 +7,16 @@ import Constants from "../../Models/Constants";
export class Mapillary extends ImageProvider {
defaultKeyPrefixes = ["mapillary","image"]
public static readonly singleton = new Mapillary();
private static readonly valuePrefix = "https://a.mapillary.com"
public static readonly valuePrefixes = [Mapillary.valuePrefix, "http://mapillary.com","https://mapillary.com","http://www.mapillary.com","https://www.mapillary.com"]
public static readonly valuePrefixes = [Mapillary.valuePrefix, "http://mapillary.com", "https://mapillary.com", "http://www.mapillary.com", "https://www.mapillary.com"]
defaultKeyPrefixes = ["mapillary", "image"]
private static ExtractKeyFromURL(value: string, failIfNoMath = false): {
key: string,
isApiv4?: boolean
} {
if (value.startsWith(Mapillary.valuePrefix)) {
const key = value.substring(0, value.lastIndexOf("?")).substring(value.lastIndexOf("/") + 1)
return {key: key, isApiv4: !isNaN(Number(key))};
@ -43,11 +42,11 @@ export class Mapillary extends ImageProvider {
if (matchApi !== null) {
return {key: matchApi[1]};
}
if(failIfNoMath){
if (failIfNoMath) {
return undefined;
}
return {key: value, isApiv4: !isNaN(Number(value))};
}
@ -59,33 +58,6 @@ export class Mapillary extends ImageProvider {
return [this.PrepareUrlAsync(key, value)]
}
private async PrepareUrlAsync(key: string, value: string): Promise<ProvidedImage> {
const failIfNoMatch = key.indexOf("mapillary") < 0
const keyV = Mapillary.ExtractKeyFromURL(value, failIfNoMatch)
if(keyV === undefined){
return undefined;
}
if (!keyV.isApiv4) {
const url = `https://images.mapillary.com/${keyV.key}/thumb-640.jpg?client_id=${Constants.mapillary_client_token_v3}`
return {
url: url,
provider: this,
key: key
}
} else {
const mapillaryId = keyV.key;
const metadataUrl = 'https://graph.mapillary.com/' + mapillaryId + '?fields=thumb_1024_url&&access_token=' + Constants.mapillary_client_token_v4;
const response = await Utils.downloadJson(metadataUrl)
const url = <string> response["thumb_1024_url"];
return {
url: url,
provider: this,
key: key
}
}
}
protected async DownloadAttribution(url: string): Promise<LicenseInfo> {
const keyV = Mapillary.ExtractKeyFromURL(url)
@ -110,4 +82,31 @@ export class Mapillary extends ImageProvider {
return license
}
private async PrepareUrlAsync(key: string, value: string): Promise<ProvidedImage> {
const failIfNoMatch = key.indexOf("mapillary") < 0
const keyV = Mapillary.ExtractKeyFromURL(value, failIfNoMatch)
if (keyV === undefined) {
return undefined;
}
if (!keyV.isApiv4) {
const url = `https://images.mapillary.com/${keyV.key}/thumb-640.jpg?client_id=${Constants.mapillary_client_token_v3}`
return {
url: url,
provider: this,
key: key
}
} else {
const mapillaryId = keyV.key;
const metadataUrl = 'https://graph.mapillary.com/' + mapillaryId + '?fields=thumb_1024_url&&access_token=' + Constants.mapillary_client_token_v4;
const response = await Utils.downloadJson(metadataUrl)
const url = <string>response["thumb_1024_url"];
return {
url: url,
provider: this,
key: key
}
}
}
}

27
Logic/ImageProviders/WikidataImageProvider.ts
View file

@ -1,4 +1,3 @@
import {Utils} from "../../Utils";
import ImageProvider, {ProvidedImage} from "./ImageProvider";
import BaseUIElement from "../../UI/BaseUIElement";
import Svg from "../../Svg";
@ -7,10 +6,6 @@ import Wikidata from "../Web/Wikidata";
export class WikidataImageProvider extends ImageProvider {
public SourceIcon(backlinkSource?: string): BaseUIElement {
throw Svg.wikidata_svg();
}
public static readonly singleton = new WikidataImageProvider()
public readonly defaultKeyPrefixes = ["wikidata"]
@ -18,17 +13,17 @@ export class WikidataImageProvider extends ImageProvider {
super()
}
protected DownloadAttribution(url: string): Promise<any> {
throw new Error("Method not implemented; shouldn't be needed!");
public SourceIcon(backlinkSource?: string): BaseUIElement {
throw Svg.wikidata_svg();
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
const entity = await Wikidata.LoadWikidataEntryAsync(value)
if(entity === undefined){
if (entity === undefined) {
return []
}
const allImages : Promise<ProvidedImage>[] = []
const allImages: Promise<ProvidedImage>[] = []
// P18 is the claim 'depicted in this image'
for (const img of Array.from(entity.claims.get("P18") ?? [])) {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, img)
@ -36,19 +31,23 @@ export class WikidataImageProvider extends ImageProvider {
}
// P373 is 'commons category'
for (let cat of Array.from(entity.claims.get("P373") ?? [])) {
if(!cat.startsWith("Category:")){
cat = "Category:"+cat
if (!cat.startsWith("Category:")) {
cat = "Category:" + cat
}
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, cat)
allImages.push(...promises)
}
const commons = entity.commons
if (commons !== undefined && (commons.startsWith("Category:") || commons.startsWith("File:"))) {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined, commons)
allImages.push(...promises)
}
return allImages
}
protected DownloadAttribution(url: string): Promise<any> {
throw new Error("Method not implemented; shouldn't be needed!");
}
}

146
Logic/ImageProviders/WikimediaImageProvider.ts
View file

@ -12,10 +12,10 @@ import Wikimedia from "../Web/Wikimedia";
export class WikimediaImageProvider extends ImageProvider {
private readonly commons_key = "wikimedia_commons"
public readonly defaultKeyPrefixes = [this.commons_key,"image"]
public static readonly singleton = new WikimediaImageProvider();
public static readonly commonsPrefixes = ["https://commons.wikimedia.org/wiki/", "https://upload.wikimedia.org", "File:"]
private readonly commons_key = "wikimedia_commons"
public readonly defaultKeyPrefixes = [this.commons_key, "image"]
private constructor() {
super();
@ -30,6 +30,40 @@ export class WikimediaImageProvider extends ImageProvider {
}
private static PrepareUrl(value: string): string {
if (value.toLowerCase().startsWith("https://commons.wikimedia.org/wiki/")) {
return value;
}
return (`https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(value)}?width=500&height=400`)
}
private static startsWithCommonsPrefix(value: string): boolean {
return WikimediaImageProvider.commonsPrefixes.some(prefix => value.startsWith(prefix))
}
private static removeCommonsPrefix(value: string): string {
if (value.startsWith("https://upload.wikimedia.org/")) {
value = value.substring(value.lastIndexOf("/") + 1)
value = decodeURIComponent(value)
if (!value.startsWith("File:")) {
value = "File:" + value
}
return value;
}
for (const prefix of WikimediaImageProvider.commonsPrefixes) {
if (value.startsWith(prefix)) {
let part = value.substr(prefix.length)
if (prefix.startsWith("http")) {
part = decodeURIComponent(part)
}
return part
}
}
return value;
}
SourceIcon(backlink: string): BaseUIElement {
const img = Svg.wikimedia_commons_white_svg()
.SetStyle("width:2em;height: 2em");
@ -44,12 +78,38 @@ export class WikimediaImageProvider extends ImageProvider {
}
private static PrepareUrl(value: string): string {
public PrepUrl(value: string): ProvidedImage {
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.toLowerCase().startsWith("https://commons.wikimedia.org/wiki/")) {
return value;
if (value.startsWith("File:")) {
return this.UrlForImage(value)
}
return (`https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(value)}?width=500&height=400`)
// We do a last effort and assume this is a file
return this.UrlForImage("File:" + value)
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
if (key !== undefined && key !== this.commons_key && !hasCommonsPrefix) {
return []
}
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("Category:")) {
const urls = await Wikimedia.GetCategoryContents(value)
return urls.filter(url => url.startsWith("File:")).map(image => Promise.resolve(this.UrlForImage(image)))
}
if (value.startsWith("File:")) {
return [Promise.resolve(this.UrlForImage(value))]
}
if (value.startsWith("http")) {
// PRobably an error
return []
}
// We do a last effort and assume this is a file
return [Promise.resolve(this.UrlForImage("File:" + value))]
}
protected async DownloadAttribution(filename: string): Promise<LicenseInfo> {
@ -66,24 +126,24 @@ export class WikimediaImageProvider extends ImageProvider {
const data = await Utils.downloadJson(url)
const licenseInfo = new LicenseInfo();
const pageInfo = data.query.pages[-1]
if(pageInfo === undefined){
if (pageInfo === undefined) {
return undefined;
}
const license = (pageInfo.imageinfo ?? [])[0]?.extmetadata;
if (license === undefined) {
console.warn("The file", filename ,"has no usable metedata or license attached... Please fix the license info file yourself!")
console.warn("The file", filename, "has no usable metedata or license attached... Please fix the license info file yourself!")
return undefined;
}
let title = pageInfo.title
if(title.startsWith("File:")){
title= title.substr("File:".length)
if (title.startsWith("File:")) {
title = title.substr("File:".length)
}
if(title.endsWith(".jpg") || title.endsWith(".png")){
if (title.endsWith(".jpg") || title.endsWith(".png")) {
title = title.substring(0, title.length - 4)
}
licenseInfo.title = title
licenseInfo.artist = license.Artist?.value;
licenseInfo.license = license.License?.value;
@ -103,66 +163,6 @@ export class WikimediaImageProvider extends ImageProvider {
}
return {url: WikimediaImageProvider.PrepareUrl(image), key: undefined, provider: this}
}
private static startsWithCommonsPrefix(value: string): boolean{
return WikimediaImageProvider.commonsPrefixes.some(prefix => value.startsWith(prefix))
}
private static removeCommonsPrefix(value: string): string{
if(value.startsWith("https://upload.wikimedia.org/")){
value = value.substring(value.lastIndexOf("/") + 1)
value = decodeURIComponent(value)
if(!value.startsWith("File:")){
value = "File:"+value
}
return value;
}
for (const prefix of WikimediaImageProvider.commonsPrefixes) {
if(value.startsWith(prefix)){
let part = value.substr(prefix.length)
if(prefix.startsWith("http")){
part = decodeURIComponent(part)
}
return part
}
}
return value;
}
public PrepUrl(value: string): ProvidedImage {
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("File:")) {
return this.UrlForImage(value)
}
// We do a last effort and assume this is a file
return this.UrlForImage("File:" + value)
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
const hasCommonsPrefix = WikimediaImageProvider.startsWithCommonsPrefix(value)
if(key !== undefined && key !== this.commons_key && !hasCommonsPrefix){
return []
}
value = WikimediaImageProvider.removeCommonsPrefix(value)
if (value.startsWith("Category:")) {
const urls = await Wikimedia.GetCategoryContents(value)
return urls.filter(url => url.startsWith("File:")).map(image => Promise.resolve(this.UrlForImage(image)))
}
if (value.startsWith("File:")) {
return [Promise.resolve(this.UrlForImage(value))]
}
if (value.startsWith("http")) {
// PRobably an error
return []
}
// We do a last effort and assume this is a file
return [Promise.resolve(this.UrlForImage("File:" + value))]
}
}