325 lines
13 KiB
Dart
325 lines
13 KiB
Dart
import 'dart:convert';
|
|
import 'dart:math';
|
|
|
|
import 'package:http/http.dart';
|
|
import 'package:intl/intl.dart';
|
|
import 'package:release_schedule/api/api_manager.dart';
|
|
import 'package:release_schedule/api/json_helper.dart';
|
|
import 'package:release_schedule/api/movie_api.dart';
|
|
import 'package:release_schedule/model/dates.dart';
|
|
import 'package:release_schedule/model/movie.dart';
|
|
|
|
class WikidataProperties {
|
|
static const String instanceOf = "P31";
|
|
static const String publicationDate = "P577";
|
|
static const String title = "P1476";
|
|
static const String partOfTheSeries = "P179";
|
|
static const String basedOn = "P144";
|
|
static const String derivativeWork = "P4969";
|
|
static const String genre = "P136";
|
|
static const String countryOfOrigin = "P496";
|
|
static const String director = "P57";
|
|
static const String castMember = "P161";
|
|
static const String distributedBy = "P750";
|
|
static const String afterAWorkBy = "P1877";
|
|
static const String duration = "P2047";
|
|
static const String reviewScore = "P444";
|
|
static const String fskFilmRating = "P1981";
|
|
static const String placeOfPublication = "P291";
|
|
static const String shortName = "P1813";
|
|
}
|
|
|
|
class WikidataEntities {
|
|
static const String film = "Q11424";
|
|
static const String filmProject = "Q18011172";
|
|
}
|
|
|
|
ApiManager _wikidataApi =
|
|
ApiManager("https://www.wikidata.org/w/api.php?origin=*");
|
|
|
|
class WikidataMovieApi implements MovieApi {
|
|
ApiManager queryApi =
|
|
ApiManager("https://query.wikidata.org/sparql?format=json&origin=*");
|
|
|
|
@override
|
|
Future<List<WikidataMovieData>> getUpcomingMovies(DateTime startDate,
|
|
[int count = 100]) async {
|
|
Response filmResponse = await queryApi.get(
|
|
"&query=${Uri.encodeComponent(_createUpcomingMovieQuery(startDate, WikidataEntities.film, count))}");
|
|
Response filmProjectResponse = await queryApi.get(
|
|
"&query=${Uri.encodeComponent(_createUpcomingMovieQuery(startDate, WikidataEntities.filmProject, count))}");
|
|
List<Response> responses = [filmResponse, filmProjectResponse];
|
|
for (var response in responses) {
|
|
if (response.statusCode != 200) {
|
|
throw Exception(
|
|
"The Wikidata request for upcoming movies failed with status ${response.statusCode} ${response.reasonPhrase}");
|
|
}
|
|
}
|
|
Iterable<Map<String, dynamic>> results =
|
|
responses.map((response) => jsonDecode(response.body));
|
|
Iterable<dynamic> entries =
|
|
results.expand((result) => result["results"]["bindings"]);
|
|
List<String> ids = entries
|
|
.map((entry) =>
|
|
RegExp(r"Q\d+$").firstMatch(entry["movie"]["value"])![0]!)
|
|
.toList();
|
|
return await _getMovieDataFromIds(ids);
|
|
}
|
|
|
|
Future<List<WikidataMovieData>> _getMovieDataFromIds(
|
|
List<String> movieIds) async {
|
|
// Wikidata limits the number of entities per request to 50
|
|
const batchSize = 50;
|
|
Map<String, dynamic> entities = {};
|
|
for (int i = 0; i < (movieIds.length / batchSize).ceil(); i++) {
|
|
final start = i * batchSize;
|
|
final end = min((i + 1) * batchSize, movieIds.length);
|
|
var response = await _wikidataApi.get(
|
|
"&action=wbgetentities&format=json&props=labels|claims|sitelinks/urls&ids=${movieIds.sublist(start, end).join("|")}");
|
|
Map<String, dynamic> result = jsonDecode(response.body);
|
|
Map<String, dynamic> batchEntities = result["entities"];
|
|
entities.addAll(batchEntities);
|
|
}
|
|
|
|
List<String> allCountryAndGenreIds = [];
|
|
// Add the country ids from the publication dates
|
|
allCountryAndGenreIds.addAll(selectInJson<String>(entities,
|
|
"*.claims.${WikidataProperties.publicationDate}.*.qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id"));
|
|
// Add the genre ids
|
|
allCountryAndGenreIds.addAll(selectInJson<String>(entities,
|
|
"*.claims.${WikidataProperties.genre}.*.mainsnak.datavalue.value.id"));
|
|
allCountryAndGenreIds = allCountryAndGenreIds.toSet().toList();
|
|
// Prefetch all labels for countries and genres
|
|
// to reduce the number of api calls,
|
|
// they will be retrieved from the cache in fromWikidataEntity
|
|
await _getLabelsForEntities(allCountryAndGenreIds);
|
|
|
|
// Get wikipedia explaintexts
|
|
Iterable<String> allWikipediaTitles =
|
|
selectInJson<String>(entities, "*.sitelinks.enwiki.url")
|
|
.map((url) => url.split("/").last);
|
|
await _getWikipediaExplainTextForTitles(allWikipediaTitles.toList());
|
|
|
|
return movieIds
|
|
.map((id) => WikidataMovieData.fromWikidataEntity(id, entities[id]))
|
|
.toList();
|
|
}
|
|
|
|
@override
|
|
Future<List<WikidataMovieData>> searchForMovies(String searchTerm) async {
|
|
String haswbstatement =
|
|
"haswbstatement:${WikidataProperties.instanceOf}=${WikidataEntities.film}|${WikidataProperties.instanceOf}=${WikidataEntities.filmProject}";
|
|
String query =
|
|
"&action=query&list=search&format=json&srsearch=${Uri.encodeComponent(searchTerm)}%20$haswbstatement";
|
|
Response result = await _wikidataApi.get(query);
|
|
Map<String, dynamic> json = jsonDecode(result.body);
|
|
List<Map<String, dynamic>> searchResults =
|
|
selectInJson<Map<String, dynamic>>(json, "query.search.*").toList();
|
|
List<String> ids = searchResults
|
|
.map((result) => result["title"] as String)
|
|
.where((title) => RegExp(r"^Q\d+$").hasMatch(title))
|
|
.toList();
|
|
return await _getMovieDataFromIds(ids);
|
|
}
|
|
}
|
|
|
|
class WikidataMovieData extends MovieData {
|
|
String entityId;
|
|
WikidataMovieData(
|
|
String title, DateWithPrecisionAndCountry releaseDate, this.entityId)
|
|
: super(title, releaseDate);
|
|
|
|
WikidataMovieData.fromEncodable(Map encodable)
|
|
: entityId = encodable["entityId"],
|
|
super.fromJsonEncodable(encodable);
|
|
|
|
@override
|
|
bool same(MovieData other) {
|
|
return other is WikidataMovieData && entityId == other.entityId;
|
|
}
|
|
|
|
@override
|
|
Map toJsonEncodable() {
|
|
return super.toJsonEncodable()..addAll({"entityId": entityId});
|
|
}
|
|
|
|
static WikidataMovieData fromWikidataEntity(
|
|
String entityId, Map<String, dynamic> entity) {
|
|
String title =
|
|
selectInJson<String>(entity, "labels.en.value").firstOrNull ??
|
|
selectInJson<String>(entity, "labels.*.value").first;
|
|
String? wikipediaTitle = selectInJson(entity, "sitelinks.enwiki.url")
|
|
.firstOrNull
|
|
?.split("/")
|
|
.last;
|
|
String? description = wikipediaTitle != null
|
|
? _getCachedWikipediaExplainTextFotTitle(wikipediaTitle)
|
|
: null;
|
|
Map<String, dynamic> claims = entity["claims"];
|
|
List<TitleInLanguage>? titles = selectInJson(
|
|
claims, "${WikidataProperties.title}.*.mainsnak.datavalue.value")
|
|
.map((value) => (
|
|
title: value["text"],
|
|
language: value["language"],
|
|
) as TitleInLanguage)
|
|
.toList();
|
|
List<DateWithPrecisionAndCountry> releaseDates =
|
|
_getReleaseDates(claims).toList();
|
|
// Sort release dates with higher precision to the beginning
|
|
releaseDates.sort((a, b) => -a.dateWithPrecision.precision.index
|
|
.compareTo(b.dateWithPrecision.precision.index));
|
|
List<String>? genres = selectInJson<String>(
|
|
claims, "${WikidataProperties.genre}.*.mainsnak.datavalue.value.id")
|
|
.map(_getCachedLabelForEntity)
|
|
.toList();
|
|
WikidataMovieData movie = WikidataMovieData(
|
|
title,
|
|
releaseDates.isNotEmpty
|
|
? releaseDates[0]
|
|
: DateWithPrecisionAndCountry(
|
|
DateTime.now(), DatePrecision.decade, "unknown location"),
|
|
entityId);
|
|
movie.setDetails(
|
|
description: description,
|
|
titles: titles,
|
|
releaseDates: releaseDates,
|
|
genres: genres,
|
|
);
|
|
return movie;
|
|
}
|
|
|
|
static Iterable<DateWithPrecisionAndCountry> _getReleaseDates(
|
|
Map<String, dynamic> claims) {
|
|
return selectInJson(claims, "${WikidataProperties.publicationDate}.*")
|
|
.where((dateClaim) => dateClaim["rank"] != "deprecated")
|
|
.expand<DateWithPrecisionAndCountry>((dateClaim) {
|
|
var value = selectInJson(dateClaim, "mainsnak.datavalue.value").first;
|
|
Iterable<String> countries = (selectInJson<String>(dateClaim,
|
|
"qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id"))
|
|
.map(_getCachedLabelForEntity);
|
|
if (countries.isEmpty) {
|
|
countries = ["unknown location"];
|
|
}
|
|
return countries.map((country) => DateWithPrecisionAndCountry(
|
|
DateTime.parse(value["time"]),
|
|
_precisionFromWikidata(value["precision"]),
|
|
country));
|
|
});
|
|
}
|
|
}
|
|
|
|
String _createUpcomingMovieQuery(
|
|
DateTime startDate, String instanceOf, int limit) {
|
|
String date = DateFormat("yyyy-MM-dd").format(startDate);
|
|
return """
|
|
SELECT
|
|
?movie
|
|
(MIN(?releaseDate) as ?minReleaseDate)
|
|
WHERE {
|
|
?movie wdt:${WikidataProperties.instanceOf} wd:$instanceOf;
|
|
wdt:${WikidataProperties.publicationDate} ?releaseDate.
|
|
?movie p:${WikidataProperties.publicationDate}/psv:${WikidataProperties.publicationDate} [wikibase:timePrecision ?precision].
|
|
FILTER (xsd:date(?releaseDate) >= xsd:date("$date"^^xsd:dateTime))
|
|
FILTER (?precision >= 10)
|
|
}
|
|
GROUP BY ?movie
|
|
ORDER BY ?minReleaseDate
|
|
LIMIT $limit""";
|
|
}
|
|
|
|
DatePrecision _precisionFromWikidata(int precision) {
|
|
return switch (precision) {
|
|
>= 13 => DatePrecision.minute,
|
|
12 => DatePrecision.hour,
|
|
11 => DatePrecision.day,
|
|
10 => DatePrecision.month,
|
|
9 => DatePrecision.year,
|
|
8 => DatePrecision.decade,
|
|
< 8 => throw Exception("The precision was too low, value: $precision"),
|
|
_ => throw Exception("Unexpected precision value: $precision"),
|
|
};
|
|
}
|
|
|
|
Map<String, String> _labelCache = {};
|
|
Future<Map<String, String>> _getLabelsForEntities(
|
|
List<String> entityIds) async {
|
|
const batchSize = 50;
|
|
Map<String, String> labels = {};
|
|
for (int i = entityIds.length - 1; i >= 0; i--) {
|
|
if (_labelCache.containsKey(entityIds[i])) {
|
|
labels[entityIds[i]] = _labelCache[entityIds[i]]!;
|
|
entityIds.removeAt(i);
|
|
}
|
|
}
|
|
for (int i = 0; i < (entityIds.length / batchSize).ceil(); i++) {
|
|
final start = i * batchSize;
|
|
final end = min((i + 1) * batchSize, entityIds.length);
|
|
Response response = await _wikidataApi.get(
|
|
"&action=wbgetentities&format=json&props=labels|claims&ids=${entityIds.sublist(start, end).join("|")}");
|
|
Map<String, dynamic> result = jsonDecode(response.body);
|
|
Map<String, dynamic> batchEntities = result["entities"];
|
|
for (String entityId in batchEntities.keys) {
|
|
String? shortName = selectInJson(batchEntities[entityId],
|
|
"claims.${WikidataProperties.shortName}.*.mainsnak.datavalue.value")
|
|
.where((value) => value["language"] == "en")
|
|
.map((value) => (value["text"] as String))
|
|
.firstOrNull;
|
|
Map<String, dynamic> responseLabels = batchEntities[entityId]["labels"];
|
|
if (shortName != null) {
|
|
_labelCache[entityId] = labels[entityId] = shortName;
|
|
continue;
|
|
}
|
|
String label = responseLabels.containsKey("en")
|
|
? responseLabels["en"]["value"]
|
|
: responseLabels[responseLabels.keys.first]["value"];
|
|
_labelCache[entityId] = labels[entityId] = label;
|
|
}
|
|
}
|
|
return labels;
|
|
}
|
|
|
|
String _getCachedLabelForEntity(String entityId) {
|
|
return _labelCache[entityId] ?? entityId;
|
|
}
|
|
|
|
ApiManager _wikipediaApi =
|
|
ApiManager("https://en.wikipedia.org/w/api.php?format=json&origin=*");
|
|
Map<String, String> _wikipediaExplainTextCache = {};
|
|
Future<Map<String, String>> _getWikipediaExplainTextForTitles(
|
|
List<String> pageTitles) async {
|
|
const batchSize = 50;
|
|
Map<String, String> explainTexts = {};
|
|
for (int i = pageTitles.length - 1; i >= 0; i--) {
|
|
if (_labelCache.containsKey(pageTitles[i])) {
|
|
explainTexts[pageTitles[i]] = _labelCache[pageTitles[i]]!;
|
|
pageTitles.removeAt(i);
|
|
}
|
|
}
|
|
for (int i = 0; i < (pageTitles.length / batchSize).ceil(); i++) {
|
|
final start = i * batchSize;
|
|
final end = min((i + 1) * batchSize, pageTitles.length);
|
|
Response response = await _wikipediaApi.get(
|
|
"&action=query&prop=extracts&exintro&explaintext&redirects=1&titles=${pageTitles.sublist(start, end).join("|")}");
|
|
Map<String, dynamic> result = jsonDecode(response.body);
|
|
List<dynamic> normalize = result["query"]["normalized"];
|
|
Map<String, dynamic> batchPages = result["query"]["pages"];
|
|
for (String pageId in batchPages.keys) {
|
|
String pageTitle = batchPages[pageId]["title"];
|
|
String originalTitle = normalize
|
|
.where((element) => element["to"] == pageTitle)
|
|
.firstOrNull?["from"] ??
|
|
pageTitle;
|
|
String? explainText = batchPages[pageId]["extract"];
|
|
if (explainText != null) {
|
|
_wikipediaExplainTextCache[originalTitle] =
|
|
explainTexts[originalTitle] = explainText;
|
|
}
|
|
}
|
|
}
|
|
return explainTexts;
|
|
}
|
|
|
|
String? _getCachedWikipediaExplainTextFotTitle(String title) {
|
|
return _wikipediaExplainTextCache[title];
|
|
}
|