2022691
/
release_schedule
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: separate file for json helpers

main
daniel-michel 2023-11-16 13:08:19 +01:00
parent ce90c7c1c9
commit 5c11b931c0
2 changed files with 171 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
lib/api/json_helper.dart 100644
View File

@ -0,0 +1,73 @@
/// Select values in nested [List] and [Map] structures using a path that may contain wildcards.
///
/// The maps must always use [String] keys.
/// The [path] is a dot-separated list of keys and indices.
/// The wildcard "*" can be used to select all elements of a list or map.
/// The wildcard "**" can be used to select all elements of a list or map and all elements of nested lists and maps.
///
/// Returns an [Iterable] of the selected values.
///
/// Also see [selectInJsonWithPath] for a version that returns the path to the selected values
Iterable<T> selectInJson<T>(dynamic json, String path) {
return selectInJsonWithPath<T>(json, path).map((e) => e.value);
}
Map<String, Iterable<dynamic>> selectMultipleInJson(
dynamic json, Map<String, String> selector) {
Map<String, Iterable<dynamic>> result = {};
for (String key in selector.keys) {
result[key] = selectInJsonWithPath(json, selector[key]!);
}
return result;
}
/// Select values in nested [List] and [Map] structures using a path that may contain wildcards.
///
/// The maps must always use [String] keys.
/// The [path] is a dot-separated list of keys and indices.
/// The wildcard "*" can be used to select all elements of a list or map.
/// The wildcard "**" can be used to select all elements of a list or map and all elements of nested lists and maps.+
///
/// Returns an [Iterable] of the selected values and their path.
Iterable<({T value, String path})> selectInJsonWithPath<T>(
dynamic json, String path) sync* {
if (path.isEmpty) {
if (json is T) {
yield (value: json, path: "");
}
return;
}
List<String> pathParts = path.split(".");
String first = pathParts.removeAt(0);
String rest = pathParts.join(".");
addFirstToPath(({T value, String path}) element) => (
value: element.value,
path: element.path.isEmpty ? first : "$first.${element.path}"
);
if (first == "*" || first == "**") {
String continueWithPath = first == "*" ? rest : path;
if (json is List) {
yield* json
.expand((e) => selectInJsonWithPath<T>(e, continueWithPath))
.map(addFirstToPath);
} else if (json is Map) {
for (String key in json.keys) {
yield* selectInJsonWithPath<T>(json[key], continueWithPath)
.map(addFirstToPath);
}
}
} else if (json is List) {
try {
int index = int.parse(first);
yield* selectInJsonWithPath<T>(json[index], rest);
} catch (e) {
// The first part of the path is not an index or out of bounds -> ignore
}
} else if (json is Map) {
dynamic value = json[first];
if (value != null) {
yield* selectInJsonWithPath<T>(value, pathParts.join("."));
}
}
}

255
lib/api/wikidata_movie_api.dart
View File

@ -4,6 +4,7 @@ import 'dart:math';
import 'package:http/http.dart';
import 'package:intl/intl.dart';
import 'package:release_schedule/api/api_manager.dart';
import 'package:release_schedule/api/json_helper.dart';
import 'package:release_schedule/api/movie_api.dart';
import 'package:release_schedule/model/movie.dart';
@ -26,162 +27,7 @@ class WikidataProperties {
static const String placeOfPublication = "P291";
}
/// Select values in nested List and Map structures using a path that may contain wildcards.
/// The maps must always use String keys.
/// The path is a dot-separated list of keys and indices.
/// The wildcard "*" can be used to select all elements of a list or map.
/// The wildcard "**" can be used to select all elements of a list or map and all elements of nested lists and maps.
Iterable<({T value, String path})> _selectInJsonWithPath<T>(
dynamic json, String path) sync* {
if (path.isEmpty) {
if (json is T) {
yield (value: json, path: "");
}
return;
}
List<String> pathParts = path.split(".");
String first = pathParts.removeAt(0);
String rest = pathParts.join(".");
addFirstToPath(({T value, String path}) element) => (
value: element.value,
path: element.path.isEmpty ? first : "$first.${element.path}"
);
if (first == "*" || first == "**") {
String continueWithPath = first == "*" ? rest : path;
if (json is List) {
yield* json
.expand((e) => _selectInJsonWithPath<T>(e, continueWithPath))
.map(addFirstToPath);
} else if (json is Map) {
for (String key in json.keys) {
yield* _selectInJsonWithPath<T>(json[key], continueWithPath)
.map(addFirstToPath);
}
}
} else if (json is List) {
try {
int index = int.parse(first);
yield* _selectInJsonWithPath<T>(json[index], rest);
} catch (e) {
// The first part of the path is not an index or out of bounds -> ignore
}
} else if (json is Map) {
dynamic value = json[first];
if (value != null) {
yield* _selectInJsonWithPath<T>(value, pathParts.join("."));
}
}
}
Iterable<T> _selectInJson<T>(dynamic json, String path) {
return _selectInJsonWithPath<T>(json, path).map((e) => e.value);
}
Map<String, Iterable<dynamic>> _selectMultipleInJson(
dynamic json, Map<String, String> selector) {
Map<String, Iterable<dynamic>> result = {};
for (String key in selector.keys) {
result[key] = _selectInJsonWithPath(json, selector[key]!);
}
return result;
}
ApiManager _wikidataApi = ApiManager("https://www.wikidata.org/w/api.php");
Map<String, String> _labelCache = {};
Future<Map<String, String>> _getLabelsForEntities(
List<String> entityIds) async {
const batchSize = 50;
Map<String, String> labels = {};
for (int i = entityIds.length - 1; i >= 0; i--) {
if (_labelCache.containsKey(entityIds[i])) {
labels[entityIds[i]] = _labelCache[entityIds[i]]!;
entityIds.removeAt(i);
}
}
for (int i = 0; i < (entityIds.length / batchSize).ceil(); i++) {
final start = i * batchSize;
final end = min((i + 1) * batchSize, entityIds.length);
Response response = await _wikidataApi.get(
"?action=wbgetentities&format=json&props=labels&ids=${entityIds.sublist(start, end).join("|")}");
Map<String, dynamic> result = jsonDecode(response.body);
Map<String, dynamic> batchEntities = result["entities"];
for (String entityId in batchEntities.keys) {
Map<String, dynamic> labels = batchEntities[entityId]["labels"];
String label = labels.containsKey("en")
? labels["en"]["value"]
: labels[labels.keys.first]["value"];
labels[entityId] = label;
_labelCache[entityId] = label;
}
}
return labels;
}
String _getCachedLabelForEntity(String entityId) {
return _labelCache[entityId] ?? entityId;
}
class WikidataMovieData extends MovieData {
String entityId;
WikidataMovieData(
String title, DateWithPrecisionAndCountry releaseDate, this.entityId)
: super(title, releaseDate);
WikidataMovieData.fromEncodable(Map encodable)
: entityId = encodable["entityId"],
super.fromJsonEncodable(encodable);
@override
bool same(MovieData other) {
return other is WikidataMovieData && entityId == other.entityId;
}
@override
Map toJsonEncodable() {
return super.toJsonEncodable()..addAll({"entityId": entityId});
}
static WikidataMovieData fromWikidataEntity(
String entityId, Map<String, dynamic> entity) {
String title =
_selectInJson<String>(entity, "labels.en.value").firstOrNull ??
_selectInJson<String>(entity, "labels.*.value").first;
Map<String, dynamic> claims = entity["claims"];
List<TitleInLanguage>? titles = _selectInJson(
claims, "${WikidataProperties.title}.*.mainsnak.datavalue.value")
.map((value) => (
title: value["text"],
language: value["language"],
) as TitleInLanguage)
.toList();
List<DateWithPrecisionAndCountry> releaseDates =
_selectInJson(claims, "${WikidataProperties.publicationDate}.*")
.map<DateWithPrecisionAndCountry>((dateClaim) {
var value = _selectInJson(dateClaim, "mainsnak.datavalue.value").first;
String country = _getCachedLabelForEntity(_selectInJson<String>(dateClaim,
"qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id")
.firstOrNull ??
"no country");
return DateWithPrecisionAndCountry(DateTime.parse(value["time"]),
_precisionFromWikidata(value["precision"]), country);
}).toList();
// Sort release dates with higher precision to the beginning
releaseDates
.sort((a, b) => -a.precision.index.compareTo(b.precision.index));
List<String>? genres = _selectInJson<String>(
claims, "${WikidataProperties.genre}.*.mainsnak.datavalue.value.id")
.map(_getCachedLabelForEntity)
.toList();
WikidataMovieData movie =
WikidataMovieData(title, releaseDates[0], entityId);
movie.setDetails(
titles: titles,
genres: genres,
);
return movie;
}
}
class WikidataMovieApi implements MovieApi {
ApiManager queryApi =
@ -228,10 +74,10 @@ class WikidataMovieApi implements MovieApi {
List<String> allCountryAndGenreIds = [];
// Add the country ids from the publication dates
allCountryAndGenreIds.addAll(_selectInJson<String>(entities,
allCountryAndGenreIds.addAll(selectInJson<String>(entities,
"*.claims.${WikidataProperties.publicationDate}.*.qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id"));
// Add the genre ids
allCountryAndGenreIds.addAll(_selectInJson<String>(entities,
allCountryAndGenreIds.addAll(selectInJson<String>(entities,
"*.claims.${WikidataProperties.genre}.*.mainsnak.datavalue.value.id"));
allCountryAndGenreIds = allCountryAndGenreIds.toSet().toList();
// Prefetch all labels for countries and genres
@ -251,6 +97,67 @@ class WikidataMovieApi implements MovieApi {
}
}
class WikidataMovieData extends MovieData {
String entityId;
WikidataMovieData(
String title, DateWithPrecisionAndCountry releaseDate, this.entityId)
: super(title, releaseDate);
WikidataMovieData.fromEncodable(Map encodable)
: entityId = encodable["entityId"],
super.fromJsonEncodable(encodable);
@override
bool same(MovieData other) {
return other is WikidataMovieData && entityId == other.entityId;
}
@override
Map toJsonEncodable() {
return super.toJsonEncodable()..addAll({"entityId": entityId});
}
static WikidataMovieData fromWikidataEntity(
String entityId, Map<String, dynamic> entity) {
String title =
selectInJson<String>(entity, "labels.en.value").firstOrNull ??
selectInJson<String>(entity, "labels.*.value").first;
Map<String, dynamic> claims = entity["claims"];
List<TitleInLanguage>? titles = selectInJson(
claims, "${WikidataProperties.title}.*.mainsnak.datavalue.value")
.map((value) => (
title: value["text"],
language: value["language"],
) as TitleInLanguage)
.toList();
List<DateWithPrecisionAndCountry> releaseDates =
selectInJson(claims, "${WikidataProperties.publicationDate}.*")
.map<DateWithPrecisionAndCountry>((dateClaim) {
var value = selectInJson(dateClaim, "mainsnak.datavalue.value").first;
String country = _getCachedLabelForEntity(selectInJson<String>(dateClaim,
"qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id")
.firstOrNull ??
"no country");
return DateWithPrecisionAndCountry(DateTime.parse(value["time"]),
_precisionFromWikidata(value["precision"]), country);
}).toList();
// Sort release dates with higher precision to the beginning
releaseDates
.sort((a, b) => -a.precision.index.compareTo(b.precision.index));
List<String>? genres = selectInJson<String>(
claims, "${WikidataProperties.genre}.*.mainsnak.datavalue.value.id")
.map(_getCachedLabelForEntity)
.toList();
WikidataMovieData movie =
WikidataMovieData(title, releaseDates[0], entityId);
movie.setDetails(
titles: titles,
genres: genres,
);
return movie;
}
}
String _createUpcomingMovieQuery(DateTime startDate, int limit) {
String date = DateFormat("yyyy-MM-dd").format(startDate);
return """
@ -287,3 +194,37 @@ DatePrecision _precisionFromWikidata(int precision) {
_ => throw Exception("Unexpected precision value: $precision"),
};
}
Map<String, String> _labelCache = {};
Future<Map<String, String>> _getLabelsForEntities(
List<String> entityIds) async {
const batchSize = 50;
Map<String, String> labels = {};
for (int i = entityIds.length - 1; i >= 0; i--) {
if (_labelCache.containsKey(entityIds[i])) {
labels[entityIds[i]] = _labelCache[entityIds[i]]!;
entityIds.removeAt(i);
}
}
for (int i = 0; i < (entityIds.length / batchSize).ceil(); i++) {
final start = i * batchSize;
final end = min((i + 1) * batchSize, entityIds.length);
Response response = await _wikidataApi.get(
"?action=wbgetentities&format=json&props=labels&ids=${entityIds.sublist(start, end).join("|")}");
Map<String, dynamic> result = jsonDecode(response.body);
Map<String, dynamic> batchEntities = result["entities"];
for (String entityId in batchEntities.keys) {
Map<String, dynamic> labels = batchEntities[entityId]["labels"];
String label = labels.containsKey("en")
? labels["en"]["value"]
: labels[labels.keys.first]["value"];
labels[entityId] = label;
_labelCache[entityId] = label;
}
}
return labels;
}
String _getCachedLabelForEntity(String entityId) {
return _labelCache[entityId] ?? entityId;
}