From 5c11b931c0a73f32332a39ac085f24f43e264160 Mon Sep 17 00:00:00 2001 From: daniel-michel <65034538+daniel-michel@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:08:19 +0100 Subject: [PATCH] refactor: separate file for json helpers --- lib/api/json_helper.dart | 73 +++++++++ lib/api/wikidata_movie_api.dart | 255 ++++++++++++-------------------- 2 files changed, 171 insertions(+), 157 deletions(-) create mode 100644 lib/api/json_helper.dart diff --git a/lib/api/json_helper.dart b/lib/api/json_helper.dart new file mode 100644 index 0000000..4fcb8e0 --- /dev/null +++ b/lib/api/json_helper.dart @@ -0,0 +1,73 @@ +/// Select values in nested [List] and [Map] structures using a path that may contain wildcards. +/// +/// The maps must always use [String] keys. +/// The [path] is a dot-separated list of keys and indices. +/// The wildcard "*" can be used to select all elements of a list or map. +/// The wildcard "**" can be used to select all elements of a list or map and all elements of nested lists and maps. +/// +/// Returns an [Iterable] of the selected values. +/// +/// Also see [selectInJsonWithPath] for a version that returns the path to the selected values +Iterable selectInJson(dynamic json, String path) { + return selectInJsonWithPath(json, path).map((e) => e.value); +} + +Map> selectMultipleInJson( + dynamic json, Map selector) { + Map> result = {}; + for (String key in selector.keys) { + result[key] = selectInJsonWithPath(json, selector[key]!); + } + return result; +} + +/// Select values in nested [List] and [Map] structures using a path that may contain wildcards. +/// +/// The maps must always use [String] keys. +/// The [path] is a dot-separated list of keys and indices. +/// The wildcard "*" can be used to select all elements of a list or map. +/// The wildcard "**" can be used to select all elements of a list or map and all elements of nested lists and maps.+ +/// +/// Returns an [Iterable] of the selected values and their path. +Iterable<({T value, String path})> selectInJsonWithPath( + dynamic json, String path) sync* { + if (path.isEmpty) { + if (json is T) { + yield (value: json, path: ""); + } + return; + } + List pathParts = path.split("."); + String first = pathParts.removeAt(0); + String rest = pathParts.join("."); + addFirstToPath(({T value, String path}) element) => ( + value: element.value, + path: element.path.isEmpty ? first : "$first.${element.path}" + ); + + if (first == "*" || first == "**") { + String continueWithPath = first == "*" ? rest : path; + if (json is List) { + yield* json + .expand((e) => selectInJsonWithPath(e, continueWithPath)) + .map(addFirstToPath); + } else if (json is Map) { + for (String key in json.keys) { + yield* selectInJsonWithPath(json[key], continueWithPath) + .map(addFirstToPath); + } + } + } else if (json is List) { + try { + int index = int.parse(first); + yield* selectInJsonWithPath(json[index], rest); + } catch (e) { + // The first part of the path is not an index or out of bounds -> ignore + } + } else if (json is Map) { + dynamic value = json[first]; + if (value != null) { + yield* selectInJsonWithPath(value, pathParts.join(".")); + } + } +} diff --git a/lib/api/wikidata_movie_api.dart b/lib/api/wikidata_movie_api.dart index 15b37d5..c638f54 100644 --- a/lib/api/wikidata_movie_api.dart +++ b/lib/api/wikidata_movie_api.dart @@ -4,6 +4,7 @@ import 'dart:math'; import 'package:http/http.dart'; import 'package:intl/intl.dart'; import 'package:release_schedule/api/api_manager.dart'; +import 'package:release_schedule/api/json_helper.dart'; import 'package:release_schedule/api/movie_api.dart'; import 'package:release_schedule/model/movie.dart'; @@ -26,162 +27,7 @@ class WikidataProperties { static const String placeOfPublication = "P291"; } -/// Select values in nested List and Map structures using a path that may contain wildcards. -/// The maps must always use String keys. -/// The path is a dot-separated list of keys and indices. -/// The wildcard "*" can be used to select all elements of a list or map. -/// The wildcard "**" can be used to select all elements of a list or map and all elements of nested lists and maps. -Iterable<({T value, String path})> _selectInJsonWithPath( - dynamic json, String path) sync* { - if (path.isEmpty) { - if (json is T) { - yield (value: json, path: ""); - } - return; - } - List pathParts = path.split("."); - String first = pathParts.removeAt(0); - String rest = pathParts.join("."); - addFirstToPath(({T value, String path}) element) => ( - value: element.value, - path: element.path.isEmpty ? first : "$first.${element.path}" - ); - - if (first == "*" || first == "**") { - String continueWithPath = first == "*" ? rest : path; - if (json is List) { - yield* json - .expand((e) => _selectInJsonWithPath(e, continueWithPath)) - .map(addFirstToPath); - } else if (json is Map) { - for (String key in json.keys) { - yield* _selectInJsonWithPath(json[key], continueWithPath) - .map(addFirstToPath); - } - } - } else if (json is List) { - try { - int index = int.parse(first); - yield* _selectInJsonWithPath(json[index], rest); - } catch (e) { - // The first part of the path is not an index or out of bounds -> ignore - } - } else if (json is Map) { - dynamic value = json[first]; - if (value != null) { - yield* _selectInJsonWithPath(value, pathParts.join(".")); - } - } -} - -Iterable _selectInJson(dynamic json, String path) { - return _selectInJsonWithPath(json, path).map((e) => e.value); -} - -Map> _selectMultipleInJson( - dynamic json, Map selector) { - Map> result = {}; - for (String key in selector.keys) { - result[key] = _selectInJsonWithPath(json, selector[key]!); - } - return result; -} - ApiManager _wikidataApi = ApiManager("https://www.wikidata.org/w/api.php"); -Map _labelCache = {}; -Future> _getLabelsForEntities( - List entityIds) async { - const batchSize = 50; - Map labels = {}; - for (int i = entityIds.length - 1; i >= 0; i--) { - if (_labelCache.containsKey(entityIds[i])) { - labels[entityIds[i]] = _labelCache[entityIds[i]]!; - entityIds.removeAt(i); - } - } - for (int i = 0; i < (entityIds.length / batchSize).ceil(); i++) { - final start = i * batchSize; - final end = min((i + 1) * batchSize, entityIds.length); - Response response = await _wikidataApi.get( - "?action=wbgetentities&format=json&props=labels&ids=${entityIds.sublist(start, end).join("|")}"); - Map result = jsonDecode(response.body); - Map batchEntities = result["entities"]; - for (String entityId in batchEntities.keys) { - Map labels = batchEntities[entityId]["labels"]; - String label = labels.containsKey("en") - ? labels["en"]["value"] - : labels[labels.keys.first]["value"]; - labels[entityId] = label; - _labelCache[entityId] = label; - } - } - return labels; -} - -String _getCachedLabelForEntity(String entityId) { - return _labelCache[entityId] ?? entityId; -} - -class WikidataMovieData extends MovieData { - String entityId; - WikidataMovieData( - String title, DateWithPrecisionAndCountry releaseDate, this.entityId) - : super(title, releaseDate); - - WikidataMovieData.fromEncodable(Map encodable) - : entityId = encodable["entityId"], - super.fromJsonEncodable(encodable); - - @override - bool same(MovieData other) { - return other is WikidataMovieData && entityId == other.entityId; - } - - @override - Map toJsonEncodable() { - return super.toJsonEncodable()..addAll({"entityId": entityId}); - } - - static WikidataMovieData fromWikidataEntity( - String entityId, Map entity) { - String title = - _selectInJson(entity, "labels.en.value").firstOrNull ?? - _selectInJson(entity, "labels.*.value").first; - Map claims = entity["claims"]; - List? titles = _selectInJson( - claims, "${WikidataProperties.title}.*.mainsnak.datavalue.value") - .map((value) => ( - title: value["text"], - language: value["language"], - ) as TitleInLanguage) - .toList(); - List releaseDates = - _selectInJson(claims, "${WikidataProperties.publicationDate}.*") - .map((dateClaim) { - var value = _selectInJson(dateClaim, "mainsnak.datavalue.value").first; - String country = _getCachedLabelForEntity(_selectInJson(dateClaim, - "qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id") - .firstOrNull ?? - "no country"); - return DateWithPrecisionAndCountry(DateTime.parse(value["time"]), - _precisionFromWikidata(value["precision"]), country); - }).toList(); - // Sort release dates with higher precision to the beginning - releaseDates - .sort((a, b) => -a.precision.index.compareTo(b.precision.index)); - List? genres = _selectInJson( - claims, "${WikidataProperties.genre}.*.mainsnak.datavalue.value.id") - .map(_getCachedLabelForEntity) - .toList(); - WikidataMovieData movie = - WikidataMovieData(title, releaseDates[0], entityId); - movie.setDetails( - titles: titles, - genres: genres, - ); - return movie; - } -} class WikidataMovieApi implements MovieApi { ApiManager queryApi = @@ -228,10 +74,10 @@ class WikidataMovieApi implements MovieApi { List allCountryAndGenreIds = []; // Add the country ids from the publication dates - allCountryAndGenreIds.addAll(_selectInJson(entities, + allCountryAndGenreIds.addAll(selectInJson(entities, "*.claims.${WikidataProperties.publicationDate}.*.qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id")); // Add the genre ids - allCountryAndGenreIds.addAll(_selectInJson(entities, + allCountryAndGenreIds.addAll(selectInJson(entities, "*.claims.${WikidataProperties.genre}.*.mainsnak.datavalue.value.id")); allCountryAndGenreIds = allCountryAndGenreIds.toSet().toList(); // Prefetch all labels for countries and genres @@ -251,6 +97,67 @@ class WikidataMovieApi implements MovieApi { } } +class WikidataMovieData extends MovieData { + String entityId; + WikidataMovieData( + String title, DateWithPrecisionAndCountry releaseDate, this.entityId) + : super(title, releaseDate); + + WikidataMovieData.fromEncodable(Map encodable) + : entityId = encodable["entityId"], + super.fromJsonEncodable(encodable); + + @override + bool same(MovieData other) { + return other is WikidataMovieData && entityId == other.entityId; + } + + @override + Map toJsonEncodable() { + return super.toJsonEncodable()..addAll({"entityId": entityId}); + } + + static WikidataMovieData fromWikidataEntity( + String entityId, Map entity) { + String title = + selectInJson(entity, "labels.en.value").firstOrNull ?? + selectInJson(entity, "labels.*.value").first; + Map claims = entity["claims"]; + List? titles = selectInJson( + claims, "${WikidataProperties.title}.*.mainsnak.datavalue.value") + .map((value) => ( + title: value["text"], + language: value["language"], + ) as TitleInLanguage) + .toList(); + List releaseDates = + selectInJson(claims, "${WikidataProperties.publicationDate}.*") + .map((dateClaim) { + var value = selectInJson(dateClaim, "mainsnak.datavalue.value").first; + String country = _getCachedLabelForEntity(selectInJson(dateClaim, + "qualifiers.${WikidataProperties.placeOfPublication}.*.datavalue.value.id") + .firstOrNull ?? + "no country"); + return DateWithPrecisionAndCountry(DateTime.parse(value["time"]), + _precisionFromWikidata(value["precision"]), country); + }).toList(); + // Sort release dates with higher precision to the beginning + releaseDates + .sort((a, b) => -a.precision.index.compareTo(b.precision.index)); + List? genres = selectInJson( + claims, "${WikidataProperties.genre}.*.mainsnak.datavalue.value.id") + .map(_getCachedLabelForEntity) + .toList(); + WikidataMovieData movie = + WikidataMovieData(title, releaseDates[0], entityId); + movie.setDetails( + titles: titles, + genres: genres, + ); + return movie; + } +} + String _createUpcomingMovieQuery(DateTime startDate, int limit) { String date = DateFormat("yyyy-MM-dd").format(startDate); return """ @@ -287,3 +194,37 @@ DatePrecision _precisionFromWikidata(int precision) { _ => throw Exception("Unexpected precision value: $precision"), }; } + +Map _labelCache = {}; +Future> _getLabelsForEntities( + List entityIds) async { + const batchSize = 50; + Map labels = {}; + for (int i = entityIds.length - 1; i >= 0; i--) { + if (_labelCache.containsKey(entityIds[i])) { + labels[entityIds[i]] = _labelCache[entityIds[i]]!; + entityIds.removeAt(i); + } + } + for (int i = 0; i < (entityIds.length / batchSize).ceil(); i++) { + final start = i * batchSize; + final end = min((i + 1) * batchSize, entityIds.length); + Response response = await _wikidataApi.get( + "?action=wbgetentities&format=json&props=labels&ids=${entityIds.sublist(start, end).join("|")}"); + Map result = jsonDecode(response.body); + Map batchEntities = result["entities"]; + for (String entityId in batchEntities.keys) { + Map labels = batchEntities[entityId]["labels"]; + String label = labels.containsKey("en") + ? labels["en"]["value"] + : labels[labels.keys.first]["value"]; + labels[entityId] = label; + _labelCache[entityId] = label; + } + } + return labels; +} + +String _getCachedLabelForEntity(String entityId) { + return _labelCache[entityId] ?? entityId; +}