diff --git a/src/org/wikipedia/WikipediaApp.java b/src/org/wikipedia/WikipediaApp.java index 91c26fc8..2b227f1d 100644 --- a/src/org/wikipedia/WikipediaApp.java +++ b/src/org/wikipedia/WikipediaApp.java @@ -18,12 +18,12 @@ import java.util.Objects; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collector; import java.util.stream.Collectors; import java.util.stream.Stream; - import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -37,8 +37,13 @@ import org.openstreetmap.josm.tools.Utils; import org.w3c.dom.Document; import org.w3c.dom.Node; +import org.wikipedia.api.wikidata_action.ApiQueryClient; +import org.wikipedia.api.wikidata_action.WikidataActionApiUrl; +import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult; +import org.wikipedia.api.wikidata_action.json.SerializationSchema; import org.wikipedia.data.WikidataEntry; import org.wikipedia.data.WikipediaEntry; +import org.wikipedia.tools.ListUtil; import org.wikipedia.tools.RegexUtil; import org.wikipedia.tools.XPath; @@ -394,47 +399,43 @@ public static String getLabelForWikidata(String wikidataId, Locale locale, Strin } static List getLabelForWikidata(List entries, Locale locale, String ... preferredLanguage) { - if (entries.size() > 50) { - return partitionList(entries, 50).stream() - .flatMap(chunk -> getLabelForWikidata(chunk, locale, preferredLanguage).stream()) - .collect(Collectors.toList()); - } else if (entries.isEmpty()) { - return Collections.emptyList(); + final Collection languages = new ArrayList<>(); + if (locale != null) { + languages.add(getMediawikiLocale(locale)); + languages.add(getMediawikiLocale(new Locale(locale.getLanguage()))); } - try { - final String url = "https://www.wikidata.org/w/api.php" + + languages.addAll(Arrays.asList(preferredLanguage)); + languages.add("en"); + languages.add(null); + + final List result = new ArrayList<>(entries.size()); + ListUtil.processInBatches(entries, 50, batch -> { + try { + final String url = "https://www.wikidata.org/w/api.php" + "?action=wbgetentities" + "&props=labels|descriptions" + "&ids=" + entries.stream().map(x -> x.article).collect(Collectors.joining("|")) + "&format=xml"; - final Collection languages = new ArrayList<>(); - if (locale != null) { - languages.add(getMediawikiLocale(locale)); - languages.add(getMediawikiLocale(new Locale(locale.getLanguage()))); - } - languages.addAll(Arrays.asList(preferredLanguage)); - languages.add("en"); - languages.add(null); - final List r = new ArrayList<>(entries.size()); - try (final InputStream in = connect(url).getContent()) { - final Document xml = newDocumentBuilder().parse(in); - for (final WikipediaEntry entry : entries) { - final Node entity = X_PATH.evaluateNode("//entity[@id='" + entry.article + "']", xml); - if (entity == null) { - continue; - } - r.add(new WikidataEntry( + try (InputStream in = connect(url).getContent()) { + final Document xml = newDocumentBuilder().parse(in); + for (final WikipediaEntry entry : entries) { + final Node entity = X_PATH.evaluateNode("//entity[@id='" + entry.article + "']", xml); + if (entity == null) { + continue; + } + result.add(new WikidataEntry( entry.article, getFirstField(languages, "label", entity), entry.coordinate, getFirstField(languages, "description", entity) - )); + )); + } } + } catch (Exception ex) { + throw new RuntimeException(ex); } - return r; - } catch (Exception ex) { - throw new RuntimeException(ex); - } + }); + return result; } private static String getFirstField(Collection languages, String field, Node entity) { diff --git a/src/org/wikipedia/gui/WikipediaToggleDialog.java b/src/org/wikipedia/gui/WikipediaToggleDialog.java index 71526e98..318e74b1 100644 --- a/src/org/wikipedia/gui/WikipediaToggleDialog.java +++ b/src/org/wikipedia/gui/WikipediaToggleDialog.java @@ -14,7 +14,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - import javax.swing.AbstractAction; import javax.swing.DefaultListCellRenderer; import javax.swing.DefaultListModel; @@ -51,6 +50,7 @@ import org.wikipedia.WikipediaApp; import org.wikipedia.actions.FetchWikidataAction; import org.wikipedia.data.WikipediaEntry; +import org.wikipedia.tools.ListUtil; public class WikipediaToggleDialog extends ToggleDialog implements ActiveLayerChangeListener, DataSetListenerAdapter.Listener { @@ -187,8 +187,8 @@ protected Void doInBackground() throws Exception { final List entries = getEntries(); entries.sort(null); publish(entries.toArray(new WikipediaEntry[entries.size()])); - WikipediaApp.partitionList(entries, 20).forEach(chunk -> { - WikipediaApp.forLanguage(chunk.get(0).lang).updateWIWOSMStatus(chunk); + ListUtil.processInBatches(entries, 20, batch -> { + WikipediaApp.forLanguage(batch.get(0).lang).updateWIWOSMStatus(batch); list.repaint(); }); return null; diff --git a/src/org/wikipedia/tools/ListUtil.java b/src/org/wikipedia/tools/ListUtil.java index 182bdc59..a03f71fc 100644 --- a/src/org/wikipedia/tools/ListUtil.java +++ b/src/org/wikipedia/tools/ListUtil.java @@ -10,6 +10,8 @@ private ListUtil() { // Private constructor to avoid instantiation } + private static final BiConsumer EMPTY_BICONSUMER = (a, b) -> {}; + /** * Splits the given list {@code fullList} into batches of a size of {@code maxBatchSize} or less and each batch is * then consumed by the given {@link Consumer} {@code processBatch}. @@ -27,5 +29,9 @@ public static void processInBatches(final List fullList, int maxBatchSize } } + public static void processInBatches(final List fullList, int maxBatchSize, final Consumer> processBatch) { + processInBatches(fullList, maxBatchSize, processBatch, EMPTY_BICONSUMER); + } + -} \ No newline at end of file +}