001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005import static org.openstreetmap.josm.tools.I18n.trn; 006 007import java.io.IOException; 008import java.io.InputStream; 009import java.net.HttpURLConnection; 010import java.util.ArrayList; 011import java.util.Collection; 012import java.util.HashSet; 013import java.util.Iterator; 014import java.util.LinkedHashSet; 015import java.util.List; 016import java.util.NoSuchElementException; 017import java.util.Set; 018import java.util.concurrent.Callable; 019import java.util.concurrent.CompletionService; 020import java.util.concurrent.ExecutionException; 021import java.util.concurrent.Executor; 022import java.util.concurrent.ExecutorCompletionService; 023import java.util.concurrent.Executors; 024import java.util.concurrent.Future; 025 026import org.openstreetmap.josm.Main; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.DataSetMerger; 029import org.openstreetmap.josm.data.osm.Node; 030import org.openstreetmap.josm.data.osm.OsmPrimitive; 031import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 032import org.openstreetmap.josm.data.osm.PrimitiveId; 033import org.openstreetmap.josm.data.osm.Relation; 034import org.openstreetmap.josm.data.osm.RelationMember; 035import org.openstreetmap.josm.data.osm.SimplePrimitiveId; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.tools.CheckParameterUtil; 040 041/** 042 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called 043 * Multi Fetch API. 044 * 045 * Usage: 046 * <pre> 047 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader() 048 * .append(2345,2334,4444) 049 * .append(new Node(72343)); 050 * reader.parseOsm(); 051 * if (!reader.getMissingPrimitives().isEmpty()) { 052 * Main.info("There are missing primitives: " + reader.getMissingPrimitives()); 053 * } 054 * if (!reader.getSkippedWays().isEmpty()) { 055 * Main.info("There are skipped ways: " + reader.getMissingPrimitives()); 056 * } 057 * </pre> 058 */ 059public class MultiFetchServerObjectReader extends OsmServerReader{ 060 /** 061 * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits, 062 * this leads to a max. request URL of ~ 1600 Bytes ((7 digits + 1 Separator) * 200), 063 * which should be safe according to the 064 * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>. 065 */ 066 private static final int MAX_IDS_PER_REQUEST = 200; 067 068 private Set<Long> nodes; 069 private Set<Long> ways; 070 private Set<Long> relations; 071 private Set<PrimitiveId> missingPrimitives; 072 private DataSet outputDataSet; 073 074 /** 075 * Constructs a {@code MultiFetchServerObjectReader}. 076 */ 077 public MultiFetchServerObjectReader() { 078 nodes = new LinkedHashSet<>(); 079 ways = new LinkedHashSet<>(); 080 relations = new LinkedHashSet<>(); 081 this.outputDataSet = new DataSet(); 082 this.missingPrimitives = new LinkedHashSet<>(); 083 } 084 085 /** 086 * Remembers an {@link OsmPrimitive}'s id. The id will 087 * later be fetched as part of a Multi Get request. 088 * 089 * Ignore the id if it represents a new primitives. 090 * 091 * @param id the id 092 */ 093 protected void remember(PrimitiveId id) { 094 if (id.isNew()) return; 095 switch(id.getType()) { 096 case NODE: nodes.add(id.getUniqueId()); break; 097 case WAY: ways.add(id.getUniqueId()); break; 098 case RELATION: relations.add(id.getUniqueId()); break; 099 } 100 } 101 102 /** 103 * remembers an {@link OsmPrimitive}'s id. <code>ds</code> must include 104 * an {@link OsmPrimitive} with id=<code>id</code>. The id will 105 * later we fetched as part of a Multi Get request. 106 * 107 * Ignore the id if it id <= 0. 108 * 109 * @param ds the dataset (must not be null) 110 * @param id the primitive id 111 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 112 * @throws IllegalArgumentException if ds is null 113 * @throws NoSuchElementException if ds does not include an {@link OsmPrimitive} with id=<code>id</code> 114 */ 115 protected void remember(DataSet ds, long id, OsmPrimitiveType type) throws IllegalArgumentException, NoSuchElementException{ 116 CheckParameterUtil.ensureParameterNotNull(ds, "ds"); 117 if (id <= 0) return; 118 OsmPrimitive primitive = ds.getPrimitiveById(id, type); 119 if (primitive == null) 120 throw new NoSuchElementException(tr("No primitive with id {0} in local dataset. Cannot infer primitive type.", id)); 121 remember(primitive.getPrimitiveId()); 122 return; 123 } 124 125 /** 126 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server. 127 * 128 * @param ds the {@link DataSet} to which the primitive belongs 129 * @param id the primitive id 130 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 131 * @return this 132 */ 133 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) { 134 OsmPrimitive p = ds.getPrimitiveById(id,type); 135 switch(type) { 136 case NODE: 137 return appendNode((Node)p); 138 case WAY: 139 return appendWay((Way)p); 140 case RELATION: 141 return appendRelation((Relation)p); 142 } 143 return this; 144 } 145 146 /** 147 * appends a {@link Node} id to the list of ids which will be fetched from the server. 148 * 149 * @param node the node (ignored, if null) 150 * @return this 151 */ 152 public MultiFetchServerObjectReader appendNode(Node node) { 153 if (node == null) return this; 154 remember(node.getPrimitiveId()); 155 return this; 156 } 157 158 /** 159 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server. 160 * 161 * @param way the way (ignored, if null) 162 * @return this 163 */ 164 public MultiFetchServerObjectReader appendWay(Way way) { 165 if (way == null) return this; 166 if (way.isNew()) return this; 167 for (Node node: way.getNodes()) { 168 if (!node.isNew()) { 169 remember(node.getPrimitiveId()); 170 } 171 } 172 remember(way.getPrimitiveId()); 173 return this; 174 } 175 176 /** 177 * appends a {@link Relation} id to the list of ids which will be fetched from the server. 178 * 179 * @param relation the relation (ignored, if null) 180 * @return this 181 */ 182 protected MultiFetchServerObjectReader appendRelation(Relation relation) { 183 if (relation == null) return this; 184 if (relation.isNew()) return this; 185 remember(relation.getPrimitiveId()); 186 for (RelationMember member : relation.getMembers()) { 187 if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) { 188 // avoid infinite recursion in case of cyclic dependencies in relations 189 // 190 if (relations.contains(member.getMember().getId())) { 191 continue; 192 } 193 } 194 if (!member.getMember().isIncomplete()) { 195 append(member.getMember()); 196 } 197 } 198 return this; 199 } 200 201 /** 202 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server. 203 * @param primitive the primitive 204 * @return this 205 */ 206 public MultiFetchServerObjectReader append(OsmPrimitive primitive) { 207 if (primitive != null) { 208 switch (OsmPrimitiveType.from(primitive)) { 209 case NODE: return appendNode((Node)primitive); 210 case WAY: return appendWay((Way)primitive); 211 case RELATION: return appendRelation((Relation)primitive); 212 } 213 } 214 return this; 215 } 216 217 /** 218 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server. 219 * 220 * @param primitives the list of primitives (ignored, if null) 221 * @return this 222 * 223 * @see #append(OsmPrimitive) 224 */ 225 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) { 226 if (primitives == null) return this; 227 for (OsmPrimitive primitive : primitives) { 228 append(primitive); 229 } 230 return this; 231 } 232 233 /** 234 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and 235 * replies the subset. The extracted subset is removed from <code>ids</code>. 236 * 237 * @param ids a set of ids 238 * @return the subset of ids 239 */ 240 protected Set<Long> extractIdPackage(Set<Long> ids) { 241 HashSet<Long> pkg = new HashSet<>(); 242 if (ids.isEmpty()) 243 return pkg; 244 if (ids.size() > MAX_IDS_PER_REQUEST) { 245 Iterator<Long> it = ids.iterator(); 246 for (int i=0; i<MAX_IDS_PER_REQUEST; i++) { 247 pkg.add(it.next()); 248 } 249 ids.removeAll(pkg); 250 } else { 251 pkg.addAll(ids); 252 ids.clear(); 253 } 254 return pkg; 255 } 256 257 /** 258 * builds the Multi Get request string for a set of ids and a given 259 * {@link OsmPrimitiveType}. 260 * 261 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 262 * @param idPackage the package of ids 263 * @return the request string 264 */ 265 protected static String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) { 266 StringBuilder sb = new StringBuilder(); 267 sb.append(type.getAPIName()).append("s?") 268 .append(type.getAPIName()).append("s="); 269 270 Iterator<Long> it = idPackage.iterator(); 271 for (int i=0; i<idPackage.size(); i++) { 272 sb.append(it.next()); 273 if (i < idPackage.size()-1) { 274 sb.append(","); 275 } 276 } 277 return sb.toString(); 278 } 279 280 /** 281 * builds the Multi Get request string for a single id and a given 282 * {@link OsmPrimitiveType}. 283 * 284 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 285 * @param id the id 286 * @return the request string 287 */ 288 protected static String buildRequestString(OsmPrimitiveType type, long id) { 289 StringBuilder sb = new StringBuilder(); 290 sb.append(type.getAPIName()).append("s?") 291 .append(type.getAPIName()).append("s=") 292 .append(id); 293 return sb.toString(); 294 } 295 296 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { 297 for (Way w: from.getWays()) { 298 if (w.hasIncompleteNodes()) { 299 for (Node n: w.getNodes()) { 300 if (n.isIncomplete()) { 301 nodes.add(n.getId()); 302 } 303 } 304 } 305 } 306 } 307 308 /** 309 * merges the dataset <code>from</code> to {@link #outputDataSet}. 310 * 311 * @param from the other dataset 312 */ 313 protected void merge(DataSet from) { 314 final DataSetMerger visitor = new DataSetMerger(outputDataSet,from); 315 visitor.merge(); 316 } 317 318 /** 319 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server 320 * 321 * @param ids the set of ids 322 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 323 * @throws OsmTransferException if an error occurs while communicating with the API server 324 */ 325 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { 326 String msg = ""; 327 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 328 switch (type) { 329 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break; 330 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break; 331 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break; 332 } 333 progressMonitor.setTicksCount(ids.size()); 334 progressMonitor.setTicks(0); 335 // The complete set containg all primitives to fetch 336 Set<Long> toFetch = new HashSet<>(ids); 337 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each. 338 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. 339 int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS); 340 threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS); 341 Executor exec = Executors.newFixedThreadPool(threadsNumber); 342 CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec); 343 List<Future<FetchResult>> jobs = new ArrayList<>(); 344 while (!toFetch.isEmpty()) { 345 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor))); 346 } 347 // Run the fetchers 348 for (int i = 0; i < jobs.size() && !isCanceled(); i++) { 349 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount()); 350 try { 351 FetchResult result = ecs.take().get(); 352 if (result.missingPrimitives != null) { 353 missingPrimitives.addAll(result.missingPrimitives); 354 } 355 if (result.dataSet != null && !isCanceled()) { 356 rememberNodesOfIncompleteWaysToLoad(result.dataSet); 357 merge(result.dataSet); 358 } 359 } catch (InterruptedException | ExecutionException e) { 360 Main.error(e); 361 } 362 } 363 // Cancel requests if the user choosed to 364 if (isCanceled()) { 365 for (Future<FetchResult> job : jobs) { 366 job.cancel(true); 367 } 368 } 369 } 370 371 /** 372 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies 373 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too! 374 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies 375 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if 376 * visible==false). 377 * 378 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been 379 * found on the server (the server response code was 404) 380 * 381 * @return the parsed data 382 * @throws OsmTransferException if an error occurs while communicating with the API server 383 * @see #getMissingPrimitives() 384 * 385 */ 386 @Override 387 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 388 int n = nodes.size() + ways.size() + relations.size(); 389 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", 390 "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl())); 391 try { 392 missingPrimitives = new HashSet<>(); 393 if (isCanceled()) return null; 394 fetchPrimitives(ways,OsmPrimitiveType.WAY, progressMonitor); 395 if (isCanceled()) return null; 396 fetchPrimitives(nodes,OsmPrimitiveType.NODE, progressMonitor); 397 if (isCanceled()) return null; 398 fetchPrimitives(relations,OsmPrimitiveType.RELATION, progressMonitor); 399 if (outputDataSet != null) { 400 outputDataSet.deleteInvisible(); 401 } 402 return outputDataSet; 403 } finally { 404 progressMonitor.finishTask(); 405 } 406 } 407 408 /** 409 * replies the set of ids of all primitives for which a fetch request to the 410 * server was submitted but which are not available from the server (the server 411 * replied a return code of 404) 412 * 413 * @return the set of ids of missing primitives 414 */ 415 public Set<PrimitiveId> getMissingPrimitives() { 416 return missingPrimitives; 417 } 418 419 /** 420 * The class holding the results given by {@link Fetcher}. 421 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded. 422 */ 423 protected static class FetchResult { 424 425 /** 426 * The resulting data set 427 */ 428 public final DataSet dataSet; 429 430 /** 431 * The collection of primitive ids that could not have been loaded 432 */ 433 public final Set<PrimitiveId> missingPrimitives; 434 435 /** 436 * Constructs a {@code FetchResult} 437 * @param dataSet The resulting data set 438 * @param missingPrimitives The collection of primitive ids that could not have been loaded 439 */ 440 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) { 441 this.dataSet = dataSet; 442 this.missingPrimitives = missingPrimitives; 443 } 444 } 445 446 /** 447 * The class that actually download data from OSM API. Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch). 448 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance. 449 * @see FetchResult 450 */ 451 protected static class Fetcher extends OsmServerReader implements Callable<FetchResult> { 452 453 private final Set<Long> pkg; 454 private final OsmPrimitiveType type; 455 private final ProgressMonitor progressMonitor; 456 457 /** 458 * Constructs a {@code Fetcher} 459 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 460 * @param idsPackage The set of primitives ids to fetch 461 * @param progressMonitor The progress monitor 462 */ 463 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) { 464 this.pkg = idsPackage; 465 this.type = type; 466 this.progressMonitor = progressMonitor; 467 } 468 469 @Override 470 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 471 // This method is implemented because of the OsmServerReader inheritance, but not used, as the main target of this class is the call() method. 472 return fetch(progressMonitor).dataSet; 473 } 474 475 @Override 476 public FetchResult call() throws Exception { 477 return fetch(progressMonitor); 478 } 479 480 /** 481 * fetches the requested primitives and updates the specified progress monitor. 482 * @param progressMonitor the progress monitor 483 * @return the {@link FetchResult} of this operation 484 * @throws OsmTransferException if an error occurs while communicating with the API server 485 */ 486 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { 487 try { 488 return multiGetIdPackage(type, pkg, progressMonitor); 489 } catch (OsmApiException e) { 490 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 491 Main.info(tr("Server replied with response code 404, retrying with an individual request for each object.")); 492 return singleGetIdPackage(type, pkg, progressMonitor); 493 } else { 494 throw e; 495 } 496 } 497 } 498 499 /** 500 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. 501 * The retrieved primitives are merged to {@link #outputDataSet}. 502 * 503 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 504 * @param pkg the package of ids 505 * @return the {@link FetchResult} of this operation 506 * @throws OsmTransferException if an error occurs while communicating with the API server 507 */ 508 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException { 509 String request = buildRequestString(type, pkg); 510 FetchResult result = null; 511 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 512 if (in == null) return null; 513 progressMonitor.subTask(tr("Downloading OSM data...")); 514 try { 515 result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); 516 } catch (Exception e) { 517 throw new OsmTransferException(e); 518 } 519 } catch (IOException ex) { 520 Main.warn(ex); 521 } 522 return result; 523 } 524 525 /** 526 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. 527 * The retrieved primitive is merged to {@link #outputDataSet}. 528 * 529 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 530 * @param id the id 531 * @return the {@link DataSet} resulting of this operation 532 * @throws OsmTransferException if an error occurs while communicating with the API server 533 */ 534 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { 535 String request = buildRequestString(type, id); 536 DataSet result = null; 537 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 538 if (in == null) return null; 539 progressMonitor.subTask(tr("Downloading OSM data...")); 540 try { 541 result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); 542 } catch (Exception e) { 543 throw new OsmTransferException(e); 544 } 545 } catch (IOException ex) { 546 Main.warn(ex); 547 } 548 return result; 549 } 550 551 /** 552 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. 553 * The retrieved primitives are merged to {@link #outputDataSet}. 554 * 555 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). 556 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. 557 * Unfortunately, the server does not provide an error header or an error body for a 404 reply. 558 * 559 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 560 * @param pkg the set of ids 561 * @return the {@link FetchResult} of this operation 562 * @throws OsmTransferException if an error occurs while communicating with the API server 563 */ 564 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException { 565 FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 566 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 567 for (long id : pkg) { 568 try { 569 String msg = ""; 570 switch (type) { 571 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; 572 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; 573 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; 574 } 575 progressMonitor.setCustomText(msg); 576 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); 577 } catch (OsmApiException e) { 578 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 579 Main.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); 580 result.missingPrimitives.add(new SimplePrimitiveId(id, type)); 581 } else { 582 throw e; 583 } 584 } 585 } 586 return result; 587 } 588 } 589}