001 // License: GPL. For details, see LICENSE file. 002 package org.openstreetmap.josm.io; 003 004 import static org.openstreetmap.josm.tools.I18n.tr; 005 import static org.openstreetmap.josm.tools.I18n.trn; 006 007 import java.io.InputStream; 008 import java.net.HttpURLConnection; 009 import java.util.ArrayList; 010 import java.util.Collection; 011 import java.util.HashSet; 012 import java.util.Iterator; 013 import java.util.LinkedHashSet; 014 import java.util.List; 015 import java.util.NoSuchElementException; 016 import java.util.Set; 017 import java.util.concurrent.Callable; 018 import java.util.concurrent.CompletionService; 019 import java.util.concurrent.ExecutionException; 020 import java.util.concurrent.Executor; 021 import java.util.concurrent.ExecutorCompletionService; 022 import java.util.concurrent.Executors; 023 import java.util.concurrent.Future; 024 025 import org.openstreetmap.josm.Main; 026 import org.openstreetmap.josm.data.osm.DataSet; 027 import org.openstreetmap.josm.data.osm.DataSetMerger; 028 import org.openstreetmap.josm.data.osm.Node; 029 import org.openstreetmap.josm.data.osm.OsmPrimitive; 030 import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 031 import org.openstreetmap.josm.data.osm.PrimitiveId; 032 import org.openstreetmap.josm.data.osm.Relation; 033 import org.openstreetmap.josm.data.osm.RelationMember; 034 import org.openstreetmap.josm.data.osm.SimplePrimitiveId; 035 import org.openstreetmap.josm.data.osm.Way; 036 import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 037 import org.openstreetmap.josm.gui.progress.ProgressMonitor; 038 import org.openstreetmap.josm.tools.CheckParameterUtil; 039 040 /** 041 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called 042 * Multi Fetch API. 043 * 044 * Usage: 045 * <pre> 046 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader() 047 * .append(2345,2334,4444) 048 * .append(new Node(72343)); 049 * reader.parseOsm(); 050 * if (!reader.getMissingPrimitives().isEmpty()) { 051 * System.out.println("There are missing primitives: " + reader.getMissingPrimitives()); 052 * } 053 * if (!reader.getSkippedWays().isEmpty()) { 054 * System.out.println("There are skipped ways: " + reader.getMissingPrimitives()); 055 * } 056 * </pre> 057 */ 058 public class MultiFetchServerObjectReader extends OsmServerReader{ 059 /** 060 * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits, 061 * this leads to a max. request URL of ~ 1600 Bytes ((7 digits + 1 Separator) * 200), 062 * which should be safe according to the 063 * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>. 064 */ 065 static private int MAX_IDS_PER_REQUEST = 200; 066 067 private Set<Long> nodes; 068 private Set<Long> ways; 069 private Set<Long> relations; 070 private Set<PrimitiveId> missingPrimitives; 071 private DataSet outputDataSet; 072 073 /** 074 * Constructs a {@code MultiFetchServerObjectReader}. 075 */ 076 public MultiFetchServerObjectReader() { 077 nodes = new LinkedHashSet<Long>(); 078 ways = new LinkedHashSet<Long>(); 079 relations = new LinkedHashSet<Long>(); 080 this.outputDataSet = new DataSet(); 081 this.missingPrimitives = new LinkedHashSet<PrimitiveId>(); 082 } 083 084 /** 085 * Remembers an {@link OsmPrimitive}'s id. The id will 086 * later be fetched as part of a Multi Get request. 087 * 088 * Ignore the id if it represents a new primitives. 089 * 090 * @param id the id 091 */ 092 protected void remember(PrimitiveId id) { 093 if (id.isNew()) return; 094 switch(id.getType()) { 095 case NODE: nodes.add(id.getUniqueId()); break; 096 case WAY: ways.add(id.getUniqueId()); break; 097 case RELATION: relations.add(id.getUniqueId()); break; 098 } 099 } 100 101 /** 102 * remembers an {@link OsmPrimitive}'s id. <code>ds</code> must include 103 * an {@link OsmPrimitive} with id=<code>id</code>. The id will 104 * later we fetched as part of a Multi Get request. 105 * 106 * Ignore the id if it id <= 0. 107 * 108 * @param ds the dataset (must not be null) 109 * @param id the primitive id 110 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 111 * @throws IllegalArgumentException if ds is null 112 * @throws NoSuchElementException if ds does not include an {@link OsmPrimitive} with id=<code>id</code> 113 */ 114 protected void remember(DataSet ds, long id, OsmPrimitiveType type) throws IllegalArgumentException, NoSuchElementException{ 115 CheckParameterUtil.ensureParameterNotNull(ds, "ds"); 116 if (id <= 0) return; 117 OsmPrimitive primitive = ds.getPrimitiveById(id, type); 118 if (primitive == null) 119 throw new NoSuchElementException(tr("No primitive with id {0} in local dataset. Cannot infer primitive type.", id)); 120 remember(primitive.getPrimitiveId()); 121 return; 122 } 123 124 /** 125 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server. 126 * 127 * @param ds the {@link DataSet} to which the primitive belongs 128 * @param id the primitive id 129 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 130 * @return this 131 */ 132 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) { 133 OsmPrimitive p = ds.getPrimitiveById(id,type); 134 switch(type) { 135 case NODE: 136 return appendNode((Node)p); 137 case WAY: 138 return appendWay((Way)p); 139 case RELATION: 140 return appendRelation((Relation)p); 141 } 142 return this; 143 } 144 145 /** 146 * appends a {@link Node} id to the list of ids which will be fetched from the server. 147 * 148 * @param node the node (ignored, if null) 149 * @return this 150 */ 151 public MultiFetchServerObjectReader appendNode(Node node) { 152 if (node == null) return this; 153 remember(node.getPrimitiveId()); 154 return this; 155 } 156 157 /** 158 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server. 159 * 160 * @param way the way (ignored, if null) 161 * @return this 162 */ 163 public MultiFetchServerObjectReader appendWay(Way way) { 164 if (way == null) return this; 165 if (way.isNew()) return this; 166 for (Node node: way.getNodes()) { 167 if (!node.isNew()) { 168 remember(node.getPrimitiveId()); 169 } 170 } 171 remember(way.getPrimitiveId()); 172 return this; 173 } 174 175 /** 176 * appends a {@link Relation} id to the list of ids which will be fetched from the server. 177 * 178 * @param relation the relation (ignored, if null) 179 * @return this 180 */ 181 protected MultiFetchServerObjectReader appendRelation(Relation relation) { 182 if (relation == null) return this; 183 if (relation.isNew()) return this; 184 remember(relation.getPrimitiveId()); 185 for (RelationMember member : relation.getMembers()) { 186 if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) { 187 // avoid infinite recursion in case of cyclic dependencies in relations 188 // 189 if (relations.contains(member.getMember().getId())) { 190 continue; 191 } 192 } 193 if (!member.getMember().isIncomplete()) { 194 append(member.getMember()); 195 } 196 } 197 return this; 198 } 199 200 /** 201 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server. 202 * @param primitive the primitive 203 * @return this 204 */ 205 public MultiFetchServerObjectReader append(OsmPrimitive primitive) { 206 if (primitive != null) { 207 switch (OsmPrimitiveType.from(primitive)) { 208 case NODE: return appendNode((Node)primitive); 209 case WAY: return appendWay((Way)primitive); 210 case RELATION: return appendRelation((Relation)primitive); 211 } 212 } 213 return this; 214 } 215 216 /** 217 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server. 218 * 219 * @param primitives the list of primitives (ignored, if null) 220 * @return this 221 * 222 * @see #append(OsmPrimitive) 223 */ 224 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) { 225 if (primitives == null) return this; 226 for (OsmPrimitive primitive : primitives) { 227 append(primitive); 228 } 229 return this; 230 } 231 232 /** 233 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and 234 * replies the subset. The extracted subset is removed from <code>ids</code>. 235 * 236 * @param ids a set of ids 237 * @return the subset of ids 238 */ 239 protected Set<Long> extractIdPackage(Set<Long> ids) { 240 HashSet<Long> pkg = new HashSet<Long>(); 241 if (ids.isEmpty()) 242 return pkg; 243 if (ids.size() > MAX_IDS_PER_REQUEST) { 244 Iterator<Long> it = ids.iterator(); 245 for (int i=0; i<MAX_IDS_PER_REQUEST; i++) { 246 pkg.add(it.next()); 247 } 248 ids.removeAll(pkg); 249 } else { 250 pkg.addAll(ids); 251 ids.clear(); 252 } 253 return pkg; 254 } 255 256 /** 257 * builds the Multi Get request string for a set of ids and a given 258 * {@link OsmPrimitiveType}. 259 * 260 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 261 * @param idPackage the package of ids 262 * @return the request string 263 */ 264 protected static String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) { 265 StringBuilder sb = new StringBuilder(); 266 sb.append(type.getAPIName()).append("s?") 267 .append(type.getAPIName()).append("s="); 268 269 Iterator<Long> it = idPackage.iterator(); 270 for (int i=0; i<idPackage.size(); i++) { 271 sb.append(it.next()); 272 if (i < idPackage.size()-1) { 273 sb.append(","); 274 } 275 } 276 return sb.toString(); 277 } 278 279 /** 280 * builds the Multi Get request string for a single id and a given 281 * {@link OsmPrimitiveType}. 282 * 283 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 284 * @param id the id 285 * @return the request string 286 */ 287 protected static String buildRequestString(OsmPrimitiveType type, long id) { 288 StringBuilder sb = new StringBuilder(); 289 sb.append(type.getAPIName()).append("s?") 290 .append(type.getAPIName()).append("s=") 291 .append(id); 292 return sb.toString(); 293 } 294 295 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { 296 for (Way w: from.getWays()) { 297 if (w.hasIncompleteNodes()) { 298 for (Node n: w.getNodes()) { 299 if (n.isIncomplete()) { 300 nodes.add(n.getId()); 301 } 302 } 303 } 304 } 305 } 306 307 /** 308 * merges the dataset <code>from</code> to {@link #outputDataSet}. 309 * 310 * @param from the other dataset 311 */ 312 protected void merge(DataSet from) { 313 final DataSetMerger visitor = new DataSetMerger(outputDataSet,from); 314 visitor.merge(); 315 } 316 317 /** 318 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server 319 * 320 * @param ids the set of ids 321 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 322 * @throws OsmTransferException if an error occurs while communicating with the API server 323 */ 324 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { 325 String msg = ""; 326 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 327 switch (type) { 328 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break; 329 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break; 330 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break; 331 } 332 progressMonitor.setTicksCount(ids.size()); 333 progressMonitor.setTicks(0); 334 // The complete set containg all primitives to fetch 335 Set<Long> toFetch = new HashSet<Long>(ids); 336 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each. 337 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. 338 int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS); 339 threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS); 340 Executor exec = Executors.newFixedThreadPool(threadsNumber); 341 CompletionService<FetchResult> ecs = new ExecutorCompletionService<FetchResult>(exec); 342 List<Future<FetchResult>> jobs = new ArrayList<Future<FetchResult>>(); 343 while (!toFetch.isEmpty()) { 344 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor))); 345 } 346 // Run the fetchers 347 for (int i = 0; i < jobs.size() && !isCanceled(); i++) { 348 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount()); 349 try { 350 FetchResult result = ecs.take().get(); 351 if (result.missingPrimitives != null) { 352 missingPrimitives.addAll(result.missingPrimitives); 353 } 354 if (result.dataSet != null && !isCanceled()) { 355 rememberNodesOfIncompleteWaysToLoad(result.dataSet); 356 merge(result.dataSet); 357 } 358 } catch (InterruptedException e) { 359 e.printStackTrace(); 360 } catch (ExecutionException e) { 361 e.printStackTrace(); 362 } 363 } 364 // Cancel requests if the user choosed to 365 if (isCanceled()) { 366 for (Future<FetchResult> job : jobs) { 367 job.cancel(true); 368 } 369 } 370 } 371 372 /** 373 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies 374 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too! 375 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies 376 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if 377 * visible==false). 378 * 379 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been 380 * found on the server (the server response code was 404) 381 * 382 * @return the parsed data 383 * @throws OsmTransferException if an error occurs while communicating with the API server 384 * @see #getMissingPrimitives() 385 * 386 */ 387 @Override 388 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 389 int n = nodes.size() + ways.size() + relations.size(); 390 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl())); 391 try { 392 missingPrimitives = new HashSet<PrimitiveId>(); 393 if (isCanceled()) return null; 394 fetchPrimitives(ways,OsmPrimitiveType.WAY, progressMonitor); 395 if (isCanceled()) return null; 396 fetchPrimitives(nodes,OsmPrimitiveType.NODE, progressMonitor); 397 if (isCanceled()) return null; 398 fetchPrimitives(relations,OsmPrimitiveType.RELATION, progressMonitor); 399 if (outputDataSet != null) { 400 outputDataSet.deleteInvisible(); 401 } 402 return outputDataSet; 403 } finally { 404 progressMonitor.finishTask(); 405 } 406 } 407 408 /** 409 * replies the set of ids of all primitives for which a fetch request to the 410 * server was submitted but which are not available from the server (the server 411 * replied a return code of 404) 412 * 413 * @return the set of ids of missing primitives 414 */ 415 public Set<PrimitiveId> getMissingPrimitives() { 416 return missingPrimitives; 417 } 418 419 /** 420 * The class holding the results given by {@link Fetcher}. 421 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded. 422 */ 423 protected static class FetchResult { 424 425 /** 426 * The resulting data set 427 */ 428 public final DataSet dataSet; 429 430 /** 431 * The collection of primitive ids that could not have been loaded 432 */ 433 public final Set<PrimitiveId> missingPrimitives; 434 435 /** 436 * Constructs a {@code FetchResult} 437 * @param dataSet The resulting data set 438 * @param missingPrimitives The collection of primitive ids that could not have been loaded 439 */ 440 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) { 441 this.dataSet = dataSet; 442 this.missingPrimitives = missingPrimitives; 443 } 444 } 445 446 /** 447 * The class that actually download data from OSM API. Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch). 448 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance. 449 * @see FetchResult 450 */ 451 protected static class Fetcher extends OsmServerReader implements Callable<FetchResult> { 452 453 private final Set<Long> pkg; 454 private final OsmPrimitiveType type; 455 private final ProgressMonitor progressMonitor; 456 457 /** 458 * Constructs a {@code Fetcher} 459 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 460 * @param idsPackage The set of primitives ids to fetch 461 * @param progressMonitor The progress monitor 462 */ 463 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) { 464 this.pkg = idsPackage; 465 this.type = type; 466 this.progressMonitor = progressMonitor; 467 } 468 469 @Override 470 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 471 // This method is implemented because of the OsmServerReader inheritance, but not used, as the main target of this class is the call() method. 472 return fetch(progressMonitor).dataSet; 473 } 474 475 @Override 476 public FetchResult call() throws Exception { 477 return fetch(progressMonitor); 478 } 479 480 /** 481 * fetches the requested primitives and updates the specified progress monitor. 482 * @param progressMonitor the progress monitor 483 * @return the {@link FetchResult} of this operation 484 * @throws OsmTransferException if an error occurs while communicating with the API server 485 */ 486 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { 487 try { 488 return multiGetIdPackage(type, pkg, progressMonitor); 489 } catch (OsmApiException e) { 490 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 491 System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object.")); 492 return singleGetIdPackage(type, pkg, progressMonitor); 493 } else { 494 throw e; 495 } 496 } 497 } 498 499 /** 500 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. 501 * The retrieved primitives are merged to {@link #outputDataSet}. 502 * 503 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 504 * @param pkg the package of ids 505 * @return the {@link FetchResult} of this operation 506 * @throws OsmTransferException if an error occurs while communicating with the API server 507 */ 508 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException { 509 String request = buildRequestString(type, pkg); 510 final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE); 511 if (in == null) return null; 512 progressMonitor.subTask(tr("Downloading OSM data...")); 513 try { 514 return new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); 515 } catch (Exception e) { 516 throw new OsmTransferException(e); 517 } 518 } 519 520 /** 521 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. 522 * The retrieved primitive is merged to {@link #outputDataSet}. 523 * 524 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 525 * @param id the id 526 * @return the {@link DataSet} resulting of this operation 527 * @throws OsmTransferException if an error occurs while communicating with the API server 528 */ 529 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { 530 String request = buildRequestString(type, id); 531 final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE); 532 if (in == null) return null; 533 progressMonitor.subTask(tr("Downloading OSM data...")); 534 try { 535 return OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); 536 } catch (Exception e) { 537 throw new OsmTransferException(e); 538 } 539 } 540 541 /** 542 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. 543 * The retrieved primitives are merged to {@link #outputDataSet}. 544 * 545 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). 546 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. 547 * Unfortunately, the server does not provide an error header or an error body for a 404 reply. 548 * 549 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 550 * @param pkg the set of ids 551 * @return the {@link FetchResult} of this operation 552 * @throws OsmTransferException if an error occurs while communicating with the API server 553 */ 554 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException { 555 FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 556 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 557 for (long id : pkg) { 558 try { 559 String msg = ""; 560 switch (type) { 561 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; 562 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; 563 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; 564 } 565 progressMonitor.setCustomText(msg); 566 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); 567 } catch (OsmApiException e) { 568 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 569 System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); 570 result.missingPrimitives.add(new SimplePrimitiveId(id, type)); 571 } else { 572 throw e; 573 } 574 } 575 } 576 return result; 577 } 578 } 579 }