001 // License: GPL. See LICENSE file for details. 002 package org.openstreetmap.josm.io; 003 004 import static org.openstreetmap.josm.tools.I18n.tr; 005 006 import java.io.InputStream; 007 import java.io.InputStreamReader; 008 import java.text.MessageFormat; 009 import java.util.ArrayList; 010 import java.util.Collection; 011 import java.util.regex.Matcher; 012 import java.util.regex.Pattern; 013 014 import javax.xml.stream.Location; 015 import javax.xml.stream.XMLInputFactory; 016 import javax.xml.stream.XMLStreamConstants; 017 import javax.xml.stream.XMLStreamException; 018 import javax.xml.stream.XMLStreamReader; 019 020 import org.openstreetmap.josm.data.Bounds; 021 import org.openstreetmap.josm.data.coor.LatLon; 022 import org.openstreetmap.josm.data.osm.Changeset; 023 import org.openstreetmap.josm.data.osm.DataSet; 024 import org.openstreetmap.josm.data.osm.DataSource; 025 import org.openstreetmap.josm.data.osm.Node; 026 import org.openstreetmap.josm.data.osm.NodeData; 027 import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 028 import org.openstreetmap.josm.data.osm.PrimitiveData; 029 import org.openstreetmap.josm.data.osm.Relation; 030 import org.openstreetmap.josm.data.osm.RelationData; 031 import org.openstreetmap.josm.data.osm.RelationMemberData; 032 import org.openstreetmap.josm.data.osm.Tagged; 033 import org.openstreetmap.josm.data.osm.User; 034 import org.openstreetmap.josm.data.osm.Way; 035 import org.openstreetmap.josm.data.osm.WayData; 036 import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 037 import org.openstreetmap.josm.gui.progress.ProgressMonitor; 038 import org.openstreetmap.josm.tools.CheckParameterUtil; 039 import org.openstreetmap.josm.tools.DateUtils; 040 041 /** 042 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it. 043 * 044 * For each xml element, there is a dedicated method. 045 * The XMLStreamReader cursor points to the start of the element, when the method is 046 * entered, and it must point to the end of the same element, when it is exited. 047 */ 048 public class OsmReader extends AbstractReader { 049 050 protected XMLStreamReader parser; 051 052 /** Used by plugins to register themselves as data postprocessors. */ 053 public static ArrayList<OsmServerReadPostprocessor> postprocessors; 054 055 /** register a new postprocessor */ 056 public static void registerPostprocessor(OsmServerReadPostprocessor pp) { 057 if (postprocessors == null) { 058 postprocessors = new ArrayList<OsmServerReadPostprocessor>(); 059 } 060 postprocessors.add(pp); 061 } 062 063 /** deregister a postprocessor previously registered with registerPostprocessor */ 064 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { 065 if (postprocessors != null) { 066 postprocessors.remove(pp); 067 } 068 } 069 070 /** 071 * constructor (for private and subclasses use only) 072 * 073 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor) 074 */ 075 protected OsmReader() { 076 } 077 078 protected void setParser(XMLStreamReader parser) { 079 this.parser = parser; 080 } 081 082 protected void throwException(String msg) throws XMLStreamException { 083 throw new OsmParsingException(msg, parser.getLocation()); 084 } 085 086 protected void parse() throws XMLStreamException { 087 int event = parser.getEventType(); 088 while (true) { 089 if (event == XMLStreamConstants.START_ELEMENT) { 090 parseRoot(); 091 } else if (event == XMLStreamConstants.END_ELEMENT) 092 return; 093 if (parser.hasNext()) { 094 event = parser.next(); 095 } else { 096 break; 097 } 098 } 099 parser.close(); 100 } 101 102 protected void parseRoot() throws XMLStreamException { 103 if (parser.getLocalName().equals("osm")) { 104 parseOsm(); 105 } else { 106 parseUnknown(); 107 } 108 } 109 110 private void parseOsm() throws XMLStreamException { 111 String v = parser.getAttributeValue(null, "version"); 112 if (v == null) { 113 throwException(tr("Missing mandatory attribute ''{0}''.", "version")); 114 } 115 if (!(v.equals("0.5") || v.equals("0.6"))) { 116 throwException(tr("Unsupported version: {0}", v)); 117 } 118 ds.setVersion(v); 119 String upload = parser.getAttributeValue(null, "upload"); 120 if (upload != null) { 121 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload)); 122 } 123 String generator = parser.getAttributeValue(null, "generator"); 124 Long uploadChangesetId = null; 125 if (parser.getAttributeValue(null, "upload-changeset") != null) { 126 uploadChangesetId = getLong("upload-changeset"); 127 } 128 while (true) { 129 int event = parser.next(); 130 if (event == XMLStreamConstants.START_ELEMENT) { 131 if (parser.getLocalName().equals("bounds")) { 132 parseBounds(generator); 133 } else if (parser.getLocalName().equals("node")) { 134 parseNode(); 135 } else if (parser.getLocalName().equals("way")) { 136 parseWay(); 137 } else if (parser.getLocalName().equals("relation")) { 138 parseRelation(); 139 } else if (parser.getLocalName().equals("changeset")) { 140 parseChangeset(uploadChangesetId); 141 } else { 142 parseUnknown(); 143 } 144 } else if (event == XMLStreamConstants.END_ELEMENT) 145 return; 146 } 147 } 148 149 private void parseBounds(String generator) throws XMLStreamException { 150 String minlon = parser.getAttributeValue(null, "minlon"); 151 String minlat = parser.getAttributeValue(null, "minlat"); 152 String maxlon = parser.getAttributeValue(null, "maxlon"); 153 String maxlat = parser.getAttributeValue(null, "maxlat"); 154 String origin = parser.getAttributeValue(null, "origin"); 155 if (minlon != null && maxlon != null && minlat != null && maxlat != null) { 156 if (origin == null) { 157 origin = generator; 158 } 159 Bounds bounds = new Bounds( 160 Double.parseDouble(minlat), Double.parseDouble(minlon), 161 Double.parseDouble(maxlat), Double.parseDouble(maxlon)); 162 if (bounds.isOutOfTheWorld()) { 163 Bounds copy = new Bounds(bounds); 164 bounds.normalize(); 165 System.out.println("Bbox " + copy + " is out of the world, normalized to " + bounds); 166 } 167 DataSource src = new DataSource(bounds, origin); 168 ds.dataSources.add(src); 169 } else { 170 throwException(tr( 171 "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.", 172 minlon, minlat, maxlon, maxlat, origin 173 )); 174 } 175 jumpToEnd(); 176 } 177 178 protected Node parseNode() throws XMLStreamException { 179 NodeData nd = new NodeData(); 180 String lat = parser.getAttributeValue(null, "lat"); 181 String lon = parser.getAttributeValue(null, "lon"); 182 if (lat != null && lon != null) { 183 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon))); 184 } 185 readCommon(nd); 186 Node n = new Node(nd.getId(), nd.getVersion()); 187 n.setVisible(nd.isVisible()); 188 n.load(nd); 189 externalIdMap.put(nd.getPrimitiveId(), n); 190 while (true) { 191 int event = parser.next(); 192 if (event == XMLStreamConstants.START_ELEMENT) { 193 if (parser.getLocalName().equals("tag")) { 194 parseTag(n); 195 } else { 196 parseUnknown(); 197 } 198 } else if (event == XMLStreamConstants.END_ELEMENT) 199 return n; 200 } 201 } 202 203 protected Way parseWay() throws XMLStreamException { 204 WayData wd = new WayData(); 205 readCommon(wd); 206 Way w = new Way(wd.getId(), wd.getVersion()); 207 w.setVisible(wd.isVisible()); 208 w.load(wd); 209 externalIdMap.put(wd.getPrimitiveId(), w); 210 211 Collection<Long> nodeIds = new ArrayList<Long>(); 212 while (true) { 213 int event = parser.next(); 214 if (event == XMLStreamConstants.START_ELEMENT) { 215 if (parser.getLocalName().equals("nd")) { 216 nodeIds.add(parseWayNode(w)); 217 } else if (parser.getLocalName().equals("tag")) { 218 parseTag(w); 219 } else { 220 parseUnknown(); 221 } 222 } else if (event == XMLStreamConstants.END_ELEMENT) { 223 break; 224 } 225 } 226 if (w.isDeleted() && nodeIds.size() > 0) { 227 System.out.println(tr("Deleted way {0} contains nodes", w.getUniqueId())); 228 nodeIds = new ArrayList<Long>(); 229 } 230 ways.put(wd.getUniqueId(), nodeIds); 231 return w; 232 } 233 234 private long parseWayNode(Way w) throws XMLStreamException { 235 if (parser.getAttributeValue(null, "ref") == null) { 236 throwException( 237 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId()) 238 ); 239 } 240 long id = getLong("ref"); 241 if (id == 0) { 242 throwException( 243 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id) 244 ); 245 } 246 jumpToEnd(); 247 return id; 248 } 249 250 protected Relation parseRelation() throws XMLStreamException { 251 RelationData rd = new RelationData(); 252 readCommon(rd); 253 Relation r = new Relation(rd.getId(), rd.getVersion()); 254 r.setVisible(rd.isVisible()); 255 r.load(rd); 256 externalIdMap.put(rd.getPrimitiveId(), r); 257 258 Collection<RelationMemberData> members = new ArrayList<RelationMemberData>(); 259 while (true) { 260 int event = parser.next(); 261 if (event == XMLStreamConstants.START_ELEMENT) { 262 if (parser.getLocalName().equals("member")) { 263 members.add(parseRelationMember(r)); 264 } else if (parser.getLocalName().equals("tag")) { 265 parseTag(r); 266 } else { 267 parseUnknown(); 268 } 269 } else if (event == XMLStreamConstants.END_ELEMENT) { 270 break; 271 } 272 } 273 if (r.isDeleted() && members.size() > 0) { 274 System.out.println(tr("Deleted relation {0} contains members", r.getUniqueId())); 275 members = new ArrayList<RelationMemberData>(); 276 } 277 relations.put(rd.getUniqueId(), members); 278 return r; 279 } 280 281 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { 282 String role = null; 283 OsmPrimitiveType type = null; 284 long id = 0; 285 String value = parser.getAttributeValue(null, "ref"); 286 if (value == null) { 287 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId())); 288 } 289 try { 290 id = Long.parseLong(value); 291 } catch(NumberFormatException e) { 292 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value)); 293 } 294 value = parser.getAttributeValue(null, "type"); 295 if (value == null) { 296 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId()))); 297 } 298 try { 299 type = OsmPrimitiveType.fromApiTypeName(value); 300 } catch(IllegalArgumentException e) { 301 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value)); 302 } 303 value = parser.getAttributeValue(null, "role"); 304 role = value; 305 306 if (id == 0) { 307 throwException(tr("Incomplete <member> specification with ref=0")); 308 } 309 jumpToEnd(); 310 return new RelationMemberData(role, type, id); 311 } 312 313 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 314 long id = getLong("id"); 315 316 if (id == uploadChangesetId) { 317 uploadChangeset = new Changeset((int) getLong("id")); 318 while (true) { 319 int event = parser.next(); 320 if (event == XMLStreamConstants.START_ELEMENT) { 321 if (parser.getLocalName().equals("tag")) { 322 parseTag(uploadChangeset); 323 } else { 324 parseUnknown(); 325 } 326 } else if (event == XMLStreamConstants.END_ELEMENT) 327 return; 328 } 329 } else { 330 jumpToEnd(false); 331 } 332 } 333 334 private void parseTag(Tagged t) throws XMLStreamException { 335 String key = parser.getAttributeValue(null, "k"); 336 String value = parser.getAttributeValue(null, "v"); 337 if (key == null || value == null) { 338 throwException(tr("Missing key or value attribute in tag.")); 339 } 340 t.put(key.intern(), value.intern()); 341 jumpToEnd(); 342 } 343 344 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 345 if (printWarning) { 346 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName())); 347 } 348 while (true) { 349 int event = parser.next(); 350 if (event == XMLStreamConstants.START_ELEMENT) { 351 parseUnknown(false); /* no more warning for inner elements */ 352 } else if (event == XMLStreamConstants.END_ELEMENT) 353 return; 354 } 355 } 356 357 protected void parseUnknown() throws XMLStreamException { 358 parseUnknown(true); 359 } 360 361 /** 362 * When cursor is at the start of an element, moves it to the end tag of that element. 363 * Nested content is skipped. 364 * 365 * This is basically the same code as parseUnknown(), except for the warnings, which 366 * are displayed for inner elements and not at top level. 367 */ 368 private void jumpToEnd(boolean printWarning) throws XMLStreamException { 369 while (true) { 370 int event = parser.next(); 371 if (event == XMLStreamConstants.START_ELEMENT) { 372 parseUnknown(printWarning); 373 } else if (event == XMLStreamConstants.END_ELEMENT) 374 return; 375 } 376 } 377 378 private void jumpToEnd() throws XMLStreamException { 379 jumpToEnd(true); 380 } 381 382 private User createUser(String uid, String name) throws XMLStreamException { 383 if (uid == null) { 384 if (name == null) 385 return null; 386 return User.createLocalUser(name); 387 } 388 try { 389 long id = Long.parseLong(uid); 390 return User.createOsmUser(id, name); 391 } catch(NumberFormatException e) { 392 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid)); 393 } 394 return null; 395 } 396 397 /** 398 * Read out the common attributes and put them into current OsmPrimitive. 399 */ 400 private void readCommon(PrimitiveData current) throws XMLStreamException { 401 current.setId(getLong("id")); 402 if (current.getUniqueId() == 0) { 403 throwException(tr("Illegal object with ID=0.")); 404 } 405 406 String time = parser.getAttributeValue(null, "timestamp"); 407 if (time != null && time.length() != 0) { 408 current.setTimestamp(DateUtils.fromString(time)); 409 } 410 411 // user attribute added in 0.4 API 412 String user = parser.getAttributeValue(null, "user"); 413 // uid attribute added in 0.6 API 414 String uid = parser.getAttributeValue(null, "uid"); 415 current.setUser(createUser(uid, user)); 416 417 // visible attribute added in 0.4 API 418 String visible = parser.getAttributeValue(null, "visible"); 419 if (visible != null) { 420 current.setVisible(Boolean.parseBoolean(visible)); 421 } 422 423 String versionString = parser.getAttributeValue(null, "version"); 424 int version = 0; 425 if (versionString != null) { 426 try { 427 version = Integer.parseInt(versionString); 428 } catch(NumberFormatException e) { 429 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString)); 430 } 431 if (ds.getVersion().equals("0.6")){ 432 if (version <= 0 && current.getUniqueId() > 0) { 433 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString)); 434 } else if (version < 0 && current.getUniqueId() <= 0) { 435 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6")); 436 version = 0; 437 } 438 } else if (ds.getVersion().equals("0.5")) { 439 if (version <= 0 && current.getUniqueId() > 0) { 440 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5")); 441 version = 1; 442 } else if (version < 0 && current.getUniqueId() <= 0) { 443 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5")); 444 version = 0; 445 } 446 } else { 447 // should not happen. API version has been checked before 448 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); 449 } 450 } else { 451 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 452 // 453 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) { 454 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); 455 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) { 456 // default version in 0.5 files for existing primitives 457 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5")); 458 version= 1; 459 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) { 460 // default version in 0.5 files for new primitives, no warning necessary. This is 461 // (was) legal in API 0.5 462 version= 0; 463 } 464 } 465 current.setVersion(version); 466 467 String action = parser.getAttributeValue(null, "action"); 468 if (action == null) { 469 // do nothing 470 } else if (action.equals("delete")) { 471 current.setDeleted(true); 472 current.setModified(current.isVisible()); 473 } else if (action.equals("modify")) { 474 current.setModified(true); 475 } 476 477 String v = parser.getAttributeValue(null, "changeset"); 478 if (v == null) { 479 current.setChangesetId(0); 480 } else { 481 try { 482 current.setChangesetId(Integer.parseInt(v)); 483 } catch(NumberFormatException e) { 484 if (current.getUniqueId() <= 0) { 485 // for a new primitive we just log a warning 486 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId())); 487 current.setChangesetId(0); 488 } else { 489 // for an existing primitive this is a problem 490 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 491 } 492 } 493 if (current.getChangesetId() <=0) { 494 if (current.getUniqueId() <= 0) { 495 // for a new primitive we just log a warning 496 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId())); 497 current.setChangesetId(0); 498 } else { 499 // for an existing primitive this is a problem 500 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 501 } 502 } 503 } 504 } 505 506 private long getLong(String name) throws XMLStreamException { 507 String value = parser.getAttributeValue(null, name); 508 if (value == null) { 509 throwException(tr("Missing required attribute ''{0}''.",name)); 510 } 511 try { 512 return Long.parseLong(value); 513 } catch(NumberFormatException e) { 514 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value)); 515 } 516 return 0; // should not happen 517 } 518 519 private static class OsmParsingException extends XMLStreamException { 520 public OsmParsingException() { 521 super(); 522 } 523 524 public OsmParsingException(String msg) { 525 super(msg); 526 } 527 528 public OsmParsingException(String msg, Location location) { 529 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */ 530 this.location = location; 531 } 532 533 public OsmParsingException(String msg, Location location, Throwable th) { 534 super(msg, th); 535 this.location = location; 536 } 537 538 public OsmParsingException(String msg, Throwable th) { 539 super(msg, th); 540 } 541 542 public OsmParsingException(Throwable th) { 543 super(th); 544 } 545 546 @Override 547 public String getMessage() { 548 String msg = super.getMessage(); 549 if (msg == null) { 550 msg = getClass().getName(); 551 } 552 if (getLocation() == null) 553 return msg; 554 msg = msg + " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber()); 555 return msg; 556 } 557 } 558 559 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 560 if (progressMonitor == null) { 561 progressMonitor = NullProgressMonitor.INSTANCE; 562 } 563 CheckParameterUtil.ensureParameterNotNull(source, "source"); 564 try { 565 progressMonitor.beginTask(tr("Prepare OSM data...", 2)); 566 progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); 567 568 InputStreamReader ir = UTFInputStreamReader.create(source, "UTF-8"); 569 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir); 570 setParser(parser); 571 parse(); 572 progressMonitor.worked(1); 573 574 progressMonitor.indeterminateSubTask(tr("Preparing data set...")); 575 prepareDataSet(); 576 progressMonitor.worked(1); 577 578 // iterate over registered postprocessors and give them each a chance 579 // to modify the dataset we have just loaded. 580 if (postprocessors != null) { 581 for (OsmServerReadPostprocessor pp : postprocessors) { 582 pp.postprocessDataSet(getDataSet(), progressMonitor); 583 } 584 } 585 return getDataSet(); 586 } catch(IllegalDataException e) { 587 throw e; 588 } catch(OsmParsingException e) { 589 throw new IllegalDataException(e.getMessage(), e); 590 } catch(XMLStreamException e) { 591 String msg = e.getMessage(); 592 Pattern p = Pattern.compile("Message: (.+)"); 593 Matcher m = p.matcher(msg); 594 if (m.find()) { 595 msg = m.group(1); 596 } 597 if (e.getLocation() != null) 598 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 599 else 600 throw new IllegalDataException(msg, e); 601 } catch(Exception e) { 602 throw new IllegalDataException(e); 603 } finally { 604 progressMonitor.finishTask(); 605 } 606 } 607 608 /** 609 * Parse the given input source and return the dataset. 610 * 611 * @param source the source input stream. Must not be null. 612 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 613 * 614 * @return the dataset with the parsed data 615 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source 616 * @throws IllegalArgumentException thrown if source is null 617 */ 618 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 619 return new OsmReader().doParseDataSet(source, progressMonitor); 620 } 621 }