001    // License: GPL. See LICENSE file for details.
002    package org.openstreetmap.josm.io;
003    
004    import static org.openstreetmap.josm.tools.I18n.tr;
005    
006    import java.io.InputStream;
007    import java.io.InputStreamReader;
008    import java.text.MessageFormat;
009    import java.util.ArrayList;
010    import java.util.Collection;
011    import java.util.regex.Matcher;
012    import java.util.regex.Pattern;
013    
014    import javax.xml.stream.Location;
015    import javax.xml.stream.XMLInputFactory;
016    import javax.xml.stream.XMLStreamConstants;
017    import javax.xml.stream.XMLStreamException;
018    import javax.xml.stream.XMLStreamReader;
019    
020    import org.openstreetmap.josm.data.Bounds;
021    import org.openstreetmap.josm.data.coor.LatLon;
022    import org.openstreetmap.josm.data.osm.Changeset;
023    import org.openstreetmap.josm.data.osm.DataSet;
024    import org.openstreetmap.josm.data.osm.DataSource;
025    import org.openstreetmap.josm.data.osm.Node;
026    import org.openstreetmap.josm.data.osm.NodeData;
027    import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
028    import org.openstreetmap.josm.data.osm.PrimitiveData;
029    import org.openstreetmap.josm.data.osm.Relation;
030    import org.openstreetmap.josm.data.osm.RelationData;
031    import org.openstreetmap.josm.data.osm.RelationMemberData;
032    import org.openstreetmap.josm.data.osm.Tagged;
033    import org.openstreetmap.josm.data.osm.User;
034    import org.openstreetmap.josm.data.osm.Way;
035    import org.openstreetmap.josm.data.osm.WayData;
036    import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
037    import org.openstreetmap.josm.gui.progress.ProgressMonitor;
038    import org.openstreetmap.josm.tools.CheckParameterUtil;
039    import org.openstreetmap.josm.tools.DateUtils;
040    
041    /**
042     * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
043     *
044     * For each xml element, there is a dedicated method.
045     * The XMLStreamReader cursor points to the start of the element, when the method is
046     * entered, and it must point to the end of the same element, when it is exited.
047     */
048    public class OsmReader extends AbstractReader {
049    
050        protected XMLStreamReader parser;
051    
052        /** Used by plugins to register themselves as data postprocessors. */
053        public static ArrayList<OsmServerReadPostprocessor> postprocessors;
054    
055        /** register a new postprocessor */
056        public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
057            if (postprocessors == null) {
058                postprocessors = new ArrayList<OsmServerReadPostprocessor>();
059            }
060            postprocessors.add(pp);
061        }
062    
063        /** deregister a postprocessor previously registered with registerPostprocessor */
064        public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
065            if (postprocessors != null) {
066                postprocessors.remove(pp);
067            }
068        }
069    
070        /**
071         * constructor (for private and subclasses use only)
072         *
073         * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
074         */
075        protected OsmReader() {
076        }
077    
078        protected void setParser(XMLStreamReader parser) {
079            this.parser = parser;
080        }
081    
082        protected void throwException(String msg) throws XMLStreamException {
083            throw new OsmParsingException(msg, parser.getLocation());
084        }
085    
086        protected void parse() throws XMLStreamException {
087            int event = parser.getEventType();
088            while (true) {
089                if (event == XMLStreamConstants.START_ELEMENT) {
090                    parseRoot();
091                } else if (event == XMLStreamConstants.END_ELEMENT)
092                    return;
093                if (parser.hasNext()) {
094                    event = parser.next();
095                } else {
096                    break;
097                }
098            }
099            parser.close();
100        }
101    
102        protected void parseRoot() throws XMLStreamException {
103            if (parser.getLocalName().equals("osm")) {
104                parseOsm();
105            } else {
106                parseUnknown();
107            }
108        }
109    
110        private void parseOsm() throws XMLStreamException {
111            String v = parser.getAttributeValue(null, "version");
112            if (v == null) {
113                throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
114            }
115            if (!(v.equals("0.5") || v.equals("0.6"))) {
116                throwException(tr("Unsupported version: {0}", v));
117            }
118            ds.setVersion(v);
119            String upload = parser.getAttributeValue(null, "upload");
120            if (upload != null) {
121                ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
122            }
123            String generator = parser.getAttributeValue(null, "generator");
124            Long uploadChangesetId = null;
125            if (parser.getAttributeValue(null, "upload-changeset") != null) {
126                uploadChangesetId = getLong("upload-changeset");
127            }
128            while (true) {
129                int event = parser.next();
130                if (event == XMLStreamConstants.START_ELEMENT) {
131                    if (parser.getLocalName().equals("bounds")) {
132                        parseBounds(generator);
133                    } else if (parser.getLocalName().equals("node")) {
134                        parseNode();
135                    } else if (parser.getLocalName().equals("way")) {
136                        parseWay();
137                    } else if (parser.getLocalName().equals("relation")) {
138                        parseRelation();
139                    } else if (parser.getLocalName().equals("changeset")) {
140                        parseChangeset(uploadChangesetId);
141                    } else {
142                        parseUnknown();
143                    }
144                } else if (event == XMLStreamConstants.END_ELEMENT)
145                    return;
146            }
147        }
148    
149        private void parseBounds(String generator) throws XMLStreamException {
150            String minlon = parser.getAttributeValue(null, "minlon");
151            String minlat = parser.getAttributeValue(null, "minlat");
152            String maxlon = parser.getAttributeValue(null, "maxlon");
153            String maxlat = parser.getAttributeValue(null, "maxlat");
154            String origin = parser.getAttributeValue(null, "origin");
155            if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
156                if (origin == null) {
157                    origin = generator;
158                }
159                Bounds bounds = new Bounds(
160                        Double.parseDouble(minlat), Double.parseDouble(minlon),
161                        Double.parseDouble(maxlat), Double.parseDouble(maxlon));
162                if (bounds.isOutOfTheWorld()) {
163                    Bounds copy = new Bounds(bounds);
164                    bounds.normalize();
165                    System.out.println("Bbox " + copy + " is out of the world, normalized to " + bounds);
166                }
167                DataSource src = new DataSource(bounds, origin);
168                ds.dataSources.add(src);
169            } else {
170                throwException(tr(
171                        "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
172                        minlon, minlat, maxlon, maxlat, origin
173                ));
174            }
175            jumpToEnd();
176        }
177    
178        protected Node parseNode() throws XMLStreamException {
179            NodeData nd = new NodeData();
180            String lat = parser.getAttributeValue(null, "lat");
181            String lon = parser.getAttributeValue(null, "lon");
182            if (lat != null && lon != null) {
183                nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
184            }
185            readCommon(nd);
186            Node n = new Node(nd.getId(), nd.getVersion());
187            n.setVisible(nd.isVisible());
188            n.load(nd);
189            externalIdMap.put(nd.getPrimitiveId(), n);
190            while (true) {
191                int event = parser.next();
192                if (event == XMLStreamConstants.START_ELEMENT) {
193                    if (parser.getLocalName().equals("tag")) {
194                        parseTag(n);
195                    } else {
196                        parseUnknown();
197                    }
198                } else if (event == XMLStreamConstants.END_ELEMENT)
199                    return n;
200            }
201        }
202    
203        protected Way parseWay() throws XMLStreamException {
204            WayData wd = new WayData();
205            readCommon(wd);
206            Way w = new Way(wd.getId(), wd.getVersion());
207            w.setVisible(wd.isVisible());
208            w.load(wd);
209            externalIdMap.put(wd.getPrimitiveId(), w);
210    
211            Collection<Long> nodeIds = new ArrayList<Long>();
212            while (true) {
213                int event = parser.next();
214                if (event == XMLStreamConstants.START_ELEMENT) {
215                    if (parser.getLocalName().equals("nd")) {
216                        nodeIds.add(parseWayNode(w));
217                    } else if (parser.getLocalName().equals("tag")) {
218                        parseTag(w);
219                    } else {
220                        parseUnknown();
221                    }
222                } else if (event == XMLStreamConstants.END_ELEMENT) {
223                    break;
224                }
225            }
226            if (w.isDeleted() && nodeIds.size() > 0) {
227                System.out.println(tr("Deleted way {0} contains nodes", w.getUniqueId()));
228                nodeIds = new ArrayList<Long>();
229            }
230            ways.put(wd.getUniqueId(), nodeIds);
231            return w;
232        }
233    
234        private long parseWayNode(Way w) throws XMLStreamException {
235            if (parser.getAttributeValue(null, "ref") == null) {
236                throwException(
237                        tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
238                );
239            }
240            long id = getLong("ref");
241            if (id == 0) {
242                throwException(
243                        tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
244                );
245            }
246            jumpToEnd();
247            return id;
248        }
249    
250        protected Relation parseRelation() throws XMLStreamException {
251            RelationData rd = new RelationData();
252            readCommon(rd);
253            Relation r = new Relation(rd.getId(), rd.getVersion());
254            r.setVisible(rd.isVisible());
255            r.load(rd);
256            externalIdMap.put(rd.getPrimitiveId(), r);
257    
258            Collection<RelationMemberData> members = new ArrayList<RelationMemberData>();
259            while (true) {
260                int event = parser.next();
261                if (event == XMLStreamConstants.START_ELEMENT) {
262                    if (parser.getLocalName().equals("member")) {
263                        members.add(parseRelationMember(r));
264                    } else if (parser.getLocalName().equals("tag")) {
265                        parseTag(r);
266                    } else {
267                        parseUnknown();
268                    }
269                } else if (event == XMLStreamConstants.END_ELEMENT) {
270                    break;
271                }
272            }
273            if (r.isDeleted() && members.size() > 0) {
274                System.out.println(tr("Deleted relation {0} contains members", r.getUniqueId()));
275                members = new ArrayList<RelationMemberData>();
276            }
277            relations.put(rd.getUniqueId(), members);
278            return r;
279        }
280    
281        private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
282            String role = null;
283            OsmPrimitiveType type = null;
284            long id = 0;
285            String value = parser.getAttributeValue(null, "ref");
286            if (value == null) {
287                throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId()));
288            }
289            try {
290                id = Long.parseLong(value);
291            } catch(NumberFormatException e) {
292                throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value));
293            }
294            value = parser.getAttributeValue(null, "type");
295            if (value == null) {
296                throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
297            }
298            try {
299                type = OsmPrimitiveType.fromApiTypeName(value);
300            } catch(IllegalArgumentException e) {
301                throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value));
302            }
303            value = parser.getAttributeValue(null, "role");
304            role = value;
305    
306            if (id == 0) {
307                throwException(tr("Incomplete <member> specification with ref=0"));
308            }
309            jumpToEnd();
310            return new RelationMemberData(role, type, id);
311        }
312    
313        private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
314            long id = getLong("id");
315    
316            if (id == uploadChangesetId) {
317                uploadChangeset = new Changeset((int) getLong("id"));
318                while (true) {
319                    int event = parser.next();
320                    if (event == XMLStreamConstants.START_ELEMENT) {
321                        if (parser.getLocalName().equals("tag")) {
322                            parseTag(uploadChangeset);
323                        } else {
324                            parseUnknown();
325                        }
326                    } else if (event == XMLStreamConstants.END_ELEMENT)
327                        return;
328                }
329            } else {
330                jumpToEnd(false);
331            }
332        }
333    
334        private void parseTag(Tagged t) throws XMLStreamException {
335            String key = parser.getAttributeValue(null, "k");
336            String value = parser.getAttributeValue(null, "v");
337            if (key == null || value == null) {
338                throwException(tr("Missing key or value attribute in tag."));
339            }
340            t.put(key.intern(), value.intern());
341            jumpToEnd();
342        }
343    
344        protected void parseUnknown(boolean printWarning) throws XMLStreamException {
345            if (printWarning) {
346                System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName()));
347            }
348            while (true) {
349                int event = parser.next();
350                if (event == XMLStreamConstants.START_ELEMENT) {
351                    parseUnknown(false); /* no more warning for inner elements */
352                } else if (event == XMLStreamConstants.END_ELEMENT)
353                    return;
354            }
355        }
356    
357        protected void parseUnknown() throws XMLStreamException {
358            parseUnknown(true);
359        }
360    
361        /**
362         * When cursor is at the start of an element, moves it to the end tag of that element.
363         * Nested content is skipped.
364         *
365         * This is basically the same code as parseUnknown(), except for the warnings, which
366         * are displayed for inner elements and not at top level.
367         */
368        private void jumpToEnd(boolean printWarning) throws XMLStreamException {
369            while (true) {
370                int event = parser.next();
371                if (event == XMLStreamConstants.START_ELEMENT) {
372                    parseUnknown(printWarning);
373                } else if (event == XMLStreamConstants.END_ELEMENT)
374                    return;
375            }
376        }
377    
378        private void jumpToEnd() throws XMLStreamException {
379            jumpToEnd(true);
380        }
381    
382        private User createUser(String uid, String name) throws XMLStreamException {
383            if (uid == null) {
384                if (name == null)
385                    return null;
386                return User.createLocalUser(name);
387            }
388            try {
389                long id = Long.parseLong(uid);
390                return User.createOsmUser(id, name);
391            } catch(NumberFormatException e) {
392                throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
393            }
394            return null;
395        }
396    
397        /**
398         * Read out the common attributes and put them into current OsmPrimitive.
399         */
400        private void readCommon(PrimitiveData current) throws XMLStreamException {
401            current.setId(getLong("id"));
402            if (current.getUniqueId() == 0) {
403                throwException(tr("Illegal object with ID=0."));
404            }
405    
406            String time = parser.getAttributeValue(null, "timestamp");
407            if (time != null && time.length() != 0) {
408                current.setTimestamp(DateUtils.fromString(time));
409            }
410    
411            // user attribute added in 0.4 API
412            String user = parser.getAttributeValue(null, "user");
413            // uid attribute added in 0.6 API
414            String uid = parser.getAttributeValue(null, "uid");
415            current.setUser(createUser(uid, user));
416    
417            // visible attribute added in 0.4 API
418            String visible = parser.getAttributeValue(null, "visible");
419            if (visible != null) {
420                current.setVisible(Boolean.parseBoolean(visible));
421            }
422    
423            String versionString = parser.getAttributeValue(null, "version");
424            int version = 0;
425            if (versionString != null) {
426                try {
427                    version = Integer.parseInt(versionString);
428                } catch(NumberFormatException e) {
429                    throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
430                }
431                if (ds.getVersion().equals("0.6")){
432                    if (version <= 0 && current.getUniqueId() > 0) {
433                        throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
434                    } else if (version < 0 && current.getUniqueId() <= 0) {
435                        System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
436                        version = 0;
437                    }
438                } else if (ds.getVersion().equals("0.5")) {
439                    if (version <= 0 && current.getUniqueId() > 0) {
440                        System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
441                        version = 1;
442                    } else if (version < 0 && current.getUniqueId() <= 0) {
443                        System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
444                        version = 0;
445                    }
446                } else {
447                    // should not happen. API version has been checked before
448                    throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
449                }
450            } else {
451                // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
452                //
453                if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
454                    throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
455                } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
456                    // default version in 0.5 files for existing primitives
457                    System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
458                    version= 1;
459                } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
460                    // default version in 0.5 files for new primitives, no warning necessary. This is
461                    // (was) legal in API 0.5
462                    version= 0;
463                }
464            }
465            current.setVersion(version);
466    
467            String action = parser.getAttributeValue(null, "action");
468            if (action == null) {
469                // do nothing
470            } else if (action.equals("delete")) {
471                current.setDeleted(true);
472                current.setModified(current.isVisible());
473            } else if (action.equals("modify")) {
474                current.setModified(true);
475            }
476    
477            String v = parser.getAttributeValue(null, "changeset");
478            if (v == null) {
479                current.setChangesetId(0);
480            } else {
481                try {
482                    current.setChangesetId(Integer.parseInt(v));
483                } catch(NumberFormatException e) {
484                    if (current.getUniqueId() <= 0) {
485                        // for a new primitive we just log a warning
486                        System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
487                        current.setChangesetId(0);
488                    } else {
489                        // for an existing primitive this is a problem
490                        throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
491                    }
492                }
493                if (current.getChangesetId() <=0) {
494                    if (current.getUniqueId() <= 0) {
495                        // for a new primitive we just log a warning
496                        System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
497                        current.setChangesetId(0);
498                    } else {
499                        // for an existing primitive this is a problem
500                        throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
501                    }
502                }
503            }
504        }
505    
506        private long getLong(String name) throws XMLStreamException {
507            String value = parser.getAttributeValue(null, name);
508            if (value == null) {
509                throwException(tr("Missing required attribute ''{0}''.",name));
510            }
511            try {
512                return Long.parseLong(value);
513            } catch(NumberFormatException e) {
514                throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
515            }
516            return 0; // should not happen
517        }
518    
519        private static class OsmParsingException extends XMLStreamException {
520            public OsmParsingException() {
521                super();
522            }
523    
524            public OsmParsingException(String msg) {
525                super(msg);
526            }
527    
528            public OsmParsingException(String msg, Location location) {
529                super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
530                this.location = location;
531            }
532    
533            public OsmParsingException(String msg, Location location, Throwable th) {
534                super(msg, th);
535                this.location = location;
536            }
537    
538            public OsmParsingException(String msg, Throwable th) {
539                super(msg, th);
540            }
541    
542            public OsmParsingException(Throwable th) {
543                super(th);
544            }
545    
546            @Override
547            public String getMessage() {
548                String msg = super.getMessage();
549                if (msg == null) {
550                    msg = getClass().getName();
551                }
552                if (getLocation() == null)
553                    return msg;
554                msg = msg + " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
555                return msg;
556            }
557        }
558    
559        protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
560            if (progressMonitor == null) {
561                progressMonitor = NullProgressMonitor.INSTANCE;
562            }
563            CheckParameterUtil.ensureParameterNotNull(source, "source");
564            try {
565                progressMonitor.beginTask(tr("Prepare OSM data...", 2));
566                progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
567    
568                InputStreamReader ir = UTFInputStreamReader.create(source, "UTF-8");
569                XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
570                setParser(parser);
571                parse();
572                progressMonitor.worked(1);
573    
574                progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
575                prepareDataSet();
576                progressMonitor.worked(1);
577    
578                // iterate over registered postprocessors and give them each a chance
579                // to modify the dataset we have just loaded.
580                if (postprocessors != null) {
581                    for (OsmServerReadPostprocessor pp : postprocessors) {
582                        pp.postprocessDataSet(getDataSet(), progressMonitor);
583                    }
584                }
585                return getDataSet();
586            } catch(IllegalDataException e) {
587                throw e;
588            } catch(OsmParsingException e) {
589                throw new IllegalDataException(e.getMessage(), e);
590            } catch(XMLStreamException e) {
591                String msg = e.getMessage();
592                Pattern p = Pattern.compile("Message: (.+)");
593                Matcher m = p.matcher(msg);
594                if (m.find()) {
595                    msg = m.group(1);
596                }
597                if (e.getLocation() != null)
598                    throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
599                else
600                    throw new IllegalDataException(msg, e);
601            } catch(Exception e) {
602                throw new IllegalDataException(e);
603            } finally {
604                progressMonitor.finishTask();
605            }
606        }
607    
608        /**
609         * Parse the given input source and return the dataset.
610         *
611         * @param source the source input stream. Must not be null.
612         * @param progressMonitor  the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
613         *
614         * @return the dataset with the parsed data
615         * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
616         * @throws IllegalArgumentException thrown if source is null
617         */
618        public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
619            return new OsmReader().doParseDataSet(source, progressMonitor);
620        }
621    }