001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.InputStream;
007import java.util.Collection;
008import java.util.Objects;
009import java.util.Set;
010import java.util.TreeSet;
011import java.util.regex.Matcher;
012import java.util.regex.Pattern;
013
014import javax.xml.stream.Location;
015import javax.xml.stream.XMLStreamConstants;
016import javax.xml.stream.XMLStreamException;
017import javax.xml.stream.XMLStreamReader;
018
019import org.openstreetmap.josm.data.osm.Changeset;
020import org.openstreetmap.josm.data.osm.DataSet;
021import org.openstreetmap.josm.data.osm.Node;
022import org.openstreetmap.josm.data.osm.PrimitiveData;
023import org.openstreetmap.josm.data.osm.Relation;
024import org.openstreetmap.josm.data.osm.RelationMemberData;
025import org.openstreetmap.josm.data.osm.Tagged;
026import org.openstreetmap.josm.data.osm.Way;
027import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
028import org.openstreetmap.josm.gui.progress.ProgressMonitor;
029import org.openstreetmap.josm.tools.Logging;
030import org.openstreetmap.josm.tools.UncheckedParseException;
031import org.openstreetmap.josm.tools.XmlUtils;
032
033/**
034 * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it.
035 *
036 * For each xml element, there is a dedicated method.
037 * The XMLStreamReader cursor points to the start of the element, when the method is
038 * entered, and it must point to the end of the same element, when it is exited.
039 */
040public class OsmReader extends AbstractReader {
041
042    protected XMLStreamReader parser;
043
044    protected boolean convertUnknownToTags;
045
046    private static final Set<String> COMMON_XML_ATTRIBUTES = new TreeSet<>();
047
048    static {
049        COMMON_XML_ATTRIBUTES.add("id");
050        COMMON_XML_ATTRIBUTES.add("timestamp");
051        COMMON_XML_ATTRIBUTES.add("user");
052        COMMON_XML_ATTRIBUTES.add("uid");
053        COMMON_XML_ATTRIBUTES.add("visible");
054        COMMON_XML_ATTRIBUTES.add("version");
055        COMMON_XML_ATTRIBUTES.add("action");
056        COMMON_XML_ATTRIBUTES.add("changeset");
057        COMMON_XML_ATTRIBUTES.add("lat");
058        COMMON_XML_ATTRIBUTES.add("lon");
059    }
060
061    /**
062     * constructor (for private and subclasses use only)
063     *
064     * @see #parseDataSet(InputStream, ProgressMonitor)
065     */
066    protected OsmReader() {
067        this(false);
068    }
069
070    /**
071     * constructor (for private and subclasses use only)
072     * @param convertUnknownToTags if true, keep unknown xml attributes as tags
073     *
074     * @see #parseDataSet(InputStream, ProgressMonitor)
075     * @since 15470
076     */
077    protected OsmReader(boolean convertUnknownToTags) {
078        // Restricts visibility
079        this.convertUnknownToTags = convertUnknownToTags;
080    }
081
082    protected void setParser(XMLStreamReader parser) {
083        this.parser = parser;
084    }
085
086    protected void throwException(Throwable th) throws XMLStreamException {
087        throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th);
088    }
089
090    protected void throwException(String msg, Throwable th) throws XMLStreamException {
091        throw new XmlStreamParsingException(msg, parser.getLocation(), th);
092    }
093
094    protected void throwException(String msg) throws XMLStreamException {
095        throw new XmlStreamParsingException(msg, parser.getLocation());
096    }
097
098    protected void parse() throws XMLStreamException {
099        int event = parser.getEventType();
100        while (true) {
101            if (event == XMLStreamConstants.START_ELEMENT) {
102                parseRoot();
103            } else if (event == XMLStreamConstants.END_ELEMENT)
104                return;
105            if (parser.hasNext()) {
106                event = parser.next();
107            } else {
108                break;
109            }
110        }
111        parser.close();
112    }
113
114    protected void parseRoot() throws XMLStreamException {
115        if ("osm".equals(parser.getLocalName())) {
116            parseOsm();
117        } else {
118            parseUnknown();
119        }
120    }
121
122    private void parseOsm() throws XMLStreamException {
123        try {
124            parseVersion(parser.getAttributeValue(null, "version"));
125            parseDownloadPolicy("download", parser.getAttributeValue(null, "download"));
126            parseUploadPolicy("upload", parser.getAttributeValue(null, "upload"));
127            parseLocked(parser.getAttributeValue(null, "locked"));
128        } catch (IllegalDataException e) {
129            throwException(e);
130        }
131        String generator = parser.getAttributeValue(null, "generator");
132        Long uploadChangesetId = null;
133        if (parser.getAttributeValue(null, "upload-changeset") != null) {
134            uploadChangesetId = getLong("upload-changeset");
135        }
136        while (parser.hasNext()) {
137            int event = parser.next();
138
139            if (cancel) {
140                cancel = false;
141                throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
142            }
143
144            if (event == XMLStreamConstants.START_ELEMENT) {
145                switch (parser.getLocalName()) {
146                case "bounds":
147                    parseBounds(generator);
148                    break;
149                case "node":
150                    parseNode();
151                    break;
152                case "way":
153                    parseWay();
154                    break;
155                case "relation":
156                    parseRelation();
157                    break;
158                case "changeset":
159                    parseChangeset(uploadChangesetId);
160                    break;
161                case "remark": // Used by Overpass API
162                    parseRemark();
163                    break;
164                default:
165                    parseUnknown();
166                }
167            } else if (event == XMLStreamConstants.END_ELEMENT) {
168                return;
169            }
170        }
171    }
172
173    private void handleIllegalDataException(IllegalDataException e) throws XMLStreamException {
174        Throwable cause = e.getCause();
175        if (cause instanceof XMLStreamException) {
176            throw (XMLStreamException) cause;
177        } else {
178            throwException(e);
179        }
180    }
181
182    private void parseRemark() throws XMLStreamException {
183        while (parser.hasNext()) {
184            int event = parser.next();
185            if (event == XMLStreamConstants.CHARACTERS) {
186                ds.setRemark(parser.getText());
187            } else if (event == XMLStreamConstants.END_ELEMENT) {
188                return;
189            }
190        }
191    }
192
193    private void parseBounds(String generator) throws XMLStreamException {
194        String minlon = parser.getAttributeValue(null, "minlon");
195        String minlat = parser.getAttributeValue(null, "minlat");
196        String maxlon = parser.getAttributeValue(null, "maxlon");
197        String maxlat = parser.getAttributeValue(null, "maxlat");
198        String origin = parser.getAttributeValue(null, "origin");
199        try {
200            parseBounds(generator, minlon, minlat, maxlon, maxlat, origin);
201        } catch (IllegalDataException e) {
202            handleIllegalDataException(e);
203        }
204        jumpToEnd();
205    }
206
207    protected Node parseNode() throws XMLStreamException {
208        String lat = parser.getAttributeValue(null, "lat");
209        String lon = parser.getAttributeValue(null, "lon");
210        try {
211            return parseNode(lat, lon, this::readCommon, this::parseNodeTags);
212        } catch (IllegalDataException e) {
213            handleIllegalDataException(e);
214        }
215        return null;
216    }
217
218    private void parseNodeTags(Node n) throws IllegalDataException {
219        try {
220            while (parser.hasNext()) {
221                int event = parser.next();
222                if (event == XMLStreamConstants.START_ELEMENT) {
223                    if ("tag".equals(parser.getLocalName())) {
224                        parseTag(n);
225                    } else {
226                        parseUnknown();
227                    }
228                } else if (event == XMLStreamConstants.END_ELEMENT) {
229                    return;
230                }
231            }
232        } catch (XMLStreamException e) {
233            throw new IllegalDataException(e);
234        }
235    }
236
237    protected Way parseWay() throws XMLStreamException {
238        try {
239            return parseWay(this::readCommon, this::parseWayNodesAndTags);
240        } catch (IllegalDataException e) {
241            handleIllegalDataException(e);
242        }
243        return null;
244    }
245
246    private void parseWayNodesAndTags(Way w, Collection<Long> nodeIds) throws IllegalDataException {
247        try {
248            while (parser.hasNext()) {
249                int event = parser.next();
250                if (event == XMLStreamConstants.START_ELEMENT) {
251                    switch (parser.getLocalName()) {
252                    case "nd":
253                        nodeIds.add(parseWayNode(w));
254                        break;
255                    case "tag":
256                        parseTag(w);
257                        break;
258                    default:
259                        parseUnknown();
260                    }
261                } else if (event == XMLStreamConstants.END_ELEMENT) {
262                    break;
263                }
264            }
265        } catch (XMLStreamException e) {
266            throw new IllegalDataException(e);
267        }
268    }
269
270    private long parseWayNode(Way w) throws XMLStreamException {
271        if (parser.getAttributeValue(null, "ref") == null) {
272            throwException(
273                    tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", Long.toString(w.getUniqueId()))
274            );
275        }
276        long id = getLong("ref");
277        if (id == 0) {
278            throwException(
279                    tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", Long.toString(id))
280            );
281        }
282        jumpToEnd();
283        return id;
284    }
285
286    protected Relation parseRelation() throws XMLStreamException {
287        try {
288            return parseRelation(this::readCommon, this::parseRelationMembersAndTags);
289        } catch (IllegalDataException e) {
290            handleIllegalDataException(e);
291        }
292        return null;
293    }
294
295    private void parseRelationMembersAndTags(Relation r, Collection<RelationMemberData> members) throws IllegalDataException {
296        try {
297            while (parser.hasNext()) {
298                int event = parser.next();
299                if (event == XMLStreamConstants.START_ELEMENT) {
300                    switch (parser.getLocalName()) {
301                    case "member":
302                        members.add(parseRelationMember(r));
303                        break;
304                    case "tag":
305                        parseTag(r);
306                        break;
307                    default:
308                        parseUnknown();
309                    }
310                } else if (event == XMLStreamConstants.END_ELEMENT) {
311                    break;
312                }
313            }
314        } catch (XMLStreamException e) {
315            throw new IllegalDataException(e);
316        }
317    }
318
319    private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
320        RelationMemberData result = null;
321        try {
322            String ref = parser.getAttributeValue(null, "ref");
323            String type = parser.getAttributeValue(null, "type");
324            String role = parser.getAttributeValue(null, "role");
325            result = parseRelationMember(r, ref, type, role);
326            jumpToEnd();
327        } catch (IllegalDataException e) {
328            handleIllegalDataException(e);
329        }
330        return result;
331    }
332
333    private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
334
335        Long id = null;
336        if (parser.getAttributeValue(null, "id") != null) {
337            id = getLong("id");
338        }
339        // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
340        if (Objects.equals(id, uploadChangesetId)) {
341            uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
342            while (true) {
343                int event = parser.next();
344                if (event == XMLStreamConstants.START_ELEMENT) {
345                    if ("tag".equals(parser.getLocalName())) {
346                        parseTag(uploadChangeset);
347                    } else {
348                        parseUnknown();
349                    }
350                } else if (event == XMLStreamConstants.END_ELEMENT)
351                    return;
352            }
353        } else {
354            jumpToEnd(false);
355        }
356    }
357
358    private void parseTag(Tagged t) throws XMLStreamException {
359        String key = parser.getAttributeValue(null, "k");
360        String value = parser.getAttributeValue(null, "v");
361        try {
362            parseTag(t, key, value);
363        } catch (IllegalDataException e) {
364            throwException(e);
365        }
366        jumpToEnd();
367    }
368
369    protected void parseUnknown(boolean printWarning) throws XMLStreamException {
370        final String element = parser.getLocalName();
371        if (printWarning && ("note".equals(element) || "meta".equals(element))) {
372            // we know that Overpass API returns those elements
373            Logging.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
374        } else if (printWarning) {
375            Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
376        }
377        while (true) {
378            int event = parser.next();
379            if (event == XMLStreamConstants.START_ELEMENT) {
380                parseUnknown(false); /* no more warning for inner elements */
381            } else if (event == XMLStreamConstants.END_ELEMENT)
382                return;
383        }
384    }
385
386    protected void parseUnknown() throws XMLStreamException {
387        parseUnknown(true);
388    }
389
390    /**
391     * When cursor is at the start of an element, moves it to the end tag of that element.
392     * Nested content is skipped.
393     *
394     * This is basically the same code as parseUnknown(), except for the warnings, which
395     * are displayed for inner elements and not at top level.
396     * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
397     * @throws XMLStreamException if there is an error processing the underlying XML source
398     */
399    protected final void jumpToEnd(boolean printWarning) throws XMLStreamException {
400        while (true) {
401            int event = parser.next();
402            if (event == XMLStreamConstants.START_ELEMENT) {
403                parseUnknown(printWarning);
404            } else if (event == XMLStreamConstants.END_ELEMENT)
405                return;
406        }
407    }
408
409    protected final void jumpToEnd() throws XMLStreamException {
410        jumpToEnd(true);
411    }
412
413    /**
414     * Read out the common attributes and put them into current OsmPrimitive.
415     * @param current primitive to update
416     * @throws IllegalDataException if there is an error processing the underlying XML source
417     */
418    private void readCommon(PrimitiveData current) throws IllegalDataException {
419        try {
420            parseId(current, getLong("id"));
421            parseTimestamp(current, parser.getAttributeValue(null, "timestamp"));
422            parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid"));
423            parseVisible(current, parser.getAttributeValue(null, "visible"));
424            parseVersion(current, parser.getAttributeValue(null, "version"));
425            parseAction(current, parser.getAttributeValue(null, "action"));
426            parseChangeset(current, parser.getAttributeValue(null, "changeset"));
427
428            if (convertUnknownToTags) {
429                for (int i = 0; i < parser.getAttributeCount(); i++) {
430                    if (!COMMON_XML_ATTRIBUTES.contains(parser.getAttributeLocalName(i))) {
431                        parseTag(current, parser.getAttributeLocalName(i), parser.getAttributeValue(i));
432                    }
433                }
434            }
435        } catch (UncheckedParseException | XMLStreamException e) {
436            throw new IllegalDataException(e);
437        }
438    }
439
440    private long getLong(String name) throws XMLStreamException {
441        String value = parser.getAttributeValue(null, name);
442        try {
443            return getLong(name, value);
444        } catch (IllegalDataException e) {
445            throwException(e);
446        }
447        return 0; // should not happen
448    }
449
450    /**
451     * Exception thrown after user cancelation.
452     */
453    private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException {
454        /**
455         * Constructs a new {@code OsmParsingCanceledException}.
456         * @param msg The error message
457         * @param location The parser location
458         */
459        OsmParsingCanceledException(String msg, Location location) {
460            super(msg, location);
461        }
462    }
463
464    @Override
465    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
466        return doParseDataSet(source, progressMonitor, ir -> {
467            try {
468                setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir));
469                parse();
470            } catch (XmlStreamParsingException | UncheckedParseException e) {
471                throw new IllegalDataException(e.getMessage(), e);
472            } catch (XMLStreamException e) {
473                String msg = e.getMessage();
474                Pattern p = Pattern.compile("Message: (.+)");
475                Matcher m = p.matcher(msg);
476                if (m.find()) {
477                    msg = m.group(1);
478                }
479                if (e.getLocation() != null)
480                    throw new IllegalDataException(tr("Line {0} column {1}: ",
481                            e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
482                else
483                    throw new IllegalDataException(msg, e);
484            }
485        });
486    }
487
488    /**
489     * Parse the given input source and return the dataset.
490     *
491     * @param source the source input stream. Must not be null.
492     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
493     *
494     * @return the dataset with the parsed data
495     * @throws IllegalDataException if an error was found while parsing the data from the source
496     * @throws IllegalArgumentException if source is null
497     */
498    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
499        return parseDataSet(source, progressMonitor, false);
500    }
501
502    /**
503     * Parse the given input source and return the dataset.
504     *
505     * @param source the source input stream. Must not be null.
506     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
507     * @param convertUnknownToTags true if unknown xml attributes should be kept as tags
508     *
509     * @return the dataset with the parsed data
510     * @throws IllegalDataException if an error was found while parsing the data from the source
511     * @throws IllegalArgumentException if source is null
512     * @since 15470
513     */
514    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor, boolean convertUnknownToTags)
515            throws IllegalDataException {
516        return new OsmReader(convertUnknownToTags).doParseDataSet(source, progressMonitor);
517    }
518}