001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.InputStream; 007import java.util.Collection; 008import java.util.Objects; 009import java.util.Set; 010import java.util.TreeSet; 011import java.util.regex.Matcher; 012import java.util.regex.Pattern; 013 014import javax.xml.stream.Location; 015import javax.xml.stream.XMLStreamConstants; 016import javax.xml.stream.XMLStreamException; 017import javax.xml.stream.XMLStreamReader; 018 019import org.openstreetmap.josm.data.osm.Changeset; 020import org.openstreetmap.josm.data.osm.DataSet; 021import org.openstreetmap.josm.data.osm.Node; 022import org.openstreetmap.josm.data.osm.PrimitiveData; 023import org.openstreetmap.josm.data.osm.Relation; 024import org.openstreetmap.josm.data.osm.RelationMemberData; 025import org.openstreetmap.josm.data.osm.Tagged; 026import org.openstreetmap.josm.data.osm.Way; 027import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 028import org.openstreetmap.josm.gui.progress.ProgressMonitor; 029import org.openstreetmap.josm.tools.Logging; 030import org.openstreetmap.josm.tools.UncheckedParseException; 031import org.openstreetmap.josm.tools.XmlUtils; 032 033/** 034 * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it. 035 * 036 * For each xml element, there is a dedicated method. 037 * The XMLStreamReader cursor points to the start of the element, when the method is 038 * entered, and it must point to the end of the same element, when it is exited. 039 */ 040public class OsmReader extends AbstractReader { 041 042 protected XMLStreamReader parser; 043 044 protected boolean convertUnknownToTags; 045 046 private static final Set<String> COMMON_XML_ATTRIBUTES = new TreeSet<>(); 047 048 static { 049 COMMON_XML_ATTRIBUTES.add("id"); 050 COMMON_XML_ATTRIBUTES.add("timestamp"); 051 COMMON_XML_ATTRIBUTES.add("user"); 052 COMMON_XML_ATTRIBUTES.add("uid"); 053 COMMON_XML_ATTRIBUTES.add("visible"); 054 COMMON_XML_ATTRIBUTES.add("version"); 055 COMMON_XML_ATTRIBUTES.add("action"); 056 COMMON_XML_ATTRIBUTES.add("changeset"); 057 COMMON_XML_ATTRIBUTES.add("lat"); 058 COMMON_XML_ATTRIBUTES.add("lon"); 059 } 060 061 /** 062 * constructor (for private and subclasses use only) 063 * 064 * @see #parseDataSet(InputStream, ProgressMonitor) 065 */ 066 protected OsmReader() { 067 this(false); 068 } 069 070 /** 071 * constructor (for private and subclasses use only) 072 * @param convertUnknownToTags if true, keep unknown xml attributes as tags 073 * 074 * @see #parseDataSet(InputStream, ProgressMonitor) 075 * @since 15470 076 */ 077 protected OsmReader(boolean convertUnknownToTags) { 078 // Restricts visibility 079 this.convertUnknownToTags = convertUnknownToTags; 080 } 081 082 protected void setParser(XMLStreamReader parser) { 083 this.parser = parser; 084 } 085 086 protected void throwException(Throwable th) throws XMLStreamException { 087 throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th); 088 } 089 090 protected void throwException(String msg, Throwable th) throws XMLStreamException { 091 throw new XmlStreamParsingException(msg, parser.getLocation(), th); 092 } 093 094 protected void throwException(String msg) throws XMLStreamException { 095 throw new XmlStreamParsingException(msg, parser.getLocation()); 096 } 097 098 protected void parse() throws XMLStreamException { 099 int event = parser.getEventType(); 100 while (true) { 101 if (event == XMLStreamConstants.START_ELEMENT) { 102 parseRoot(); 103 } else if (event == XMLStreamConstants.END_ELEMENT) 104 return; 105 if (parser.hasNext()) { 106 event = parser.next(); 107 } else { 108 break; 109 } 110 } 111 parser.close(); 112 } 113 114 protected void parseRoot() throws XMLStreamException { 115 if ("osm".equals(parser.getLocalName())) { 116 parseOsm(); 117 } else { 118 parseUnknown(); 119 } 120 } 121 122 private void parseOsm() throws XMLStreamException { 123 try { 124 parseVersion(parser.getAttributeValue(null, "version")); 125 parseDownloadPolicy("download", parser.getAttributeValue(null, "download")); 126 parseUploadPolicy("upload", parser.getAttributeValue(null, "upload")); 127 parseLocked(parser.getAttributeValue(null, "locked")); 128 } catch (IllegalDataException e) { 129 throwException(e); 130 } 131 String generator = parser.getAttributeValue(null, "generator"); 132 Long uploadChangesetId = null; 133 if (parser.getAttributeValue(null, "upload-changeset") != null) { 134 uploadChangesetId = getLong("upload-changeset"); 135 } 136 while (parser.hasNext()) { 137 int event = parser.next(); 138 139 if (cancel) { 140 cancel = false; 141 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation()); 142 } 143 144 if (event == XMLStreamConstants.START_ELEMENT) { 145 switch (parser.getLocalName()) { 146 case "bounds": 147 parseBounds(generator); 148 break; 149 case "node": 150 parseNode(); 151 break; 152 case "way": 153 parseWay(); 154 break; 155 case "relation": 156 parseRelation(); 157 break; 158 case "changeset": 159 parseChangeset(uploadChangesetId); 160 break; 161 case "remark": // Used by Overpass API 162 parseRemark(); 163 break; 164 default: 165 parseUnknown(); 166 } 167 } else if (event == XMLStreamConstants.END_ELEMENT) { 168 return; 169 } 170 } 171 } 172 173 private void handleIllegalDataException(IllegalDataException e) throws XMLStreamException { 174 Throwable cause = e.getCause(); 175 if (cause instanceof XMLStreamException) { 176 throw (XMLStreamException) cause; 177 } else { 178 throwException(e); 179 } 180 } 181 182 private void parseRemark() throws XMLStreamException { 183 while (parser.hasNext()) { 184 int event = parser.next(); 185 if (event == XMLStreamConstants.CHARACTERS) { 186 ds.setRemark(parser.getText()); 187 } else if (event == XMLStreamConstants.END_ELEMENT) { 188 return; 189 } 190 } 191 } 192 193 private void parseBounds(String generator) throws XMLStreamException { 194 String minlon = parser.getAttributeValue(null, "minlon"); 195 String minlat = parser.getAttributeValue(null, "minlat"); 196 String maxlon = parser.getAttributeValue(null, "maxlon"); 197 String maxlat = parser.getAttributeValue(null, "maxlat"); 198 String origin = parser.getAttributeValue(null, "origin"); 199 try { 200 parseBounds(generator, minlon, minlat, maxlon, maxlat, origin); 201 } catch (IllegalDataException e) { 202 handleIllegalDataException(e); 203 } 204 jumpToEnd(); 205 } 206 207 protected Node parseNode() throws XMLStreamException { 208 String lat = parser.getAttributeValue(null, "lat"); 209 String lon = parser.getAttributeValue(null, "lon"); 210 try { 211 return parseNode(lat, lon, this::readCommon, this::parseNodeTags); 212 } catch (IllegalDataException e) { 213 handleIllegalDataException(e); 214 } 215 return null; 216 } 217 218 private void parseNodeTags(Node n) throws IllegalDataException { 219 try { 220 while (parser.hasNext()) { 221 int event = parser.next(); 222 if (event == XMLStreamConstants.START_ELEMENT) { 223 if ("tag".equals(parser.getLocalName())) { 224 parseTag(n); 225 } else { 226 parseUnknown(); 227 } 228 } else if (event == XMLStreamConstants.END_ELEMENT) { 229 return; 230 } 231 } 232 } catch (XMLStreamException e) { 233 throw new IllegalDataException(e); 234 } 235 } 236 237 protected Way parseWay() throws XMLStreamException { 238 try { 239 return parseWay(this::readCommon, this::parseWayNodesAndTags); 240 } catch (IllegalDataException e) { 241 handleIllegalDataException(e); 242 } 243 return null; 244 } 245 246 private void parseWayNodesAndTags(Way w, Collection<Long> nodeIds) throws IllegalDataException { 247 try { 248 while (parser.hasNext()) { 249 int event = parser.next(); 250 if (event == XMLStreamConstants.START_ELEMENT) { 251 switch (parser.getLocalName()) { 252 case "nd": 253 nodeIds.add(parseWayNode(w)); 254 break; 255 case "tag": 256 parseTag(w); 257 break; 258 default: 259 parseUnknown(); 260 } 261 } else if (event == XMLStreamConstants.END_ELEMENT) { 262 break; 263 } 264 } 265 } catch (XMLStreamException e) { 266 throw new IllegalDataException(e); 267 } 268 } 269 270 private long parseWayNode(Way w) throws XMLStreamException { 271 if (parser.getAttributeValue(null, "ref") == null) { 272 throwException( 273 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", Long.toString(w.getUniqueId())) 274 ); 275 } 276 long id = getLong("ref"); 277 if (id == 0) { 278 throwException( 279 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", Long.toString(id)) 280 ); 281 } 282 jumpToEnd(); 283 return id; 284 } 285 286 protected Relation parseRelation() throws XMLStreamException { 287 try { 288 return parseRelation(this::readCommon, this::parseRelationMembersAndTags); 289 } catch (IllegalDataException e) { 290 handleIllegalDataException(e); 291 } 292 return null; 293 } 294 295 private void parseRelationMembersAndTags(Relation r, Collection<RelationMemberData> members) throws IllegalDataException { 296 try { 297 while (parser.hasNext()) { 298 int event = parser.next(); 299 if (event == XMLStreamConstants.START_ELEMENT) { 300 switch (parser.getLocalName()) { 301 case "member": 302 members.add(parseRelationMember(r)); 303 break; 304 case "tag": 305 parseTag(r); 306 break; 307 default: 308 parseUnknown(); 309 } 310 } else if (event == XMLStreamConstants.END_ELEMENT) { 311 break; 312 } 313 } 314 } catch (XMLStreamException e) { 315 throw new IllegalDataException(e); 316 } 317 } 318 319 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { 320 RelationMemberData result = null; 321 try { 322 String ref = parser.getAttributeValue(null, "ref"); 323 String type = parser.getAttributeValue(null, "type"); 324 String role = parser.getAttributeValue(null, "role"); 325 result = parseRelationMember(r, ref, type, role); 326 jumpToEnd(); 327 } catch (IllegalDataException e) { 328 handleIllegalDataException(e); 329 } 330 return result; 331 } 332 333 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 334 335 Long id = null; 336 if (parser.getAttributeValue(null, "id") != null) { 337 id = getLong("id"); 338 } 339 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value 340 if (Objects.equals(id, uploadChangesetId)) { 341 uploadChangeset = new Changeset(id != null ? id.intValue() : 0); 342 while (true) { 343 int event = parser.next(); 344 if (event == XMLStreamConstants.START_ELEMENT) { 345 if ("tag".equals(parser.getLocalName())) { 346 parseTag(uploadChangeset); 347 } else { 348 parseUnknown(); 349 } 350 } else if (event == XMLStreamConstants.END_ELEMENT) 351 return; 352 } 353 } else { 354 jumpToEnd(false); 355 } 356 } 357 358 private void parseTag(Tagged t) throws XMLStreamException { 359 String key = parser.getAttributeValue(null, "k"); 360 String value = parser.getAttributeValue(null, "v"); 361 try { 362 parseTag(t, key, value); 363 } catch (IllegalDataException e) { 364 throwException(e); 365 } 366 jumpToEnd(); 367 } 368 369 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 370 final String element = parser.getLocalName(); 371 if (printWarning && ("note".equals(element) || "meta".equals(element))) { 372 // we know that Overpass API returns those elements 373 Logging.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 374 } else if (printWarning) { 375 Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 376 } 377 while (true) { 378 int event = parser.next(); 379 if (event == XMLStreamConstants.START_ELEMENT) { 380 parseUnknown(false); /* no more warning for inner elements */ 381 } else if (event == XMLStreamConstants.END_ELEMENT) 382 return; 383 } 384 } 385 386 protected void parseUnknown() throws XMLStreamException { 387 parseUnknown(true); 388 } 389 390 /** 391 * When cursor is at the start of an element, moves it to the end tag of that element. 392 * Nested content is skipped. 393 * 394 * This is basically the same code as parseUnknown(), except for the warnings, which 395 * are displayed for inner elements and not at top level. 396 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met 397 * @throws XMLStreamException if there is an error processing the underlying XML source 398 */ 399 protected final void jumpToEnd(boolean printWarning) throws XMLStreamException { 400 while (true) { 401 int event = parser.next(); 402 if (event == XMLStreamConstants.START_ELEMENT) { 403 parseUnknown(printWarning); 404 } else if (event == XMLStreamConstants.END_ELEMENT) 405 return; 406 } 407 } 408 409 protected final void jumpToEnd() throws XMLStreamException { 410 jumpToEnd(true); 411 } 412 413 /** 414 * Read out the common attributes and put them into current OsmPrimitive. 415 * @param current primitive to update 416 * @throws IllegalDataException if there is an error processing the underlying XML source 417 */ 418 private void readCommon(PrimitiveData current) throws IllegalDataException { 419 try { 420 parseId(current, getLong("id")); 421 parseTimestamp(current, parser.getAttributeValue(null, "timestamp")); 422 parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid")); 423 parseVisible(current, parser.getAttributeValue(null, "visible")); 424 parseVersion(current, parser.getAttributeValue(null, "version")); 425 parseAction(current, parser.getAttributeValue(null, "action")); 426 parseChangeset(current, parser.getAttributeValue(null, "changeset")); 427 428 if (convertUnknownToTags) { 429 for (int i = 0; i < parser.getAttributeCount(); i++) { 430 if (!COMMON_XML_ATTRIBUTES.contains(parser.getAttributeLocalName(i))) { 431 parseTag(current, parser.getAttributeLocalName(i), parser.getAttributeValue(i)); 432 } 433 } 434 } 435 } catch (UncheckedParseException | XMLStreamException e) { 436 throw new IllegalDataException(e); 437 } 438 } 439 440 private long getLong(String name) throws XMLStreamException { 441 String value = parser.getAttributeValue(null, name); 442 try { 443 return getLong(name, value); 444 } catch (IllegalDataException e) { 445 throwException(e); 446 } 447 return 0; // should not happen 448 } 449 450 /** 451 * Exception thrown after user cancelation. 452 */ 453 private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException { 454 /** 455 * Constructs a new {@code OsmParsingCanceledException}. 456 * @param msg The error message 457 * @param location The parser location 458 */ 459 OsmParsingCanceledException(String msg, Location location) { 460 super(msg, location); 461 } 462 } 463 464 @Override 465 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 466 return doParseDataSet(source, progressMonitor, ir -> { 467 try { 468 setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir)); 469 parse(); 470 } catch (XmlStreamParsingException | UncheckedParseException e) { 471 throw new IllegalDataException(e.getMessage(), e); 472 } catch (XMLStreamException e) { 473 String msg = e.getMessage(); 474 Pattern p = Pattern.compile("Message: (.+)"); 475 Matcher m = p.matcher(msg); 476 if (m.find()) { 477 msg = m.group(1); 478 } 479 if (e.getLocation() != null) 480 throw new IllegalDataException(tr("Line {0} column {1}: ", 481 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 482 else 483 throw new IllegalDataException(msg, e); 484 } 485 }); 486 } 487 488 /** 489 * Parse the given input source and return the dataset. 490 * 491 * @param source the source input stream. Must not be null. 492 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 493 * 494 * @return the dataset with the parsed data 495 * @throws IllegalDataException if an error was found while parsing the data from the source 496 * @throws IllegalArgumentException if source is null 497 */ 498 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 499 return parseDataSet(source, progressMonitor, false); 500 } 501 502 /** 503 * Parse the given input source and return the dataset. 504 * 505 * @param source the source input stream. Must not be null. 506 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 507 * @param convertUnknownToTags true if unknown xml attributes should be kept as tags 508 * 509 * @return the dataset with the parsed data 510 * @throws IllegalDataException if an error was found while parsing the data from the source 511 * @throws IllegalArgumentException if source is null 512 * @since 15470 513 */ 514 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor, boolean convertUnknownToTags) 515 throws IllegalDataException { 516 return new OsmReader(convertUnknownToTags).doParseDataSet(source, progressMonitor); 517 } 518}