001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.ByteArrayInputStream; 007import java.io.IOException; 008import java.net.SocketTimeoutException; 009import java.net.URL; 010import java.nio.charset.StandardCharsets; 011import java.util.HashSet; 012import java.util.List; 013import java.util.Map; 014import java.util.Map.Entry; 015import java.util.Optional; 016import java.util.Set; 017import java.util.concurrent.ConcurrentHashMap; 018import java.util.concurrent.ConcurrentMap; 019import java.util.concurrent.ThreadPoolExecutor; 020import java.util.concurrent.TimeUnit; 021import java.util.regex.Matcher; 022import java.util.regex.Pattern; 023 024import org.apache.commons.jcs.access.behavior.ICacheAccess; 025import org.openstreetmap.gui.jmapviewer.Tile; 026import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 027import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 028import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 029import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; 030import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; 031import org.openstreetmap.josm.data.cache.CacheEntry; 032import org.openstreetmap.josm.data.cache.CacheEntryAttributes; 033import org.openstreetmap.josm.data.cache.ICachedLoaderListener; 034import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; 035import org.openstreetmap.josm.data.preferences.LongProperty; 036import org.openstreetmap.josm.tools.HttpClient; 037import org.openstreetmap.josm.tools.Logging; 038import org.openstreetmap.josm.tools.Utils; 039 040/** 041 * Class bridging TMS requests to JCS cache requests 042 * 043 * @author Wiktor Niesiobędzki 044 * @since 8168 045 */ 046public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener { 047 /** General maximum expires for tiles. Might be overridden by imagery settings */ 048 public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30)); 049 /** General minimum expires for tiles. Might be overridden by imagery settings */ 050 public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1)); 051 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+"); 052 static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*"); 053 static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}"); 054 protected final Tile tile; 055 private volatile URL url; 056 private final TileJobOptions options; 057 058 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created 059 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints 060 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>(); 061 062 /** 063 * Constructor for creating a job, to get a specific tile from cache 064 * @param listener Tile loader listener 065 * @param tile to be fetched from cache 066 * @param cache object 067 * @param options for job (such as http headers, timeouts etc.) 068 * @param downloadExecutor that will be executing the jobs 069 */ 070 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, 071 ICacheAccess<String, BufferedImageCacheEntry> cache, 072 TileJobOptions options, 073 ThreadPoolExecutor downloadExecutor) { 074 super(cache, options, downloadExecutor); 075 this.tile = tile; 076 this.options = options; 077 if (listener != null) { 078 inProgress.computeIfAbsent(getCacheKey(), k -> new HashSet<>()).add(listener); 079 } 080 } 081 082 @Override 083 public String getCacheKey() { 084 if (tile != null) { 085 TileSource tileSource = tile.getTileSource(); 086 return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':' 087 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile()); 088 } 089 return null; 090 } 091 092 /* 093 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution 094 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching 095 * data from cache, that's why URL creation is postponed until it's needed 096 * 097 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different 098 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection 099 * 100 */ 101 @Override 102 public URL getUrl() throws IOException { 103 if (url == null) { 104 synchronized (this) { 105 if (url == null) { 106 String sUrl = tile.getUrl(); 107 if (!"".equals(sUrl)) { 108 url = new URL(sUrl); 109 } 110 } 111 } 112 } 113 return url; 114 } 115 116 @Override 117 public boolean isObjectLoadable() { 118 if (cacheData != null) { 119 byte[] content = cacheData.getContent(); 120 try { 121 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom(); 122 } catch (IOException e) { 123 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}", 124 tile.getKey(), e.getMessage()); 125 } 126 } 127 return false; 128 } 129 130 @Override 131 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) { 132 attributes.setMetadata(tile.getTileSource().getMetadata(headers)); 133 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) { 134 attributes.setNoTileAtZoom(true); 135 return false; // do no try to load data from no-tile at zoom, cache empty object instead 136 } 137 return super.isResponseLoadable(headers, statusCode, content); 138 } 139 140 @Override 141 protected boolean cacheAsEmpty() { 142 return isNoTileAtZoom() || super.cacheAsEmpty(); 143 } 144 145 @Override 146 public void submit(boolean force) { 147 tile.initLoading(); 148 try { 149 super.submit(this, force); 150 } catch (IOException | IllegalArgumentException e) { 151 // if we fail to submit the job, mark tile as loaded and set error message 152 Logging.log(Logging.LEVEL_WARN, e); 153 tile.finishLoading(); 154 tile.setError(e.getMessage()); 155 } 156 } 157 158 @Override 159 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) { 160 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along 161 Set<TileLoaderListener> listeners = inProgress.remove(getCacheKey()); 162 boolean status = result == LoadResult.SUCCESS; 163 164 try { 165 tile.finishLoading(); // whatever happened set that loading has finished 166 // set tile metadata 167 if (this.attributes != null) { 168 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) { 169 tile.putValue(e.getKey(), e.getValue()); 170 } 171 } 172 173 switch(result) { 174 case SUCCESS: 175 handleNoTileAtZoom(); 176 if (attributes != null) { 177 int httpStatusCode = attributes.getResponseCode(); 178 if (httpStatusCode >= 400 && !isNoTileAtZoom()) { 179 status = false; 180 handleError(attributes); 181 } 182 } 183 status &= tryLoadTileImage(object); //try to keep returned image as background 184 break; 185 case FAILURE: 186 handleError(attributes); 187 tryLoadTileImage(object); 188 break; 189 case CANCELED: 190 tile.loadingCanceled(); 191 // do nothing 192 } 193 194 // always check, if there is some listener interested in fact, that tile has finished loading 195 if (listeners != null) { // listeners might be null, if some other thread notified already about success 196 for (TileLoaderListener l: listeners) { 197 l.tileLoadingFinished(tile, status); 198 } 199 } 200 } catch (IOException e) { 201 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage()); 202 tile.setError(e); 203 tile.setLoaded(false); 204 if (listeners != null) { // listeners might be null, if some other thread notified already about success 205 for (TileLoaderListener l: listeners) { 206 l.tileLoadingFinished(tile, false); 207 } 208 } 209 } 210 } 211 212 private void handleError(CacheEntryAttributes attributes) { 213 if (attributes != null) { 214 int httpStatusCode = attributes.getResponseCode(); 215 if (attributes.getErrorMessage() == null) { 216 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode)); 217 } else { 218 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage())); 219 } 220 if (httpStatusCode >= 500 && httpStatusCode != 599) { 221 // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException 222 tile.setLoaded(false); // treat 500 errors as temporary and try to load it again 223 } 224 // treat SocketTimeoutException as transient error 225 attributes.getException() 226 .filter(x -> x.isAssignableFrom(SocketTimeoutException.class)) 227 .ifPresent(x -> tile.setLoaded(false)); 228 } else { 229 tile.setError(tr("Problem loading tile")); 230 // treat unknown errors as permanent and do not try to load tile again 231 } 232 } 233 234 /** 235 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers) 236 * 237 * @return base URL of TMS or server url as defined in super class 238 */ 239 @Override 240 protected String getServerKey() { 241 TileSource ts = tile.getSource(); 242 if (ts instanceof AbstractTMSTileSource) { 243 return ((AbstractTMSTileSource) ts).getBaseUrl(); 244 } 245 return super.getServerKey(); 246 } 247 248 @Override 249 protected BufferedImageCacheEntry createCacheEntry(byte[] content) { 250 return new BufferedImageCacheEntry(content); 251 } 252 253 @Override 254 public void submit() { 255 submit(false); 256 } 257 258 @Override 259 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) { 260 CacheEntryAttributes ret = super.parseHeaders(urlConn); 261 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles 262 // at least for some short period of time, but not too long 263 if (ret.getExpirationTime() < now + Math.max(MINIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) { 264 ret.setExpirationTime(now + Math.max(MINIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()))); 265 } 266 if (ret.getExpirationTime() > now + Math.max(MAXIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) { 267 ret.setExpirationTime(now + Math.max(MAXIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()))); 268 } 269 return ret; 270 } 271 272 private boolean handleNoTileAtZoom() { 273 if (isNoTileAtZoom()) { 274 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); 275 tile.setError(tr("No tiles at this zoom level")); 276 tile.putValue("tile-info", "no-tile"); 277 return true; 278 } 279 return false; 280 } 281 282 private boolean isNoTileAtZoom() { 283 if (attributes == null) { 284 Logging.warn("Cache attributes are null"); 285 } 286 return attributes != null && attributes.isNoTileAtZoom(); 287 } 288 289 private boolean tryLoadTileImage(CacheEntry object) throws IOException { 290 if (object != null) { 291 byte[] content = object.getContent(); 292 if (content.length > 0) { 293 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) { 294 tile.loadImage(in); 295 if (tile.getImage() == null) { 296 String s = new String(content, StandardCharsets.UTF_8); 297 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s); 298 if (m.matches()) { 299 String message = Utils.strip(m.group(1)); 300 tile.setError(message); 301 Logging.error(message); 302 Logging.debug(s); 303 } else { 304 tile.setError(tr("Could not load image from tile server")); 305 } 306 return false; 307 } 308 } catch (UnsatisfiedLinkError | SecurityException e) { 309 throw new IOException(e); 310 } 311 } 312 } 313 return true; 314 } 315 316 @Override 317 public String detectErrorMessage(String data) { 318 Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data); 319 Matcher json = JSON_PATTERN.matcher(data); 320 return xml.matches() ? removeCdata(Utils.strip(xml.group(1))) 321 : json.matches() ? Utils.strip(json.group(1)) 322 : super.detectErrorMessage(data); 323 } 324 325 private static String removeCdata(String msg) { 326 Matcher m = CDATA_PATTERN.matcher(msg); 327 return m.matches() ? Utils.strip(m.group(1)) : msg; 328 } 329}