001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.ByteArrayInputStream;
007import java.io.IOException;
008import java.net.SocketTimeoutException;
009import java.net.URL;
010import java.nio.charset.StandardCharsets;
011import java.util.HashSet;
012import java.util.List;
013import java.util.Map;
014import java.util.Map.Entry;
015import java.util.Optional;
016import java.util.Set;
017import java.util.concurrent.ConcurrentHashMap;
018import java.util.concurrent.ConcurrentMap;
019import java.util.concurrent.ThreadPoolExecutor;
020import java.util.concurrent.TimeUnit;
021import java.util.regex.Matcher;
022import java.util.regex.Pattern;
023
024import org.apache.commons.jcs.access.behavior.ICacheAccess;
025import org.openstreetmap.gui.jmapviewer.Tile;
026import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
027import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
028import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
029import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
030import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
031import org.openstreetmap.josm.data.cache.CacheEntry;
032import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
033import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
034import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
035import org.openstreetmap.josm.data.preferences.LongProperty;
036import org.openstreetmap.josm.tools.HttpClient;
037import org.openstreetmap.josm.tools.Logging;
038import org.openstreetmap.josm.tools.Utils;
039
040/**
041 * Class bridging TMS requests to JCS cache requests
042 *
043 * @author Wiktor Niesiobędzki
044 * @since 8168
045 */
046public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
047    /** General maximum expires for tiles. Might be overridden by imagery settings */
048    public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
049    /** General minimum expires for tiles. Might be overridden by imagery settings */
050    public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
051    static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
052    static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*");
053    static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}");
054    protected final Tile tile;
055    private volatile URL url;
056    private final TileJobOptions options;
057
058    // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
059    // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
060    private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
061
062    /**
063     * Constructor for creating a job, to get a specific tile from cache
064     * @param listener Tile loader listener
065     * @param tile to be fetched from cache
066     * @param cache object
067     * @param options for job (such as http headers, timeouts etc.)
068     * @param downloadExecutor that will be executing the jobs
069     */
070    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
071            ICacheAccess<String, BufferedImageCacheEntry> cache,
072            TileJobOptions options,
073            ThreadPoolExecutor downloadExecutor) {
074        super(cache, options, downloadExecutor);
075        this.tile = tile;
076        this.options = options;
077        if (listener != null) {
078            inProgress.computeIfAbsent(getCacheKey(), k -> new HashSet<>()).add(listener);
079        }
080    }
081
082    @Override
083    public String getCacheKey() {
084        if (tile != null) {
085            TileSource tileSource = tile.getTileSource();
086            return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':'
087                    + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
088        }
089        return null;
090    }
091
092    /*
093     *  this doesn't needs to be synchronized, as it's not that costly to keep only one execution
094     *  in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
095     *  data from cache, that's why URL creation is postponed until it's needed
096     *
097     *  We need to have static url value for TileLoaderJob, as for some TileSources we might get different
098     *  URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
099     *
100     */
101    @Override
102    public URL getUrl() throws IOException {
103        if (url == null) {
104            synchronized (this) {
105                if (url == null) {
106                    String sUrl = tile.getUrl();
107                    if (!"".equals(sUrl)) {
108                        url = new URL(sUrl);
109                    }
110                }
111            }
112        }
113        return url;
114    }
115
116    @Override
117    public boolean isObjectLoadable() {
118        if (cacheData != null) {
119            byte[] content = cacheData.getContent();
120            try {
121                return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
122            } catch (IOException e) {
123                Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
124                        tile.getKey(), e.getMessage());
125            }
126        }
127        return false;
128    }
129
130    @Override
131    protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
132        attributes.setMetadata(tile.getTileSource().getMetadata(headers));
133        if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
134            attributes.setNoTileAtZoom(true);
135            return false; // do no try to load data from no-tile at zoom, cache empty object instead
136        }
137        return super.isResponseLoadable(headers, statusCode, content);
138    }
139
140    @Override
141    protected boolean cacheAsEmpty() {
142        return isNoTileAtZoom() || super.cacheAsEmpty();
143    }
144
145    @Override
146    public void submit(boolean force) {
147        tile.initLoading();
148        try {
149            super.submit(this, force);
150        } catch (IOException | IllegalArgumentException e) {
151            // if we fail to submit the job, mark tile as loaded and set error message
152            Logging.log(Logging.LEVEL_WARN, e);
153            tile.finishLoading();
154            tile.setError(e.getMessage());
155        }
156    }
157
158    @Override
159    public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
160        this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
161        Set<TileLoaderListener> listeners = inProgress.remove(getCacheKey());
162        boolean status = result == LoadResult.SUCCESS;
163
164        try {
165            tile.finishLoading(); // whatever happened set that loading has finished
166            // set tile metadata
167            if (this.attributes != null) {
168                for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
169                    tile.putValue(e.getKey(), e.getValue());
170                }
171            }
172
173            switch(result) {
174            case SUCCESS:
175                handleNoTileAtZoom();
176                if (attributes != null) {
177                    int httpStatusCode = attributes.getResponseCode();
178                    if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
179                        status = false;
180                        handleError(attributes);
181                    }
182                }
183                status &= tryLoadTileImage(object); //try to keep returned image as background
184                break;
185            case FAILURE:
186                handleError(attributes);
187                tryLoadTileImage(object);
188                break;
189            case CANCELED:
190                tile.loadingCanceled();
191                // do nothing
192            }
193
194            // always check, if there is some listener interested in fact, that tile has finished loading
195            if (listeners != null) { // listeners might be null, if some other thread notified already about success
196                for (TileLoaderListener l: listeners) {
197                    l.tileLoadingFinished(tile, status);
198                }
199            }
200        } catch (IOException e) {
201            Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage());
202            tile.setError(e);
203            tile.setLoaded(false);
204            if (listeners != null) { // listeners might be null, if some other thread notified already about success
205                for (TileLoaderListener l: listeners) {
206                    l.tileLoadingFinished(tile, false);
207                }
208            }
209        }
210    }
211
212    private void handleError(CacheEntryAttributes attributes) {
213        if (attributes != null) {
214            int httpStatusCode = attributes.getResponseCode();
215            if (attributes.getErrorMessage() == null) {
216                tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
217            } else {
218                tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
219            }
220            if (httpStatusCode >= 500 && httpStatusCode != 599) {
221                // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException
222                tile.setLoaded(false); // treat 500 errors as temporary and try to load it again
223            }
224            // treat SocketTimeoutException as transient error
225            attributes.getException()
226            .filter(x -> x.isAssignableFrom(SocketTimeoutException.class))
227            .ifPresent(x -> tile.setLoaded(false));
228        } else {
229            tile.setError(tr("Problem loading tile"));
230            // treat unknown errors as permanent and do not try to load tile again
231        }
232    }
233
234    /**
235     * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
236     *
237     * @return base URL of TMS or server url as defined in super class
238     */
239    @Override
240    protected String getServerKey() {
241        TileSource ts = tile.getSource();
242        if (ts instanceof AbstractTMSTileSource) {
243            return ((AbstractTMSTileSource) ts).getBaseUrl();
244        }
245        return super.getServerKey();
246    }
247
248    @Override
249    protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
250        return new BufferedImageCacheEntry(content);
251    }
252
253    @Override
254    public void submit() {
255        submit(false);
256    }
257
258    @Override
259    protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
260        CacheEntryAttributes ret = super.parseHeaders(urlConn);
261        // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
262        // at least for some short period of time, but not too long
263        if (ret.getExpirationTime() < now + Math.max(MINIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) {
264            ret.setExpirationTime(now + Math.max(MINIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime())));
265        }
266        if (ret.getExpirationTime() > now + Math.max(MAXIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) {
267            ret.setExpirationTime(now + Math.max(MAXIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime())));
268        }
269        return ret;
270    }
271
272    private boolean handleNoTileAtZoom() {
273        if (isNoTileAtZoom()) {
274            Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
275            tile.setError(tr("No tiles at this zoom level"));
276            tile.putValue("tile-info", "no-tile");
277            return true;
278        }
279        return false;
280    }
281
282    private boolean isNoTileAtZoom() {
283        if (attributes == null) {
284            Logging.warn("Cache attributes are null");
285        }
286        return attributes != null && attributes.isNoTileAtZoom();
287    }
288
289    private boolean tryLoadTileImage(CacheEntry object) throws IOException {
290        if (object != null) {
291            byte[] content = object.getContent();
292            if (content.length > 0) {
293                try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
294                    tile.loadImage(in);
295                    if (tile.getImage() == null) {
296                        String s = new String(content, StandardCharsets.UTF_8);
297                        Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
298                        if (m.matches()) {
299                            String message = Utils.strip(m.group(1));
300                            tile.setError(message);
301                            Logging.error(message);
302                            Logging.debug(s);
303                        } else {
304                            tile.setError(tr("Could not load image from tile server"));
305                        }
306                        return false;
307                    }
308                } catch (UnsatisfiedLinkError | SecurityException e) {
309                    throw new IOException(e);
310                }
311            }
312        }
313        return true;
314    }
315
316    @Override
317    public String detectErrorMessage(String data) {
318        Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data);
319        Matcher json = JSON_PATTERN.matcher(data);
320        return xml.matches() ? removeCdata(Utils.strip(xml.group(1)))
321            : json.matches() ? Utils.strip(json.group(1))
322            : super.detectErrorMessage(data);
323    }
324
325    private static String removeCdata(String msg) {
326        Matcher m = CDATA_PATTERN.matcher(msg);
327        return m.matches() ? Utils.strip(m.group(1)) : msg;
328    }
329}