001// License: GPL. For details, see Readme.txt file. 002package org.openstreetmap.gui.jmapviewer; 003 004import java.io.BufferedReader; 005import java.io.ByteArrayInputStream; 006import java.io.ByteArrayOutputStream; 007import java.io.File; 008import java.io.FileInputStream; 009import java.io.FileNotFoundException; 010import java.io.FileOutputStream; 011import java.io.IOException; 012import java.io.InputStream; 013import java.io.InputStreamReader; 014import java.io.OutputStreamWriter; 015import java.io.PrintWriter; 016import java.net.HttpURLConnection; 017import java.net.URL; 018import java.net.URLConnection; 019import java.nio.charset.Charset; 020import java.util.HashMap; 021import java.util.Map; 022import java.util.Map.Entry; 023import java.util.Random; 024import java.util.logging.Level; 025import java.util.logging.Logger; 026 027import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; 028import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController; 029import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 030import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; 031import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 032import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 033import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate; 034 035/** 036 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and 037 * saves all loaded files in a directory located in the temporary directory. 038 * If a tile is present in this file cache it will not be loaded from OSM again. 039 * 040 * @author Jan Peter Stotz 041 * @author Stefan Zeller 042 */ 043public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader { 044 045 private static final Logger log = FeatureAdapter.getLogger(OsmFileCacheTileLoader.class.getName()); 046 047 protected static final String TAGS_FILE_EXT = "tags"; 048 049 private static final Charset TAGS_CHARSET = Charset.forName("UTF-8"); 050 051 public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24; 052 public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7; 053 054 protected String cacheDirBase; 055 056 protected final Map<TileSource, File> sourceCacheDirMap; 057 058 protected long maxCacheFileAge = Long.MAX_VALUE; // max. age not limited 059 protected long recheckAfter = FILE_AGE_ONE_WEEK; 060 061 public static File getDefaultCacheDir() throws SecurityException { 062 String tempDir = null; 063 String userName = System.getProperty("user.name"); 064 try { 065 tempDir = System.getProperty("java.io.tmpdir"); 066 } catch (SecurityException e) { 067 log.log(Level.WARNING, 068 "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: " 069 + e.toString()); 070 throw e; // rethrow 071 } 072 try { 073 if (tempDir == null) 074 throw new IOException("No temp directory set"); 075 String subDirName = "JMapViewerTiles"; 076 // On Linux/Unix systems we do not have a per user tmp directory. 077 // Therefore we add the user name for getting a unique dir name. 078 if (userName != null && userName.length() > 0) { 079 subDirName += "_" + userName; 080 } 081 File cacheDir = new File(tempDir, subDirName); 082 return cacheDir; 083 } catch (Exception e) { 084 } 085 return null; 086 } 087 088 /** 089 * Create a OSMFileCacheTileLoader with given cache directory. 090 * If cacheDir is not set or invalid, IOException will be thrown. 091 * @param map the listener checking for tile load events (usually the map for display) 092 * @param cacheDir directory to store cached tiles 093 */ 094 public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException { 095 super(map); 096 if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs())) 097 throw new IOException("Cannot access cache directory"); 098 099 log.finest("Tile cache directory: " + cacheDir); 100 cacheDirBase = cacheDir.getAbsolutePath(); 101 sourceCacheDirMap = new HashMap<>(); 102 } 103 104 /** 105 * Create a OSMFileCacheTileLoader with system property temp dir. 106 * If not set an IOException will be thrown. 107 * @param map the listener checking for tile load events (usually the map for display) 108 */ 109 public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException { 110 this(map, getDefaultCacheDir()); 111 } 112 113 @Override 114 public TileJob createTileLoaderJob(final Tile tile) { 115 return new FileLoadJob(tile); 116 } 117 118 protected File getSourceCacheDir(TileSource source) { 119 File dir = sourceCacheDirMap.get(source); 120 if (dir == null) { 121 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_")); 122 if (!dir.exists()) { 123 dir.mkdirs(); 124 } 125 } 126 return dir; 127 } 128 129 protected class FileLoadJob implements TileJob { 130 InputStream input = null; 131 132 Tile tile; 133 File tileCacheDir; 134 File tileFile = null; 135 Long fileMtime = null; 136 Long now = null; // current time in milliseconds (keep consistent value for the whole run) 137 138 public FileLoadJob(Tile tile) { 139 this.tile = tile; 140 } 141 142 @Override 143 public Tile getTile() { 144 return tile; 145 } 146 147 @Override 148 public void run() { 149 synchronized (tile) { 150 if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading()) 151 return; 152 tile.loaded = false; 153 tile.error = false; 154 tile.loading = true; 155 } 156 now = System.currentTimeMillis(); 157 tileCacheDir = getSourceCacheDir(tile.getSource()); 158 159 if (loadTileFromFile(recheckAfter)) { 160 log.log(Level.FINE, "TMS - found in tile cache: {0}", tile); 161 tile.setLoaded(true); 162 listener.tileLoadingFinished(tile, true); 163 return; 164 } 165 TileJob job = new TileJob() { 166 167 @Override 168 public void run() { 169 if (loadOrUpdateTile()) { 170 tile.setLoaded(true); 171 listener.tileLoadingFinished(tile, true); 172 } else { 173 // failed to download - use old cache file if available 174 if (loadTileFromFile(maxCacheFileAge)) { 175 tile.setLoaded(true); 176 tile.error = false; 177 listener.tileLoadingFinished(tile, true); 178 log.log(Level.FINE, "TMS - found stale tile in cache: {0}", tile); 179 } else { 180 // failed completely 181 tile.setLoaded(true); 182 listener.tileLoadingFinished(tile, false); 183 } 184 } 185 } 186 @Override 187 public Tile getTile() { 188 return tile; 189 } 190 }; 191 JobDispatcher.getInstance().addJob(job); 192 } 193 194 protected boolean loadOrUpdateTile() { 195 try { 196 URLConnection urlConn = loadTileFromOsm(tile); 197 if (fileMtime != null && now - fileMtime <= maxCacheFileAge) { 198 switch (tile.getSource().getTileUpdate()) { 199 case IfModifiedSince: 200 urlConn.setIfModifiedSince(fileMtime); 201 break; 202 case LastModified: 203 if (!isOsmTileNewer(fileMtime)) { 204 log.log(Level.FINE, "TMS - LastModified test: local version is up to date: {0}", tile); 205 tileFile.setLastModified(now); 206 return true; 207 } 208 break; 209 default: 210 break; 211 } 212 } 213 if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) { 214 String fileETag = tile.getValue("etag"); 215 if (fileETag != null) { 216 switch (tile.getSource().getTileUpdate()) { 217 case IfNoneMatch: 218 urlConn.addRequestProperty("If-None-Match", fileETag); 219 break; 220 case ETag: 221 if (hasOsmTileETag(fileETag)) { 222 log.log(Level.FINE, "TMS - ETag test: local version is up to date: {0}", tile); 223 tileFile.setLastModified(now); 224 return true; 225 } 226 default: 227 break; 228 } 229 } 230 tile.putValue("etag", urlConn.getHeaderField("ETag")); 231 } 232 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { 233 // If we are isModifiedSince or If-None-Match has been set 234 // and the server answers with a HTTP 304 = "Not Modified" 235 switch (tile.getSource().getTileUpdate()) { 236 case IfModifiedSince: 237 log.log(Level.FINE, "TMS - IfModifiedSince test: local version is up to date: {0}", tile); 238 break; 239 case IfNoneMatch: 240 log.log(Level.FINE, "TMS - IfNoneMatch test: local version is up to date: {0}", tile); 241 break; 242 default: 243 break; 244 } 245 if (loadTileFromFile(maxCacheFileAge)) { 246 tileFile.setLastModified(now); 247 return true; 248 } 249 } 250 251 loadTileMetadata(tile, urlConn); 252 saveTagsToFile(); 253 254 if ("no-tile".equals(tile.getValue("tile-info"))) 255 { 256 log.log(Level.FINE, "TMS - No tile: tile-info=no-tile: {0}", tile); 257 tile.setError("No tile at this zoom level"); 258 return true; 259 } else { 260 for (int i = 0; i < 5; ++i) { 261 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) { 262 Thread.sleep(5000+(new Random()).nextInt(5000)); 263 continue; 264 } 265 byte[] buffer = loadTileInBuffer(urlConn); 266 if (buffer != null) { 267 tile.loadImage(new ByteArrayInputStream(buffer)); 268 saveTileToFile(buffer); 269 log.log(Level.FINE, "TMS - downloaded tile from server: {0}", tile.getUrl()); 270 return true; 271 } 272 } 273 } 274 } catch (Exception e) { 275 tile.setError(e.getMessage()); 276 if (input == null) { 277 try { 278 log.log(Level.WARNING, "TMS - Failed downloading {0}: {1}", new Object[]{tile.getUrl(), e.getMessage()}); 279 return false; 280 } catch(IOException i) { 281 } 282 } 283 } 284 log.log(Level.WARNING, "TMS - Failed downloading tile: {0}", tile); 285 return false; 286 } 287 288 protected boolean loadTileFromFile(long maxAge) { 289 try { 290 tileFile = getTileFile(); 291 if (!tileFile.exists()) 292 return false; 293 loadTagsFromFile(); 294 295 fileMtime = tileFile.lastModified(); 296 if (now - fileMtime > maxAge) 297 return false; 298 299 if ("no-tile".equals(tile.getValue("tile-info"))) { 300 tile.setError("No tile at this zoom level"); 301 if (tileFile.exists()) { 302 tileFile.delete(); 303 } 304 tileFile = null; 305 } else { 306 try (FileInputStream fin = new FileInputStream(tileFile)) { 307 if (fin.available() == 0) 308 throw new IOException("File empty"); 309 tile.loadImage(fin); 310 } 311 } 312 return true; 313 314 } catch (Exception e) { 315 log.log(Level.WARNING, "TMS - Error while loading image from tile cache: {0}; {1}", new Object[]{e.getMessage(), tile}); 316 tileFile.delete(); 317 tileFile = null; 318 fileMtime = null; 319 } 320 return false; 321 } 322 323 protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException { 324 input = urlConn.getInputStream(); 325 try { 326 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available()); 327 byte[] buffer = new byte[2048]; 328 boolean finished = false; 329 do { 330 int read = input.read(buffer); 331 if (read >= 0) { 332 bout.write(buffer, 0, read); 333 } else { 334 finished = true; 335 } 336 } while (!finished); 337 if (bout.size() == 0) 338 return null; 339 return bout.toByteArray(); 340 } finally { 341 input.close(); 342 input = null; 343 } 344 } 345 346 /** 347 * Performs a <code>HEAD</code> request for retrieving the 348 * <code>LastModified</code> header value. 349 * 350 * Note: This does only work with servers providing the 351 * <code>LastModified</code> header: 352 * <ul> 353 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.CycleMap} - supported</li> 354 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.Mapnik} - not supported</li> 355 * </ul> 356 * 357 * @param fileAge time of the 358 * @return <code>true</code> if the tile on the server is newer than the 359 * file 360 * @throws IOException 361 */ 362 protected boolean isOsmTileNewer(long fileAge) throws IOException { 363 URL url; 364 url = new URL(tile.getUrl()); 365 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 366 prepareHttpUrlConnection(urlConn); 367 urlConn.setRequestMethod("HEAD"); 368 urlConn.setReadTimeout(30000); // 30 seconds read timeout 369 // System.out.println("Tile age: " + new 370 // Date(urlConn.getLastModified()) + " / " 371 // + new Date(fileMtime)); 372 long lastModified = urlConn.getLastModified(); 373 if (lastModified == 0) 374 return true; // no LastModified time returned 375 return (lastModified > fileAge); 376 } 377 378 protected boolean hasOsmTileETag(String eTag) throws IOException { 379 URL url; 380 url = new URL(tile.getUrl()); 381 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 382 prepareHttpUrlConnection(urlConn); 383 urlConn.setRequestMethod("HEAD"); 384 urlConn.setReadTimeout(30000); // 30 seconds read timeout 385 // System.out.println("Tile age: " + new 386 // Date(urlConn.getLastModified()) + " / " 387 // + new Date(fileMtime)); 388 String osmETag = urlConn.getHeaderField("ETag"); 389 if (osmETag == null) 390 return true; 391 return (osmETag.equals(eTag)); 392 } 393 394 protected File getTileFile() { 395 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." 396 + tile.getSource().getTileType()); 397 } 398 399 protected File getTagsFile() { 400 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." 401 + TAGS_FILE_EXT); 402 } 403 404 protected void saveTileToFile(byte[] rawData) { 405 File file = getTileFile(); 406 file.getParentFile().mkdirs(); 407 try ( 408 FileOutputStream f = new FileOutputStream(file) 409 ) { 410 f.write(rawData); 411 } catch (Exception e) { 412 log.log(Level.SEVERE, "Failed to save tile content: {0}", e.getLocalizedMessage()); 413 } 414 } 415 416 protected void saveTagsToFile() { 417 File tagsFile = getTagsFile(); 418 tagsFile.getParentFile().mkdirs(); 419 if (tile.getMetadata() == null) { 420 tagsFile.delete(); 421 return; 422 } 423 try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) { 424 for (Entry<String, String> entry : tile.getMetadata().entrySet()) { 425 f.println(entry.getKey() + "=" + entry.getValue()); 426 } 427 } catch (Exception e) { 428 System.err.println("Failed to save tile tags: " + e.getLocalizedMessage()); 429 } 430 } 431 432 protected void loadTagsFromFile() { 433 File tagsFile = getTagsFile(); 434 try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) { 435 for (String line = f.readLine(); line != null; line = f.readLine()) { 436 final int i = line.indexOf('='); 437 if (i == -1 || i == 0) { 438 System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line); 439 continue; 440 } 441 tile.putValue(line.substring(0,i),line.substring(i+1)); 442 } 443 } catch (FileNotFoundException e) { 444 } catch (Exception e) { 445 System.err.println("Failed to load tile tags: " + e.getLocalizedMessage()); 446 } 447 } 448 } 449 450 public long getMaxFileAge() { 451 return maxCacheFileAge; 452 } 453 454 /** 455 * Sets the maximum age of the local cached tile in the file system. If a 456 * local tile is older than the specified file age 457 * {@link OsmFileCacheTileLoader} will connect to the tile server and check 458 * if a newer tile is available using the mechanism specified for the 459 * selected tile source/server. 460 * 461 * @param maxFileAge 462 * maximum age in milliseconds 463 * @see #FILE_AGE_ONE_DAY 464 * @see #FILE_AGE_ONE_WEEK 465 * @see TileSource#getTileUpdate() 466 */ 467 public void setCacheMaxFileAge(long maxFileAge) { 468 this.maxCacheFileAge = maxFileAge; 469 } 470 471 public String getCacheDirBase() { 472 return cacheDirBase; 473 } 474 475 public void setTileCacheDir(String tileCacheDir) { 476 File dir = new File(tileCacheDir); 477 dir.mkdirs(); 478 this.cacheDirBase = dir.getAbsolutePath(); 479 } 480 481 @Override 482 public void clearCache(TileSource source) { 483 clearCache(source, null); 484 } 485 486 @Override 487 public void clearCache(TileSource source, TileClearController controller) { 488 File dir = getSourceCacheDir(source); 489 if (dir != null) { 490 if (controller != null) controller.initClearDir(dir); 491 if (dir.isDirectory()) { 492 File[] files = dir.listFiles(); 493 if (controller != null) controller.initClearFiles(files); 494 for (File file : files) { 495 if (controller != null && controller.cancel()) return; 496 file.delete(); 497 if (controller != null) controller.fileDeleted(file); 498 } 499 } 500 dir.delete(); 501 } 502 if (controller != null) controller.clearFinished(); 503 } 504}