001 package org.openstreetmap.gui.jmapviewer; 002 003 //License: GPL. Copyright 2008 by Jan Peter Stotz 004 005 import java.io.BufferedReader; 006 import java.io.ByteArrayInputStream; 007 import java.io.ByteArrayOutputStream; 008 import java.io.File; 009 import java.io.FileInputStream; 010 import java.io.FileNotFoundException; 011 import java.io.FileOutputStream; 012 import java.io.IOException; 013 import java.io.InputStream; 014 import java.io.InputStreamReader; 015 import java.io.OutputStreamWriter; 016 import java.io.PrintWriter; 017 import java.lang.Thread; 018 import java.net.HttpURLConnection; 019 import java.net.URL; 020 import java.net.URLConnection; 021 import java.nio.charset.Charset; 022 import java.util.HashMap; 023 import java.util.Map; 024 import java.util.Map.Entry; 025 import java.util.Random; 026 import java.util.logging.Level; 027 import java.util.logging.Logger; 028 029 import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 030 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; 031 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 032 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 033 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate; 034 035 /** 036 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and 037 * saves all loaded files in a directory located in the temporary directory. 038 * If a tile is present in this file cache it will not be loaded from OSM again. 039 * 040 * @author Jan Peter Stotz 041 * @author Stefan Zeller 042 */ 043 public class OsmFileCacheTileLoader extends OsmTileLoader { 044 045 private static final Logger log = Logger.getLogger(OsmFileCacheTileLoader.class.getName()); 046 047 private static final String ETAG_FILE_EXT = ".etag"; 048 private static final String TAGS_FILE_EXT = ".tags"; 049 050 private static final Charset TAGS_CHARSET = Charset.forName("UTF-8"); 051 052 public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24; 053 public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7; 054 055 protected String cacheDirBase; 056 057 protected final Map<TileSource, File> sourceCacheDirMap; 058 059 protected long maxCacheFileAge = FILE_AGE_ONE_WEEK; 060 protected long recheckAfter = FILE_AGE_ONE_DAY; 061 062 public static File getDefaultCacheDir() throws SecurityException { 063 String tempDir = null; 064 String userName = System.getProperty("user.name"); 065 try { 066 tempDir = System.getProperty("java.io.tmpdir"); 067 } catch (SecurityException e) { 068 log.log(Level.WARNING, 069 "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: " 070 + e.toString()); 071 throw e; // rethrow 072 } 073 try { 074 if (tempDir == null) 075 throw new IOException("No temp directory set"); 076 String subDirName = "JMapViewerTiles"; 077 // On Linux/Unix systems we do not have a per user tmp directory. 078 // Therefore we add the user name for getting a unique dir name. 079 if (userName != null && userName.length() > 0) { 080 subDirName += "_" + userName; 081 } 082 File cacheDir = new File(tempDir, subDirName); 083 return cacheDir; 084 } catch (Exception e) { 085 } 086 return null; 087 } 088 089 /** 090 * Create a OSMFileCacheTileLoader with given cache directory. 091 * If cacheDir is not set or invalid, IOException will be thrown. 092 * @param map the listener checking for tile load events (usually the map for display) 093 * @param cacheDir directory to store cached tiles 094 */ 095 public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException { 096 super(map); 097 if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs())) 098 throw new IOException("Cannot access cache directory"); 099 100 log.finest("Tile cache directory: " + cacheDir); 101 cacheDirBase = cacheDir.getAbsolutePath(); 102 sourceCacheDirMap = new HashMap<TileSource, File>(); 103 } 104 105 /** 106 * Create a OSMFileCacheTileLoader with system property temp dir. 107 * If not set an IOException will be thrown. 108 * @param map the listener checking for tile load events (usually the map for display) 109 */ 110 public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException { 111 this(map, getDefaultCacheDir()); 112 } 113 114 @Override 115 public TileJob createTileLoaderJob(final Tile tile) { 116 return new FileLoadJob(tile); 117 } 118 119 protected File getSourceCacheDir(TileSource source) { 120 File dir = sourceCacheDirMap.get(source); 121 if (dir == null) { 122 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_")); 123 if (!dir.exists()) { 124 dir.mkdirs(); 125 } 126 } 127 return dir; 128 } 129 130 protected class FileLoadJob implements TileJob { 131 InputStream input = null; 132 133 Tile tile; 134 File tileCacheDir; 135 File tileFile = null; 136 long fileAge = 0; 137 boolean fileTilePainted = false; 138 139 public FileLoadJob(Tile tile) { 140 this.tile = tile; 141 } 142 143 public Tile getTile() { 144 return tile; 145 } 146 147 public void run() { 148 synchronized (tile) { 149 if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading()) 150 return; 151 tile.loaded = false; 152 tile.error = false; 153 tile.loading = true; 154 } 155 tileCacheDir = getSourceCacheDir(tile.getSource()); 156 if (loadTileFromFile()) { 157 return; 158 } 159 if (fileTilePainted) { 160 TileJob job = new TileJob() { 161 162 public void run() { 163 loadOrUpdateTile(); 164 } 165 public Tile getTile() { 166 return tile; 167 } 168 }; 169 JobDispatcher.getInstance().addJob(job); 170 } else { 171 loadOrUpdateTile(); 172 } 173 } 174 175 protected void loadOrUpdateTile() { 176 try { 177 URLConnection urlConn = loadTileFromOsm(tile); 178 if (tileFile != null) { 179 switch (tile.getSource().getTileUpdate()) { 180 case IfModifiedSince: 181 urlConn.setIfModifiedSince(fileAge); 182 break; 183 case LastModified: 184 if (!isOsmTileNewer(fileAge)) { 185 log.finest("LastModified test: local version is up to date: " + tile); 186 tile.setLoaded(true); 187 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter); 188 return; 189 } 190 break; 191 } 192 } 193 if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) { 194 String fileETag = tile.getValue("etag"); 195 if (fileETag != null) { 196 switch (tile.getSource().getTileUpdate()) { 197 case IfNoneMatch: 198 urlConn.addRequestProperty("If-None-Match", fileETag); 199 break; 200 case ETag: 201 if (hasOsmTileETag(fileETag)) { 202 tile.setLoaded(true); 203 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge 204 + recheckAfter); 205 return; 206 } 207 } 208 } 209 tile.putValue("etag", urlConn.getHeaderField("ETag")); 210 } 211 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { 212 // If we are isModifiedSince or If-None-Match has been set 213 // and the server answers with a HTTP 304 = "Not Modified" 214 log.finest("ETag test: local version is up to date: " + tile); 215 tile.setLoaded(true); 216 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter); 217 return; 218 } 219 220 loadTileMetadata(tile, urlConn); 221 saveTagsToFile(); 222 223 if ("no-tile".equals(tile.getValue("tile-info"))) 224 { 225 tile.setError("No tile at this zoom level"); 226 listener.tileLoadingFinished(tile, true); 227 } else { 228 for(int i = 0; i < 5; ++i) { 229 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) { 230 Thread.sleep(5000+(new Random()).nextInt(5000)); 231 continue; 232 } 233 byte[] buffer = loadTileInBuffer(urlConn); 234 if (buffer != null) { 235 tile.loadImage(new ByteArrayInputStream(buffer)); 236 tile.setLoaded(true); 237 listener.tileLoadingFinished(tile, true); 238 saveTileToFile(buffer); 239 break; 240 } 241 } 242 } 243 } catch (Exception e) { 244 tile.setError(e.getMessage()); 245 listener.tileLoadingFinished(tile, false); 246 if (input == null) { 247 try { 248 System.err.println("Failed loading " + tile.getUrl() +": " + e.getMessage()); 249 } catch(IOException i) { 250 } 251 } 252 } finally { 253 tile.loading = false; 254 tile.setLoaded(true); 255 } 256 } 257 258 protected boolean loadTileFromFile() { 259 FileInputStream fin = null; 260 try { 261 tileFile = getTileFile(); 262 if (!tileFile.exists()) 263 return false; 264 265 loadTagsFromFile(); 266 if ("no-tile".equals(tile.getValue("tile-info"))) 267 { 268 tile.setError("No tile at this zoom level"); 269 if (tileFile.exists()) { 270 tileFile.delete(); 271 } 272 tileFile = getTagsFile(); 273 } else { 274 fin = new FileInputStream(tileFile); 275 if (fin.available() == 0) 276 throw new IOException("File empty"); 277 tile.loadImage(fin); 278 fin.close(); 279 } 280 281 fileAge = tileFile.lastModified(); 282 boolean oldTile = System.currentTimeMillis() - fileAge > maxCacheFileAge; 283 if (!oldTile) { 284 tile.setLoaded(true); 285 listener.tileLoadingFinished(tile, true); 286 fileTilePainted = true; 287 return true; 288 } 289 listener.tileLoadingFinished(tile, true); 290 fileTilePainted = true; 291 } catch (Exception e) { 292 try { 293 if (fin != null) { 294 fin.close(); 295 tileFile.delete(); 296 } 297 } catch (Exception e1) { 298 } 299 tileFile = null; 300 fileAge = 0; 301 } 302 return false; 303 } 304 305 protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException { 306 input = urlConn.getInputStream(); 307 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available()); 308 byte[] buffer = new byte[2048]; 309 boolean finished = false; 310 do { 311 int read = input.read(buffer); 312 if (read >= 0) { 313 bout.write(buffer, 0, read); 314 } else { 315 finished = true; 316 } 317 } while (!finished); 318 if (bout.size() == 0) 319 return null; 320 return bout.toByteArray(); 321 } 322 323 /** 324 * Performs a <code>HEAD</code> request for retrieving the 325 * <code>LastModified</code> header value. 326 * 327 * Note: This does only work with servers providing the 328 * <code>LastModified</code> header: 329 * <ul> 330 * <li>{@link tilesources.OsmTileSource.CycleMap} - supported</li> 331 * <li>{@link tilesources.OsmTileSource.Mapnik} - not supported</li> 332 * </ul> 333 * 334 * @param fileAge time of the 335 * @return <code>true</code> if the tile on the server is newer than the 336 * file 337 * @throws IOException 338 */ 339 protected boolean isOsmTileNewer(long fileAge) throws IOException { 340 URL url; 341 url = new URL(tile.getUrl()); 342 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 343 prepareHttpUrlConnection(urlConn); 344 urlConn.setRequestMethod("HEAD"); 345 urlConn.setReadTimeout(30000); // 30 seconds read timeout 346 // System.out.println("Tile age: " + new 347 // Date(urlConn.getLastModified()) + " / " 348 // + new Date(fileAge)); 349 long lastModified = urlConn.getLastModified(); 350 if (lastModified == 0) 351 return true; // no LastModified time returned 352 return (lastModified > fileAge); 353 } 354 355 protected boolean hasOsmTileETag(String eTag) throws IOException { 356 URL url; 357 url = new URL(tile.getUrl()); 358 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 359 prepareHttpUrlConnection(urlConn); 360 urlConn.setRequestMethod("HEAD"); 361 urlConn.setReadTimeout(30000); // 30 seconds read timeout 362 // System.out.println("Tile age: " + new 363 // Date(urlConn.getLastModified()) + " / " 364 // + new Date(fileAge)); 365 String osmETag = urlConn.getHeaderField("ETag"); 366 if (osmETag == null) 367 return true; 368 return (osmETag.equals(eTag)); 369 } 370 371 protected File getTileFile() { 372 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." 373 + tile.getSource().getTileType()); 374 } 375 376 protected File getTagsFile() { 377 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() 378 + TAGS_FILE_EXT); 379 } 380 381 protected void saveTileToFile(byte[] rawData) { 382 try { 383 FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() 384 + "_" + tile.getYtile() + "." + tile.getSource().getTileType()); 385 f.write(rawData); 386 f.close(); 387 // System.out.println("Saved tile to file: " + tile); 388 } catch (Exception e) { 389 System.err.println("Failed to save tile content: " + e.getLocalizedMessage()); 390 } 391 } 392 393 protected void saveTagsToFile() { 394 File tagsFile = getTagsFile(); 395 if (tile.getMetadata() == null) { 396 tagsFile.delete(); 397 return; 398 } 399 try { 400 final PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), 401 TAGS_CHARSET)); 402 for (Entry<String, String> entry : tile.getMetadata().entrySet()) { 403 f.println(entry.getKey() + "=" + entry.getValue()); 404 } 405 f.close(); 406 } catch (Exception e) { 407 System.err.println("Failed to save tile tags: " + e.getLocalizedMessage()); 408 } 409 } 410 411 /** Load backward-compatiblity .etag file and if it exists move it to new .tags file*/ 412 private void loadOldETagfromFile() { 413 File etagFile = new File(tileCacheDir, tile.getZoom() + "_" 414 + tile.getXtile() + "_" + tile.getYtile() + ETAG_FILE_EXT); 415 if (!etagFile.exists()) return; 416 try { 417 FileInputStream f = new FileInputStream(etagFile); 418 byte[] buf = new byte[f.available()]; 419 f.read(buf); 420 f.close(); 421 String etag = new String(buf, TAGS_CHARSET.name()); 422 tile.putValue("etag", etag); 423 if (etagFile.delete()) { 424 saveTagsToFile(); 425 } 426 } catch (IOException e) { 427 System.err.println("Failed to load compatiblity etag: " + e.getLocalizedMessage()); 428 } 429 } 430 431 protected void loadTagsFromFile() { 432 loadOldETagfromFile(); 433 File tagsFile = getTagsFile(); 434 try { 435 final BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), 436 TAGS_CHARSET)); 437 for (String line = f.readLine(); line != null; line = f.readLine()) { 438 final int i = line.indexOf('='); 439 if (i == -1 || i == 0) { 440 System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line); 441 continue; 442 } 443 tile.putValue(line.substring(0,i),line.substring(i+1)); 444 } 445 f.close(); 446 } catch (FileNotFoundException e) { 447 } catch (Exception e) { 448 System.err.println("Failed to load tile tags: " + e.getLocalizedMessage()); 449 } 450 } 451 452 } 453 454 public long getMaxFileAge() { 455 return maxCacheFileAge; 456 } 457 458 /** 459 * Sets the maximum age of the local cached tile in the file system. If a 460 * local tile is older than the specified file age 461 * {@link OsmFileCacheTileLoader} will connect to the tile server and check 462 * if a newer tile is available using the mechanism specified for the 463 * selected tile source/server. 464 * 465 * @param maxFileAge 466 * maximum age in milliseconds 467 * @see #FILE_AGE_ONE_DAY 468 * @see #FILE_AGE_ONE_WEEK 469 * @see TileSource#getTileUpdate() 470 */ 471 public void setCacheMaxFileAge(long maxFileAge) { 472 this.maxCacheFileAge = maxFileAge; 473 } 474 475 public String getCacheDirBase() { 476 return cacheDirBase; 477 } 478 479 public void setTileCacheDir(String tileCacheDir) { 480 File dir = new File(tileCacheDir); 481 dir.mkdirs(); 482 this.cacheDirBase = dir.getAbsolutePath(); 483 } 484 485 public static interface TileClearController { 486 487 void initClearDir(File dir); 488 489 void initClearFiles(File[] files); 490 491 boolean cancel(); 492 493 void fileDeleted(File file); 494 495 void clearFinished(); 496 } 497 498 public void clearCache(TileSource source) { 499 clearCache(source, null); 500 } 501 502 public void clearCache(TileSource source, TileClearController controller) { 503 File dir = getSourceCacheDir(source); 504 if (dir != null) { 505 if (controller != null) controller.initClearDir(dir); 506 if (dir.isDirectory()) { 507 File[] files = dir.listFiles(); 508 if (controller != null) controller.initClearFiles(files); 509 for (File file : files) { 510 if (controller != null && controller.cancel()) return; 511 file.delete(); 512 if (controller != null) controller.fileDeleted(file); 513 } 514 } 515 dir.delete(); 516 } 517 if (controller != null) controller.clearFinished(); 518 } 519 }