001 // License: GPL. For details, see LICENSE file. 002 package org.openstreetmap.josm.data.imagery; 003 004 005 import java.awt.Graphics2D; 006 import java.awt.image.BufferedImage; 007 import java.io.BufferedOutputStream; 008 import java.io.File; 009 import java.io.FileInputStream; 010 import java.io.FileNotFoundException; 011 import java.io.FileOutputStream; 012 import java.io.IOException; 013 import java.io.InputStream; 014 import java.io.OutputStream; 015 import java.lang.ref.SoftReference; 016 import java.net.URLConnection; 017 import java.text.SimpleDateFormat; 018 import java.util.ArrayList; 019 import java.util.Calendar; 020 import java.util.Collections; 021 import java.util.Comparator; 022 import java.util.HashMap; 023 import java.util.HashSet; 024 import java.util.Iterator; 025 import java.util.List; 026 import java.util.Map; 027 import java.util.Properties; 028 import java.util.Set; 029 030 import javax.imageio.ImageIO; 031 import javax.xml.bind.JAXBContext; 032 import javax.xml.bind.Marshaller; 033 import javax.xml.bind.Unmarshaller; 034 035 import org.openstreetmap.josm.Main; 036 import org.openstreetmap.josm.data.ProjectionBounds; 037 import org.openstreetmap.josm.data.coor.EastNorth; 038 import org.openstreetmap.josm.data.coor.LatLon; 039 import org.openstreetmap.josm.data.imagery.types.EntryType; 040 import org.openstreetmap.josm.data.imagery.types.ProjectionType; 041 import org.openstreetmap.josm.data.imagery.types.WmsCacheType; 042 import org.openstreetmap.josm.data.preferences.StringProperty; 043 import org.openstreetmap.josm.data.projection.Projection; 044 import org.openstreetmap.josm.gui.NavigatableComponent; 045 import org.openstreetmap.josm.tools.Utils; 046 047 048 049 public class WmsCache { 050 //TODO Property for maximum cache size 051 //TODO Property for maximum age of tile, automatically remove old tiles 052 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache 053 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load 054 055 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms"); 056 private static final String INDEX_FILENAME = "index.xml"; 057 private static final String LAYERS_INDEX_FILENAME = "layers.properties"; 058 059 private static class CacheEntry { 060 final double pixelPerDegree; 061 final double east; 062 final double north; 063 final ProjectionBounds bounds; 064 final String filename; 065 066 long lastUsed; 067 long lastModified; 068 069 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) { 070 this.pixelPerDegree = pixelPerDegree; 071 this.east = east; 072 this.north = north; 073 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree); 074 this.filename = filename; 075 } 076 } 077 078 private static class ProjectionEntries { 079 final String projection; 080 final String cacheDirectory; 081 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>(); 082 083 ProjectionEntries(String projection, String cacheDirectory) { 084 this.projection = projection; 085 this.cacheDirectory = cacheDirectory; 086 } 087 } 088 089 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>(); 090 private final File cacheDir; 091 private final int tileSize; // Should be always 500 092 private int totalFileSize; 093 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated 094 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found 095 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>(); 096 private Set<ProjectionBounds> areaToCache; 097 098 protected String cacheDirPath() { 099 String cPath = PROP_CACHE_PATH.get(); 100 if (!(new File(cPath).isAbsolute())) { 101 cPath = Main.pref.getCacheDirectory() + File.separator + cPath; 102 } 103 // Migrate to new cache directory. Remove 2012-06 104 { 105 File oldPath = new File(Main.pref.getPreferencesDirFile(), "wms-cache"); 106 File newPath = new File(cPath); 107 if (oldPath.exists() && !newPath.exists()) { 108 oldPath.renameTo(newPath); 109 } 110 } 111 return cPath; 112 } 113 114 public WmsCache(String url, int tileSize) { 115 File globalCacheDir = new File(cacheDirPath()); 116 globalCacheDir.mkdirs(); 117 cacheDir = new File(globalCacheDir, getCacheDirectory(url)); 118 cacheDir.mkdirs(); 119 this.tileSize = tileSize; 120 } 121 122 private String getCacheDirectory(String url) { 123 String cacheDirName = null; 124 InputStream fis = null; 125 OutputStream fos = null; 126 try { 127 Properties layersIndex = new Properties(); 128 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME); 129 try { 130 fis = new FileInputStream(layerIndexFile); 131 layersIndex.load(fis); 132 } catch (FileNotFoundException e) { 133 System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)"); 134 } catch (IOException e) { 135 System.err.println("Unable to load layers index for wms cache"); 136 e.printStackTrace(); 137 } 138 139 for (Object propKey: layersIndex.keySet()) { 140 String s = (String)propKey; 141 if (url.equals(layersIndex.getProperty(s))) { 142 cacheDirName = s; 143 break; 144 } 145 } 146 147 if (cacheDirName == null) { 148 int counter = 0; 149 while (true) { 150 counter++; 151 if (!layersIndex.keySet().contains(String.valueOf(counter))) { 152 break; 153 } 154 } 155 cacheDirName = String.valueOf(counter); 156 layersIndex.setProperty(cacheDirName, url); 157 try { 158 fos = new FileOutputStream(layerIndexFile); 159 layersIndex.store(fos, ""); 160 } catch (IOException e) { 161 System.err.println("Unable to save layer index for wms cache"); 162 e.printStackTrace(); 163 } 164 } 165 } finally { 166 try { 167 if (fis != null) { 168 fis.close(); 169 } 170 if (fos != null) { 171 fos.close(); 172 } 173 } catch (IOException e) { 174 e.printStackTrace(); 175 } 176 } 177 178 return cacheDirName; 179 } 180 181 private ProjectionEntries getProjectionEntries(Projection projection) { 182 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName()); 183 } 184 185 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) { 186 ProjectionEntries result = entries.get(projection); 187 if (result == null) { 188 result = new ProjectionEntries(projection, cacheDirectory); 189 entries.put(projection, result); 190 } 191 192 return result; 193 } 194 195 public synchronized void loadIndex() { 196 File indexFile = new File(cacheDir, INDEX_FILENAME); 197 try { 198 JAXBContext context = JAXBContext.newInstance( 199 WmsCacheType.class.getPackage().getName(), 200 WmsCacheType.class.getClassLoader()); 201 Unmarshaller unmarshaller = context.createUnmarshaller(); 202 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile)); 203 totalFileSize = cacheEntries.getTotalFileSize(); 204 if (cacheEntries.getTileSize() != tileSize) { 205 System.out.println("Cache created with different tileSize, cache will be discarded"); 206 return; 207 } 208 for (ProjectionType projectionType: cacheEntries.getProjection()) { 209 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory()); 210 for (EntryType entry: projectionType.getEntry()) { 211 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename()); 212 ce.lastUsed = entry.getLastUsed().getTimeInMillis(); 213 ce.lastModified = entry.getLastModified().getTimeInMillis(); 214 projection.entries.add(ce); 215 } 216 } 217 } catch (Exception e) { 218 if (indexFile.exists()) { 219 e.printStackTrace(); 220 System.out.println("Unable to load index for wms-cache, new file will be created"); 221 } else { 222 System.out.println("Index for wms-cache doesn't exist, new file will be created"); 223 } 224 } 225 226 removeNonReferencedFiles(); 227 } 228 229 private void removeNonReferencedFiles() { 230 231 Set<String> usedProjections = new HashSet<String>(); 232 233 for (ProjectionEntries projectionEntries: entries.values()) { 234 235 usedProjections.add(projectionEntries.cacheDirectory); 236 237 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory); 238 if (projectionDir.exists()) { 239 Set<String> referencedFiles = new HashSet<String>(); 240 241 for (CacheEntry ce: projectionEntries.entries) { 242 referencedFiles.add(ce.filename); 243 } 244 245 for (File file: projectionDir.listFiles()) { 246 if (!referencedFiles.contains(file.getName())) { 247 file.delete(); 248 } 249 } 250 } 251 } 252 253 for (File projectionDir: cacheDir.listFiles()) { 254 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) { 255 Utils.deleteDirectory(projectionDir); 256 } 257 } 258 } 259 260 private int calculateTotalFileSize() { 261 int result = 0; 262 for (ProjectionEntries projectionEntries: entries.values()) { 263 Iterator<CacheEntry> it = projectionEntries.entries.iterator(); 264 while (it.hasNext()) { 265 CacheEntry entry = it.next(); 266 File imageFile = getImageFile(projectionEntries, entry); 267 if (!imageFile.exists()) { 268 it.remove(); 269 } else { 270 result += imageFile.length(); 271 } 272 } 273 } 274 return result; 275 } 276 277 public synchronized void saveIndex() { 278 WmsCacheType index = new WmsCacheType(); 279 280 if (totalFileSizeDirty) { 281 totalFileSize = calculateTotalFileSize(); 282 } 283 284 index.setTileSize(tileSize); 285 index.setTotalFileSize(totalFileSize); 286 for (ProjectionEntries projectionEntries: entries.values()) { 287 if (projectionEntries.entries.size() > 0) { 288 ProjectionType projectionType = new ProjectionType(); 289 projectionType.setName(projectionEntries.projection); 290 projectionType.setCacheDirectory(projectionEntries.cacheDirectory); 291 index.getProjection().add(projectionType); 292 for (CacheEntry ce: projectionEntries.entries) { 293 EntryType entry = new EntryType(); 294 entry.setPixelPerDegree(ce.pixelPerDegree); 295 entry.setEast(ce.east); 296 entry.setNorth(ce.north); 297 Calendar c = Calendar.getInstance(); 298 c.setTimeInMillis(ce.lastUsed); 299 entry.setLastUsed(c); 300 c = Calendar.getInstance(); 301 c.setTimeInMillis(ce.lastModified); 302 entry.setLastModified(c); 303 entry.setFilename(ce.filename); 304 projectionType.getEntry().add(entry); 305 } 306 } 307 } 308 try { 309 JAXBContext context = JAXBContext.newInstance( 310 WmsCacheType.class.getPackage().getName(), 311 WmsCacheType.class.getClassLoader()); 312 Marshaller marshaller = context.createMarshaller(); 313 marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME))); 314 } catch (Exception e) { 315 System.err.println("Failed to save wms-cache file"); 316 e.printStackTrace(); 317 } 318 } 319 320 private File getImageFile(ProjectionEntries projection, CacheEntry entry) { 321 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename); 322 } 323 324 325 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException { 326 327 synchronized (this) { 328 entry.lastUsed = System.currentTimeMillis(); 329 330 SoftReference<BufferedImage> memCache = memoryCache.get(entry); 331 if (memCache != null) { 332 BufferedImage result = memCache.get(); 333 if (result != null) 334 return result; 335 } 336 } 337 338 try { 339 // Reading can't be in synchronized section, it's too slow 340 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry)); 341 synchronized (this) { 342 if (result == null) { 343 projectionEntries.entries.remove(entry); 344 totalFileSizeDirty = true; 345 } 346 return result; 347 } 348 } catch (IOException e) { 349 synchronized (this) { 350 projectionEntries.entries.remove(entry); 351 totalFileSizeDirty = true; 352 throw e; 353 } 354 } 355 } 356 357 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) { 358 for (CacheEntry entry: projectionEntries.entries) { 359 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north) 360 return entry; 361 } 362 return null; 363 } 364 365 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) { 366 ProjectionEntries projectionEntries = getProjectionEntries(projection); 367 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north); 368 return (entry != null); 369 } 370 371 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) { 372 CacheEntry entry = null; 373 ProjectionEntries projectionEntries = null; 374 synchronized (this) { 375 projectionEntries = getProjectionEntries(projection); 376 entry = findEntry(projectionEntries, pixelPerDegree, east, north); 377 } 378 if (entry != null) { 379 try { 380 return loadImage(projectionEntries, entry); 381 } catch (IOException e) { 382 System.err.println("Unable to load file from wms cache"); 383 e.printStackTrace(); 384 return null; 385 } 386 } 387 return null; 388 } 389 390 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) { 391 ProjectionEntries projectionEntries; 392 List<CacheEntry> matches; 393 synchronized (this) { 394 matches = new ArrayList<WmsCache.CacheEntry>(); 395 396 double minPPD = pixelPerDegree / 5; 397 double maxPPD = pixelPerDegree * 5; 398 projectionEntries = getProjectionEntries(projection); 399 400 double size2 = tileSize / pixelPerDegree; 401 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly 402 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border, 403 east + size2 - border, north + size2 - border); 404 405 //TODO Do not load tile if it is completely overlapped by other tile with better ppd 406 for (CacheEntry entry: projectionEntries.entries) { 407 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) { 408 entry.lastUsed = System.currentTimeMillis(); 409 matches.add(entry); 410 } 411 } 412 413 if (matches.isEmpty()) 414 return null; 415 416 417 Collections.sort(matches, new Comparator<CacheEntry>() { 418 @Override 419 public int compare(CacheEntry o1, CacheEntry o2) { 420 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree); 421 } 422 }); 423 } 424 425 //TODO Use alpha layer only when enabled on wms layer 426 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR); 427 Graphics2D g = result.createGraphics(); 428 429 430 boolean drawAtLeastOnce = false; 431 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>(); 432 for (CacheEntry ce: matches) { 433 BufferedImage img; 434 try { 435 img = loadImage(projectionEntries, ce); 436 localCache.put(ce, new SoftReference<BufferedImage>(img)); 437 } catch (IOException e) { 438 continue; 439 } 440 441 drawAtLeastOnce = true; 442 443 int xDiff = (int)((ce.east - east) * pixelPerDegree); 444 int yDiff = (int)((ce.north - north) * pixelPerDegree); 445 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize); 446 447 int x = xDiff; 448 int y = -size + tileSize - yDiff; 449 450 g.drawImage(img, x, y, size, size, null); 451 } 452 453 if (drawAtLeastOnce) { 454 synchronized (this) { 455 memoryCache.putAll(localCache); 456 } 457 return result; 458 } else 459 return null; 460 } 461 462 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) { 463 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north)); 464 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north)); 465 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree)); 466 467 double deltaLat = Math.abs(ll3.lat() - ll1.lat()); 468 double deltaLon = Math.abs(ll3.lon() - ll1.lon()); 469 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1); 470 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1); 471 472 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2)); 473 String extension; 474 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) { 475 extension = "jpg"; 476 } else if ("image/png".equals(mimeType)) { 477 extension = "png"; 478 } else if ("image/gif".equals(mimeType)) { 479 extension = "gif"; 480 } else { 481 extension = "dat"; 482 } 483 484 int counter = 0; 485 FILENAME_LOOP: 486 while (true) { 487 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension); 488 for (CacheEntry entry: projectionEntries.entries) { 489 if (entry.filename.equals(result)) { 490 counter++; 491 continue FILENAME_LOOP; 492 } 493 } 494 return result; 495 } 496 } 497 498 /** 499 * 500 * @param img Used only when overlapping is used, when not used, used raw from imageData 501 * @param imageData 502 * @param projection 503 * @param pixelPerDegree 504 * @param east 505 * @param north 506 * @throws IOException 507 */ 508 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException { 509 ProjectionEntries projectionEntries = getProjectionEntries(projection); 510 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north); 511 File imageFile; 512 if (entry == null) { 513 514 String mimeType; 515 if (img != null) { 516 mimeType = "image/png"; 517 } else { 518 mimeType = URLConnection.guessContentTypeFromStream(imageData); 519 } 520 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType)); 521 entry.lastUsed = System.currentTimeMillis(); 522 entry.lastModified = entry.lastUsed; 523 projectionEntries.entries.add(entry); 524 imageFile = getImageFile(projectionEntries, entry); 525 } else { 526 imageFile = getImageFile(projectionEntries, entry); 527 totalFileSize -= imageFile.length(); 528 } 529 530 imageFile.getParentFile().mkdirs(); 531 532 if (img != null) { 533 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType()); 534 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null); 535 ImageIO.write(copy, "png", imageFile); 536 totalFileSize += imageFile.length(); 537 } else { 538 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile)); 539 try { 540 totalFileSize += Utils.copyStream(imageData, os); 541 } finally { 542 os.close(); 543 } 544 } 545 } 546 547 public synchronized void cleanSmallFiles(int size) { 548 for (ProjectionEntries projectionEntries: entries.values()) { 549 Iterator<CacheEntry> it = projectionEntries.entries.iterator(); 550 while (it.hasNext()) { 551 File file = getImageFile(projectionEntries, it.next()); 552 long length = file.length(); 553 if (length <= size) { 554 if (length == 0) { 555 totalFileSizeDirty = true; // File probably doesn't exist 556 } 557 totalFileSize -= size; 558 file.delete(); 559 it.remove(); 560 } 561 } 562 } 563 } 564 565 public static String printDate(Calendar c) { 566 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime()); 567 } 568 569 private boolean isInsideAreaToCache(CacheEntry cacheEntry) { 570 for (ProjectionBounds b: areaToCache) { 571 if (cacheEntry.bounds.intersects(b)) 572 return true; 573 } 574 return false; 575 } 576 577 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) { 578 this.areaToCache = areaToCache; 579 Iterator<CacheEntry> it = memoryCache.keySet().iterator(); 580 while (it.hasNext()) { 581 if (!isInsideAreaToCache(it.next())) { 582 it.remove(); 583 } 584 } 585 } 586 }