001    // License: GPL. For details, see LICENSE file.
002    package org.openstreetmap.josm.data.imagery;
003    
004    
005    import java.awt.Graphics2D;
006    import java.awt.image.BufferedImage;
007    import java.io.BufferedOutputStream;
008    import java.io.File;
009    import java.io.FileInputStream;
010    import java.io.FileNotFoundException;
011    import java.io.FileOutputStream;
012    import java.io.IOException;
013    import java.io.InputStream;
014    import java.io.OutputStream;
015    import java.lang.ref.SoftReference;
016    import java.net.URLConnection;
017    import java.text.SimpleDateFormat;
018    import java.util.ArrayList;
019    import java.util.Calendar;
020    import java.util.Collections;
021    import java.util.Comparator;
022    import java.util.HashMap;
023    import java.util.HashSet;
024    import java.util.Iterator;
025    import java.util.List;
026    import java.util.Map;
027    import java.util.Properties;
028    import java.util.Set;
029    
030    import javax.imageio.ImageIO;
031    import javax.xml.bind.JAXBContext;
032    import javax.xml.bind.Marshaller;
033    import javax.xml.bind.Unmarshaller;
034    
035    import org.openstreetmap.josm.Main;
036    import org.openstreetmap.josm.data.ProjectionBounds;
037    import org.openstreetmap.josm.data.coor.EastNorth;
038    import org.openstreetmap.josm.data.coor.LatLon;
039    import org.openstreetmap.josm.data.imagery.types.EntryType;
040    import org.openstreetmap.josm.data.imagery.types.ProjectionType;
041    import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
042    import org.openstreetmap.josm.data.preferences.StringProperty;
043    import org.openstreetmap.josm.data.projection.Projection;
044    import org.openstreetmap.josm.gui.NavigatableComponent;
045    import org.openstreetmap.josm.tools.Utils;
046    
047    
048    
049    public class WmsCache {
050        //TODO Property for maximum cache size
051        //TODO Property for maximum age of tile, automatically remove old tiles
052        //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
053        //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
054    
055        private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
056        private static final String INDEX_FILENAME = "index.xml";
057        private static final String LAYERS_INDEX_FILENAME = "layers.properties";
058    
059        private static class CacheEntry {
060            final double pixelPerDegree;
061            final double east;
062            final double north;
063            final ProjectionBounds bounds;
064            final String filename;
065    
066            long lastUsed;
067            long lastModified;
068    
069            CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
070                this.pixelPerDegree = pixelPerDegree;
071                this.east = east;
072                this.north = north;
073                this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
074                this.filename = filename;
075            }
076        }
077    
078        private static class ProjectionEntries {
079            final String projection;
080            final String cacheDirectory;
081            final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
082    
083            ProjectionEntries(String projection, String cacheDirectory) {
084                this.projection = projection;
085                this.cacheDirectory = cacheDirectory;
086            }
087        }
088    
089        private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
090        private final File cacheDir;
091        private final int tileSize; // Should be always 500
092        private int totalFileSize;
093        private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
094        // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
095        private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
096        private Set<ProjectionBounds> areaToCache;
097    
098        protected String cacheDirPath() {
099            String cPath = PROP_CACHE_PATH.get();
100            if (!(new File(cPath).isAbsolute())) {
101                cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
102            }
103            // Migrate to new cache directory. Remove 2012-06
104            {
105                File oldPath = new File(Main.pref.getPreferencesDirFile(), "wms-cache");
106                File newPath = new File(cPath);
107                if (oldPath.exists() && !newPath.exists()) {
108                    oldPath.renameTo(newPath);
109                }
110            }
111            return cPath;
112        }
113    
114        public WmsCache(String url, int tileSize) {
115            File globalCacheDir = new File(cacheDirPath());
116            globalCacheDir.mkdirs();
117            cacheDir = new File(globalCacheDir, getCacheDirectory(url));
118            cacheDir.mkdirs();
119            this.tileSize = tileSize;
120        }
121    
122        private String getCacheDirectory(String url) {
123            String cacheDirName = null;
124            InputStream fis = null;
125            OutputStream fos = null;
126            try {
127                Properties layersIndex = new Properties();
128                File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
129                try {
130                    fis = new FileInputStream(layerIndexFile);
131                    layersIndex.load(fis);
132                } catch (FileNotFoundException e) {
133                    System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
134                } catch (IOException e) {
135                    System.err.println("Unable to load layers index for wms cache");
136                    e.printStackTrace();
137                }
138    
139                for (Object propKey: layersIndex.keySet()) {
140                    String s = (String)propKey;
141                    if (url.equals(layersIndex.getProperty(s))) {
142                        cacheDirName = s;
143                        break;
144                    }
145                }
146    
147                if (cacheDirName == null) {
148                    int counter = 0;
149                    while (true) {
150                        counter++;
151                        if (!layersIndex.keySet().contains(String.valueOf(counter))) {
152                            break;
153                        }
154                    }
155                    cacheDirName = String.valueOf(counter);
156                    layersIndex.setProperty(cacheDirName, url);
157                    try {
158                        fos = new FileOutputStream(layerIndexFile);
159                        layersIndex.store(fos, "");
160                    } catch (IOException e) {
161                        System.err.println("Unable to save layer index for wms cache");
162                        e.printStackTrace();
163                    }
164                }
165            } finally {
166                try {
167                    if (fis != null) {
168                        fis.close();
169                    }
170                    if (fos != null) {
171                        fos.close();
172                    }
173                } catch (IOException e) {
174                    e.printStackTrace();
175                }
176            }
177    
178            return cacheDirName;
179        }
180    
181        private ProjectionEntries getProjectionEntries(Projection projection) {
182            return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
183        }
184    
185        private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
186            ProjectionEntries result = entries.get(projection);
187            if (result == null) {
188                result = new ProjectionEntries(projection, cacheDirectory);
189                entries.put(projection, result);
190            }
191    
192            return result;
193        }
194    
195        public synchronized void loadIndex() {
196            File indexFile = new File(cacheDir, INDEX_FILENAME);
197            try {
198                JAXBContext context = JAXBContext.newInstance(
199                        WmsCacheType.class.getPackage().getName(),
200                        WmsCacheType.class.getClassLoader());
201                Unmarshaller unmarshaller = context.createUnmarshaller();
202                WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile));
203                totalFileSize = cacheEntries.getTotalFileSize();
204                if (cacheEntries.getTileSize() != tileSize) {
205                    System.out.println("Cache created with different tileSize, cache will be discarded");
206                    return;
207                }
208                for (ProjectionType projectionType: cacheEntries.getProjection()) {
209                    ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
210                    for (EntryType entry: projectionType.getEntry()) {
211                        CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
212                        ce.lastUsed = entry.getLastUsed().getTimeInMillis();
213                        ce.lastModified = entry.getLastModified().getTimeInMillis();
214                        projection.entries.add(ce);
215                    }
216                }
217            } catch (Exception e) {
218                if (indexFile.exists()) {
219                    e.printStackTrace();
220                    System.out.println("Unable to load index for wms-cache, new file will be created");
221                } else {
222                    System.out.println("Index for wms-cache doesn't exist, new file will be created");
223                }
224            }
225    
226            removeNonReferencedFiles();
227        }
228    
229        private void removeNonReferencedFiles() {
230    
231            Set<String> usedProjections = new HashSet<String>();
232    
233            for (ProjectionEntries projectionEntries: entries.values()) {
234    
235                usedProjections.add(projectionEntries.cacheDirectory);
236    
237                File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
238                if (projectionDir.exists()) {
239                    Set<String> referencedFiles = new HashSet<String>();
240    
241                    for (CacheEntry ce: projectionEntries.entries) {
242                        referencedFiles.add(ce.filename);
243                    }
244    
245                    for (File file: projectionDir.listFiles()) {
246                        if (!referencedFiles.contains(file.getName())) {
247                            file.delete();
248                        }
249                    }
250                }
251            }
252    
253            for (File projectionDir: cacheDir.listFiles()) {
254                if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
255                    Utils.deleteDirectory(projectionDir);
256                }
257            }
258        }
259    
260        private int calculateTotalFileSize() {
261            int result = 0;
262            for (ProjectionEntries projectionEntries: entries.values()) {
263                Iterator<CacheEntry> it = projectionEntries.entries.iterator();
264                while (it.hasNext()) {
265                    CacheEntry entry = it.next();
266                    File imageFile = getImageFile(projectionEntries, entry);
267                    if (!imageFile.exists()) {
268                        it.remove();
269                    } else {
270                        result += imageFile.length();
271                    }
272                }
273            }
274            return result;
275        }
276    
277        public synchronized void saveIndex() {
278            WmsCacheType index = new WmsCacheType();
279    
280            if (totalFileSizeDirty) {
281                totalFileSize = calculateTotalFileSize();
282            }
283    
284            index.setTileSize(tileSize);
285            index.setTotalFileSize(totalFileSize);
286            for (ProjectionEntries projectionEntries: entries.values()) {
287                if (projectionEntries.entries.size() > 0) {
288                    ProjectionType projectionType = new ProjectionType();
289                    projectionType.setName(projectionEntries.projection);
290                    projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
291                    index.getProjection().add(projectionType);
292                    for (CacheEntry ce: projectionEntries.entries) {
293                        EntryType entry = new EntryType();
294                        entry.setPixelPerDegree(ce.pixelPerDegree);
295                        entry.setEast(ce.east);
296                        entry.setNorth(ce.north);
297                        Calendar c = Calendar.getInstance();
298                        c.setTimeInMillis(ce.lastUsed);
299                        entry.setLastUsed(c);
300                        c = Calendar.getInstance();
301                        c.setTimeInMillis(ce.lastModified);
302                        entry.setLastModified(c);
303                        entry.setFilename(ce.filename);
304                        projectionType.getEntry().add(entry);
305                    }
306                }
307            }
308            try {
309                JAXBContext context = JAXBContext.newInstance(
310                        WmsCacheType.class.getPackage().getName(),
311                        WmsCacheType.class.getClassLoader());
312                Marshaller marshaller = context.createMarshaller();
313                marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME)));
314            } catch (Exception e) {
315                System.err.println("Failed to save wms-cache file");
316                e.printStackTrace();
317            }
318        }
319    
320        private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
321            return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
322        }
323    
324    
325        private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
326    
327            synchronized (this) {
328                entry.lastUsed = System.currentTimeMillis();
329    
330                SoftReference<BufferedImage> memCache = memoryCache.get(entry);
331                if (memCache != null) {
332                    BufferedImage result = memCache.get();
333                    if (result != null)
334                        return result;
335                }
336            }
337    
338            try {
339                // Reading can't be in synchronized section, it's too slow
340                BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
341                synchronized (this) {
342                    if (result == null) {
343                        projectionEntries.entries.remove(entry);
344                        totalFileSizeDirty = true;
345                    }
346                    return result;
347                }
348            } catch (IOException e) {
349                synchronized (this) {
350                    projectionEntries.entries.remove(entry);
351                    totalFileSizeDirty = true;
352                    throw e;
353                }
354            }
355        }
356    
357        private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
358            for (CacheEntry entry: projectionEntries.entries) {
359                if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
360                    return entry;
361            }
362            return null;
363        }
364    
365        public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
366            ProjectionEntries projectionEntries = getProjectionEntries(projection);
367            CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
368            return (entry != null);
369        }
370    
371        public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
372            CacheEntry entry = null;
373            ProjectionEntries projectionEntries = null;
374            synchronized (this) {
375                projectionEntries = getProjectionEntries(projection);
376                entry = findEntry(projectionEntries, pixelPerDegree, east, north);
377            }
378            if (entry != null) {
379                try {
380                    return loadImage(projectionEntries, entry);
381                } catch (IOException e) {
382                    System.err.println("Unable to load file from wms cache");
383                    e.printStackTrace();
384                    return null;
385                }
386            }
387            return null;
388        }
389    
390        public  BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
391            ProjectionEntries projectionEntries;
392            List<CacheEntry> matches;
393            synchronized (this) {
394                matches = new ArrayList<WmsCache.CacheEntry>();
395    
396                double minPPD = pixelPerDegree / 5;
397                double maxPPD = pixelPerDegree * 5;
398                projectionEntries = getProjectionEntries(projection);
399    
400                double size2 = tileSize / pixelPerDegree;
401                double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
402                ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
403                        east + size2 - border, north + size2 - border);
404    
405                //TODO Do not load tile if it is completely overlapped by other tile with better ppd
406                for (CacheEntry entry: projectionEntries.entries) {
407                    if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
408                        entry.lastUsed = System.currentTimeMillis();
409                        matches.add(entry);
410                    }
411                }
412    
413                if (matches.isEmpty())
414                    return null;
415    
416    
417                Collections.sort(matches, new Comparator<CacheEntry>() {
418                    @Override
419                    public int compare(CacheEntry o1, CacheEntry o2) {
420                        return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
421                    }
422                });
423            }
424    
425            //TODO Use alpha layer only when enabled on wms layer
426            BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
427            Graphics2D g = result.createGraphics();
428    
429    
430            boolean drawAtLeastOnce = false;
431            Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
432            for (CacheEntry ce: matches) {
433                BufferedImage img;
434                try {
435                    img = loadImage(projectionEntries, ce);
436                    localCache.put(ce, new SoftReference<BufferedImage>(img));
437                } catch (IOException e) {
438                    continue;
439                }
440    
441                drawAtLeastOnce = true;
442    
443                int xDiff = (int)((ce.east - east) * pixelPerDegree);
444                int yDiff = (int)((ce.north - north) * pixelPerDegree);
445                int size = (int)(pixelPerDegree / ce.pixelPerDegree  * tileSize);
446    
447                int x = xDiff;
448                int y = -size + tileSize - yDiff;
449    
450                g.drawImage(img, x, y, size, size, null);
451            }
452    
453            if (drawAtLeastOnce) {
454                synchronized (this) {
455                    memoryCache.putAll(localCache);
456                }
457                return result;
458            } else
459                return null;
460        }
461    
462        private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
463            LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
464            LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
465            LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
466    
467            double deltaLat = Math.abs(ll3.lat() - ll1.lat());
468            double deltaLon = Math.abs(ll3.lon() - ll1.lon());
469            int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
470            int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
471    
472            String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
473            String extension;
474            if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
475                extension = "jpg";
476            } else if ("image/png".equals(mimeType)) {
477                extension = "png";
478            } else if ("image/gif".equals(mimeType)) {
479                extension = "gif";
480            } else {
481                extension = "dat";
482            }
483    
484            int counter = 0;
485            FILENAME_LOOP:
486                while (true) {
487                    String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
488                    for (CacheEntry entry: projectionEntries.entries) {
489                        if (entry.filename.equals(result)) {
490                            counter++;
491                            continue FILENAME_LOOP;
492                        }
493                    }
494                    return result;
495                }
496        }
497    
498        /**
499         *
500         * @param img Used only when overlapping is used, when not used, used raw from imageData
501         * @param imageData
502         * @param projection
503         * @param pixelPerDegree
504         * @param east
505         * @param north
506         * @throws IOException
507         */
508        public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
509            ProjectionEntries projectionEntries = getProjectionEntries(projection);
510            CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
511            File imageFile;
512            if (entry == null) {
513    
514                String mimeType;
515                if (img != null) {
516                    mimeType = "image/png";
517                } else {
518                    mimeType = URLConnection.guessContentTypeFromStream(imageData);
519                }
520                entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
521                entry.lastUsed = System.currentTimeMillis();
522                entry.lastModified = entry.lastUsed;
523                projectionEntries.entries.add(entry);
524                imageFile = getImageFile(projectionEntries, entry);
525            } else {
526                imageFile = getImageFile(projectionEntries, entry);
527                totalFileSize -= imageFile.length();
528            }
529    
530            imageFile.getParentFile().mkdirs();
531    
532            if (img != null) {
533                BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
534                copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
535                ImageIO.write(copy, "png", imageFile);
536                totalFileSize += imageFile.length();
537            } else {
538                OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
539                try {
540                    totalFileSize += Utils.copyStream(imageData, os);
541                } finally {
542                    os.close();
543                }
544            }
545        }
546    
547        public synchronized void cleanSmallFiles(int size) {
548            for (ProjectionEntries projectionEntries: entries.values()) {
549                Iterator<CacheEntry> it = projectionEntries.entries.iterator();
550                while (it.hasNext()) {
551                    File file = getImageFile(projectionEntries, it.next());
552                    long length = file.length();
553                    if (length <= size) {
554                        if (length == 0) {
555                            totalFileSizeDirty = true; // File probably doesn't exist
556                        }
557                        totalFileSize -= size;
558                        file.delete();
559                        it.remove();
560                    }
561                }
562            }
563        }
564    
565        public static String printDate(Calendar c) {
566            return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
567        }
568    
569        private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
570            for (ProjectionBounds b: areaToCache) {
571                if (cacheEntry.bounds.intersects(b))
572                    return true;
573            }
574            return false;
575        }
576    
577        public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
578            this.areaToCache = areaToCache;
579            Iterator<CacheEntry> it = memoryCache.keySet().iterator();
580            while (it.hasNext()) {
581                if (!isInsideAreaToCache(it.next())) {
582                    it.remove();
583                }
584            }
585        }
586    }