001    package org.openstreetmap.gui.jmapviewer;
002    
003    //License: GPL. Copyright 2008 by Jan Peter Stotz
004    
005    import java.io.BufferedReader;
006    import java.io.ByteArrayInputStream;
007    import java.io.ByteArrayOutputStream;
008    import java.io.File;
009    import java.io.FileInputStream;
010    import java.io.FileNotFoundException;
011    import java.io.FileOutputStream;
012    import java.io.IOException;
013    import java.io.InputStream;
014    import java.io.InputStreamReader;
015    import java.io.OutputStreamWriter;
016    import java.io.PrintWriter;
017    import java.lang.Thread;
018    import java.net.HttpURLConnection;
019    import java.net.URL;
020    import java.net.URLConnection;
021    import java.nio.charset.Charset;
022    import java.util.HashMap;
023    import java.util.Map;
024    import java.util.Map.Entry;
025    import java.util.Random;
026    import java.util.logging.Level;
027    import java.util.logging.Logger;
028    
029    import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
030    import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
031    import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
032    import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
033    import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate;
034    
035    /**
036     * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and
037     * saves all loaded files in a directory located in the temporary directory.
038     * If a tile is present in this file cache it will not be loaded from OSM again.
039     *
040     * @author Jan Peter Stotz
041     * @author Stefan Zeller
042     */
043    public class OsmFileCacheTileLoader extends OsmTileLoader {
044    
045        private static final Logger log = Logger.getLogger(OsmFileCacheTileLoader.class.getName());
046    
047        private static final String ETAG_FILE_EXT = ".etag";
048        private static final String TAGS_FILE_EXT = ".tags";
049    
050        private static final Charset TAGS_CHARSET = Charset.forName("UTF-8");
051    
052        public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24;
053        public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7;
054    
055        protected String cacheDirBase;
056        
057        protected final Map<TileSource, File> sourceCacheDirMap;
058    
059        protected long maxCacheFileAge = FILE_AGE_ONE_WEEK;
060        protected long recheckAfter = FILE_AGE_ONE_DAY;
061    
062        public static File getDefaultCacheDir() throws SecurityException {
063            String tempDir = null;
064            String userName = System.getProperty("user.name");
065            try {
066                tempDir = System.getProperty("java.io.tmpdir");
067            } catch (SecurityException e) {
068                log.log(Level.WARNING,
069                        "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: "
070                        + e.toString());
071                throw e; // rethrow
072            }
073            try {
074                if (tempDir == null)
075                    throw new IOException("No temp directory set");
076                String subDirName = "JMapViewerTiles";
077                // On Linux/Unix systems we do not have a per user tmp directory.
078                // Therefore we add the user name for getting a unique dir name.
079                if (userName != null && userName.length() > 0) {
080                    subDirName += "_" + userName;
081                }
082                File cacheDir = new File(tempDir, subDirName);
083                return cacheDir;
084            } catch (Exception e) {
085            }
086            return null;
087        }
088    
089        /**
090         * Create a OSMFileCacheTileLoader with given cache directory.
091         * If cacheDir is not set or invalid, IOException will be thrown.
092         * @param map the listener checking for tile load events (usually the map for display)
093         * @param cacheDir directory to store cached tiles
094         */
095        public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException  {
096            super(map);
097            if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs()))
098                throw new IOException("Cannot access cache directory");
099    
100            log.finest("Tile cache directory: " + cacheDir);
101            cacheDirBase = cacheDir.getAbsolutePath();
102            sourceCacheDirMap = new HashMap<TileSource, File>();
103        }
104    
105        /**
106         * Create a OSMFileCacheTileLoader with system property temp dir.
107         * If not set an IOException will be thrown.
108         * @param map the listener checking for tile load events (usually the map for display)
109         */
110        public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException {
111            this(map, getDefaultCacheDir());
112        }
113    
114        @Override
115        public TileJob createTileLoaderJob(final Tile tile) {
116            return new FileLoadJob(tile);
117        }
118    
119        protected File getSourceCacheDir(TileSource source) {
120            File dir = sourceCacheDirMap.get(source);
121            if (dir == null) {
122                dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));
123                if (!dir.exists()) {
124                    dir.mkdirs();
125                }
126            }
127            return dir;
128        }
129        
130        protected class FileLoadJob implements TileJob {
131            InputStream input = null;
132    
133            Tile tile;
134            File tileCacheDir;
135            File tileFile = null;
136            long fileAge = 0;
137            boolean fileTilePainted = false;
138    
139            public FileLoadJob(Tile tile) {
140                this.tile = tile;
141            }
142    
143            public Tile getTile() {
144                return tile;
145            }
146    
147            public void run() {
148                synchronized (tile) {
149                    if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading())
150                        return;
151                    tile.loaded = false;
152                    tile.error = false;
153                    tile.loading = true;
154                }
155                tileCacheDir = getSourceCacheDir(tile.getSource());
156                if (loadTileFromFile()) {
157                    return;
158                }
159                if (fileTilePainted) {
160                    TileJob job = new TileJob() {
161    
162                        public void run() {
163                            loadOrUpdateTile();
164                        }
165                        public Tile getTile() {
166                            return tile;
167                        }
168                    };
169                    JobDispatcher.getInstance().addJob(job);
170                } else {
171                    loadOrUpdateTile();
172                }
173            }
174    
175            protected void loadOrUpdateTile() {
176                try {
177                    URLConnection urlConn = loadTileFromOsm(tile);
178                    if (tileFile != null) {
179                        switch (tile.getSource().getTileUpdate()) {
180                        case IfModifiedSince:
181                            urlConn.setIfModifiedSince(fileAge);
182                            break;
183                        case LastModified:
184                            if (!isOsmTileNewer(fileAge)) {
185                                log.finest("LastModified test: local version is up to date: " + tile);
186                                tile.setLoaded(true);
187                                tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
188                                return;
189                            }
190                            break;
191                        }
192                    }
193                    if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) {
194                        String fileETag = tile.getValue("etag");
195                        if (fileETag != null) {
196                            switch (tile.getSource().getTileUpdate()) {
197                            case IfNoneMatch:
198                                urlConn.addRequestProperty("If-None-Match", fileETag);
199                                break;
200                            case ETag:
201                                if (hasOsmTileETag(fileETag)) {
202                                    tile.setLoaded(true);
203                                    tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge
204                                            + recheckAfter);
205                                    return;
206                                }
207                            }
208                        }
209                        tile.putValue("etag", urlConn.getHeaderField("ETag"));
210                    }
211                    if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {
212                        // If we are isModifiedSince or If-None-Match has been set
213                        // and the server answers with a HTTP 304 = "Not Modified"
214                        log.finest("ETag test: local version is up to date: " + tile);
215                        tile.setLoaded(true);
216                        tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
217                        return;
218                    }
219    
220                    loadTileMetadata(tile, urlConn);
221                    saveTagsToFile();
222    
223                    if ("no-tile".equals(tile.getValue("tile-info")))
224                    {
225                        tile.setError("No tile at this zoom level");
226                        listener.tileLoadingFinished(tile, true);
227                    } else {
228                        for(int i = 0; i < 5; ++i) {
229                            if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {
230                                Thread.sleep(5000+(new Random()).nextInt(5000));
231                                continue;
232                            }
233                            byte[] buffer = loadTileInBuffer(urlConn);
234                            if (buffer != null) {
235                                tile.loadImage(new ByteArrayInputStream(buffer));
236                                tile.setLoaded(true);
237                                listener.tileLoadingFinished(tile, true);
238                                saveTileToFile(buffer);
239                                break;
240                            }
241                        }
242                    }
243                } catch (Exception e) {
244                    tile.setError(e.getMessage());
245                    listener.tileLoadingFinished(tile, false);
246                    if (input == null) {
247                        try {
248                            System.err.println("Failed loading " + tile.getUrl() +": " + e.getMessage());
249                        } catch(IOException i) {
250                        }
251                    }
252                } finally {
253                    tile.loading = false;
254                    tile.setLoaded(true);
255                }
256            }
257    
258            protected boolean loadTileFromFile() {
259                FileInputStream fin = null;
260                try {
261                    tileFile = getTileFile();
262                    if (!tileFile.exists())
263                        return false;
264    
265                    loadTagsFromFile();
266                    if ("no-tile".equals(tile.getValue("tile-info")))
267                    {
268                        tile.setError("No tile at this zoom level");
269                        if (tileFile.exists()) {
270                            tileFile.delete();
271                        }
272                        tileFile = getTagsFile();
273                    } else {
274                        fin = new FileInputStream(tileFile);
275                        if (fin.available() == 0)
276                            throw new IOException("File empty");
277                        tile.loadImage(fin);
278                        fin.close();
279                    }
280    
281                    fileAge = tileFile.lastModified();
282                    boolean oldTile = System.currentTimeMillis() - fileAge > maxCacheFileAge;
283                    if (!oldTile) {
284                        tile.setLoaded(true);
285                        listener.tileLoadingFinished(tile, true);
286                        fileTilePainted = true;
287                        return true;
288                    }
289                    listener.tileLoadingFinished(tile, true);
290                    fileTilePainted = true;
291                } catch (Exception e) {
292                    try {
293                        if (fin != null) {
294                            fin.close();
295                            tileFile.delete();
296                        }
297                    } catch (Exception e1) {
298                    }
299                    tileFile = null;
300                    fileAge = 0;
301                }
302                return false;
303            }
304    
305            protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException {
306                input = urlConn.getInputStream();
307                ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());
308                byte[] buffer = new byte[2048];
309                boolean finished = false;
310                do {
311                    int read = input.read(buffer);
312                    if (read >= 0) {
313                        bout.write(buffer, 0, read);
314                    } else {
315                        finished = true;
316                    }
317                } while (!finished);
318                if (bout.size() == 0)
319                    return null;
320                return bout.toByteArray();
321            }
322    
323            /**
324             * Performs a <code>HEAD</code> request for retrieving the
325             * <code>LastModified</code> header value.
326             *
327             * Note: This does only work with servers providing the
328             * <code>LastModified</code> header:
329             * <ul>
330             * <li>{@link tilesources.OsmTileSource.CycleMap} - supported</li>
331             * <li>{@link tilesources.OsmTileSource.Mapnik} - not supported</li>
332             * </ul>
333             *
334             * @param fileAge time of the 
335             * @return <code>true</code> if the tile on the server is newer than the
336             *         file
337             * @throws IOException
338             */
339            protected boolean isOsmTileNewer(long fileAge) throws IOException {
340                URL url;
341                url = new URL(tile.getUrl());
342                HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
343                prepareHttpUrlConnection(urlConn);
344                urlConn.setRequestMethod("HEAD");
345                urlConn.setReadTimeout(30000); // 30 seconds read timeout
346                // System.out.println("Tile age: " + new
347                // Date(urlConn.getLastModified()) + " / "
348                // + new Date(fileAge));
349                long lastModified = urlConn.getLastModified();
350                if (lastModified == 0)
351                    return true; // no LastModified time returned
352                return (lastModified > fileAge);
353            }
354    
355            protected boolean hasOsmTileETag(String eTag) throws IOException {
356                URL url;
357                url = new URL(tile.getUrl());
358                HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
359                prepareHttpUrlConnection(urlConn);
360                urlConn.setRequestMethod("HEAD");
361                urlConn.setReadTimeout(30000); // 30 seconds read timeout
362                // System.out.println("Tile age: " + new
363                // Date(urlConn.getLastModified()) + " / "
364                // + new Date(fileAge));
365                String osmETag = urlConn.getHeaderField("ETag");
366                if (osmETag == null)
367                    return true;
368                return (osmETag.equals(eTag));
369            }
370    
371            protected File getTileFile() {
372                return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."
373                        + tile.getSource().getTileType());
374            }
375    
376            protected File getTagsFile() {
377                return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile()
378                        + TAGS_FILE_EXT);
379            }
380    
381            protected void saveTileToFile(byte[] rawData) {
382                try {
383                    FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile()
384                            + "_" + tile.getYtile() + "." + tile.getSource().getTileType());
385                    f.write(rawData);
386                    f.close();
387                    // System.out.println("Saved tile to file: " + tile);
388                } catch (Exception e) {
389                    System.err.println("Failed to save tile content: " + e.getLocalizedMessage());
390                }
391            }
392    
393            protected void saveTagsToFile() {
394                File tagsFile = getTagsFile();
395                if (tile.getMetadata() == null) {
396                    tagsFile.delete();
397                    return;
398                }
399                try {
400                    final PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile),
401                            TAGS_CHARSET));
402                    for (Entry<String, String> entry : tile.getMetadata().entrySet()) {
403                        f.println(entry.getKey() + "=" + entry.getValue());
404                    }
405                    f.close();
406                } catch (Exception e) {
407                    System.err.println("Failed to save tile tags: " + e.getLocalizedMessage());
408                }
409            }
410    
411            /** Load backward-compatiblity .etag file and if it exists move it to new .tags file*/
412            private void loadOldETagfromFile() {
413                File etagFile = new File(tileCacheDir, tile.getZoom() + "_"
414                        + tile.getXtile() + "_" + tile.getYtile() + ETAG_FILE_EXT);
415                if (!etagFile.exists()) return;
416                try {
417                    FileInputStream f = new FileInputStream(etagFile);
418                    byte[] buf = new byte[f.available()];
419                    f.read(buf);
420                    f.close();
421                    String etag = new String(buf, TAGS_CHARSET.name());
422                    tile.putValue("etag", etag);
423                    if (etagFile.delete()) {
424                        saveTagsToFile();
425                    }
426                } catch (IOException e) {
427                    System.err.println("Failed to load compatiblity etag: " + e.getLocalizedMessage());
428                }
429            }
430    
431            protected void loadTagsFromFile() {
432                loadOldETagfromFile();
433                File tagsFile = getTagsFile();
434                try {
435                    final BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile),
436                            TAGS_CHARSET));
437                    for (String line = f.readLine(); line != null; line = f.readLine()) {
438                        final int i = line.indexOf('=');
439                        if (i == -1 || i == 0) {
440                            System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line);
441                            continue;
442                        }
443                        tile.putValue(line.substring(0,i),line.substring(i+1));
444                    }
445                    f.close();
446                } catch (FileNotFoundException e) {
447                } catch (Exception e) {
448                    System.err.println("Failed to load tile tags: " + e.getLocalizedMessage());
449                }
450            }
451    
452        }
453    
454        public long getMaxFileAge() {
455            return maxCacheFileAge;
456        }
457    
458        /**
459         * Sets the maximum age of the local cached tile in the file system. If a
460         * local tile is older than the specified file age
461         * {@link OsmFileCacheTileLoader} will connect to the tile server and check
462         * if a newer tile is available using the mechanism specified for the
463         * selected tile source/server.
464         *
465         * @param maxFileAge
466         *            maximum age in milliseconds
467         * @see #FILE_AGE_ONE_DAY
468         * @see #FILE_AGE_ONE_WEEK
469         * @see TileSource#getTileUpdate()
470         */
471        public void setCacheMaxFileAge(long maxFileAge) {
472            this.maxCacheFileAge = maxFileAge;
473        }
474    
475        public String getCacheDirBase() {
476            return cacheDirBase;
477        }
478    
479        public void setTileCacheDir(String tileCacheDir) {
480            File dir = new File(tileCacheDir);
481            dir.mkdirs();
482            this.cacheDirBase = dir.getAbsolutePath();
483        }
484        
485        public static interface TileClearController {
486    
487            void initClearDir(File dir);
488    
489            void initClearFiles(File[] files);
490    
491            boolean cancel();
492    
493            void fileDeleted(File file);
494    
495            void clearFinished();
496        }
497        
498        public void clearCache(TileSource source) {
499            clearCache(source, null);
500        }
501        
502        public void clearCache(TileSource source, TileClearController controller) {
503            File dir = getSourceCacheDir(source);
504            if (dir != null) {
505                if (controller != null) controller.initClearDir(dir);
506                if (dir.isDirectory()) {
507                    File[] files = dir.listFiles();
508                    if (controller != null) controller.initClearFiles(files);
509                    for (File file : files) {
510                        if (controller != null && controller.cancel()) return;
511                        file.delete();
512                        if (controller != null) controller.fileDeleted(file);
513                    }
514                }
515                dir.delete();
516            }
517            if (controller != null) controller.clearFinished();
518        }
519    }