001    // License: GPL. See LICENSE file for details.
002    package org.openstreetmap.josm.data.validation.tests;
003    
004    import static org.openstreetmap.josm.tools.I18n.marktr;
005    import static org.openstreetmap.josm.tools.I18n.tr;
006    
007    import java.awt.Dimension;
008    import java.awt.GridBagConstraints;
009    import java.awt.GridBagLayout;
010    import java.awt.event.ActionEvent;
011    import java.awt.event.ActionListener;
012    import java.io.BufferedReader;
013    import java.io.FileNotFoundException;
014    import java.io.IOException;
015    import java.io.InputStreamReader;
016    import java.io.UnsupportedEncodingException;
017    import java.text.MessageFormat;
018    import java.util.ArrayList;
019    import java.util.Arrays;
020    import java.util.Collection;
021    import java.util.Collections;
022    import java.util.HashMap;
023    import java.util.List;
024    import java.util.Map;
025    import java.util.Map.Entry;
026    import java.util.Set;
027    import java.util.regex.Matcher;
028    import java.util.regex.Pattern;
029    import java.util.regex.PatternSyntaxException;
030    
031    import javax.swing.DefaultListModel;
032    import javax.swing.JButton;
033    import javax.swing.JCheckBox;
034    import javax.swing.JLabel;
035    import javax.swing.JList;
036    import javax.swing.JOptionPane;
037    import javax.swing.JPanel;
038    import javax.swing.JScrollPane;
039    
040    import org.openstreetmap.josm.Main;
041    import org.openstreetmap.josm.command.ChangePropertyCommand;
042    import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
043    import org.openstreetmap.josm.command.Command;
044    import org.openstreetmap.josm.command.SequenceCommand;
045    import org.openstreetmap.josm.data.osm.Node;
046    import org.openstreetmap.josm.data.osm.OsmPrimitive;
047    import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
048    import org.openstreetmap.josm.data.osm.OsmUtils;
049    import org.openstreetmap.josm.data.osm.Relation;
050    import org.openstreetmap.josm.data.osm.Way;
051    import org.openstreetmap.josm.data.validation.Severity;
052    import org.openstreetmap.josm.data.validation.Test;
053    import org.openstreetmap.josm.data.validation.TestError;
054    import org.openstreetmap.josm.data.validation.util.Entities;
055    import org.openstreetmap.josm.gui.preferences.ValidatorPreference;
056    import org.openstreetmap.josm.gui.preferences.map.TaggingPresetPreference;
057    import org.openstreetmap.josm.gui.progress.ProgressMonitor;
058    import org.openstreetmap.josm.gui.tagging.TaggingPreset;
059    import org.openstreetmap.josm.io.MirroredInputStream;
060    import org.openstreetmap.josm.tools.GBC;
061    import org.openstreetmap.josm.tools.MultiMap;
062    
063    /**
064     * Check for misspelled or wrong properties
065     *
066     * @author frsantos
067     */
068    public class TagChecker extends Test
069    {
070        /** The default data files */
071        public static final String DATA_FILE = "resource://data/tagchecker.cfg";
072        public static final String IGNORE_FILE = "resource://data/ignoretags.cfg";
073        public static final String SPELL_FILE = "resource://data/words.cfg";
074    
075        /** The spell check key substitutions: the key should be substituted by the value */
076        protected static Map<String, String> spellCheckKeyData;
077        /** The spell check preset values */
078        protected static MultiMap<String, String> presetsValueData;
079        /** The TagChecker data */
080        protected static final List<CheckerData> checkerData = new ArrayList<CheckerData>();
081        protected static final List<String> ignoreDataStartsWith = new ArrayList<String>();
082        protected static final List<String> ignoreDataEquals = new ArrayList<String>();
083        protected static final List<String> ignoreDataEndsWith = new ArrayList<String>();
084        protected static final List<IgnoreKeyPair> ignoreDataKeyPair = new ArrayList<IgnoreKeyPair>();
085    
086        /** The preferences prefix */
087        protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
088    
089        public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
090        public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
091        public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
092        public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
093    
094        public static final String PREF_SOURCES = PREFIX + ".sources";
095        public static final String PREF_USE_DATA_FILE = PREFIX + ".usedatafile";
096        public static final String PREF_USE_IGNORE_FILE = PREFIX + ".useignorefile";
097        public static final String PREF_USE_SPELL_FILE = PREFIX + ".usespellfile";
098    
099        public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
100        public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
101        public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
102        public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
103    
104        protected boolean checkKeys = false;
105        protected boolean checkValues = false;
106        protected boolean checkComplex = false;
107        protected boolean checkFixmes = false;
108    
109        protected JCheckBox prefCheckKeys;
110        protected JCheckBox prefCheckValues;
111        protected JCheckBox prefCheckComplex;
112        protected JCheckBox prefCheckFixmes;
113        protected JCheckBox prefCheckPaint;
114    
115        protected JCheckBox prefCheckKeysBeforeUpload;
116        protected JCheckBox prefCheckValuesBeforeUpload;
117        protected JCheckBox prefCheckComplexBeforeUpload;
118        protected JCheckBox prefCheckFixmesBeforeUpload;
119        protected JCheckBox prefCheckPaintBeforeUpload;
120    
121        protected JCheckBox prefUseDataFile;
122        protected JCheckBox prefUseIgnoreFile;
123        protected JCheckBox prefUseSpellFile;
124    
125        protected JButton addSrcButton;
126        protected JButton editSrcButton;
127        protected JButton deleteSrcButton;
128    
129        protected static final int EMPTY_VALUES      = 1200;
130        protected static final int INVALID_KEY       = 1201;
131        protected static final int INVALID_VALUE     = 1202;
132        protected static final int FIXME             = 1203;
133        protected static final int INVALID_SPACE     = 1204;
134        protected static final int INVALID_KEY_SPACE = 1205;
135        protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
136        protected static final int LONG_VALUE        = 1208;
137        protected static final int LONG_KEY          = 1209;
138        protected static final int LOW_CHAR_VALUE    = 1210;
139        protected static final int LOW_CHAR_KEY      = 1211;
140        /** 1250 and up is used by tagcheck */
141    
142        /** List of sources for spellcheck data */
143        protected JList sourcesList;
144    
145        protected static final Entities entities = new Entities();
146    
147        /**
148         * Constructor
149         */
150        public TagChecker() {
151            super(tr("Properties checker :"),
152                    tr("This plugin checks for errors in property keys and values."));
153        }
154    
155        @Override
156        public void initialize() throws IOException {
157            initializeData();
158            initializePresets();
159        }
160    
161        /**
162         * Reads the spellcheck file into a HashMap.
163         * The data file is a list of words, beginning with +/-. If it starts with +,
164         * the word is valid, but if it starts with -, the word should be replaced
165         * by the nearest + word before this.
166         *
167         * @throws FileNotFoundException
168         * @throws IOException
169         */
170        private static void initializeData() throws IOException {
171            checkerData.clear();
172            ignoreDataStartsWith.clear();
173            ignoreDataEquals.clear();
174            ignoreDataEndsWith.clear();
175            ignoreDataKeyPair.clear();
176    
177            spellCheckKeyData = new HashMap<String, String>();
178            String sources = Main.pref.get( PREF_SOURCES, "");
179            if (Main.pref.getBoolean(PREF_USE_DATA_FILE, true)) {
180                if (sources == null || sources.length() == 0) {
181                    sources = DATA_FILE;
182                } else {
183                    sources = DATA_FILE + ";" + sources;
184                }
185            }
186            if (Main.pref.getBoolean(PREF_USE_IGNORE_FILE, true)) {
187                if (sources == null || sources.length() == 0) {
188                    sources = IGNORE_FILE;
189                } else {
190                    sources = IGNORE_FILE + ";" + sources;
191                }
192            }
193            if (Main.pref.getBoolean(PREF_USE_SPELL_FILE, true)) {
194                if( sources == null || sources.length() == 0) {
195                    sources = SPELL_FILE;
196                } else {
197                    sources = SPELL_FILE + ";" + sources;
198                }
199            }
200    
201            String errorSources = "";
202            if (sources.length() == 0)
203                return;
204            for (String source : sources.split(";")) {
205                try {
206                    MirroredInputStream s = new MirroredInputStream(source);
207                    InputStreamReader r;
208                    try {
209                        r = new InputStreamReader(s, "UTF-8");
210                    } catch (UnsupportedEncodingException e) {
211                        r = new InputStreamReader(s);
212                    }
213                    BufferedReader reader = new BufferedReader(r);
214    
215                    String okValue = null;
216                    boolean tagcheckerfile = false;
217                    boolean ignorefile = false;
218                    String line;
219                    while ((line = reader.readLine()) != null && (tagcheckerfile || line.length() != 0)) {
220                        if (line.startsWith("#")) {
221                            if (line.startsWith("# JOSM TagChecker")) {
222                                tagcheckerfile = true;
223                            }
224                            if (line.startsWith("# JOSM IgnoreTags")) {
225                                ignorefile = true;
226                            }
227                            continue;
228                        } else if (ignorefile) {
229                            line = line.trim();
230                            if (line.length() < 4) {
231                                continue;
232                            }
233    
234                            String key = line.substring(0, 2);
235                            line = line.substring(2);
236    
237                            if (key.equals("S:")) {
238                                ignoreDataStartsWith.add(line);
239                            } else if (key.equals("E:")) {
240                                ignoreDataEquals.add(line);
241                            } else if (key.equals("F:")) {
242                                ignoreDataEndsWith.add(line);
243                            } else if (key.equals("K:")) {
244                                IgnoreKeyPair tmp = new IgnoreKeyPair();
245                                int mid = line.indexOf("=");
246                                tmp.key = line.substring(0, mid);
247                                tmp.value = line.substring(mid+1);
248                                ignoreDataKeyPair.add(tmp);
249                            }
250                            continue;
251                        } else if (tagcheckerfile) {
252                            if (line.length() > 0) {
253                                CheckerData d = new CheckerData();
254                                String err = d.getData(line);
255    
256                                if (err == null) {
257                                    checkerData.add(d);
258                                } else {
259                                    System.err.println(tr("Invalid tagchecker line - {0}: {1}", err, line));
260                                }
261                            }
262                        } else if (line.charAt(0) == '+') {
263                            okValue = line.substring(1);
264                        } else if (line.charAt(0) == '-' && okValue != null) {
265                            spellCheckKeyData.put(line.substring(1), okValue);
266                        } else {
267                            System.err.println(tr("Invalid spellcheck line: {0}", line));
268                        }
269                    }
270                } catch (IOException e) {
271                    errorSources += source + "\n";
272                }
273            }
274    
275            if (errorSources.length() > 0)
276                throw new IOException( tr("Could not access data file(s):\n{0}", errorSources) );
277        }
278    
279        /**
280         * Reads the presets data.
281         *
282         */
283        public static void initializePresets() {
284    
285            if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
286                return;
287    
288            Collection<TaggingPreset> presets = TaggingPresetPreference.taggingPresets;
289            if (presets != null) {
290                presetsValueData = new MultiMap<String, String>();
291                for (String a : OsmPrimitive.getUninterestingKeys()) {
292                    presetsValueData.putVoid(a);
293                }
294                // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
295                /*  for(String a : OsmPrimitive.getDirectionKeys())
296                    presetsValueData.add(a);
297                 */
298                for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
299                        Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
300                    presetsValueData.putVoid(a);
301                }
302                for (TaggingPreset p : presets) {
303                    for (TaggingPreset.Item i : p.data) {
304                        if (i instanceof TaggingPreset.KeyedItem) {
305                            TaggingPreset.KeyedItem ky = (TaggingPreset.KeyedItem) i;
306                            presetsValueData.putAll(ky.key, ky.getValues());
307                        }
308                    }
309                }
310            }
311        }
312    
313        @Override
314        public void visit(Node n) {
315            checkPrimitive(n);
316        }
317    
318        @Override
319        public void visit(Relation n) {
320            checkPrimitive(n);
321        }
322    
323        @Override
324        public void visit(Way w) {
325            checkPrimitive(w);
326        }
327    
328        /**
329         * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
330         * @param s string to check
331         */
332        private boolean containsLow(String s) {
333            if (s == null)
334                return false;
335            for (int i = 0; i < s.length(); i++) {
336                if (s.charAt(i) < 0x20)
337                    return true;
338            }
339            return false;
340        }
341    
342        /**
343         * Checks the primitive properties
344         * @param p The primitive to check
345         */
346        private void checkPrimitive(OsmPrimitive p) {
347            // Just a collection to know if a primitive has been already marked with error
348            MultiMap<OsmPrimitive, String> withErrors = new MultiMap<OsmPrimitive, String>();
349    
350            if (checkComplex) {
351                Map<String, String> keys = p.getKeys();
352                for (CheckerData d : checkerData) {
353                    if (d.match(p, keys)) {
354                        errors.add( new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"),
355                                d.getDescription(), d.getDescriptionOrig(), d.getCode(), p) );
356                        withErrors.put(p, "TC");
357                    }
358                }
359            }
360    
361            Map<String, String> props = (p.getKeys() == null) ? Collections.<String, String>emptyMap() : p.getKeys();
362            for (Entry<String, String> prop : props.entrySet()) {
363                String s = marktr("Key ''{0}'' invalid.");
364                String key = prop.getKey();
365                String value = prop.getValue();
366                if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
367                    errors.add( new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"),
368                            tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p) );
369                    withErrors.put(p, "ICV");
370                }
371                if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
372                    errors.add( new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"),
373                            tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p) );
374                    withErrors.put(p, "ICK");
375                }
376                if (checkValues && (value!=null && value.length() > 255) && !withErrors.contains(p, "LV")) {
377                    errors.add( new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"),
378                            tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p) );
379                    withErrors.put(p, "LV");
380                }
381                if (checkKeys && (key!=null && key.length() > 255) && !withErrors.contains(p, "LK")) {
382                    errors.add( new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"),
383                            tr(s, key), MessageFormat.format(s, key), LONG_KEY, p) );
384                    withErrors.put(p, "LK");
385                }
386                if (checkValues && (value==null || value.trim().length() == 0) && !withErrors.contains(p, "EV")) {
387                    errors.add( new TestError(this, Severity.WARNING, tr("Tags with empty values"),
388                            tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p) );
389                    withErrors.put(p, "EV");
390                }
391                if (checkKeys && spellCheckKeyData.containsKey(key) && !withErrors.contains(p, "IPK")) {
392                    errors.add( new TestError(this, Severity.WARNING, tr("Invalid property key"),
393                            tr(s, key), MessageFormat.format(s, key), INVALID_KEY, p) );
394                    withErrors.put(p, "IPK");
395                }
396                if (checkKeys && key.indexOf(" ") >= 0 && !withErrors.contains(p, "IPK")) {
397                    errors.add( new TestError(this, Severity.WARNING, tr("Invalid white space in property key"),
398                            tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p) );
399                    withErrors.put(p, "IPK");
400                }
401                if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
402                    errors.add( new TestError(this, Severity.OTHER, tr("Property values start or end with white space"),
403                            tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p) );
404                    withErrors.put(p, "SPACE");
405                }
406                if (checkValues && value != null && !value.equals(entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
407                    errors.add( new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"),
408                            tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p) );
409                    withErrors.put(p, "HTML");
410                }
411                if (checkValues && value != null && value.length() > 0 && presetsValueData != null) {
412                    final Set<String> values = presetsValueData.get(key);
413                    final boolean keyInPresets = values != null;  
414                    final boolean tagInPresets = values != null && (values.isEmpty() || values.contains(prop.getValue()));
415                    
416                    boolean ignore = false;
417                    for (String a : ignoreDataStartsWith) {
418                        if (key.startsWith(a)) {
419                            ignore = true;
420                        }
421                    }
422                    for (String a : ignoreDataEquals) {
423                        if(key.equals(a)) {
424                            ignore = true;
425                        }
426                    }
427                    for (String a : ignoreDataEndsWith) {
428                        if(key.endsWith(a)) {
429                            ignore = true;
430                        }
431                    }
432                    
433                    if (!tagInPresets) {
434                        for (IgnoreKeyPair a : ignoreDataKeyPair) {
435                            if (key.equals(a.key) && value.equals(a.value)) {
436                                ignore = true;
437                            }
438                        }
439                    }
440                    
441                    if (!ignore) {
442                        if (!keyInPresets) {
443                            String i = marktr("Key ''{0}'' not in presets.");
444                            errors.add( new TestError(this, Severity.OTHER, tr("Presets do not contain property key"),
445                                    tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p) );
446                            withErrors.put(p, "UPK");
447                        } else if (!tagInPresets) {
448                            String i = marktr("Value ''{0}'' for key ''{1}'' not in presets.");
449                            errors.add( new TestError(this, Severity.OTHER, tr("Presets do not contain property value"),
450                                    tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p) );
451                            withErrors.put(p, "UPV");
452                        }
453                    }
454                }
455                if (checkFixmes && value != null && value.length() > 0) {
456                    if ((value.toLowerCase().contains("fixme")
457                            || value.contains("check and delete")
458                            || key.contains("todo") || key.toLowerCase().contains("fixme"))
459                            && !withErrors.contains(p, "FIXME")) {
460                        errors.add(new TestError(this, Severity.OTHER,
461                                tr("FIXMES"), FIXME, p));
462                        withErrors.put(p, "FIXME");
463                    }
464                }
465            }
466        }
467    
468        @Override
469        public void startTest(ProgressMonitor monitor) {
470            super.startTest(monitor);
471            checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
472            if (isBeforeUpload) {
473                checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
474            }
475    
476            checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
477            if (isBeforeUpload) {
478                checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
479            }
480    
481            checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
482            if (isBeforeUpload) {
483                checkComplex = checkValues && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
484            }
485    
486            checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
487            if (isBeforeUpload) {
488                checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
489            }
490        }
491    
492        @Override
493        public void visit(Collection<OsmPrimitive> selection) {
494            if (checkKeys || checkValues || checkComplex || checkFixmes) {
495                super.visit(selection);
496            }
497        }
498    
499        @Override
500        public void addGui(JPanel testPanel) {
501            GBC a = GBC.eol();
502            a.anchor = GridBagConstraints.EAST;
503    
504            testPanel.add(new JLabel(name), GBC.eol().insets(3,0,0,0));
505    
506            prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
507            prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
508            testPanel.add(prefCheckKeys, GBC.std().insets(20,0,0,0));
509    
510            prefCheckKeysBeforeUpload = new JCheckBox();
511            prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
512            testPanel.add(prefCheckKeysBeforeUpload, a);
513    
514            prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
515            prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
516            testPanel.add(prefCheckComplex, GBC.std().insets(20,0,0,0));
517    
518            prefCheckComplexBeforeUpload = new JCheckBox();
519            prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
520            testPanel.add(prefCheckComplexBeforeUpload, a);
521    
522            sourcesList = new JList(new DefaultListModel());
523    
524            String sources = Main.pref.get( PREF_SOURCES );
525            if (sources != null && sources.length() > 0) {
526                for (String source : sources.split(";")) {
527                    ((DefaultListModel)sourcesList.getModel()).addElement(source);
528                }
529            }
530    
531            addSrcButton = new JButton(tr("Add"));
532            addSrcButton.addActionListener(new ActionListener() {
533                @Override
534                public void actionPerformed(ActionEvent e) {
535                    String source = JOptionPane.showInputDialog(
536                            Main.parent,
537                            tr("TagChecker source"),
538                            tr("TagChecker source"),
539                            JOptionPane.QUESTION_MESSAGE);
540                    if (source != null) {
541                        ((DefaultListModel)sourcesList.getModel()).addElement(source);
542                    }
543                    sourcesList.clearSelection();
544                }
545            });
546    
547            editSrcButton = new JButton(tr("Edit"));
548            editSrcButton.addActionListener(new ActionListener() {
549                @Override
550                public void actionPerformed(ActionEvent e) {
551                    int row = sourcesList.getSelectedIndex();
552                    if (row == -1 && sourcesList.getModel().getSize() == 1) {
553                        sourcesList.setSelectedIndex(0);
554                        row = 0;
555                    }
556                    if (row == -1) {
557                        if (sourcesList.getModel().getSize() == 0) {
558                            String source = JOptionPane.showInputDialog(Main.parent, tr("TagChecker source"), tr("TagChecker source"), JOptionPane.QUESTION_MESSAGE);
559                            if (source != null) {
560                                ((DefaultListModel)sourcesList.getModel()).addElement(source);
561                            }
562                        } else {
563                            JOptionPane.showMessageDialog(
564                                    Main.parent,
565                                    tr("Please select the row to edit."),
566                                    tr("Information"),
567                                    JOptionPane.INFORMATION_MESSAGE
568                                    );
569                        }
570                    } else {
571                        String source = (String)JOptionPane.showInputDialog(Main.parent,
572                                tr("TagChecker source"),
573                                tr("TagChecker source"),
574                                JOptionPane.QUESTION_MESSAGE, null, null,
575                                sourcesList.getSelectedValue());
576                        if (source != null) {
577                            ((DefaultListModel)sourcesList.getModel()).setElementAt(source, row);
578                        }
579                    }
580                    sourcesList.clearSelection();
581                }
582            });
583    
584            deleteSrcButton = new JButton(tr("Delete"));
585            deleteSrcButton.addActionListener(new ActionListener() {
586                @Override
587                public void actionPerformed(ActionEvent e) {
588                    if (sourcesList.getSelectedIndex() == -1) {
589                        JOptionPane.showMessageDialog(Main.parent, tr("Please select the row to delete."), tr("Information"), JOptionPane.QUESTION_MESSAGE);
590                    } else {
591                        ((DefaultListModel)sourcesList.getModel()).remove(sourcesList.getSelectedIndex());
592                    }
593                }
594            });
595            sourcesList.setMinimumSize(new Dimension(300,50));
596            sourcesList.setVisibleRowCount(3);
597    
598            sourcesList.setToolTipText(tr("The sources (URL or filename) of spell check (see http://wiki.openstreetmap.org/index.php/User:JLS/speller) or tag checking data files."));
599            addSrcButton.setToolTipText(tr("Add a new source to the list."));
600            editSrcButton.setToolTipText(tr("Edit the selected source."));
601            deleteSrcButton.setToolTipText(tr("Delete the selected source from the list."));
602    
603            testPanel.add(new JLabel(tr("Data sources")), GBC.eol().insets(23,0,0,0));
604            testPanel.add(new JScrollPane(sourcesList), GBC.eol().insets(23,0,0,0).fill(GridBagConstraints.HORIZONTAL));
605            final JPanel buttonPanel = new JPanel(new GridBagLayout());
606            testPanel.add(buttonPanel, GBC.eol().fill(GridBagConstraints.HORIZONTAL));
607            buttonPanel.add(addSrcButton, GBC.std().insets(0,5,0,0));
608            buttonPanel.add(editSrcButton, GBC.std().insets(5,5,5,0));
609            buttonPanel.add(deleteSrcButton, GBC.std().insets(0,5,0,0));
610    
611            ActionListener disableCheckActionListener = new ActionListener() {
612                @Override
613                public void actionPerformed(ActionEvent e) {
614                    handlePrefEnable();
615                }
616            };
617            prefCheckKeys.addActionListener(disableCheckActionListener);
618            prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
619            prefCheckComplex.addActionListener(disableCheckActionListener);
620            prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
621    
622            handlePrefEnable();
623    
624            prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
625            prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
626            testPanel.add(prefCheckValues, GBC.std().insets(20,0,0,0));
627    
628            prefCheckValuesBeforeUpload = new JCheckBox();
629            prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
630            testPanel.add(prefCheckValuesBeforeUpload, a);
631    
632            prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
633            prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
634            testPanel.add(prefCheckFixmes, GBC.std().insets(20,0,0,0));
635    
636            prefCheckFixmesBeforeUpload = new JCheckBox();
637            prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
638            testPanel.add(prefCheckFixmesBeforeUpload, a);
639    
640            prefUseDataFile = new JCheckBox(tr("Use default data file."), Main.pref.getBoolean(PREF_USE_DATA_FILE, true));
641            prefUseDataFile.setToolTipText(tr("Use the default data file (recommended)."));
642            testPanel.add(prefUseDataFile, GBC.eol().insets(20,0,0,0));
643    
644            prefUseIgnoreFile = new JCheckBox(tr("Use default tag ignore file."), Main.pref.getBoolean(PREF_USE_IGNORE_FILE, true));
645            prefUseIgnoreFile.setToolTipText(tr("Use the default tag ignore file (recommended)."));
646            testPanel.add(prefUseIgnoreFile, GBC.eol().insets(20,0,0,0));
647    
648            prefUseSpellFile = new JCheckBox(tr("Use default spellcheck file."), Main.pref.getBoolean(PREF_USE_SPELL_FILE, true));
649            prefUseSpellFile.setToolTipText(tr("Use the default spellcheck file (recommended)."));
650            testPanel.add(prefUseSpellFile, GBC.eol().insets(20,0,0,0));
651        }
652    
653        public void handlePrefEnable() {
654            boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
655                    || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
656            sourcesList.setEnabled( selected );
657            addSrcButton.setEnabled(selected);
658            editSrcButton.setEnabled(selected);
659            deleteSrcButton.setEnabled(selected);
660        }
661    
662        @Override
663        public boolean ok()
664        {
665            enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
666            testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
667                    || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
668    
669            Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
670            Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
671            Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
672            Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
673            Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
674            Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
675            Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
676            Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
677            Main.pref.put(PREF_USE_DATA_FILE, prefUseDataFile.isSelected());
678            Main.pref.put(PREF_USE_IGNORE_FILE, prefUseIgnoreFile.isSelected());
679            Main.pref.put(PREF_USE_SPELL_FILE, prefUseSpellFile.isSelected());
680            String sources = "";
681            if (sourcesList.getModel().getSize() > 0) {
682                String sb = "";
683                for (int i = 0; i < sourcesList.getModel().getSize(); ++i) {
684                    sb += ";"+sourcesList.getModel().getElementAt(i);
685                }
686                sources = sb.substring(1);
687            }
688            if (sources.length() == 0) {
689                sources = null;
690            }
691            return Main.pref.put(PREF_SOURCES, sources);
692        }
693    
694        @Override
695        public Command fixError(TestError testError) {
696    
697            List<Command> commands = new ArrayList<Command>(50);
698    
699            Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
700            for (OsmPrimitive p : primitives) {
701                Map<String, String> tags = p.getKeys();
702                if (tags == null || tags.isEmpty()) {
703                    continue;
704                }
705    
706                for (Entry<String, String> prop: tags.entrySet()) {
707                    String key = prop.getKey();
708                    String value = prop.getValue();
709                    if (value == null || value.trim().length() == 0) {
710                        commands.add(new ChangePropertyCommand(Collections.singleton(p), key, null));
711                    } else if (value.startsWith(" ") || value.endsWith(" ")) {
712                        commands.add(new ChangePropertyCommand(Collections.singleton(p), key, value.trim()));
713                    } else if (key.startsWith(" ") || key.endsWith(" ")) {
714                        commands.add(new ChangePropertyKeyCommand(Collections.singleton(p), key, key.trim()));
715                    } else {
716                        String evalue = entities.unescape(value);
717                        if (!evalue.equals(value)) {
718                            commands.add(new ChangePropertyCommand(Collections.singleton(p), key, evalue));
719                        } else {
720                            String replacementKey = spellCheckKeyData.get(key);
721                            if (replacementKey != null) {
722                                commands.add(new ChangePropertyKeyCommand(Collections.singleton(p),
723                                        key, replacementKey));
724                            }
725                        }
726                    }
727                }
728            }
729    
730            if (commands.isEmpty())
731                return null;
732            if (commands.size() == 1)
733                return commands.get(0);
734    
735            return new SequenceCommand(tr("Fix properties"), commands);
736        }
737    
738        @Override
739        public boolean isFixable(TestError testError) {
740    
741            if (testError.getTester() instanceof TagChecker) {
742                int code = testError.getCode();
743                return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || code == INVALID_KEY_SPACE || code == INVALID_HTML;
744            }
745    
746            return false;
747        }
748    
749        protected static class IgnoreKeyPair {
750            public String key;
751            public String value;
752        }
753    
754        protected static class CheckerData {
755            private String description;
756            private List<CheckerElement> data = new ArrayList<CheckerElement>();
757            private OsmPrimitiveType type;
758            private int code;
759            protected Severity severity;
760            protected static final int TAG_CHECK_ERROR  = 1250;
761            protected static final int TAG_CHECK_WARN   = 1260;
762            protected static final int TAG_CHECK_INFO   = 1270;
763    
764            private static class CheckerElement {
765                public Object tag;
766                public Object value;
767                public boolean noMatch;
768                public boolean tagAll = false;
769                public boolean valueAll = false;
770                public boolean valueBool = false;
771    
772                private Pattern getPattern(String str) throws IllegalStateException, PatternSyntaxException {
773                    if (str.endsWith("/i"))
774                        return Pattern.compile(str.substring(1,str.length()-2), Pattern.CASE_INSENSITIVE);
775                    if (str.endsWith("/"))
776                        return Pattern.compile(str.substring(1,str.length()-1));
777    
778                    throw new IllegalStateException();
779                }
780                public CheckerElement(String exp) throws IllegalStateException, PatternSyntaxException {
781                    Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
782                    m.matches();
783    
784                    String n = m.group(1).trim();
785    
786                    if(n.equals("*")) {
787                        tagAll = true;
788                    } else {
789                        tag = n.startsWith("/") ? getPattern(n) : n;
790                        noMatch = m.group(2).equals("!=");
791                        n = m.group(3).trim();
792                        if (n.equals("*")) {
793                            valueAll = true;
794                        } else if (n.equals("BOOLEAN_TRUE")) {
795                            valueBool = true;
796                            value = OsmUtils.trueval;
797                        } else if (n.equals("BOOLEAN_FALSE")) {
798                            valueBool = true;
799                            value = OsmUtils.falseval;
800                        } else {
801                            value = n.startsWith("/") ? getPattern(n) : n;
802                        }
803                    }
804                }
805    
806                public boolean match(OsmPrimitive osm, Map<String, String> keys) {
807                    for (Entry<String, String> prop: keys.entrySet()) {
808                        String key = prop.getKey();
809                        String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
810                        if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
811                                && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
812                            return !noMatch;
813                    }
814                    return noMatch;
815                }
816            };
817    
818            public String getData(String str) {
819                Matcher m = Pattern.compile(" *# *([^#]+) *$").matcher(str);
820                str = m.replaceFirst("").trim();
821                try {
822                    description = m.group(1);
823                    if (description != null && description.length() == 0) {
824                        description = null;
825                    }
826                } catch (IllegalStateException e) {
827                    description = null;
828                }
829                String[] n = str.split(" *: *", 3);
830                if (n[0].equals("way")) {
831                    type = OsmPrimitiveType.WAY;
832                } else if (n[0].equals("node")) {
833                    type = OsmPrimitiveType.NODE;
834                } else if (n[0].equals("relation")) {
835                    type = OsmPrimitiveType.RELATION;
836                } else if (n[0].equals("*")) {
837                    type = null;
838                } else
839                    return tr("Could not find element type");
840                if (n.length != 3)
841                    return tr("Incorrect number of parameters");
842    
843                if (n[1].equals("W")) {
844                    severity = Severity.WARNING;
845                    code = TAG_CHECK_WARN;
846                } else if (n[1].equals("E")) {
847                    severity = Severity.ERROR;
848                    code = TAG_CHECK_ERROR;
849                } else if(n[1].equals("I")) {
850                    severity = Severity.OTHER;
851                    code = TAG_CHECK_INFO;
852                } else
853                    return tr("Could not find warning level");
854                for (String exp: n[2].split(" *&& *")) {
855                    try {
856                        data.add(new CheckerElement(exp));
857                    } catch (IllegalStateException e) {
858                        return tr("Illegal expression ''{0}''", exp);
859                    }
860                    catch (PatternSyntaxException e) {
861                        return tr("Illegal regular expression ''{0}''", exp);
862                    }
863                }
864                return null;
865            }
866    
867            public boolean match(OsmPrimitive osm, Map<String, String> keys) {
868                if (type != null && OsmPrimitiveType.from(osm) != type)
869                    return false;
870    
871                for (CheckerElement ce : data) {
872                    if (!ce.match(osm, keys))
873                        return false;
874                }
875                return true;
876            }
877    
878            public String getDescription() {
879                return tr(description);
880            }
881    
882            public String getDescriptionOrig() {
883                return description;
884            }
885    
886            public Severity getSeverity() {
887                return severity;
888            }
889    
890            public int getCode() {
891                if (type == null)
892                    return code;
893    
894                return code + type.ordinal() + 1;
895            }
896        }
897    }