Merge pull request #50 from eb4j/topic/miurahr/ignore-comment-line

Loader: ignore comment line

Author: miurahr

src/main/java/io/github/eb4j/dsl/impl/EntriesLoaderImpl.java

@@ -45,6 +45,7 @@ public class EntriesLoaderImpl implements AutoCloseable {
     private final byte[] lf;
     private final byte[] tab;
     private final byte[] space;
+    private final byte[] commentStart;
 
     public EntriesLoaderImpl(final Path path, final boolean isDictZip, final Charset charset, final byte[] eol)
             throws IOException {
@@ -64,6 +65,7 @@ public EntriesLoaderImpl(final Path path, final boolean isDictZip, final Charset
         lf = "\n".getBytes(charset);
         tab = "\t".getBytes(charset);
         space = " ".getBytes(charset);
+        commentStart = "{{".getBytes(charset);
     }
 
     public void close() throws IOException {
@@ -81,13 +83,19 @@ public List<DslIndex.Entry> load() throws IOException {
         String headWords;
         cardStart = entryStartSearch();
         while (true) {
+            // skip comment
+            long next = skipComment();
             // check multiple head words
             long headWordLen = eolSearch();
             if (headWordLen == -1) {
                 break;
             }
             while (!isSpaceOrTab()) {
-                headWordLen += eolSearch();
+                next = eolSearch();
+                if (next == -1) {
+                    break;
+                }
+                headWordLen += next;
             }
             seek(cardStart);
             byte[] headWordBytes = new byte[(int) headWordLen];
@@ -151,6 +159,33 @@ private long position() throws IOException {
         }
     }
 
+    private long skipComment() throws IOException {
+        byte[] b = new byte[commentStart.length];
+        InputStream is;
+        if (isDictZip) {
+            is = dzis;
+        } else {
+            is = rais;
+        }
+        is.mark(commentStart.length);
+        if (is.read(b) > 0) {
+            if (Arrays.equals(commentStart, b)) {
+                // a line looks comment
+                long next = eolSearch();
+                if (next == -1) {
+                    return -1;
+                }
+                // we should check end of comment, but now we ignore line.
+                return next;
+            }
+            is.reset();
+            return 0;
+        }
+        // got EOF
+        return -1;
+
+    }
+
     private long entryStartSearch() throws IOException {
         byte[] b = new byte[eol.length];
         InputStream is;

src/test/java/io/github/eb4j/dsl/DslDictionaryTest.java

@@ -283,4 +283,14 @@ void loadUTF16LE_BOM_CRLF_EL_DZ() throws URISyntaxException, IOException {
                 "[m2]to [ref]abandon \\[price\\] control[/ref][/m]\n" +
                 "[m2]to [ref]abandon a right[/ref][/m]\n", entry.getValue());
     }
+
+    @Test
+    void loadUtf8_Comment() throws URISyntaxException, IOException {
+        URL utf8 = this.getClass().getResource("/utf8_comment.dsl");
+        DslDictionary dictionary = DslDictionary.loadDictionary(new File(utf8.toURI()));
+        DumpDslVisitor dumper = new DumpDslVisitor();
+        DslResult results = dictionary.lookup("About");
+        Map.Entry<String, String> entry = results.getEntries(dumper).get(0);
+        assertEquals("Version: 0.0\n", entry.getValue());
+    }
 }

src/test/resources/utf8_comment.dsl

@@ -0,0 +1,18 @@
+#NAME	"test (En-Ru)"
+#INDEX_LANGUAGE	"English"
+#CONTENTS_LANGUAGE	"Russian"
+
+ABC
+	[m1]\[[t]ABC[/t]\] [p]n[/p][/m]
+Foo
+	[m1]\[[t]Boo[/t]\] [p]pl[/p][/m]
+
+{{ The End }}
+
+{{ Техническая часть }}
+
+About
+**
+        Version: 0.0
+
+{{ Техническая часть }}