Merge pull request #217 from structuredllm/fix9

Fix minor issues in Java Grammar

Author: shubhamugare

README.md

@@ -44,7 +44,7 @@ Define your own grammar using simple EBNF syntax. Check out our [notebooks direc
 | 🔥 Fast grammar-guided generation (as little as 10% generation overhead with Python and Go!)                        |
 | 🤖 Seamlessly work with any HuggingFace Language Model, including Code, Chat, and Instruct models                   |
 | 🖍️ Pass in any CFG in the EBNF format (even large grammars for programming languages like Python and Go!)            |
-| 📝 Built-in CFGs for **Python, Go, SQL, Math, JSON**, and more!                                                              |
+| 📝 Built-in CFGs for **Python, Go, Java, SQL, Math, JSON**, and more!                                                              |
 | 🎲 Sample with any existing decoding strategy (eg. greedy, beam search, nucleus sampling)                           |
 
 
@@ -148,7 +148,7 @@ print(f"SynCode output:\n{output}")
 # }
 ```
 
-Check more examples of using Python, Go, and other grammars in <a href="#-example-usage">Notebooks</a> and a quick example at 
+Check more examples of using Python, Go, Java and other grammars in <a href="#-example-usage">Notebooks</a> and a quick example at 
 &nbsp; [<img align="center" src="https://colab.research.google.com/assets/colab-badge.svg" />](https://colab.research.google.com/drive/1rYm8iehx_qYdtgWmqLkmhIjizhUVTb9E?usp=sharing)
 
 #### Instuct-tuned Models

syncode/parsers/grammars/java.lark

@@ -1,21 +1,9 @@
 start: compilation_unit 
 
-%import common.CNAME
-%import common.DIGIT
-%import common.WS
-
-%ignore WS
-
-LINE_COMMENT: /\/\/[^\n\r]*/
-BLOCK_COMMENT: /\/\*[\s\S]*?\*\//
-
 type_parameters: "<" type_parameter ("," type_parameter)* ">"
 type_parameter: CNAME type_bound?
 type_bound: "extends" type ("&" type)*
 
-%ignore LINE_COMMENT
-%ignore BLOCK_COMMENT
-
 compilation_unit: package_declaration? import_declarations? type_declarations?
 
 package_declaration: "package" name ";"
@@ -188,7 +176,7 @@ floating_point_literal: DIGIT+ "." DIGIT+
 
 boolean_literal: "true" | "false"
 
-character_literal: "'" /[^\\'\n\r]/ "'"
+character_literal: /'([^'\r\n\\]|\\([btnfr"'\\0-7]|[0-3]?[0-7]{2})|\\u[0-9a-fA-f]{4})'/
 
 string_literal: /".*?"/
 
@@ -206,8 +194,6 @@ dim_expr: "[" expression "]"
 
 dims: "[" "]"+
 
-
-
 field_access: primary "." CNAME | "super" "." CNAME
 
 method_invocation: name "(" argument_list? ")" | primary "." CNAME "(" argument_list? ")" | "super" "." CNAME "(" argument_list? ")"
@@ -289,3 +275,12 @@ reference_type: class_or_interface_type | array_type
 
 array_type: primitive_type dims | name dims | array_type dims
 
+LINE_COMMENT: /\/\/[^\n\r]*/
+BLOCK_COMMENT: /\/\*[\s\S]*?\*\//
+
+%import common.CNAME
+%import common.DIGIT
+%import common.WS
+%ignore WS
+%ignore LINE_COMMENT
+%ignore BLOCK_COMMENT

tests/parser/test_grammar_java.py

@@ -388,4 +388,8 @@ def test_java_parser26(self):
 public """
         r = inc_parser.get_acceptable_next_terminals(partial_code)
         assert AcceptSequence(['CLASS']) in r.accept_sequences
-        assert r.remainder_state == RemainderState.COMPLETE
+        assert r.remainder_state == RemainderState.COMPLETE
+
+if __name__ == '__main__':
+    unittest.main()
+