Merge pull request #55 from bloriot97/position-patch
Error of token position of lines preceded by inline comments (issue #54)
diff --git a/javalang/test/test_tokenizer.py b/javalang/test/test_tokenizer.py
index edf438b..a6a632b 100644
--- a/javalang/test/test_tokenizer.py
+++ b/javalang/test/test_tokenizer.py
@@ -81,6 +81,24 @@
# Then
self.assertEqual(len(tokens), 14)
+ def test_tokenize_token_position_after_comment(self):
+ # Given
+ code = """
+public int function() {
+ int a = 10;
+ // some comment
+ int b = 10;
+}
+ """
+
+ # When
+ tokens = list(tokenizer.tokenize(code))
+
+ # Then
+ # both token 6 and 11 are the "int" tokens of line 2 and 4
+ self.assertEqual(tokens[6].position[1], 5)
+ self.assertEqual(tokens[6].position[1], tokens[11].position[1])
+
def test_tokenize_hex_float_integer_at_end(self):
# Given
code = "nextKey = new BlockKey(serialNo, System.currentTimeMillis() + 0x3.2p2"
@@ -103,3 +121,6 @@
# Then
self.assertEqual(len(tokens), 8)
+
+if __name__=="__main__":
+ unittest.main()
diff --git a/javalang/tokenizer.py b/javalang/tokenizer.py
index 031b7e9..8e6c81e 100644
--- a/javalang/tokenizer.py
+++ b/javalang/tokenizer.py
@@ -249,10 +249,7 @@
self.i = self.length
return
- i += 1
-
self.start_of_line = i
- self.current_line += 1
self.i = i
else:
