[backport] SI-6478 Fixing JavaTokenParser ident

Backport of 256934160007079f473131469af2df4d023c2cfc from PR
https://github.com/scala/scala/pull/1466

3 files changed, 56 insertions, 3 deletions

diff --git a/src/library/scala/util/parsing/combinator/JavaTokenParsers.scala b/src/library/scala/util/parsing/combinator/JavaTokenParsers.scala
index 78817cfb67..89832d3fb2 100644
--- a/src/library/scala/util/parsing/combinator/JavaTokenParsers.scala
+++ b/src/library/scala/util/parsing/combinator/JavaTokenParsers.scala
@@ -21,11 +21,12 @@ import scala.annotation.migration
  *  - `floatingPointNumber`
  */
 trait JavaTokenParsers extends RegexParsers {
-  /** Anything starting with an ASCII alphabetic character or underscore,
-   *  followed by zero or more repetitions of regex's `\w`.
+  /** Anything that is a valid Java identifier, according to
+   * <a href="http://docs.oracle.com/javase/specs/jls/se7/html/jls-3.html#jls-3.8">The Java Language Spec</a>.
+   * Generally, this means a letter, followed by zero or more letters or numbers.
    */
   def ident: Parser[String] =
-    """[a-zA-Z_]\w*""".r
+    """\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*""".r
   /** An integer, without sign or with a negative sign. */
   def wholeNumber: Parser[String] =
     """-?\d+""".r
diff --git a/test/files/run/parserJavaIdent.check b/test/files/run/parserJavaIdent.check
new file mode 100644
index 0000000000..597ddbee47
--- /dev/null
+++ b/test/files/run/parserJavaIdent.check
@@ -0,0 +1,26 @@
+[1.7] parsed: simple
+[1.8] parsed: with123
+[1.6] parsed: with$
+[1.10] parsed: withøßöèæ
+[1.6] parsed: with_
+[1.6] parsed: _with
+[1.1] failure: java identifier expected
+
+3start
+^
+[1.1] failure: java identifier expected
+
+-start
+^
+[1.5] failure: java identifier expected
+
+with-s
+    ^
+[1.3] failure: java identifier expected
+
+we♥scala
+  ^
+[1.6] failure: java identifier expected
+
+with space
+     ^
diff --git a/test/files/run/parserJavaIdent.scala b/test/files/run/parserJavaIdent.scala
new file mode 100644
index 0000000000..c068075e4e
--- /dev/null
+++ b/test/files/run/parserJavaIdent.scala
@@ -0,0 +1,26 @@
+object Test extends scala.util.parsing.combinator.JavaTokenParsers {
+
+    def test[A](s: String) {
+      val res = parseAll(ident, s) match {
+        case Failure(_, in) => Failure("java identifier expected", in)
+        case o => o
+      }
+      println(res)
+    }
+
+    def main(args: Array[String]) {
+      // Happy tests
+      test("simple")
+      test("with123")
+      test("with$")
+      test("withøßöèæ")
+      test("with_")
+      test("_with")
+      // Sad tests
+      test("3start")
+      test("-start")
+      test("with-s")
+      test("we♥scala")
+      test("with space")
+    }
+}