Added two options to tools/tokens, --sliding an...

Added two options to tools/tokens, --sliding and --freq. Using both:

% tools/tokens --sliding 10 --freq
src/compiler/scala/tools/nsc/typechecker | head -5 34 'if' '(' settings
'.' debug '.' value ')' log '(' 17 scala '.' tools '.' nsc ; 'package'
typechecker ;; 'import' 17 'package' scala '.' tools '.' nsc ; 'package'
typechecker ;;

16 '}' 'package' scala '.' tools '.' nsc ; 'package' typechecker
15 ')' '{' 'if' '(' settings '.' debug '.' value ')'

No review.

3 files changed, 38 insertions, 16 deletions

diff --git a/src/compiler/scala/tools/cmd/program/Tokens.scala b/src/compiler/scala/tools/cmd/program/Tokens.scala
index 01ca306c48..36786aa2b7 100644
--- a/src/compiler/scala/tools/cmd/program/Tokens.scala
+++ b/src/compiler/scala/tools/cmd/program/Tokens.scala
@@ -8,6 +8,7 @@ package cmd
 package program
 
 import nsc._
+import util.Chars.char2uescape
 import io._
 import ast.parser.Tokens._
 
@@ -16,12 +17,19 @@ import ast.parser.Tokens._
  */
 object Tokens {
   private val tokensUsage = "Usage: tokens [options] <path1 path2 ...>\n\nOptions:"
-  private val tokensOptions = List(
+  private val tokensUnary = List(
     "verbose" -> "be more verbose",
+    "freq"    -> "combine token lists and sort by frequency",
     "stats"   -> "output some stats"
   )
+  private val tokensBinary = List(
+    "sliding" -> "print tokens in groups of given size"
+  )
   private val tokensInfo = Simple.scalaProgramInfo("tokens", tokensUsage)
-  private lazy val TokensSpec = Simple(tokensInfo, tokensOptions, Nil, null)
+  private lazy val TokensSpec = Simple(tokensInfo, tokensUnary, tokensBinary, null)
+
+  def sanitize(x: Any): String = sanitize(x.toString)
+  def sanitize(str: String): String = str flatMap (x => if (x.isControl) char2uescape(x) else x.toString)
 
   def main(args0: Array[String]): Unit = {
     if (args0.isEmpty)
@@ -37,9 +45,23 @@ object Tokens {
     if (parsed isSet "--stats")
       println("Stats not yet implemented.")
 
-    files flatMap fromScalaSource foreach println
+    def raw = files flatMap fromScalaSource
+    def tokens: List[Any] =
+      if (parsed isSet "--sliding") raw sliding parsed("--sliding").toInt map (_ map sanitize mkString " ") toList
+      else raw
+
+    def output =
+      if (parsed isSet "--freq")
+        (tokens groupBy (x => x) mapValues (_.length)).toList sortBy (-_._2) map (x => x._2 + " " + x._1)
+      else
+        tokens
+
+    output foreach println
   }
 
+  def fromPaths(paths: String*): List[Any] =
+    (paths.toList flatMap walk).distinct flatMap fromScalaSource
+
   /** Given a path, returns all .scala files underneath it.
    */
   private def walk(arg: String): List[File] = {
diff --git a/src/compiler/scala/tools/nsc/Interpreter.scala b/src/compiler/scala/tools/nsc/Interpreter.scala
index 0cc0f65640..d3f51541bf 100644
--- a/src/compiler/scala/tools/nsc/Interpreter.scala
+++ b/src/compiler/scala/tools/nsc/Interpreter.scala
@@ -27,7 +27,7 @@ import scala.util.control.Exception.{ Catcher, catching, ultimately, unwrapping
 import io.{ PlainFile, VirtualDirectory }
 import reporters.{ ConsoleReporter, Reporter }
 import symtab.{ Flags, Names }
-import util.{ SourceFile, BatchSourceFile, ClassPath }
+import util.{ SourceFile, BatchSourceFile, ClassPath, Chars }
 import scala.reflect.NameTransformer
 import scala.tools.nsc.{ InterpreterResults => IR }
 import interpreter._
@@ -1275,21 +1275,10 @@ object Interpreter {
    *  This requires replacing all special characters by escape
    *  codes. It does not add the surrounding " marks.  */
   def string2code(str: String): String = {
-    /** Convert a character to a backslash-u escape */
-    def char2uescape(c: Char): String = {
-      var rest = c.toInt
-      val buf = new StringBuilder
-      for (i <- 1 to 4) {
-        buf ++= (rest % 16).toHexString
-        rest = rest / 16
-      }
-      "\\u" + buf.toString.reverse
-    }
-
     val res = new StringBuilder
     for (c <- str) c match {
       case '"' | '\'' | '\\'  => res += '\\' ; res += c
-      case _ if c.isControl   => res ++= char2uescape(c)
+      case _ if c.isControl   => res ++= Chars.char2uescape(c)
       case _                  => res += c
     }
     res.toString
diff --git a/src/compiler/scala/tools/nsc/util/Chars.scala b/src/compiler/scala/tools/nsc/util/Chars.scala
index 5a64f36eb4..562806b6eb 100755
--- a/src/compiler/scala/tools/nsc/util/Chars.scala
+++ b/src/compiler/scala/tools/nsc/util/Chars.scala
@@ -34,6 +34,17 @@ object Chars {
       -1
   }
 
+  /** Convert a character to a backslash-u escape */
+  def char2uescape(c: Char): String = {
+    var rest = c.toInt
+    val buf = new StringBuilder
+    for (i <- 1 to 4) {
+      buf ++= (rest % 16).toHexString
+      rest = rest / 16
+    }
+    "\\u" + buf.toString.reverse
+  }
+
   /** Is character a line break? */
   @inline def isLineBreakChar(c: Char) = (c: @switch) match {
     case LF|FF|CR|SU  => true