diff options
author | Paul Phillips <paulp@improving.org> | 2010-04-21 23:11:43 +0000 |
---|---|---|
committer | Paul Phillips <paulp@improving.org> | 2010-04-21 23:11:43 +0000 |
commit | 3861a3a42e4347c733a3c15a7cd85fa86d1d1011 (patch) | |
tree | 0f50b09137f53a2025788a0820ec56f3166b64f2 /src | |
parent | c8266ce2b587e495179d8ce50207e27f3352821d (diff) | |
download | scala-3861a3a42e4347c733a3c15a7cd85fa86d1d1011.tar.gz scala-3861a3a42e4347c733a3c15a7cd85fa86d1d1011.tar.bz2 scala-3861a3a42e4347c733a3c15a7cd85fa86d1d1011.zip |
Added two options to tools/tokens, --sliding an...
Added two options to tools/tokens, --sliding and --freq. Using both:
% tools/tokens --sliding 10 --freq
src/compiler/scala/tools/nsc/typechecker | head -5 34 'if' '(' settings
'.' debug '.' value ')' log '(' 17 scala '.' tools '.' nsc ; 'package'
typechecker ;; 'import' 17 'package' scala '.' tools '.' nsc ; 'package'
typechecker ;;
16 '}' 'package' scala '.' tools '.' nsc ; 'package' typechecker
15 ')' '{' 'if' '(' settings '.' debug '.' value ')'
No review.
Diffstat (limited to 'src')
-rw-r--r-- | src/compiler/scala/tools/cmd/program/Tokens.scala | 28 | ||||
-rw-r--r-- | src/compiler/scala/tools/nsc/Interpreter.scala | 15 | ||||
-rwxr-xr-x | src/compiler/scala/tools/nsc/util/Chars.scala | 11 |
3 files changed, 38 insertions, 16 deletions
diff --git a/src/compiler/scala/tools/cmd/program/Tokens.scala b/src/compiler/scala/tools/cmd/program/Tokens.scala index 01ca306c48..36786aa2b7 100644 --- a/src/compiler/scala/tools/cmd/program/Tokens.scala +++ b/src/compiler/scala/tools/cmd/program/Tokens.scala @@ -8,6 +8,7 @@ package cmd package program import nsc._ +import util.Chars.char2uescape import io._ import ast.parser.Tokens._ @@ -16,12 +17,19 @@ import ast.parser.Tokens._ */ object Tokens { private val tokensUsage = "Usage: tokens [options] <path1 path2 ...>\n\nOptions:" - private val tokensOptions = List( + private val tokensUnary = List( "verbose" -> "be more verbose", + "freq" -> "combine token lists and sort by frequency", "stats" -> "output some stats" ) + private val tokensBinary = List( + "sliding" -> "print tokens in groups of given size" + ) private val tokensInfo = Simple.scalaProgramInfo("tokens", tokensUsage) - private lazy val TokensSpec = Simple(tokensInfo, tokensOptions, Nil, null) + private lazy val TokensSpec = Simple(tokensInfo, tokensUnary, tokensBinary, null) + + def sanitize(x: Any): String = sanitize(x.toString) + def sanitize(str: String): String = str flatMap (x => if (x.isControl) char2uescape(x) else x.toString) def main(args0: Array[String]): Unit = { if (args0.isEmpty) @@ -37,9 +45,23 @@ object Tokens { if (parsed isSet "--stats") println("Stats not yet implemented.") - files flatMap fromScalaSource foreach println + def raw = files flatMap fromScalaSource + def tokens: List[Any] = + if (parsed isSet "--sliding") raw sliding parsed("--sliding").toInt map (_ map sanitize mkString " ") toList + else raw + + def output = + if (parsed isSet "--freq") + (tokens groupBy (x => x) mapValues (_.length)).toList sortBy (-_._2) map (x => x._2 + " " + x._1) + else + tokens + + output foreach println } + def fromPaths(paths: String*): List[Any] = + (paths.toList flatMap walk).distinct flatMap fromScalaSource + /** Given a path, returns all .scala files underneath it. */ private def walk(arg: String): List[File] = { diff --git a/src/compiler/scala/tools/nsc/Interpreter.scala b/src/compiler/scala/tools/nsc/Interpreter.scala index 0cc0f65640..d3f51541bf 100644 --- a/src/compiler/scala/tools/nsc/Interpreter.scala +++ b/src/compiler/scala/tools/nsc/Interpreter.scala @@ -27,7 +27,7 @@ import scala.util.control.Exception.{ Catcher, catching, ultimately, unwrapping import io.{ PlainFile, VirtualDirectory } import reporters.{ ConsoleReporter, Reporter } import symtab.{ Flags, Names } -import util.{ SourceFile, BatchSourceFile, ClassPath } +import util.{ SourceFile, BatchSourceFile, ClassPath, Chars } import scala.reflect.NameTransformer import scala.tools.nsc.{ InterpreterResults => IR } import interpreter._ @@ -1275,21 +1275,10 @@ object Interpreter { * This requires replacing all special characters by escape * codes. It does not add the surrounding " marks. */ def string2code(str: String): String = { - /** Convert a character to a backslash-u escape */ - def char2uescape(c: Char): String = { - var rest = c.toInt - val buf = new StringBuilder - for (i <- 1 to 4) { - buf ++= (rest % 16).toHexString - rest = rest / 16 - } - "\\u" + buf.toString.reverse - } - val res = new StringBuilder for (c <- str) c match { case '"' | '\'' | '\\' => res += '\\' ; res += c - case _ if c.isControl => res ++= char2uescape(c) + case _ if c.isControl => res ++= Chars.char2uescape(c) case _ => res += c } res.toString diff --git a/src/compiler/scala/tools/nsc/util/Chars.scala b/src/compiler/scala/tools/nsc/util/Chars.scala index 5a64f36eb4..562806b6eb 100755 --- a/src/compiler/scala/tools/nsc/util/Chars.scala +++ b/src/compiler/scala/tools/nsc/util/Chars.scala @@ -34,6 +34,17 @@ object Chars { -1 } + /** Convert a character to a backslash-u escape */ + def char2uescape(c: Char): String = { + var rest = c.toInt + val buf = new StringBuilder + for (i <- 1 to 4) { + buf ++= (rest % 16).toHexString + rest = rest / 16 + } + "\\u" + buf.toString.reverse + } + /** Is character a line break? */ @inline def isLineBreakChar(c: Char) = (c: @switch) match { case LF|FF|CR|SU => true |