Two new command line programs in ~/tools: scmp ...

Two new command line programs in ~/tools: scmp and tokens.

1) scmp: will need a bit more fleshing out to be super useful, but here
is what you can do right now:

  // This means run the given command line first with the options
  // given to p1 and then without, and show the diff in output.
  % tools/scmp --p1 '-no-specialization -nowarn' scalac -Ydebug
  src/library/scala/Function1.scala

Upcoming features will involve seeing diffs of such things as the
pickled signatures of generated files and the javap disassembly.

2) tokens: tokenizes all the scala files found under any given paths and
prints one token per line.

Example: the five most frequently used tokens under scala/util.

  % tools/tokens src/library/scala/util |sort | uniq -c | sort -r | head
  -5 598 ')' 598 '(' 347 ; 294 '=' 278 ,

Good to see those parens are balanced.

Example: number of appearances of an identifier called x:

  % tools/tokens src/library/scala/util | grep ^x$ | wc 137

Way to go, x.  Review by community.

2 files changed, 134 insertions, 0 deletions

diff --git a/src/compiler/scala/tools/cmd/program/Scmp.scala b/src/compiler/scala/tools/cmd/program/Scmp.scala
new file mode 100644
index 0000000000..ff4fa11eaf
--- /dev/null
+++ b/src/compiler/scala/tools/cmd/program/Scmp.scala
@@ -0,0 +1,59 @@
+/* NEST (New Scala Test)
+ * Copyright 2007-2010 LAMP/EPFL
+ * @author  Paul Phillips
+ */
+
+package scala.tools
+package cmd
+package program
+
+import nsc.io._
+
+object Scmp {
+  private val scmpUsage = """
+    |Usage: scmp [options] <cmd line>
+    |Example: scmp --p1 '-no-specialization -Ydebug' scalac src/library/scala/Function1.scala
+    |
+    |Note: the command line must start with a path to scalac.
+    |""".stripMargin
+  private val scmpOptions = List(
+    "p1" -> "options for the first run only",
+    "p2" -> "options for the second run only"
+  )
+  private val scmpInfo = Simple.scalaProgramInfo("scmp", scmpUsage)
+  lazy val ScmpSpec = Simple(scmpInfo, Nil, scmpOptions, x => returning(x)(_.onlyKnownOptions = false))
+
+  def main(args0: Array[String]): Unit = {
+    if (args0.isEmpty)
+      return println(scmpUsage)
+
+    val runner = ScmpSpec instance args0
+    import runner._
+
+    val p1args = parsed.getOrElse("--p1", "")
+    val p2args = parsed.getOrElse("--p2", "")
+
+    if (p1args.isEmpty && p2args.isEmpty)
+      return println("At least one of --p1 and --p2 must be given.")
+    if (residualArgs.isEmpty)
+      return println("There is no command to run.")
+
+    def createCmd(extras: String) =
+      fromArgs(residualArgs.patch(1, toArgs(extras), 0))
+
+    def runCmd(cmd: String) = {
+      val output = Process(cmd, redirect = true).slurp()
+
+      returning(File.makeTemp())(_ writeAll output)
+    }
+
+    val cmds = List(p1args, p2args) map createCmd
+    println(cmds.mkString("Running command lines:\n  ", "\n  ", ""))
+
+    val files = cmds map runCmd map (_.path)
+    val diff = Process("diff %s %s".format(files: _*)).slurp()
+
+    if (diff.isEmpty) println("No differences.")
+    else println(diff)
+  }
+}
diff --git a/src/compiler/scala/tools/cmd/program/Tokens.scala b/src/compiler/scala/tools/cmd/program/Tokens.scala
new file mode 100644
index 0000000000..30502213f6
--- /dev/null
+++ b/src/compiler/scala/tools/cmd/program/Tokens.scala
@@ -0,0 +1,75 @@
+/* NSC -- new Scala compiler
+ * Copyright 2005-2010 LAMP/EPFL
+ * @author Paul Phillips
+ */
+
+package scala.tools
+package cmd
+package program
+
+import nsc._
+import io._
+import ast.parser.Tokens._
+
+/** Given paths on the command line, tokenizes any scala files found
+ *  and prints one token per line.
+ */
+object Tokens {
+  private val tokensUsage = "Usage: tokens [options] <path1 path2 ...>\n\nOptions:"
+  private val tokensOptions = List(
+    "verbose" -> "be more verbose",
+    "stats"   -> "output some stats"
+  )
+  private val tokensInfo = Simple.scalaProgramInfo("tokens", tokensUsage)
+  private lazy val TokensSpec = Simple(tokensInfo, tokensOptions, Nil, null)
+
+  def main(args0: Array[String]): Unit = {
+    if (args0.isEmpty)
+      return println(TokensSpec.helpMsg)
+
+    val runner = TokensSpec instance args0
+    import runner._
+
+    val files = (residualArgs flatMap walk).distinct
+    if (parsed isSet "verbose")
+      println("Tokenizing: " + (files map (_.name) mkString " "))
+
+    if (parsed isSet "stats")
+      println("Stats not yet implemented.")
+
+    files flatMap fromScalaSource foreach println
+  }
+
+  /** Given a path, returns all .scala files underneath it.
+   */
+  private def walk(arg: String): List[File] =
+    Path(arg).walkFilter(x => x.isFile && x.hasExtension("scala")) map (_.toFile) toList
+
+  /** Tokenizes a single scala file.
+   */
+  def fromScalaSource(file: Path): List[Any] = fromScalaSource(file.path)
+  def fromScalaSource(file: String): List[Any] = {
+    val global = new Global(new Settings())
+    import global._
+    import syntaxAnalyzer.{ UnitScanner, token2string }
+
+    val in = new UnitScanner(new CompilationUnit(getSourceFile(file)))
+    in.init()
+
+    Iterator continually {
+      val token = in.token match {
+        case IDENTIFIER | BACKQUOTED_IDENT  => in.name
+        case CHARLIT | INTLIT | LONGLIT     => in.intVal
+        case DOUBLELIT | FLOATLIT           => in.floatVal
+        case STRINGLIT                      => "\"" + in.strVal + "\""
+        case SEMI | NEWLINE                 => ";"
+        case NEWLINES                       => ";;"
+        case COMMA                          => ","
+        case EOF                            => null
+        case x                              => token2string(x)
+      }
+      in.nextToken()
+      token
+    } takeWhile (_ != null) toList
+  }
+}