Tired of ugly-printing in the repl, I sort of f...

Tired of ugly-printing in the repl, I sort of finished some old code for
pretty printing token streams. It is at least a lot prettier than it
once was, and I threw in some power mode helpers. Now you can do this.

% scala -Dscala.repl.power
Welcome to Scala version 2.10.0.r25427-b20110801144412 (Java HotSpot(TM) 64-Bit Server VM, Java 1.6.0_26).

// .u turns a string into an URL like .r does into
a regex, and .pp pretty prints the url scala>
"https://raw.github.com/scalaz/scalaz/master/example/src/main/scala/scal
az/example/ExampleIteratee.scala".u.pp package scalaz.example

object ExampleIteratee {
  def main (args: Array[String]) = run

  import scalaz._
  import Scalaz._
  import IterV._
  [etc it's all there in real life]
}

No review.

6 files changed, 426 insertions, 15 deletions

diff --git a/src/compiler/scala/tools/nsc/interpreter/ExprTyper.scala b/src/compiler/scala/tools/nsc/interpreter/ExprTyper.scala
index e47eefa85c..3e7593cd8c 100644
--- a/src/compiler/scala/tools/nsc/interpreter/ExprTyper.scala
+++ b/src/compiler/scala/tools/nsc/interpreter/ExprTyper.scala
@@ -7,13 +7,15 @@ package scala.tools.nsc
 package interpreter
 
 import util.BatchSourceFile
-import ast.parser.Tokens.EOF
+import scala.tools.nsc.ast.parser.Tokens.EOF
 
 trait ExprTyper {
   val repl: IMain
+
   import repl._
-  import global.{ reporter => _, _ }
-  import syntaxAnalyzer.UnitParser
+  import replTokens.{ Tokenizer }
+  import global.{ reporter => _, Import => _, _ }
+  import syntaxAnalyzer.{ UnitParser, UnitScanner, token2name }
   import naming.freshInternalVarName
 
   object codeParser extends { val global: repl.global.type = repl.global } with CodeHandlers[Tree] {
@@ -22,11 +24,20 @@ trait ExprTyper {
       val unit    = new CompilationUnit(new BatchSourceFile("<console>", code))
       val scanner = new UnitParser(unit)
       val result  = rule(scanner)
+
       if (!reporter.hasErrors)
         scanner.accept(EOF)
 
       result
     }
+    def tokens(code: String) = {
+      reporter.reset()
+      val unit = new CompilationUnit(new BatchSourceFile("<tokens>", code))
+      val in   = new UnitScanner(unit)
+      in.init()
+
+      new Tokenizer(in) tokenIterator
+    }
 
     def decl(code: String)  = CodeHandlers.fail("todo")
     def defn(code: String)  = CodeHandlers.fail("todo")
@@ -50,6 +61,7 @@ trait ExprTyper {
       else Some(trees)
     }
   }
+  def tokens(line: String) = codeParser.tokens(line)
 
   // TODO: integrate these into a CodeHandler[Type].
 
diff --git a/src/compiler/scala/tools/nsc/interpreter/IMain.scala b/src/compiler/scala/tools/nsc/interpreter/IMain.scala
index b84bf178a0..207def1eb2 100644
--- a/src/compiler/scala/tools/nsc/interpreter/IMain.scala
+++ b/src/compiler/scala/tools/nsc/interpreter/IMain.scala
@@ -19,7 +19,7 @@ import reporters._
 import symtab.Flags
 import scala.reflect.internal.Names
 import scala.tools.util.PathResolver
-import scala.tools.nsc.util.{ ScalaClassLoader, Exceptional }
+import scala.tools.nsc.util.{ ScalaClassLoader, Exceptional, Indenter }
 import ScalaClassLoader.URLClassLoader
 import Exceptional.unwrap
 import scala.collection.{ mutable, immutable }
@@ -1007,11 +1007,20 @@ class IMain(val settings: Settings, protected val out: JPrintWriter) extends Imp
     }
   }
 
-  private object exprTyper extends { val repl: IMain.this.type = imain } with ExprTyper { }
+  object replTokens extends {
+    val global: imain.global.type = imain.global
+  } with ReplTokens { }
+
+  private object exprTyper extends {
+    val repl: IMain.this.type = imain
+  } with ExprTyper { }
+
   def parse(line: String): Option[List[Tree]] = exprTyper.parse(line)
   def typeOfExpression(expr: String, silent: Boolean = true): Option[Type] = {
     exprTyper.typeOfExpression(expr, silent)
   }
+  def prettyPrint(code: String) =
+    replTokens.prettyPrint(exprTyper tokens code)
 
   protected def onlyTerms(xs: List[Name]) = xs collect { case x: TermName => x }
   protected def onlyTypes(xs: List[Name]) = xs collect { case x: TypeName => x }
@@ -1090,9 +1099,20 @@ class IMain(val settings: Settings, protected val out: JPrintWriter) extends Imp
     /** Secret bookcase entrance for repl debuggers: end the line
      *  with "// show" and see what's going on.
      */
-    if (repllog.isTrace || (code.lines exists (_.trim endsWith "// show"))) {
-      echo(code)
-      parse(code) foreach (ts => ts foreach (t => withoutUnwrapping(repldbg(asCompactString(t)))))
+    def isShow    = code.lines exists (_.trim endsWith "// show")
+    def isShowRaw = code.lines exists (_.trim endsWith "// raw")
+
+    // checking for various debug signals
+    if (isShowRaw)
+      replTokens withRawTokens prettyPrint(code)
+    else if (repllog.isTrace || isShow)
+      prettyPrint(code)
+
+    // old style
+    parse(code) foreach { ts =>
+      ts foreach { t =>
+        withoutUnwrapping(repldbg(asCompactString(t)))
+      }
     }
   }
 
diff --git a/src/compiler/scala/tools/nsc/interpreter/Power.scala b/src/compiler/scala/tools/nsc/interpreter/Power.scala
index cfef2c5e87..477803ed5b 100644
--- a/src/compiler/scala/tools/nsc/interpreter/Power.scala
+++ b/src/compiler/scala/tools/nsc/interpreter/Power.scala
@@ -317,10 +317,13 @@ abstract class Power(
   }
 
   class RichReplString(s: String) {
+    // pretty print the string
+    def pp { intp.prettyPrint(s) }
+    // make an url out of the string
     def u: URL = (
-      if (s contains ":") new java.net.URL(s)
-      else if (new java.io.File(s) exists) new java.io.File(s).toURI.toURL
-      else new java.net.URL("http://" + s)
+      if (s contains ":") new URL(s)
+      else if (new JFile(s) exists) new JFile(s).toURI.toURL
+      else new URL("http://" + s)
     )
   }
   class RichInputStream(in: InputStream)(implicit codec: Codec) {
@@ -328,6 +331,10 @@ abstract class Power(
     def slurp(): String       = io.Streamable.slurp(in)
     def <<(): String          = slurp()
   }
+  class RichReplURL(url: URL)(implicit codec: Codec) {
+    def slurp(): String = io.Streamable.slurp(url)
+    def pp { intp prettyPrint slurp() }
+  }
 
   protected trait Implicits1 {
     // fallback
@@ -355,8 +362,9 @@ abstract class Power(
       new MultiPrettifierClass[T](xs.toSeq)
     implicit def replPrettifier[T] : Prettifier[T] = Prettifier.default[T]
     implicit def replTypeApplication(sym: Symbol): RichSymbol = new RichSymbol(sym)
+
     implicit def replInputStream(in: InputStream)(implicit codec: Codec) = new RichInputStream(in)
-    implicit def replInputStreamURL(url: URL)(implicit codec: Codec) = new RichInputStream(url.openStream())
+    implicit def replEnhancedURLs(url: URL)(implicit codec: Codec): RichReplURL = new RichReplURL(url)(codec)
   }
   object Implicits extends Implicits2 { }
 
diff --git a/src/compiler/scala/tools/nsc/interpreter/ReplTokens.scala b/src/compiler/scala/tools/nsc/interpreter/ReplTokens.scala
new file mode 100644
index 0000000000..2663eef897
--- /dev/null
+++ b/src/compiler/scala/tools/nsc/interpreter/ReplTokens.scala
@@ -0,0 +1,286 @@
+/* NSC -- new Scala compiler
+ * Copyright 2005-2011 LAMP/EPFL
+ * @author  Paul Phillips
+ */
+
+package scala.tools.nsc
+package interpreter
+
+import util.{ BatchSourceFile, Indenter }
+import scala.tools.nsc.ast.parser.Tokens._
+import java.lang.Integer.toOctalString
+
+/** This began as an attempt at a completely minimal
+ *  pretty printer for a token stream, but as it turns out
+ *  it's "minimal, pretty, scala: pick any two." So
+ *  now it's an unattractive hybrid between minimalism
+ *  and other things.  Still, it's a big improvement on the
+ *  way I was printing source in the repl, so in it goes.
+ *
+ *  @author   Paul Phillips
+ */
+abstract class ReplTokens {
+  val global: Global
+
+  import global._
+  import syntaxAnalyzer.{ UnitScanner, token2name }
+
+  // Mostly, this means print <NL> so we can see where
+  // semicolon inference took place.
+  private var rawTokens: Boolean = false
+  def withRawTokens[T](body: => T): T = {
+    rawTokens = true
+    try body
+    finally rawTokens = false
+  }
+  // There's the seed of a good idea in here, but you wouldn't
+  // know it from the current implementation.  The objects are
+  // trying to depict what feelings of coziness a given token
+  // might have toward its immediate neighbors.  But it lacks
+  // sufficient granularity and a good resolution mechanism.
+  sealed abstract class Cozy(l: => Boolean, r: => Boolean) {
+    def left = l
+    def right = r
+  }
+  object Cozy {
+    def unapply(x: Cozy) = Some((x.left, x.right))
+  }
+  case object  |--*--|  extends Cozy(false, false)
+  case object    <*--|  extends Cozy(true, false)
+  case object  |--*>    extends Cozy(false, true)
+  case object    <*>    extends Cozy(true, true)
+
+  @annotation.switch def escapedChar(ch: Char): String = ch match {
+    case '\b' => "\\b"
+    case '\t' => "\\t"
+    case '\n' => "\\n"
+    case '\f' => "\\f"
+    case '\r' => "\\r"
+    case '"'  => "\\\""
+    case '\'' => "\\\'"
+    case '\\' => "\\\\"
+    case _    => String.valueOf(ch)
+  }
+  def escape(text: String): String = {
+    text map { ch =>
+      if (ch.isControl) "\\0" + toOctalString(ch)
+      else escapedChar(ch)
+    } mkString ""
+  }
+  private class Arrow(code: Int) {
+    def ->(str: String): (Int, ReplToken)     = (code, Token(code)(str))
+    def ->(tok: ReplToken): (Int, ReplToken)  = (code, tok)
+  }
+  private val symbolTokenMap = {
+    implicit def liftToken(code: Int): Arrow = new Arrow(code)
+
+    Map[Int, ReplToken](
+      AT         -> At,
+      CASECLASS  -> "case class",
+      CASEOBJECT -> "case object",
+      COLON      -> Colon,
+      COMMA      -> Comma,
+      DOT        -> Dot,
+      EOF        -> Eof,
+      ERROR      -> "<error>",
+      FALSE      -> False,
+      IMPORT     -> Import,
+      LBRACE     -> LBrace,
+      LBRACKET   -> LBracket,
+      LPAREN     -> LParen,
+      NEWLINE    -> Newline,
+      NEWLINES   -> Newlines,
+      NULL       -> Null,
+      RBRACE     -> RBrace,
+      RBRACKET   -> RBracket,
+      RPAREN     -> RParen,
+      SEMI       -> Semi,
+      SUBTYPE    -> Subtype,
+      SUPERTYPE  -> Supertype,
+      TRUE       -> True,
+      VIEWBOUND  -> ViewBound,
+      XMLSTART   -> "<xmlstart>"
+    )
+  }
+  def isAlphaId(t: ReplToken) = t match {
+    case Id(name) => name forall (ch => ch.isDigit || ch.isLetter || ch == '_')
+    case _ => false
+  }
+  def isOperatorId(t: ReplToken) = t match {
+    case Id(name) => !isAlphaId(t)
+    case _        => false
+  }
+
+  sealed abstract class ReplToken(val tokenString: String, val cozy: Cozy) {
+    def this(str: String) = this(str, |--*--| )
+
+    def insistsOnSpace = false
+    def cozyRight(other: ReplToken) = (cozy.right || other.cozy.left)
+    def cozyLeft(other: ReplToken)  = (cozy.left || other.cozy.right)
+
+    final def <--?-->(other: ReplToken) = {
+      !(insistsOnSpace || other.insistsOnSpace) && (
+        (this cozyRight other) ||
+        (other cozyLeft this)
+      )
+    }
+
+    // to show invisibles
+    def rawString = tokenString
+    override def toString = (
+      if (rawTokens) rawString
+      else tokenString
+    )
+  }
+  trait InsistCozyRight extends ReplToken {
+    final override def cozyRight(other: ReplToken) = true
+  }
+  trait InsistCozyLeft extends ReplToken {
+    final override def cozyLeft(other: ReplToken) = true
+  }
+  trait InsistCozy extends InsistCozyLeft with InsistCozyRight { }
+  trait InsistSpaced extends ReplToken {
+    final override def insistsOnSpace = true
+  }
+  trait CozyWithLetters extends ReplToken {
+    override def cozyRight(other: ReplToken) = isAlphaId(other) || super.cozyRight(other)
+    override def cozyLeft(other: ReplToken) = isAlphaId(other) || super.cozyLeft(other)
+  }
+  trait Brackets extends ReplToken {
+    private def isCozyToken(t: ReplToken)    = t == LBracket || t == RBracket || isAlphaId(t)
+    override def cozyRight(other: ReplToken) = isCozyToken(other) || super.cozyRight(other)
+    override def cozyLeft(other: ReplToken)  = isCozyToken(other) || super.cozyLeft(other)
+  }
+
+  case class Token(value: Int)(str: String) extends ReplToken(str) { }
+  case class Id(name: String) extends ReplToken(name) { }
+  case class Lit[T](value: T) extends ReplToken(value match {
+    case s: String  => "\"" + s + "\""
+    case _          => "" + value
+  })
+  case object At extends ReplToken("@") with InsistCozyRight { }
+  case object Colon extends ReplToken(":", <*--|)
+  case object Comma extends ReplToken(",", <*--|) with InsistCozyLeft { }
+  case object Dot extends ReplToken(".", <*>) with InsistCozy { }
+  case object Eof extends ReplToken("EOF")
+  case object ErrorToken extends ReplToken("<internal error>")
+  case object False extends ReplToken("false")
+  case object Import extends ReplToken("import")
+  case object LBrace extends ReplToken("{") with InsistSpaced { }
+  case object LBracket extends ReplToken("[") with Brackets { }
+  case object LParen extends ReplToken("(", |--*>)
+  case object Newline extends ReplToken("\n", <*>) with InsistCozy { override def rawString = "<NL>\n" }
+  case object Newlines extends ReplToken("\n\n", <*>) with InsistCozy { override def rawString = "<NLS>\n\n" }
+  case object Null extends ReplToken("null")
+  case object RBrace extends ReplToken("}", |--*>) with InsistSpaced { }
+  case object RBracket extends ReplToken("]") with Brackets { }
+  case object RParen extends ReplToken(")", <*--|)
+  case object Semi extends ReplToken(";", <*--|)
+  case object Subtype extends ReplToken("<:") with InsistSpaced { }
+  case object Supertype extends ReplToken(">:") with InsistSpaced { }
+  case object True extends ReplToken("true")
+  case object ViewBound extends ReplToken("<%") with InsistSpaced { }
+
+  class Tokenizer(in: UnitScanner) {
+    private def translate(tokenCode: Int): ReplToken = tokenCode match {
+      case IDENTIFIER | BACKQUOTED_IDENT => Id("" + in.name)
+      case CHARLIT | INTLIT | LONGLIT    => Lit(in.intVal)
+      case DOUBLELIT | FLOATLIT          => Lit(in.floatVal)
+      case STRINGLIT                     => Lit(escape(in.strVal))
+      case SYMBOLLIT                     => Lit(scala.Symbol(in.strVal))
+      case _ =>
+        symbolTokenMap.getOrElse(
+          tokenCode,
+          token2name get tokenCode match {
+            case Some(name) => Token(tokenCode)("" + name)
+            case _          => Token(tokenCode)("<unknown: " + tokenCode + ">")
+          }
+        )
+    }
+    def tokenIterator: Iterator[ReplToken] = (
+      Iterator continually {
+        try translate(in.token)
+        finally in.nextToken()
+      } takeWhile (_ ne Eof)
+    )
+  }
+
+  def prettyPrintRaw(tokens: TraversableOnce[ReplToken]) {
+    withRawTokens(prettyPrint(tokens))
+  }
+
+  def prettyPrint(tokens: TraversableOnce[ReplToken]) {
+    new TokenPrinter prettyPrint tokens
+  }
+
+  private class TokenPrinter {
+    type TokenTriple = (ReplToken, ReplToken, ReplToken)
+    val writer = new Indenter
+    var prev: List[ReplToken] = Nil
+
+    def isIdentPart(t: ReplToken) = t match {
+      case Dot | Id(_)  => true
+      case _            => false
+    }
+    def prevNonIdent = prev dropWhile isIdentPart match {
+      case Nil    => ErrorToken
+      case t :: _ => t
+    }
+    def inImport = prevNonIdent == Import
+
+    def printToken(left: ReplToken, token: ReplToken) = token match {
+      case LBrace =>
+        writer openIndent (
+          if (writer.atStartOfLine) token
+          else " " + token
+        )
+      case RBrace             =>
+        writer.closeIndent(token)
+      case tok @ (Newline | Newlines) =>
+        writer.nextIndent(tok)
+      case _                  =>
+        writer print (
+          if (writer.atStartOfLine) token
+          else if (left <--?--> token) token
+          else " " + token
+        )
+    }
+
+    def prettyPrint(tokens: TraversableOnce[ReplToken]) {
+      val it = Iterator(Newline) ++ tokens.toIterator ++ Iterator(Newline) sliding 3 map { x =>
+        (x: @unchecked) match {
+          case List(x1, x2, x3) => ((x1, x2, x3))
+        }
+      }
+      prettyPrint(it)
+    }
+    def prettyPrint(it: Iterator[TokenTriple]) {
+      while (it.hasNext) it.next match {
+        // special casing to avoid newline on empty blocks
+        case (left, LBrace, RBrace)        =>
+          it.next
+          writer print " { }"
+        // special casing to avoid newlines on import x.{ y, z, q }
+        case (left, LBrace, _) if inImport =>
+          writer print LBrace
+          def loop() {
+            if (it.hasNext) {
+              val (_, tok, _) = it.next
+              if (tok != Comma) {
+                writer print " "
+              }
+              writer print tok
+              if (tok != RBrace)
+                loop()
+            }
+          }
+          loop()
+        case (left, token, right) =>
+          printToken(left, token)
+
+          if (it.hasNext) prev ::= token
+          else printToken(token, right)
+      }
+    }
+  }
+}
diff --git a/src/compiler/scala/tools/nsc/util/Indenter.scala b/src/compiler/scala/tools/nsc/util/Indenter.scala
new file mode 100644
index 0000000000..f9ddc4a194
--- /dev/null
+++ b/src/compiler/scala/tools/nsc/util/Indenter.scala
@@ -0,0 +1,85 @@
+package scala.tools.nsc
+package util
+
+import java.io.PrintStream
+
+class Indenter(var stringFn: Any => String) {
+  def this() = this("" + _)
+  def out: PrintStream = System.out
+
+  var indentSpaces = 2
+  var isSorted     = false
+  var openString   = ""
+  var closeString  = ""
+
+  def braces: this.type = {
+    openString = " {"
+    closeString = "}"
+    this
+  }
+  def sorted: this.type = { isSorted = true ; this }
+  def stringify(fn: Any => String): this.type = {
+    stringFn = fn
+    this
+  }
+
+  def atStartOfLine = startOfLine
+  private var indentLevel = 0
+  private var startOfLine = true
+  def indent: this.type = { indentLevel += 1 ; this }
+  def undent: this.type = { indentLevel -= 1 ; this }
+  def currentIndent = " " * indentLevel * indentSpaces
+  def printIndent() = {
+    out.print(currentIndent)
+    startOfLine = true
+  }
+
+  // Execute the given body indented one level further.
+  def >>[T](body: => T): T = {
+    indentLevel += 1
+    try body
+    finally indentLevel -= 1
+  }
+
+  def openIndent(token: Any) {
+    print(token + "\n")
+    indent
+    printIndent()
+  }
+  def closeIndent(token: Any) {
+    print("\n")
+    undent
+    printIndent()
+    print(token)
+  }
+  def finishLine(token: Any) {
+    print(token)
+    printIndent()
+  }
+  def nextIndent(endOfLine: Any) = finishLine(endOfLine)
+
+  def block(label: String)(body: => Unit) {
+    if (label != "" || openString != "")
+      pp(label + openString)
+
+    this >> body
+
+    if (closeString != "")
+      pp(closeString)
+  }
+  def print(x: Any) = {
+    out print stringFn(x)
+    out.flush()
+    startOfLine = false
+  }
+  def pps(xs: TraversableOnce[Any]) {
+    if (isSorted) xs.toSeq.sortBy("" + _) foreach pp
+    else xs foreach pp
+  }
+  def pp(x: Any) {
+    printIndent()
+    out println stringFn(x)
+    out.flush()
+    startOfLine = false
+  }
+}
diff --git a/src/scalap/scala/tools/scalap/Main.scala b/src/scalap/scala/tools/scalap/Main.scala
index 24f26e2b60..7254b00480 100644
--- a/src/scalap/scala/tools/scalap/Main.scala
+++ b/src/scalap/scala/tools/scalap/Main.scala
@@ -131,9 +131,9 @@ class Main {
     def asClasspathString = ""
 
     val context     = DefaultJavaContext
-    val classes     = IndexedSeq[ClassRep]()
-    val packages    = IndexedSeq[ClassPath[AbstractFile]]()
-    val sourcepaths = IndexedSeq[AbstractFile]()
+    val classes     = IndexedSeq()
+    val packages    = IndexedSeq()
+    val sourcepaths = IndexedSeq()
   }
 }