Finished new position unpickling code.

Pickling still has to be written.

6 files changed, 97 insertions, 118 deletions

diff --git a/src/dotty/tools/dotc/core/pickling/DottyUnpickler.scala b/src/dotty/tools/dotc/core/pickling/DottyUnpickler.scala
index 9dbbec01e..f4a6f2421 100644
--- a/src/dotty/tools/dotc/core/pickling/DottyUnpickler.scala
+++ b/src/dotty/tools/dotc/core/pickling/DottyUnpickler.scala
@@ -6,6 +6,8 @@ package pickling
 import Contexts._, SymDenotations._
 import dotty.tools.dotc.ast.tpd
 import TastyUnpickler._, TastyBuffer._
+import util.Positions._
+import PositionUnpickler._
 
 object DottyUnpickler {
 
@@ -22,21 +24,25 @@ class DottyUnpickler(bytes: Array[Byte], readPositions: Boolean = false)(implici
   import tpd._
 
   val unpickler = new TastyUnpickler(bytes)
-
+  
   def result: List[Tree] = {
-    val trees = unpickler.unpickle(new TreeSectionUnpickler()).getOrElse(Nil)
-    if (readPositions)
-      unpickler.unpickle(new PositionsSectionUnpickler(trees))
-    trees
+    val (totalRange, positions) = 
+      if (readPositions) 
+        unpickler.unpickle(new PositionsSectionUnpickler())
+          .getOrElse((NoPosition, null))
+      else (NoPosition, null)
+    unpickler.unpickle(new TreeSectionUnpickler(totalRange, positions))
+      .getOrElse(Nil)
   }
 
-  class TreeSectionUnpickler()(implicit ctx: Context) extends SectionUnpickler[List[Tree]]("ASTs") {
+  class TreeSectionUnpickler(totalRange: Position, positions: AddrToPosition)(implicit ctx: Context) 
+      extends SectionUnpickler[List[Tree]]("ASTs") {
     def unpickle(reader: TastyReader, tastyName: TastyName.Table): List[Tree] =
-      new TreeUnpickler(reader, tastyName, readPositions).unpickle()
+      new TreeUnpickler(reader, tastyName, totalRange, positions).unpickle()
   }
   
-  class PositionsSectionUnpickler(trees: List[Tree])(implicit ctx: Context) extends SectionUnpickler[Unit]("Positions") {
-    def unpickle(reader: TastyReader, tastyName: TastyName.Table): Unit =
-      new PositionReader(reader).unpickle(trees)
+  class PositionsSectionUnpickler()(implicit ctx: Context) extends SectionUnpickler[(Position, AddrToPosition)]("Positions") {
+    def unpickle(reader: TastyReader, tastyName: TastyName.Table) =
+      new PositionUnpickler(reader).unpickle()
   }
 }
diff --git a/src/dotty/tools/dotc/core/pickling/PickleFormat.scala b/src/dotty/tools/dotc/core/pickling/PickleFormat.scala
index 1c8555430..557a45f8c 100644
--- a/src/dotty/tools/dotc/core/pickling/PickleFormat.scala
+++ b/src/dotty/tools/dotc/core/pickling/PickleFormat.scala
@@ -185,8 +185,8 @@ Standard Section: "Positions" sourceLength_Nat Assoc*
 
   Assoc         = addr_Delta offset_Delta offset_Delta?
                                             // addr_Delta      :
-                                            //    Difference of address to last recorded node. Always > 0
-                                            //    (The initial base is -1, so a a first node of 0 would have a delta of 1).
+                                            //    Difference of address to last recorded node.
+                                            //    All but the first addr_Deltas are > 0, the first is >= 0.
                                             // 2nd offset_Delta:
                                             //    Difference of end offset of addressed node vs parent node. Always <= 0
                                             // 1st offset Delta, if delta >= 0 or 2nd offset delta exists
diff --git a/src/dotty/tools/dotc/core/pickling/PositionReader.scala b/src/dotty/tools/dotc/core/pickling/PositionReader.scala
deleted file mode 100644
index 5e35fa397..000000000
--- a/src/dotty/tools/dotc/core/pickling/PositionReader.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-package dotty.tools
-package dotc
-package core
-package pickling
-
-import java.io.IOException
-
-import Contexts._
-import ast.tpd
-import ast.Trees.{Lazy,WithLazyField}
-import TastyBuffer._
-import Traversals._
-import util.Positions._
-
-/** Unpickler for start or end offset of tree positions */
-class PositionReader(reader: TastyReader, startOffset: Int, initNextOffset: Int, initNextAddr: Int) {
-  import reader._
-  import tpd._
-      
-  def this(reader: TastyReader) = {
-    this(reader, 0, 0, 0)
-    next()
-  }
-  
-  var lastOffset: Int = startOffset
-  var nextOffset = initNextOffset
-  var nextAddr = initNextAddr
-  
-  private def fork = 
-    new PositionReader(subReader(currentAddr, endAddr), lastOffset, nextOffset, nextAddr)
-  
-  private def next() = {
-    lastOffset = nextOffset
-    if (!isAtEnd) {
-      nextOffset += readInt()
-      nextAddr += readInt()
-    }
-  }  
-
-  def readStart(tree: Tree) = 
-    if (tree.pos.exists) { // TODO assign positions of tree nodes with NoPosition afterwards based on the context.
-      val addr = tree.pos.end
-      if (addr == nextAddr) next()
-      tree.setPosUnchecked(tree.pos.withStart(lastOffset))
-    }
-  
-  def readEnd(tree: Tree) = {
-    if (tree.pos.exists) {
-      val addr = tree.pos.end
-      tree match {
-        case tree: WithLazyField[_] =>
-          tree.unforced match {
-            case rdr: TreeUnpickler#LazyReader[_] =>
-              //rdr.posReader = Some(fork)
-              while (addr != nextAddr) next()
-            case _ =>
-          }
-        case _ =>
-      }
-      if (addr == nextAddr) next()
-      tree.setPosUnchecked(tree.pos.withEnd(lastOffset))
-    }
-  }
-  
-  def unpickle(x: Any) =
-    traverse(x, readStart, readEnd)
-}
\ No newline at end of file
diff --git a/src/dotty/tools/dotc/core/pickling/PositionUnpickler.scala b/src/dotty/tools/dotc/core/pickling/PositionUnpickler.scala
new file mode 100644
index 000000000..c341d58cc
--- /dev/null
+++ b/src/dotty/tools/dotc/core/pickling/PositionUnpickler.scala
@@ -0,0 +1,37 @@
+package dotty.tools
+package dotc
+package core
+package pickling
+
+import util.Positions._
+import collection.mutable
+import TastyBuffer.Addr
+
+object PositionUnpickler {
+  type AddrToPosition = mutable.HashMap[Addr, Position]
+}
+
+/** Unpickler for tree positions */
+class PositionUnpickler(reader: TastyReader) {
+  import PositionUnpickler._
+  import reader._
+
+  def unpickle(): (Position, AddrToPosition) = {
+    val positions = new mutable.HashMap[Addr, Position] // Dotty deviation: Can't use new AddrToPosition here. TODO: fix this!
+    val sourceLength = readNat()
+    def readDelta() = if (isAtEnd) 0 else readInt()
+    var curIndex: Addr = Addr(readDelta())
+    while (!isAtEnd) {
+      val delta1 = readDelta()
+      val delta2 = readDelta()
+      val (startDelta, endDelta, indexDelta) = 
+        if (delta2 <= 0) (delta1, -delta2, readDelta())
+        else if (delta1 < 0) (0, -delta1, delta2)
+        else (delta1, 0, delta2)
+      positions(curIndex) = Position(startDelta, endDelta, 0) 
+        // make non-synthetic position; will be made synthetic by normalization.
+      curIndex += indexDelta
+    }
+    (Position(0, sourceLength), positions)
+  }
+}
\ No newline at end of file
diff --git a/src/dotty/tools/dotc/core/pickling/TastyPrinter.scala b/src/dotty/tools/dotc/core/pickling/TastyPrinter.scala
index 751642a51..83b53abb3 100644
--- a/src/dotty/tools/dotc/core/pickling/TastyPrinter.scala
+++ b/src/dotty/tools/dotc/core/pickling/TastyPrinter.scala
@@ -6,6 +6,9 @@ import Contexts._, Decorators._
 import printing.Texts._
 import TastyName._
 import TastyUnpickler._
+import TastyBuffer.Addr
+import util.Positions.Position
+import collection.mutable
 
 class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) {
   
@@ -33,11 +36,11 @@ class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) {
     println("Names:")
     printNames()
     println("Trees:")
-    unpickle(new TreeUnpickler)
-    unpickle(new PositionUnpickler)
+    unpickle(new TreeSectionUnpickler)
+    unpickle(new PositionSectionUnpickler)
   }
   
-  class TreeUnpickler extends SectionUnpickler[Unit]("ASTs") {
+  class TreeSectionUnpickler extends SectionUnpickler[Unit]("ASTs") {
     import PickleFormat._
     def unpickle(reader: TastyReader, tastyName: TastyName.Table): Unit = {
       import reader._
@@ -103,16 +106,12 @@ class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) {
     }
   }
 
-  class PositionUnpickler extends SectionUnpickler[Unit]("Positions") {
+  class PositionSectionUnpickler extends SectionUnpickler[Unit]("Positions") {
     def unpickle(reader: TastyReader, tastyName: TastyName.Table): Unit = {
-      import reader._
-      var lastOffset = 0
-      var lastAddr = 0
-      while (!isAtEnd) {
-        lastOffset += readInt()
-        lastAddr += readInt()
-        println(s"$lastOffset: $lastAddr")
-      }
+      val (totalRange, positions) = new PositionUnpickler(reader).unpickle()
+      println(s"Positions in $totalRange:")
+      val sorted = positions.toSeq.sortBy(_._1.index)
+      for ((addr, pos) <- sorted) println(s"$addr: $pos")
     }
   }
 }
\ No newline at end of file
diff --git a/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala b/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala
index 72020012d..ceb1f7db1 100644
--- a/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala
+++ b/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala
@@ -17,22 +17,28 @@ import typer.Mode
 /** Unpickler for typed trees
  *  @param reader         the reader from which to unpickle
  *  @param tastyName      the nametable
- *  @param roots          a set of pre-existing symbols whose attributes should be overwritten
- *                        instead of creating a new symbol.
- *  @param readPositions  a flag indicating whether positions should be read
+ *  @param totalRange     the range position enclosing all returned trees,
+                          or NoPosition if positions should not be unpickled
+ *  @param positions      A map from 
  */
-class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositions: Boolean) {
+class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, 
+    totalRange: Position, positions: collection.Map[Addr, Position]) {
   import dotty.tools.dotc.core.pickling.PickleFormat._
   import TastyName._
   import tpd._
+  
+  def readPositions = totalRange.exists
 
   private val symAtAddr  = new mutable.HashMap[Addr, Symbol]
   private val treeAtAddr = new mutable.HashMap[Addr, Tree]
   
   private val typeAtAddr = new mutable.HashMap[Addr, Type] // currently populated only for types that are known to be SHAREd.
 
-  def unpickle()(implicit ctx: Context): List[Tree] =
-    new TreeReader(reader).readTopLevelStats()
+  def unpickle()(implicit ctx: Context): List[Tree] = {
+    val stats = new TreeReader(reader).readTopLevelStats()
+    normalizePos(stats, totalRange)
+    stats
+  }
     
   def toTermName(tname: TastyName): TermName = tname match {
     case Simple(name) => name
@@ -475,7 +481,7 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
         else Modifiers(annotations = sym.annotations.map(_.tree))
       tree.withMods(mods) // record annotations in tree so that tree positions can be filled in.
       skipTo(end)
-      addAddr(start, tree)
+      setPos(start, tree)
     }
     
     private def readTemplate(implicit ctx: Context): Template = {
@@ -502,7 +508,7 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
       fork.indexStats(end)
       val constr = readIndexedDef().asInstanceOf[DefDef]
       val lazyStats = readLater(end, rdr => ctx => rdr.readIndexedStats(localDummy, end)(ctx))
-      addAddr(start,
+      setPos(start,
         untpd.Template(constr, parents, self, lazyStats)
           .withType(localDummy.nonMemberTermRef))
     }
@@ -625,14 +631,14 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
         finally assert(currentAddr == end, s"$currentAddr $end ${astTagToString(tag)}")
       }
 
-      addAddr(start, 
+      setPos(start, 
         if (tag < firstLengthTreeTag) readSimpleTerm()
         else readLengthTerm())
     }
     
     def readTpt()(implicit ctx: Context) = {
       val start = currentAddr
-      addAddr(start, TypeTree(readType()))
+      setPos(start, TypeTree(readType()))
     }
 
     def readCases()(implicit ctx: Context): List[CaseDef] = 
@@ -643,7 +649,7 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
         val pat = readTerm()
         val rhs = readTerm()
         val guard = ifBefore(end)(readTerm(), EmptyTree)
-        addAddr(start, CaseDef(pat, guard, rhs))
+        setPos(start, CaseDef(pat, guard, rhs))
       }
 
     def readTopLevelStats()(implicit ctx: Context): List[Tree] = {
@@ -654,7 +660,7 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
           readByte()
           val end = readEnd()
           val pid = ref(readTermRef()).asInstanceOf[RefTree]
-          addAddr(start,
+          setPos(start,
             PackageDef(pid, readStats(NoSymbol, end)(ctx.fresh.setOwner(pid.symbol.moduleClass))))     
         }
         else readIndexedStat(ctx.owner)
@@ -668,20 +674,15 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
     }
     
 // ------ Hooks for positions ------------------------------------------------
-    
-    /** A temporary position encoding.
-     *  Start and end fields are each given the address of bytes from which tree is unpickled
-     *  These are later overridden with the actual offsets taken from the Positions section.
+          
+    /** Record address from which tree was created as a temporary position in the tree. 
+     *  The temporary position contains deltas relative to the position of the (as yet unknown)
+     *  parent node. It is marked as a non-synthetic source position. 
      */
-    def indexPosition(addr: Addr): Position = {
-      assert(addr.index < MaxOffset)
-      Position(addr.index, addr.index, 0)
-    }
-      
-    /** Record address from which tree was created as a temporary position in the tree. */
-    def addAddr[T <: Tree](addr: Addr, tree: T): T = {
-      if (readPositions) { tree.setPosUnchecked(indexPosition(addr)); tree }
-      else tree
+    def setPos[T <: Tree](addr: Addr, tree: T): T = {
+      if (readPositions)
+        tree.setPosUnchecked(positions.getOrElse(addr, Position(0, 0, 0)))
+      tree
     }
   }
   
@@ -696,8 +697,11 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
           assert(x.pos.exists)
           val absPos = Position(parentPos.start + x.pos.start, parentPos.end - x.pos.end)
           x.setPosUnchecked(absPos)
-          for (child <- x.productIterator) 
-            normalizePos(child, absPos)
+          x match {
+            case x: MemberDef => normalizePos(x.symbol.annotations, absPos)
+            case _ =>
+          }
+          normalizePos(x.productIterator, absPos)
         case x: DeferredPosition =>
           x.parentPos = parentPos
         case xs: List[_] =>