Halfway to yet another scheme for handling positions

The previous attempt was very fragile, and did not play
well with lazy trees, despite best intentions. It did
not work correctly for normal trees, and it seemed anyeway
very hard to extend this to annotations.

The new scheme, as outlined in PicklerFormat, is
simpler and naturally handles lazy trees and annotations.

4 files changed, 57 insertions, 20 deletions

diff --git a/src/dotty/tools/dotc/core/Annotations.scala b/src/dotty/tools/dotc/core/Annotations.scala
index 79aa058ef..d264483e6 100644
--- a/src/dotty/tools/dotc/core/Annotations.scala
+++ b/src/dotty/tools/dotc/core/Annotations.scala
@@ -33,12 +33,13 @@ object Annotations {
     def tree(implicit ctx: Context): Tree = t
   }
 
-  case class LazyAnnotation(sym: Symbol)(treeFn: Context => Tree) extends Annotation {
+  abstract case class LazyAnnotation(sym: Symbol) extends Annotation {
     private var myTree: Tree = null
     def tree(implicit ctx: Context) = {
-      if (myTree == null) myTree = treeFn(ctx)
+      if (myTree == null) myTree = complete(ctx)
       myTree
     }
+    def complete(implicit ctx: Context): Tree
     override def symbol(implicit ctx: Context): Symbol = sym
   }
 
@@ -74,7 +75,9 @@ object Annotations {
     }
 
     def deferred(sym: Symbol, treeFn: Context => Tree)(implicit ctx: Context): Annotation =
-      new LazyAnnotation(sym)(treeFn)
+      new LazyAnnotation(sym) {
+        def complete(implicit ctx: Context) = treeFn(ctx)
+      }
 
     def deferred(atp: Type, args: List[Tree])(implicit ctx: Context): Annotation =
       deferred(atp.classSymbol, implicit ctx => New(atp, args))
diff --git a/src/dotty/tools/dotc/core/pickling/PickleFormat.scala b/src/dotty/tools/dotc/core/pickling/PickleFormat.scala
index 9e864fc43..1c8555430 100644
--- a/src/dotty/tools/dotc/core/pickling/PickleFormat.scala
+++ b/src/dotty/tools/dotc/core/pickling/PickleFormat.scala
@@ -181,15 +181,21 @@ Note: Tree tags are grouped into 4 categories that determine what follows, and t
 	Category 3 (tags 100-127):	tag Nat AST
 	Category 4 (tags 128-255):  tag Length <payload>
 
-Standard Section: "Positions" Assoc*
-
-  Assoc         = offset_Delta addr_Delta   // offsets and addresses determined by a tree traversal:
-                                            // For each node with positions <start..end>
-                                            //   start <encoding of children> end
+Standard Section: "Positions" sourceLength_Nat Assoc*
+
+  Assoc         = addr_Delta offset_Delta offset_Delta?
+                                            // addr_Delta      :
+                                            //    Difference of address to last recorded node. Always > 0
+                                            //    (The initial base is -1, so a a first node of 0 would have a delta of 1).
+                                            // 2nd offset_Delta:
+                                            //    Difference of end offset of addressed node vs parent node. Always <= 0
+                                            // 1st offset Delta, if delta >= 0 or 2nd offset delta exists
+                                            //    Difference of start offset of addressed node vs parent node.
+                                            // 1st offset Delta, if delta < 0 and 2nd offset delta does not exist:
+                                            //    Difference of end offset of addressed node vs parent node.
                                             // Offsets and addresses are difference encoded.
-                                            // Entries with same offset as previous entry are omitted.
+                                            // Nodes which have the same positions as their parents are omitted.
   Delta         = Int                 	    // Difference between consecutive offsets / tree addresses,
-                                            // First offset/address is always assumed to be 0
 
 **************************************************************************************/
 
@@ -330,9 +336,9 @@ object PickleFormat {
     case VALDEF | DEFDEF | TYPEDEF | TYPEPARAM | PARAM => true
     case _ => false
   }
-  
-  def isParamTag(tag: Int) = tag == PARAM || tag == TYPEPARAM 
-   
+
+  def isParamTag(tag: Int) = tag == PARAM || tag == TYPEPARAM
+
   def nameTagToString(tag: Int): String = tag match {
     case UTF8 => "UTF8"
     case QUALIFIED => "QUALIFIED"
diff --git a/src/dotty/tools/dotc/core/pickling/PositionReader.scala b/src/dotty/tools/dotc/core/pickling/PositionReader.scala
index 5acb3c6d8..5e35fa397 100644
--- a/src/dotty/tools/dotc/core/pickling/PositionReader.scala
+++ b/src/dotty/tools/dotc/core/pickling/PositionReader.scala
@@ -51,7 +51,7 @@ class PositionReader(reader: TastyReader, startOffset: Int, initNextOffset: Int,
         case tree: WithLazyField[_] =>
           tree.unforced match {
             case rdr: TreeUnpickler#LazyReader[_] =>
-              rdr.posReader = Some(fork)
+              //rdr.posReader = Some(fork)
               while (addr != nextAddr) next()
             case _ =>
           }
diff --git a/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala b/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala
index c037b4336..c6c099f2e 100644
--- a/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala
+++ b/src/dotty/tools/dotc/core/pickling/TreeUnpickler.scala
@@ -279,7 +279,7 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
       if (ctx.mode.is(Mode.InSuperCall) && !flags.is(ParamOrAccessor)) flags |= InSuperCall
       if (ctx.owner.isClass) {
         if (tag == TYPEPARAM) flags |= Param
-        else if (tag == PARAM) flags |= ParamAccessor // TODO: try to unify param and paramaccessor
+        else if (tag == PARAM) flags |= ParamAccessor
       }
       else if (isParamTag(tag)) flags |= Param
       val nameMatches = (_: Denotation).symbol.name == name
@@ -659,7 +659,7 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
       }
     }
 
-    def readLater[T](end: Addr, op: TreeReader => T): Trees.Lazy[T] = {
+    def readLater[T](end: Addr, op: TreeReader => T)(implicit ctx: Context): Trees.Lazy[T] = {
       val localReader = fork
       skipTo(end)
       new LazyReader(localReader, op)
@@ -682,14 +682,42 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, readPositio
       else tree
     }
   }
+  
+  trait DeferredPosition {
+    var parentPos: Position = NoPosition
+  }
+  
+  def normalizePos(x: Any, parentPos: Position)(implicit ctx: Context): Unit = {
+    if (parentPos.exists) 
+      x match {
+        case x: Tree @unchecked if !x.pos.isSynthetic =>
+          assert(x.pos.exists)
+          val absPos = Position(parentPos.start + x.pos.start, parentPos.end - x.pos.end)
+          x.setPosUnchecked(absPos)
+          for (child <- x.productIterator) 
+            normalizePos(child, absPos)
+        case x: DeferredPosition =>
+          x.parentPos = parentPos
+        case xs: List[_] =>
+          xs.foreach(normalizePos(_, parentPos))
+        case _ =>
+      }
+  }
 
-  class LazyReader[T](reader: TreeReader, op: TreeReader => T) extends Trees.Lazy[T] {
-    var posReader: Option[PositionReader] = None
+  class LazyReader[T](reader: TreeReader, op: TreeReader => T)(implicit ctx: Context) extends Trees.Lazy[T] with DeferredPosition {
     def complete: T = {
       val res = op(reader)
-      posReader.foreach(_.unpickle(res))
+      normalizePos(res, parentPos)
       res
     }  
   }
-    
+  
+  class LazyAnnotationReader(sym: Symbol, reader: TreeReader) 
+      extends LazyAnnotation(sym) with DeferredPosition {
+    def complete(implicit ctx: Context) = {
+      val res = reader.readTerm()
+      normalizePos(res, parentPos)
+      res
+    }
+  }
 }