diff options
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/NameBuffer.scala | 30 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/PickleFormat.scala | 176 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TastyBuffer.scala | 20 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TastyName.scala | 21 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TastyPickler.scala | 6 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TastyPrinter.scala | 107 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TastyReader.scala | 78 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TastyUnpickler.scala | 84 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TreeBuffer.scala | 54 | ||||
-rw-r--r-- | src/dotty/tools/dotc/core/pickling/TreePickler.scala | 92 | ||||
-rw-r--r-- | src/dotty/tools/dotc/transform/Pickler.scala | 29 |
11 files changed, 602 insertions, 95 deletions
diff --git a/src/dotty/tools/dotc/core/pickling/NameBuffer.scala b/src/dotty/tools/dotc/core/pickling/NameBuffer.scala index c9994ecb5..70d6b9ee1 100644 --- a/src/dotty/tools/dotc/core/pickling/NameBuffer.scala +++ b/src/dotty/tools/dotc/core/pickling/NameBuffer.scala @@ -13,18 +13,18 @@ import PickleFormat._ class NameBuffer extends TastyBuffer(100000) { - private val nameRefs = new mutable.LinkedHashMap[TastyName, Ref] + private val nameRefs = new mutable.LinkedHashMap[TastyName, NameRef] - def nameIndex(name: TastyName): Ref = nameRefs.get(name) match { + def nameIndex(name: TastyName): NameRef = nameRefs.get(name) match { case Some(ref) => ref case None => - val ref = new Ref(nameRefs.size) + val ref = NameRef(nameRefs.size) nameRefs(name) = ref ref } - def nameIndex(name: Name): Ref = nameIndex(Simple(name.toTermName)) - def nameIndex(str: String): Ref = nameIndex(str.toTermName) + def nameIndex(name: Name): NameRef = nameIndex(Simple(name.toTermName)) + def nameIndex(str: String): NameRef = nameIndex(str.toTermName) private def withLength(op: => Unit): Unit = { val lengthAddr = currentAddr @@ -35,32 +35,34 @@ class NameBuffer extends TastyBuffer(100000) { putNat(lengthAddr, length, 1) } - def writeRef(ref: Ref) = writeNat(ref.index) + def writeNameRef(ref: NameRef) = writeNat(ref.index) def pickleName(name: TastyName): Unit = name match { case Simple(name) => - val bytes = Codec.toUTF8(chrs, name.start, name.length) + val bytes = + if (name.length == 0) new Array[Byte](0) + else Codec.toUTF8(chrs, name.start, name.length) writeByte(UTF8) writeNat(bytes.length) writeBytes(bytes, bytes.length) case Qualified(qualified, selector) => writeByte(QUALIFIED) - withLength { writeRef(qualified); writeRef(selector) } + withLength { writeNameRef(qualified); writeNameRef(selector) } case Signed(original, params, result) => writeByte(SIGNED) - withLength { writeRef(original); writeRef(result); params.foreach(writeRef) } + withLength { writeNameRef(original); writeNameRef(result); params.foreach(writeNameRef) } case Expanded(original) => writeByte(EXPANDED) - withLength { writeRef(original) } + withLength { writeNameRef(original) } case ModuleClass(module) => writeByte(MODULECLASS) - withLength { writeRef(module) } + withLength { writeNameRef(module) } case SuperAccessor(accessed) => writeByte(SUPERACCESSOR) - withLength { writeRef(accessed) } - case DefaultGetter(method, paramNumer) => + withLength { writeNameRef(accessed) } + case DefaultGetter(method, paramNumber) => writeByte(DEFAULTGETTER) - withLength { writeRef(method); writeNat(paramNumer) } + withLength { writeNameRef(method); writeNat(paramNumber) } } override def assemble(): Unit = { diff --git a/src/dotty/tools/dotc/core/pickling/PickleFormat.scala b/src/dotty/tools/dotc/core/pickling/PickleFormat.scala index 16356718c..6769cd016 100644 --- a/src/dotty/tools/dotc/core/pickling/PickleFormat.scala +++ b/src/dotty/tools/dotc/core/pickling/PickleFormat.scala @@ -95,6 +95,7 @@ Standard-Section: "ASTs" Tree* Path = Constant TERMREFdirect sym_ASTRef + TERMREFstatic fullyQualified_NameRef TERMREFsymbol qual_Type sym_ASTRef TERMREF qual_Type possiblySigned_NameRef THIS Length clsRef_Type @@ -123,6 +124,7 @@ Standard-Section: "ASTs" Tree* Type = Path TYPEREFdirect sym_ASTRef + TYPEREFstatic fullyQualified_NameRef TYPEREFsymbol qual_Type sym_ASTRef TYPEREF qual_Type possiblySigned_NameRef SUPERtype Length this_Type underlying_Type @@ -135,9 +137,14 @@ Standard-Section: "ASTs" Tree* ANDtype Length left_Type right_Type ORtype Length left_Type right_Type BYNAMEtype Length underlying_Type + POLYtype Length result_Type NamesTypes // needed for refinements + METHODtype Length result_Type NamesTypes // needed for refinements + PARAMtype Length binder_ASTref paramNum_Nat // needed for refinements NOTYPE NOPREFIX SHARED type_ASTRef + NamesTypes = ParamType* + NameType = paramName_NameRef typeOrBounds_ASTRef Modifier = PRIVATE INTERNAL // package private @@ -189,10 +196,10 @@ object PickleFormat { final val header = "5CA1AB1F" final val MajorVersion = 0 - final val MinorVersion = 1 - + final val MinorVersion = 2 + // Name tags - + final val UTF8 = 1 final val QUALIFIED = 2 final val SIGNED = 3 @@ -200,7 +207,7 @@ object PickleFormat { final val MODULECLASS = 5 final val SUPERACCESSOR = 6 final val DEFAULTGETTER = 7 - + // AST tags final val EMPTYTREE = 0 @@ -241,20 +248,22 @@ object PickleFormat { final val SHARED = 96 final val TERMREFdirect = 97 final val TYPEREFdirect = 98 - final val BYTEconst = 99 - final val BYTEneg = 100 - final val SHORTconst = 101 - final val SHORTneg = 102 - final val CHARconst = 103 - final val INTconst = 104 - final val INTneg = 105 - final val LONGconst = 106 - final val LONGneg = 107 - final val FLOATconst = 108 - final val DOUBLEconst = 109 - final val STRINGconst = 110 - final val PRIVATEqualified = 111 - final val PROTECTEDqualified = 112 + final val TERMREFstatic = 99 + final val TYPEREFstatic = 100 + final val BYTEconst = 101 + final val BYTEneg = 102 + final val SHORTconst = 103 + final val SHORTneg = 104 + final val CHARconst = 105 + final val INTconst = 106 + final val INTneg = 107 + final val LONGconst = 108 + final val LONGneg = 109 + final val FLOATconst = 110 + final val DOUBLEconst = 111 + final val STRINGconst = 112 + final val PRIVATEqualified = 113 + final val PROTECTEDqualified = 114 final val SELECT = 128 final val TERMREFsymbol = 129 @@ -308,10 +317,137 @@ object PickleFormat { final val ANDtype = 203 final val ORtype = 204 final val BYNAMEtype = 205 - final val IMPLICITARG = 206 - + final val METHODtype = 206 + final val POLYtype = 207 + final val PARAMtype = 208 + final val IMPLICITARG = 209 + final val firstSimpleTreeTag = EMPTYTREE final val firstNatTreeTag = SHARED final val firstTreeNatTreeTag = SELECT final val firstLengthTreeTag = PACKAGE + + def nameTagToString(tag: Int): String = tag match { + case UTF8 => "UTF8" + case QUALIFIED => "QUALIFIED" + case SIGNED => "SIGNED" + case EXPANDED => "EXPANDED" + case MODULECLASS => "MODULECLASS" + case SUPERACCESSOR => "SUPERACCESSOR" + case DEFAULTGETTER => "DEFAULTGETTER" + } + + def astTagToString(tag: Int): String = tag match { + case EMPTYTREE => "EMPTYTREE" + case NOTYPE => "NOTYPE" + case NOPREFIX => "NOPREFIX" + case UNITconst => "UNITconst" + case FALSEconst => "FALSEconst" + case TRUEconst => "TRUEconst" + case NULLconst => "NULLconst" + case PRIVATE => "PRIVATE" + case INTERNAL => "INTERNAL" + case PROTECTED => "PROTECTED" + case ABSTRACT => "ABSTRACT" + case FINAL => "FINAL" + case SEALED => "SEALED" + case CASE => "CASE" + case IMPLICIT => "IMPLICIT" + case LAZY => "LAZY" + case OVERRIDE => "OVERRIDE" + case INLINE => "INLINE" + case ABSOVERRIDE => "ABSOVERRIDE" + case STATIC => "STATIC" + case MODULE => "MODULE" + case LOCAL => "LOCAL" + case SYNTHETIC => "SYNTHETIC" + case ARTIFACT => "ARTIFACT" + case MUTABLE => "MUTABLE" + case LABEL => "LABEL" + case FIELDaccessor => "FIELDaccessor" + case PARAMaccessor => "PARAMaccessor" + case CASEaccessor => "CASEaccessor" + case COVARIANT => "COVARIANT" + case CONTRAVARIANT => "CONTRAVARIANT" + case SCALA2X => "SCALA2X" + case DEFAULTparameterized => "DEFAULTparameterized" + case DEFAULTinit => "DEFAULTinit" + + case SHARED => "SHARED" + case TERMREFdirect => "TERMREFdirect" + case TYPEREFdirect => "TYPEREFdirect" + case TERMREFstatic => "TERMREFstatic" + case TYPEREFstatic => "TYPEREFstatic" + case BYTEconst => "BYTEconst" + case BYTEneg => "BYTEneg" + case SHORTconst => "SHORTconst" + case SHORTneg => "SHORTneg" + case CHARconst => "CHARconst" + case INTconst => "INTconst" + case INTneg => "INTneg" + case LONGconst => "LONGconst" + case LONGneg => "LONGneg" + case FLOATconst => "FLOATconst" + case DOUBLEconst => "DOUBLEconst" + case STRINGconst => "STRINGconst" + case PRIVATEqualified => "PRIVATEqualified" + case PROTECTEDqualified => "PROTECTEDqualified" + + case SELECT => "SELECT" + case TERMREFsymbol => "TERMREFsymbol" + case TERMREF => "TERMREF" + case TYPEREFsymbol => "TYPEREFsymbol" + case TYPEREF => "TYPEREF" + + case PACKAGE => "PACKAGE" + case VALDEF => "VALDEF" + case DEFDEF => "DEFDEF" + case TYPEDEF => "TYPEDEF" + case IMPORT => "IMPORT" + case TYPEPARAM => "TYPEPARAM" + case PARAMS => "PARAMS" + case PARAM => "PARAM" + case IMPORTED => "IMPORTED" + case RENAMED => "RENAMED" + case APPLY => "APPLY" + case TYPEAPPLY => "TYPEAPPLY" + case NEW => "NEW" + case PAIR => "PAIR" + case TYPED => "TYPED" + case NAMEDARG => "NAMEDARG" + case ASSIGN => "ASSIGN" + case BLOCK => "BLOCK" + case IF => "IF" + case CLOSURE => "CLOSURE" + case MATCH => "MATCH" + case RETURN => "RETURN" + case TRY => "TRY" + case THROW => "THROW" + case SEQLITERAL => "SEQLITERAL" + case JSEQLITERAL => "JSEQLITERAL" + case BIND => "BIND" + case ALTERNATIVE => "ALTERNATIVE" + case UNAPPLY => "UNAPPLY" + case ANNOTATED => "ANNOTATED" + case CASEDEF => "CASEDEF" + case IMPLICITarg => "IMPLICITarg" + case TEMPLATE => "TEMPLATE" + case THIS => "THIS" + case SUPER => "SUPER" + case CLASSconst => "CLASSconst" + case ENUMconst => "ENUMconst" + case SUPERtype => "SUPERtype" + case SKOLEMtype => "SKOLEMtype" + case REFINEDtype => "REFINEDtype" + case APPLIEDtype => "APPLIEDtype" + case TYPEBOUNDS => "TYPEBOUNDS" + case TYPEALIAS => "TYPEALIAS" + case ANDtype => "ANDtype" + case ORtype => "ORtype" + case BYNAMEtype => "BYNAMEtype" + case POLYtype => "POLYtype" + case METHODtype => "METHODtype" + case PARAMtype => "PARAMtype" + case IMPLICITARG => "IMPLICITARG" + } } diff --git a/src/dotty/tools/dotc/core/pickling/TastyBuffer.scala b/src/dotty/tools/dotc/core/pickling/TastyBuffer.scala index f6a7a17b4..0e44dbd76 100644 --- a/src/dotty/tools/dotc/core/pickling/TastyBuffer.scala +++ b/src/dotty/tools/dotc/core/pickling/TastyBuffer.scala @@ -12,9 +12,9 @@ object TastyBuffer { if (nat < 128) 1 else natSize(nat >>> 7) + 1 /** An address pointing to an index in a Tasty buffer's byte array */ - class Addr(val index: Int) extends AnyVal { - def -(delta: Int): Addr = new Addr(this.index - delta) - def +(delta: Int): Addr = new Addr(this.index + delta) + case class Addr(val index: Int) extends AnyVal { + def -(delta: Int): Addr = Addr(this.index - delta) + def +(delta: Int): Addr = Addr(this.index + delta) def relativeTo(base: Addr): Addr = this - base.index - AddrWidth } @@ -49,7 +49,7 @@ class TastyBuffer(initialSize: Int) { /** Write the first `n` bytes of `data`. */ def writeBytes(data: Array[Byte], n: Int): Unit = { - while (bytes.length < length + data.length) bytes = dble(bytes) + while (bytes.length < length + n) bytes = dble(bytes) Array.copy(data, 0, bytes, length, n) length += n } @@ -71,11 +71,11 @@ class TastyBuffer(initialSize: Int) { def writeNatPrefix(x: Long): Unit = { val y = x >>> 7 if (y != 0L) writeNatPrefix(y) - writeByte(((x & 0x7f) | 0x80).toInt) + writeByte((x & 0x7f).toInt) } val y = x >>> 7 if (y != 0L) writeNatPrefix(y) - writeByte((x & 0x7f).toInt) + writeByte(((x & 0x7f) | 0x80).toInt) } /** Write the `nbytes` least significant bytes of `x` in big endian format */ @@ -119,14 +119,14 @@ class TastyBuffer(initialSize: Int) { var idx = at.index do { b = bytes(idx) - x = (x << 7) + (b & 0x7f) + x = (x << 7) | (b & 0x7f) idx += 1 - } while ((b & 0x80) != 0L) + } while ((b & 0x80) == 0) x } /** The address (represented as a natural number) at address `at` */ - def getAddr(at: Addr) = new Addr(getNat(at)) + def getAddr(at: Addr) = Addr(getNat(at)) /** The smallest address equal to or following `at` which points to a non-zero byte */ final def skipZeroes(at: Addr): Addr = @@ -139,7 +139,7 @@ class TastyBuffer(initialSize: Int) { } /** The address referring to the end of data written so far */ - def currentAddr: Addr = new Addr(length) + def currentAddr: Addr = Addr(length) /** Reserve `AddrWidth` bytes to write an address into */ def reserveAddr(): Addr = { diff --git a/src/dotty/tools/dotc/core/pickling/TastyName.scala b/src/dotty/tools/dotc/core/pickling/TastyName.scala index 911d4c0cd..581a87901 100644 --- a/src/dotty/tools/dotc/core/pickling/TastyName.scala +++ b/src/dotty/tools/dotc/core/pickling/TastyName.scala @@ -4,19 +4,26 @@ package core package pickling import core.Names.TermName +import collection.mutable abstract class TastyName object TastyName { - class Ref(val index: Int) extends AnyVal + case class NameRef(val index: Int) extends AnyVal case class Simple(name: TermName) extends TastyName - case class Qualified(qualified: Ref, selector: Ref) extends TastyName - case class Signed(original: Ref, params: List[Ref], result: Ref) extends TastyName - case class Expanded(original: Ref) extends TastyName - case class ModuleClass(module: Ref) extends TastyName - case class SuperAccessor(accessed: Ref) extends TastyName - case class DefaultGetter(method: Ref, num: Int) extends TastyName + case class Qualified(qualified: NameRef, selector: NameRef) extends TastyName + case class Signed(original: NameRef, params: List[NameRef], result: NameRef) extends TastyName + case class Expanded(original: NameRef) extends TastyName + case class ModuleClass(module: NameRef) extends TastyName + case class SuperAccessor(accessed: NameRef) extends TastyName + case class DefaultGetter(method: NameRef, num: Int) extends TastyName + class Table extends (NameRef => TastyName) { + private val names = new mutable.ArrayBuffer[TastyName] + def add(name: TastyName) = names += name + def apply(ref: NameRef) = names(ref.index) + def contents: Iterable[TastyName] = names + } } diff --git a/src/dotty/tools/dotc/core/pickling/TastyPickler.scala b/src/dotty/tools/dotc/core/pickling/TastyPickler.scala index 2ae6848e0..32cc1ae43 100644 --- a/src/dotty/tools/dotc/core/pickling/TastyPickler.scala +++ b/src/dotty/tools/dotc/core/pickling/TastyPickler.scala @@ -9,7 +9,7 @@ import TastyBuffer._ class TastyPickler { - private val sections = new mutable.ArrayBuffer[(TastyName.Ref, TastyBuffer)] + private val sections = new mutable.ArrayBuffer[(TastyName.NameRef, TastyBuffer)] private val headerBuffer = { val buf = new TastyBuffer(16) @@ -24,7 +24,7 @@ class TastyPickler { def newSection(name: String, buf: TastyBuffer) = sections += ((nameBuffer.nameIndex(name), buf)) - def assembleParts: Array[Byte] = { + def assembleParts(): Array[Byte] = { def lengthWithLength(buf: TastyBuffer) = { buf.assemble() buf.length + natSize(buf.length) @@ -44,7 +44,7 @@ class TastyPickler { all.writeNat(buf.length) all.writeBytes(buf.bytes, buf.length) } - assert(all.length == totalSize && all.bytes.length == totalSize) + assert(all.length == totalSize && all.bytes.length == totalSize, s"totalSize = $totalSize, all.length = ${all.length}, all.bytes.length = ${all.bytes.length}") all.bytes } } diff --git a/src/dotty/tools/dotc/core/pickling/TastyPrinter.scala b/src/dotty/tools/dotc/core/pickling/TastyPrinter.scala new file mode 100644 index 000000000..3e583beb0 --- /dev/null +++ b/src/dotty/tools/dotc/core/pickling/TastyPrinter.scala @@ -0,0 +1,107 @@ +package dotty.tools.dotc +package core +package pickling + +import Contexts._, Decorators._ +import printing.Texts._ +import TastyName._ +import TastyUnpickler._ + +class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) { + + val reader = new TastyReader(bytes) + val unpickler = new TastyUnpickler(reader) + import unpickler.{tastyName, unpickled} + + def nameToString(name: TastyName): String = name match { + case Simple(name) => name.toString + case Qualified(qual, name) => nameRefToString(qual) + "." + nameRefToString(name) + case Signed(original, params, result) => + i"${nameRefToString(original)}@${params.map(nameRefToString)}%,%:${nameRefToString(result)}" + case Expanded(original) => nameRefToString(original) + "/EXPANDED" + case ModuleClass(original) => nameRefToString(original) + "/MODULECLASS" + case SuperAccessor(accessed) => nameRefToString(accessed) + "/SUPERACCESSOR" + case DefaultGetter(meth, num) => nameRefToString(meth) + "/DEFAULTGETTER" + num + } + + def nameRefToString(ref: NameRef): String = nameToString(tastyName(ref)) + + def printNames() = + for ((name, idx) <- tastyName.contents.zipWithIndex) + println(f"$idx%4d: " + nameToString(name)) + + def printContents(): Unit = { + println("Names:") + printNames() + println("Trees:") + unpickled(new TreeUnpickler) + } + + class TreeUnpickler extends SectionUnpickler[Text]("ASTs") { + import PickleFormat._ + def unpickle(reader: TastyReader, tastyName: TastyName.Table): Text = { + import reader._ + val sb = new StringBuilder(s"${reader.end.index - reader.from.index} bytes of AST:") + var indent = 0 + def newLine() = print(f"\n ${currentAddr.index - from.index}%5d:" + " " * indent) + def printNat() = print(" " + readNat()) + def printName() = { + val idx = readNat() + print(" ") ;print(idx); print("["); print(nameRefToString(NameRef(idx))); print("]") + } + def printTree(): Unit = { + newLine() + val tag = readByte() + print(" ");print(astTagToString(tag)) + indent += 2 + if (tag >= firstLengthTreeTag) { + val len = readNat() + print(s"($len)") + val end = currentAddr + len + def printTrees() = until(end)(printTree()) + tag match { + case IMPORTED => + printName() + case RENAMED => + printName(); printName() + case VALDEF | DEFDEF | TYPEDEF | TYPEPARAM | PARAM | NAMEDARG | BIND | REFINEDtype => + printName(); printTrees() + case RETURN => + printNat(); printTrees() + case METHODtype | POLYtype => + printTree() + until(end) { printName(); printTree() } + case PARAMtype => + printNat(); printNat() + case _ => + printTrees() + } + if (currentAddr != end) { + println(s"incomplete read, current = $currentAddr, end = $end") + skipTo(currentAddr) + } + } + else if (tag >= firstTreeNatTreeTag) { + printTree() + newLine() + tag match { + case SELECT | TERMREF | TYPEREF => printName() + case _ => printNat() + } + } + else if (tag >= firstNatTreeTag) + tag match { + case TERMREFstatic | TYPEREFstatic | STRINGconst => printName() + case _ => printNat() + } + indent -= 2 + } + println(s"base = $currentAddr") + while (!atEnd) { + printTree() + newLine() + } + sb.toString + } + } +}
\ No newline at end of file diff --git a/src/dotty/tools/dotc/core/pickling/TastyReader.scala b/src/dotty/tools/dotc/core/pickling/TastyReader.scala new file mode 100644 index 000000000..659eb8977 --- /dev/null +++ b/src/dotty/tools/dotc/core/pickling/TastyReader.scala @@ -0,0 +1,78 @@ +package dotty.tools +package dotc +package core +package pickling + + +import TastyBuffer._ +import TastyName.NameRef +import collection.mutable + +/** A byte array bufferfer that can be filled with bytes or natural numbers in TASTY format, + * and that supports reading and patching addresses represented as natural numbers. + */ +class TastyReader(val bytes: Array[Byte], val from: Addr, val end: Addr) { + + def this(bytes: Array[Byte]) = this(bytes, Addr(0), Addr(bytes.length)) + + private var bp: Int = from.index + + def currentAddr: Addr = Addr(bp) + + def atEnd: Boolean = bp == end.index + + /** Read a byte of data. */ + def readByte(): Int = { + val result = bytes(bp) & 0xff + bp += 1 + result + } + + /** Read the next `n` bytes of `data`. */ + def readBytes(n: Int): Array[Byte] = { + val result = new Array[Byte](n) + Array.copy(bytes, bp, result, 0, n) + bp += n + result + } + + /** Read a natural number fitting in an Int in big endian format, base 128. + * All but the last digits have bit 0x80 set. + */ + def readNat(): Int = readLongNat.toInt + + /** Read a natural number fitting in a Long in big endian format, base 128. + * All but the last digits have bit 0x80 set. + */ + def readLongNat(): Long = { + var b = 0L + var x = 0L + do { + b = bytes(bp) + x = (x << 7) | (b & 0x7f) + bp += 1 + } while ((b & 0x80) == 0) + x + } + + /** Read `nbytes` bytes in big endian format into a Long */ + def readRaw(nbytes: Int): Unit = { + def recur(x: Long, n: Int): Long = + if (n == 0) x else recur((x << 8) | (readByte & 0xff), n - 1) + recur(0, nbytes) + } + + def readNameRef() = NameRef(readNat()) + + def readEnd(): Addr = Addr(readNat() + bp) + + def skipTo(addr: Addr): Unit = + bp = addr.index + + def until[T](end: Addr)(op: => T): List[T] = { + val buf = new mutable.ListBuffer[T] + while (bp < end.index) buf += op + assert(bp == end.index) + buf.toList + } +} diff --git a/src/dotty/tools/dotc/core/pickling/TastyUnpickler.scala b/src/dotty/tools/dotc/core/pickling/TastyUnpickler.scala new file mode 100644 index 000000000..f107c7c83 --- /dev/null +++ b/src/dotty/tools/dotc/core/pickling/TastyUnpickler.scala @@ -0,0 +1,84 @@ +package dotty.tools.dotc +package core +package pickling + +import scala.collection.mutable +import PickleFormat._ +import Names.{Name, termName} + +object TastyUnpickler { + class UnpickleException(msg: String) extends Exception(msg) + + abstract class SectionUnpickler[R](val name: String) { + def unpickle(reader: TastyReader, tastyName: TastyName.Table): R + } +} + +import TastyUnpickler._ + +class TastyUnpickler(reader: TastyReader) { + import reader._ + + private val sectionReader = new mutable.HashMap[String, TastyReader] + val tastyName = new TastyName.Table + + def check(cond: Boolean, msg: => String) = + if (!cond) throw new UnpickleException(msg) + + def readString(): String = { + val TastyName.Simple(name) = tastyName(readNameRef()) + name.toString + } + + def readName(): TastyName = { + import TastyName._ + val tag = readByte() + val length = readNat() + val start = currentAddr + val end = start + length + val result = tag match { + case UTF8 => + skipTo(end) + Simple(termName(bytes, start.index, length)) + case QUALIFIED => + Qualified(readNameRef(), readNameRef()) + case SIGNED => + val original = readNameRef() + val result = readNameRef() + val params = until(end)(readNameRef()) + Signed(original, params, result) + case EXPANDED => + Expanded(readNameRef()) + case MODULECLASS => + ModuleClass(readNameRef()) + case SUPERACCESSOR => + SuperAccessor(readNameRef()) + case DEFAULTGETTER => + DefaultGetter(readNameRef(), readNat()) + } + assert(currentAddr == end, s"bad name $result $start $currentAddr $end") + result + } + + locally { + val magic = readBytes(8) + check(magic.map(_.toChar).mkString == header, "not a TASTy file") + val major = readNat() + val minor = readNat() + check(major == MajorVersion && (major != 0 || minor == MinorVersion), + s"""TASTy signature has wrong version. + | expected: $MajorVersion.$MinorVersion + | found : $major.$minor""".stripMargin) + until(readEnd()) { tastyName.add(readName()) } + while (!atEnd) { + val secName = readString() + val secEnd = readEnd() + sectionReader(secName) = new TastyReader(bytes, currentAddr, secEnd) + skipTo(secEnd) + } + } + + def unpickled[R](sec: SectionUnpickler[R]): Option[R] = + for (reader <- sectionReader.get(sec.name)) yield + sec.unpickle(reader, tastyName) +} diff --git a/src/dotty/tools/dotc/core/pickling/TreeBuffer.scala b/src/dotty/tools/dotc/core/pickling/TreeBuffer.scala index 73b944b92..a09b50119 100644 --- a/src/dotty/tools/dotc/core/pickling/TreeBuffer.scala +++ b/src/dotty/tools/dotc/core/pickling/TreeBuffer.scala @@ -16,7 +16,7 @@ class TreeBuffer extends TastyBuffer(1000000) { private var delta: Array[Int] = _ private var numOffsets = 0 - private def offset(i: Int): Addr = new Addr(offsets(i)) + private def offset(i: Int): Addr = Addr(offsets(i)) private def keepOffset(relative: Boolean): Unit = { if (numOffsets == offsets.length) { @@ -37,18 +37,20 @@ class TreeBuffer extends TastyBuffer(1000000) { def writeRef(target: Addr) = { keepOffset(relative = false) - writeNat(target.index) + fillAddr(reserveAddr(), target) } def fillRef(at: Addr, target: Addr, relative: Boolean) = { val addr = if (relative) target.relativeTo(at) else target fillAddr(at, addr) } - - def adjusted(x: Addr): Addr = { - val idx = bestFit(offsets, numOffsets, x.index - 1) - if (idx < 0) x else x - delta(idx) + + def deltaAt(at: Addr): Int = { + val idx = bestFit(offsets, numOffsets, at.index - 1) + if (idx < 0) 0 else delta(idx) } + + def adjusted(x: Addr): Addr = x - deltaAt(x) private def computeDeltas() = { delta = new Array[Int](numOffsets) @@ -65,20 +67,23 @@ class TreeBuffer extends TastyBuffer(1000000) { } } - private def adjustedOffset(at: Addr, isRelative: Boolean): Addr = { + private def adjustedOffset(i: Int): Addr = { + val at = offset(i) val original = getAddr(at) - if (isRelative) { - val start = skipNat(at).index - adjusted(original + start) - start + if (isRelative(i)) { + val start = skipNat(at) + val len1 = original + delta(i) - deltaAt(original + start.index) + val len2 = adjusted(original + start.index) - adjusted(start).index + assert(len1 == len2, + s"adjusting offset #$i: $at, original = $original, len1 = $len1, len2 = $len2") + len1 } else adjusted(original) } private def adjustOffsets(): Unit = { for (i <- 0 until numOffsets) { - val off = offset(i) - val original = getAddr(off) - val corrected = adjustedOffset(off, isRelative(i)) - fillAddr(off, corrected) + val corrected = adjustedOffset(i) + fillAddr(offset(i), corrected) } } @@ -87,7 +92,7 @@ class TreeBuffer extends TastyBuffer(1000000) { var lastDelta = 0 var i = 0 while (i < numOffsets) { - val corrected = adjustedOffset(offset(i), isRelative(i)) + val corrected = adjustedOffset(i) lastDelta += AddrWidth - TastyBuffer.natSize(corrected.index) delta1(i) = lastDelta i += 1 @@ -104,16 +109,19 @@ class TreeBuffer extends TastyBuffer(1000000) { var start = 0 var i = 0 var wasted = 0 + def shift(end: Int) = + Array.copy(bytes, start, bytes, start - lastDelta, end - start) while (i < numOffsets) { val next = offsets(i) - Array.copy(bytes, start, bytes, start - lastDelta, next - start) + shift(next) start = next + delta(i) - lastDelta - val pastZeroes = skipZeroes(new Addr(next)).index + val pastZeroes = skipZeroes(Addr(next)).index assert(pastZeroes >= start, s"something's wrong: eliminated non-zero") wasted += (pastZeroes - start) lastDelta = delta(i) i += 1 } + shift(length) length -= lastDelta wasted } @@ -121,12 +129,16 @@ class TreeBuffer extends TastyBuffer(1000000) { override def assemble(): Unit = { val origLength = length computeDeltas() - adjustOffsets() - if (false) { + //println(s"offsets: ${offsets.take(numOffsets).deep}") + //println(s"deltas: ${delta.take(numOffsets).deep}") + if (true) { var saved = 0 - do saved = adjustDeltas() - while (saved > 0 && length / saved < 100) + do { + saved = adjustDeltas() + println(s"adjusting deltas, saved = $saved") + } while (saved > 0 && length / saved < 100) } + adjustOffsets() val wasted = compress() println(s"original length: $origLength, compressed to: $length, wasted: $wasted") } diff --git a/src/dotty/tools/dotc/core/pickling/TreePickler.scala b/src/dotty/tools/dotc/core/pickling/TreePickler.scala index 8c92e2ed8..cc5a8b8f2 100644 --- a/src/dotty/tools/dotc/core/pickling/TreePickler.scala +++ b/src/dotty/tools/dotc/core/pickling/TreePickler.scala @@ -3,7 +3,6 @@ package dotc package core package pickling -import util.Util.{bestFit, dble} import ast.Trees._ import PickleFormat._ import core._ @@ -55,6 +54,10 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { def pickle(tree: Tree)(implicit ctx: Context) = { + def qualifiedName(sym: Symbol): TastyName = + if (sym.isRoot || sym.owner.isRoot) TastyName.Simple(sym.name.toTermName) + else TastyName.Qualified(nameIndex(qualifiedName(sym.owner)), nameIndex(sym.name)) + def pickleConstant(c: Constant): Unit = { def pickleNum(nonNegTag: Int, negTag: Int) = { val x = c.longValue @@ -103,12 +106,13 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { } } - def pickleType(tpe: Type): Unit = { + def pickleType(tpe0: Type, richTypes: Boolean = false): Unit = { + val tpe = tpe0.stripTypeVar val prev = pickledTypes.get(tpe) if (prev == null) { val addr = currentAddr - pickleNewType(tpe) pickledTypes.put(tpe, addr) + pickleNewType(tpe, richTypes) } else { writeByte(SHARED) @@ -116,10 +120,15 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { } } - def pickleNewType(tpe: Type)= tpe match { - case ConstantType(value) => pickleConstant(value) + def pickleNewType(tpe: Type, richTypes: Boolean): Unit = tpe match { + case ConstantType(value) => + pickleConstant(value) case tpe: WithFixedSym => - if (tpe.prefix == NoPrefix) { + if (tpe.symbol.isStatic) { + writeByte(if (tpe.isType) TYPEREFstatic else TERMREFstatic) + pickleName(qualifiedName(tpe.symbol)) + } + else if (tpe.prefix == NoPrefix) { writeByte(if (tpe.isType) TYPEREFdirect else TERMREFdirect) pickleSym(tpe.symbol) } @@ -135,7 +144,7 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { pickleType(tpe.prefix); pickleName(tpe.name) case tpe: ThisType => writeByte(THIS) - pickleType(tpe.tref) + withLength { pickleType(tpe.tref) } case tpe: SuperType => writeByte(SUPERtype) withLength { pickleType(tpe.thistpe); pickleType(tpe.supertpe)} @@ -146,11 +155,11 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { val args = tpe.argInfos(interpolate = false) if (args.isEmpty) { writeByte(REFINEDtype) - withLength { pickleName(tpe.refinedName); pickleType(tpe.refinedInfo) } + withLength { pickleName(tpe.refinedName); pickleType(tpe.refinedInfo, richTypes = true) } } else { writeByte(APPLIEDtype) - withLength { pickleType(tpe.withoutArgs(args)); args.foreach(pickleType) } + withLength { pickleType(tpe.withoutArgs(args)); args.foreach(pickleType(_)) } } case tpe: TypeAlias => writeByte(TYPEALIAS) @@ -167,22 +176,56 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { case tpe: ExprType => writeByte(BYNAMEtype) withLength { pickleType(tpe.underlying) } + case tpe: MethodType if richTypes => + writeByte(METHODtype) + pickleMethodic(tpe.resultType, tpe.paramNames, tpe.paramTypes) + case tpe: PolyType if richTypes => + writeByte(POLYtype) + pickleMethodic(tpe.resultType, tpe.paramNames, tpe.paramBounds) + case tpe: PolyParam => + if (!pickleParamType(tpe)) + // TODO figure out why this case arises in e.g. pickling AbstractFileReader. + ctx.typerState.constraint.entry(tpe) match { + case TypeBounds(lo, hi) if lo eq hi => pickleNewType(lo, richTypes) + } + case tpe: MethodParam => + assert(pickleParamType(tpe), "method parameter in wrong position") case NoType => writeByte(NOTYPE) // case NoPrefix => // not sure we need this! // writeByte(NOPREFIX) } + def pickleMethodic(result: Type, names: List[Name], types: List[Type]) = + withLength { + pickleType(result, richTypes = true) + (names, types).zipped.foreach { (name, tpe) => + pickleName(name); pickleType(tpe) + } + } + + def pickleParamType(tpe: ParamType): Boolean = { + val binder = pickledTypes.get(tpe.binder) + val pickled = binder != null + if (pickled) { + writeByte(PARAMtype) + withLength { writeRef(binder.asInstanceOf[Addr]); writeNat(tpe.paramNum) } + } + pickled + } + def pickleTpt(tpt: Tree): Unit = pickleType(tpt.tpe) // TODO correlate with original when generating positions def pickleTreeIfNonEmpty(tree: Tree): Unit = if (!tree.isEmpty) pickleTree(tree) - def pickleTree(tree: Tree): Unit = tree match { + def pickleTree(tree: Tree): Unit = { + tree match { case Ident(_) | This(_) => pickleType(tree.tpe) case Select(qual, name) => writeByte(SELECT) + pickleTree(qual) val sig = tree.tpe.signature if (sig == Signature.NotAMethod) pickleName(name) else pickleNameAndSig(name, sig) @@ -271,26 +314,29 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { patterns.foreach(pickleTree) } case tree: ValDef => - pickleDef(VALDEF, tree.symbol, tree.rhs) + pickleDef(VALDEF, tree.symbol, tree.tpt, tree.rhs) case tree: DefDef => def pickleParams = { - for (tparam <- tree.tparams) pickleDef(TYPEPARAM, tparam.symbol, EmptyTree) + for (tparam <- tree.tparams) + pickleDef(TYPEPARAM, tparam.symbol, tparam.rhs, EmptyTree) for (vparams <- tree.vparamss) { writeByte(PARAMS) withLength { - for (vparam <- vparams) pickleDef(PARAM, vparam.symbol, EmptyTree) + for (vparam <- vparams) + pickleDef(PARAM, vparam.symbol, vparam.tpt, EmptyTree) } } } - pickleDef(DEFDEF, tree.symbol, tree.rhs, pickleParams) + pickleDef(DEFDEF, tree.symbol, tree.tpt, tree.rhs, pickleParams) case tree: TypeDef => - pickleDef(TYPEDEF, tree.symbol, tree.rhs) + pickleDef(TYPEDEF, tree.symbol, tree.rhs, EmptyTree) case tree: Template => + registerDef(tree.symbol) writeByte(TEMPLATE) withLength { tree.parents.foreach(pickleTree) if (!tree.self.isEmpty) - pickleDef(PARAM, tree.self.symbol, EmptyTree) + pickleDef(PARAM, tree.self.symbol, tree.self.tpt, EmptyTree) pickleTreeIfNonEmpty(tree.constr) tree.body.foreach(pickleTree) } @@ -313,16 +359,21 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { case Annotated(annot, arg) => writeByte(ANNOTATED) withLength { pickleTree(annot); pickleTree(arg) } - } + case EmptyTree => + writeByte(EMPTYTREE) + }} - def pickleDef(tag: Int, sym: Symbol, rhs: Tree, pickleParams: => Unit = ()) = { + def pickleDef(tag: Int, sym: Symbol, tpt: Tree, rhs: Tree, pickleParams: => Unit = ()) = { registerDef(sym) writeByte(tag) withLength { pickleName(sym.name) pickleParams - if (tag != TYPEDEF) pickleType(sym.info.finalResultType) - if (tag != PARAM && tag != TYPEPARAM) pickleTree(rhs) + tpt match { + case tpt: TypeTree => pickleTpt(tpt) + case _ => pickleTree(tpt) + } + if (tag == VALDEF || tag == DEFDEF) pickleTree(rhs) pickleModifiers(sym) } } @@ -367,5 +418,6 @@ class TreePickler(pickler: TastyPickler, picklePositions: Boolean) { } pickleTree(tree) + assert(forwardSymRefs.isEmpty, i"unresolved symbols: ${forwardSymRefs.keySet.toList}%, %") } } diff --git a/src/dotty/tools/dotc/transform/Pickler.scala b/src/dotty/tools/dotc/transform/Pickler.scala new file mode 100644 index 000000000..644a66784 --- /dev/null +++ b/src/dotty/tools/dotc/transform/Pickler.scala @@ -0,0 +1,29 @@ +package dotty.tools.dotc +package transform + +import core._ +import TreeTransforms._ +import Contexts.Context +import Decorators._ +import pickling._ + +/** This miniphase pickles trees */ +class Pickler extends MiniPhaseTransform { thisTransform => + import ast.tpd._ + + override def phaseName: String = "pickler" + + + override def transformUnit(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = { + val pickler = new TastyPickler + new TreePickler(pickler, picklePositions = false).pickle(tree) + val bytes = pickler.assembleParts() + def rawBytes = + bytes.iterator.grouped(10).toList.zipWithIndex.map { + case (row, i) => s"${i}0: ${row.mkString(" ")}" + } + //println(s"written:\n${rawBytes.mkString("\n")}") + new TastyPrinter(bytes).printContents() + tree + } +}
\ No newline at end of file |