From 3d58ebc52f9cffe97470f30f588d0182c372b227 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Fri, 18 Nov 2016 17:17:39 +0100 Subject: Don't keep full picklers around until backend. The memory footprint captured by pickler seems to be about 1/3rd of total footprint. So we gain a lot by not making this die sooner rather than later. --- src/dotty/tools/dotc/CompilationUnit.scala | 8 ++------ src/dotty/tools/dotc/transform/Pickler.scala | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) (limited to 'src/dotty/tools/dotc') diff --git a/src/dotty/tools/dotc/CompilationUnit.scala b/src/dotty/tools/dotc/CompilationUnit.scala index 16a59250b..491c2bd9b 100644 --- a/src/dotty/tools/dotc/CompilationUnit.scala +++ b/src/dotty/tools/dotc/CompilationUnit.scala @@ -17,12 +17,8 @@ class CompilationUnit(val source: SourceFile) { def isJava = source.file.name.endsWith(".java") - /** - * Picklers used to create TASTY sections, indexed by toplevel class to which they belong. - * Sections: Header, ASTs and Positions are populated by `pickler` phase. - * Subsequent phases can add new sections. - */ - var picklers: Map[ClassSymbol, TastyPickler] = Map() + /** Pickled TASTY binaries, indexed by class. */ + var pickled: Map[ClassSymbol, Array[Byte]] = Map() var unpicklers: Map[ClassSymbol, TastyUnpickler] = Map() } diff --git a/src/dotty/tools/dotc/transform/Pickler.scala b/src/dotty/tools/dotc/transform/Pickler.scala index fc70ac4f2..71cb355df 100644 --- a/src/dotty/tools/dotc/transform/Pickler.scala +++ b/src/dotty/tools/dotc/transform/Pickler.scala @@ -25,7 +25,9 @@ class Pickler extends Phase { s.close } + // Maps that keep a record if -Ytest-pickler is set. private val beforePickling = new mutable.HashMap[ClassSymbol, String] + private val picklers = new mutable.HashMap[ClassSymbol, TastyPickler] /** Drop any elements of this list that are linked module classes of other elements in the list */ private def dropCompanionModuleClasses(clss: List[ClassSymbol])(implicit ctx: Context): List[ClassSymbol] = { @@ -40,9 +42,11 @@ class Pickler extends Phase { for { cls <- dropCompanionModuleClasses(topLevelClasses(unit.tpdTree)) tree <- sliceTopLevel(unit.tpdTree, cls) } { - if (ctx.settings.YtestPickler.value) beforePickling(cls) = tree.show val pickler = new TastyPickler() - unit.picklers += (cls -> pickler) + if (ctx.settings.YtestPickler.value) { + beforePickling(cls) = tree.show + picklers(cls) = pickler + } val treePkl = pickler.treePkl treePkl.pickle(tree :: Nil) treePkl.compactify() @@ -51,8 +55,12 @@ class Pickler extends Phase { if (tree.pos.exists) new PositionPickler(pickler, treePkl.buf.addrsOfTree).picklePositions(tree :: Nil) + // other pickle sections go here. + val pickled = pickler.assembleParts() + unit.pickled += (cls -> pickled) + def rawBytes = // not needed right now, but useful to print raw format. - pickler.assembleParts().iterator.grouped(10).toList.zipWithIndex.map { + pickled.iterator.grouped(10).toList.zipWithIndex.map { case (row, i) => s"${i}0: ${row.mkString(" ")}" } // println(i"rawBytes = \n$rawBytes%\n%") // DEBUG @@ -66,18 +74,18 @@ class Pickler extends Phase { override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = { val result = super.runOn(units) if (ctx.settings.YtestPickler.value) - testUnpickler(units)( + testUnpickler( ctx.fresh .setPeriod(Period(ctx.runId + 1, FirstPhaseId)) .addMode(Mode.ReadPositions)) result } - private def testUnpickler(units: List[CompilationUnit])(implicit ctx: Context): Unit = { + private def testUnpickler(implicit ctx: Context): Unit = { pickling.println(i"testing unpickler at run ${ctx.runId}") ctx.initialize() val unpicklers = - for (unit <- units; (cls, pickler) <- unit.picklers) yield { + for ((cls, pickler) <- picklers) yield { val unpickler = new DottyUnpickler(pickler.assembleParts()) unpickler.enter(roots = Set()) cls -> unpickler -- cgit v1.2.3 From 5979e36d7c206beb6abf53302462091fad834c76 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Fri, 18 Nov 2016 17:19:20 +0100 Subject: Say what is compiled under -verbose --- src/dotty/tools/dotc/typer/FrontEnd.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/dotty/tools/dotc') diff --git a/src/dotty/tools/dotc/typer/FrontEnd.scala b/src/dotty/tools/dotc/typer/FrontEnd.scala index 7bbf0169b..e24d1f25d 100644 --- a/src/dotty/tools/dotc/typer/FrontEnd.scala +++ b/src/dotty/tools/dotc/typer/FrontEnd.scala @@ -64,8 +64,10 @@ class FrontEnd extends Phase { unit.isJava || firstTopLevelDef(unit.tpdTree :: Nil).isPrimitiveValueClass override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = { - val unitContexts = for (unit <- units) yield + val unitContexts = for (unit <- units) yield { + ctx.inform(s"compiling ${unit.source}") ctx.fresh.setCompilationUnit(unit).setFreshNames(new FreshNameCreator.Default) + } unitContexts foreach (parse(_)) record("parsedTrees", ast.Trees.ntrees) unitContexts foreach (enterSyms(_)) -- cgit v1.2.3 From 9bada400b36834e92484e416a5b9b41b10cbd5a5 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Fri, 18 Nov 2016 17:19:59 +0100 Subject: More extensive stats about generated trees --- src/dotty/tools/dotc/Run.scala | 3 +++ src/dotty/tools/dotc/typer/FrontEnd.scala | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src/dotty/tools/dotc') diff --git a/src/dotty/tools/dotc/Run.scala b/src/dotty/tools/dotc/Run.scala index f5ba56a7e..0f652ff0b 100644 --- a/src/dotty/tools/dotc/Run.scala +++ b/src/dotty/tools/dotc/Run.scala @@ -82,6 +82,9 @@ class Run(comp: Compiler)(implicit ctx: Context) { ctx.informTime(s"$phase ", start) } if (!ctx.reporter.hasErrors) Rewrites.writeBack() + for (unit <- units) + Stats.record("retained typed trees at end", unit.tpdTree.treeSize) + Stats.record("total trees at end", ast.Trees.ntrees) } private sealed trait PrintedTree diff --git a/src/dotty/tools/dotc/typer/FrontEnd.scala b/src/dotty/tools/dotc/typer/FrontEnd.scala index e24d1f25d..4f8bc55d8 100644 --- a/src/dotty/tools/dotc/typer/FrontEnd.scala +++ b/src/dotty/tools/dotc/typer/FrontEnd.scala @@ -49,8 +49,8 @@ class FrontEnd extends Phase { val unit = ctx.compilationUnit unit.tpdTree = ctx.typer.typedExpr(unit.untpdTree) typr.println("typed: " + unit.source) - record("retainedUntypedTrees", unit.untpdTree.treeSize) - record("retainedTypedTrees", unit.tpdTree.treeSize) + record("retained untyped trees", unit.untpdTree.treeSize) + record("retained typed trees after typer", unit.tpdTree.treeSize) } private def firstTopLevelDef(trees: List[tpd.Tree])(implicit ctx: Context): Symbol = trees match { @@ -72,7 +72,7 @@ class FrontEnd extends Phase { record("parsedTrees", ast.Trees.ntrees) unitContexts foreach (enterSyms(_)) unitContexts foreach (typeCheck(_)) - record("totalTrees", ast.Trees.ntrees) + record("total trees after typer", ast.Trees.ntrees) unitContexts.map(_.compilationUnit).filterNot(discardAfterTyper) } -- cgit v1.2.3 From ddec688b6e165c68cdd9a33dac968ee4a09b447a Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 19 Nov 2016 00:56:32 +0100 Subject: Fix memory leak The lazy val `dummyTree` acquires a type because of copy-on-write and that type can refer via lastDenotation to a context base. --- src/dotty/tools/dotc/typer/ProtoTypes.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/dotty/tools/dotc') diff --git a/src/dotty/tools/dotc/typer/ProtoTypes.scala b/src/dotty/tools/dotc/typer/ProtoTypes.scala index 08f566d49..5eef096db 100644 --- a/src/dotty/tools/dotc/typer/ProtoTypes.scala +++ b/src/dotty/tools/dotc/typer/ProtoTypes.scala @@ -477,7 +477,7 @@ object ProtoTypes { def apply(tp: Type) = wildApprox(tp, this) } - @sharable private lazy val dummyTree = untpd.Literal(Constant(null)) + @sharable private def dummyTree = untpd.Literal(Constant(null)) /** Dummy tree to be used as an argument of a FunProto or ViewProto type */ object dummyTreeOfType { -- cgit v1.2.3 From ca8def4b01369ffbb367b9bb6531dfc1ae4e7936 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 19 Nov 2016 13:42:36 +0100 Subject: Plug another space leak. `initInfo` was retained in Symbols. When called from `Namer`, `initInfo` referred to a completer, which referred to a context. With this space leak plugged, we can now compile 1000 times core/Comments.scala (460lines) with -Xmx400M. There still seems to be a space leak on the order of 200KB per run, though. But that seems to have to do with symbols, not contexts. --- src/dotty/tools/dotc/core/SymDenotations.scala | 6 +++--- src/dotty/tools/dotc/typer/ProtoTypes.scala | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) (limited to 'src/dotty/tools/dotc') diff --git a/src/dotty/tools/dotc/core/SymDenotations.scala b/src/dotty/tools/dotc/core/SymDenotations.scala index fb6a40100..8b7c28e19 100644 --- a/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/src/dotty/tools/dotc/core/SymDenotations.scala @@ -105,7 +105,7 @@ object SymDenotations { ownerIfExists: Symbol, final val name: Name, initFlags: FlagSet, - final val initInfo: Type, + initInfo: Type, initPrivateWithin: Symbol = NoSymbol) extends SingleDenotation(symbol) { //assert(symbol.id != 4940, name) @@ -232,7 +232,7 @@ object SymDenotations { case _ => } */ - if (Config.checkNoSkolemsInInfo) assertNoSkolems(initInfo) + if (Config.checkNoSkolemsInInfo) assertNoSkolems(tp) myInfo = tp } @@ -751,7 +751,7 @@ object SymDenotations { // def isOverridable: Boolean = !!! need to enforce that classes cannot be redefined def isSkolem: Boolean = name == nme.SKOLEM - def isInlineMethod(implicit ctx: Context): Boolean = is(InlineMethod, butNot = Accessor) + def isInlineMethod(implicit ctx: Context): Boolean = is(InlineMethod, butNot = Accessor) // ------ access to related symbols --------------------------------- diff --git a/src/dotty/tools/dotc/typer/ProtoTypes.scala b/src/dotty/tools/dotc/typer/ProtoTypes.scala index 5eef096db..9a20a452e 100644 --- a/src/dotty/tools/dotc/typer/ProtoTypes.scala +++ b/src/dotty/tools/dotc/typer/ProtoTypes.scala @@ -477,11 +477,9 @@ object ProtoTypes { def apply(tp: Type) = wildApprox(tp, this) } - @sharable private def dummyTree = untpd.Literal(Constant(null)) - /** Dummy tree to be used as an argument of a FunProto or ViewProto type */ object dummyTreeOfType { - def apply(tp: Type): Tree = dummyTree withTypeUnchecked tp + def apply(tp: Type): Tree = untpd.Literal(Constant(null)) withTypeUnchecked tp def unapply(tree: Tree): Option[Type] = tree match { case Literal(Constant(null)) => Some(tree.typeOpt) case _ => None -- cgit v1.2.3 From 8932d98c4dcb6eb840cf640bc636982236613a16 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 19 Nov 2016 17:51:08 +0100 Subject: Recreate FreshNameCreator for each run. Previously only the FrontEnd got a fresh FreshNameCreator for each run, the other phases used a global one instead. This means that compiling the same file several times would create different synthetic names and classes on each run. --- src/dotty/tools/dotc/Compiler.scala | 4 +++- src/dotty/tools/dotc/typer/FrontEnd.scala | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'src/dotty/tools/dotc') diff --git a/src/dotty/tools/dotc/Compiler.scala b/src/dotty/tools/dotc/Compiler.scala index 178cba7c4..4bc5263e9 100644 --- a/src/dotty/tools/dotc/Compiler.scala +++ b/src/dotty/tools/dotc/Compiler.scala @@ -11,6 +11,7 @@ import typer.{FrontEnd, Typer, ImportInfo, RefChecks} import reporting.{Reporter, ConsoleReporter} import Phases.Phase import transform._ +import util.FreshNameCreator import transform.TreeTransforms.{TreeTransform, TreeTransformer} import core.DenotTransformers.DenotTransformer import core.Denotations.SingleDenotation @@ -140,7 +141,8 @@ class Compiler { .setTyper(new Typer) .setMode(Mode.ImplicitsEnabled) .setTyperState(new MutableTyperState(ctx.typerState, ctx.typerState.reporter, isCommittable = true)) - ctx.initialize()(start) // re-initialize the base context with start + .setFreshNames(new FreshNameCreator.Default) + ctx.initialize()(start) // re-initialize the base context with start def addImport(ctx: Context, refFn: () => TermRef) = ctx.fresh.setImportInfo(ImportInfo.rootImport(refFn)(ctx)) (start.setRunInfo(new RunInfo(start)) /: defn.RootImportFns)(addImport) diff --git a/src/dotty/tools/dotc/typer/FrontEnd.scala b/src/dotty/tools/dotc/typer/FrontEnd.scala index 4f8bc55d8..c444631ae 100644 --- a/src/dotty/tools/dotc/typer/FrontEnd.scala +++ b/src/dotty/tools/dotc/typer/FrontEnd.scala @@ -12,7 +12,6 @@ import config.Printers.{typr, default} import util.Stats._ import scala.util.control.NonFatal import ast.Trees._ -import util.FreshNameCreator class FrontEnd extends Phase { @@ -66,7 +65,7 @@ class FrontEnd extends Phase { override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = { val unitContexts = for (unit <- units) yield { ctx.inform(s"compiling ${unit.source}") - ctx.fresh.setCompilationUnit(unit).setFreshNames(new FreshNameCreator.Default) + ctx.fresh.setCompilationUnit(unit) } unitContexts foreach (parse(_)) record("parsedTrees", ast.Trees.ntrees) -- cgit v1.2.3