From 22d907c1e97a608063f94b87aff1279482e2e58a Mon Sep 17 00:00:00 2001 From: Miguel Garcia Date: Wed, 21 Aug 2013 17:22:07 +0200 Subject: GenBCode: decouple ClassNode building from classfile writing This commit and two follow-up commits (recognizable because of the starting words "decouple this from that" in the commit message) are baby steps towards pipelining of class building and class writing, ie allowing class building (which requires typer) and "the rest" which doesn't) to run in parallel. The commits have been broken up following a previous review comment. --- .../scala/tools/nsc/backend/jvm/GenBCode.scala | 108 +++++++++++++++++---- 1 file changed, 90 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala b/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala index e55a3baed0..6a2ba43f4c 100644 --- a/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala +++ b/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala @@ -52,6 +52,38 @@ abstract class GenBCode extends BCodeSyncAndTry { private var needsOutFolder = false // whether getOutFolder(claszSymbol) should be invoked for each claszSymbol + /* ---------------- q3 ---------------- */ + + /* + * An item of queue-3 (the last queue before serializing to disk) contains three of these + * (one for each of mirror, plain, and bean classes). + * + * @param jclassName internal name of the class + * @param jclassBytes bytecode emitted for the class SubItem3 represents + */ + case class SubItem3( + jclassName: String, + jclassBytes: Array[Byte] + ) + + case class Item3(arrivalPos: Int, + mirror: SubItem3, + plain: SubItem3, + bean: SubItem3, + outFolder: scala.tools.nsc.io.AbstractFile) { + + def isPoison = { arrivalPos == Int.MaxValue } + } + private val i3comparator = new java.util.Comparator[Item3] { + override def compare(a: Item3, b: Item3) = { + if (a.arrivalPos < b.arrivalPos) -1 + else if (a.arrivalPos == b.arrivalPos) 0 + else 1 + } + } + private val poison3 = Item3(Int.MaxValue, null, null, null, null) + private val q3 = new java.util.PriorityQueue[Item3](1000, i3comparator) + val caseInsensitively = mutable.Map.empty[String, Symbol] /* @@ -102,7 +134,19 @@ abstract class GenBCode extends BCodeSyncAndTry { ) } else null - // ----------- serialize classfiles to disk + // ----------- hand over to next pipeline + + addToQ3(arrivalPos, + mirrorC, plainC, beanC, + outF) + + } // end of method visit() + + private def addToQ3(arrivalPos: Int, + mirror: asm.tree.ClassNode, + plain: asm.tree.ClassNode, + bean: asm.tree.ClassNode, + outFolder: scala.tools.nsc.io.AbstractFile) { def getByteArray(cn: asm.tree.ClassNode): Array[Byte] = { val cw = new CClassWriter(extraProc) @@ -110,15 +154,13 @@ abstract class GenBCode extends BCodeSyncAndTry { cw.toByteArray } - if (mirrorC != null) { - sendToDisk(mirrorC.name, getByteArray(mirrorC), outF) - } - sendToDisk(plainC.name, getByteArray(plainC), outF) - if (beanC != null) { - sendToDisk(beanC.name, getByteArray(beanC), outF) - } + val mirrorC = if (mirror == null) null else SubItem3(mirror.name, getByteArray(mirror)) + val plainC = SubItem3(plain.name, getByteArray(plain)) + val beanC = if (bean == null) null else SubItem3(bean.name, getByteArray(bean)) - } // end of method visit() + q3 add Item3(arrivalPos, mirrorC, plainC, beanC, outFolder) + + } var arrivalPos = 0 @@ -147,6 +189,8 @@ abstract class GenBCode extends BCodeSyncAndTry { needsOutFolder = bytecodeWriter.isInstanceOf[ClassBytecodeWriter] super.run() + q3 add poison3 + drainQ3() // closing output files. bytecodeWriter.close() @@ -170,17 +214,45 @@ abstract class GenBCode extends BCodeSyncAndTry { clearBCodeTypes() } - def sendToDisk(jclassName: String, jclassBytes: Array[Byte], outFolder: _root_.scala.tools.nsc.io.AbstractFile) { - try { - val outFile = - if (outFolder == null) null - else getFileForClassfile(outFolder, jclassName, ".class") - bytecodeWriter.writeClass(jclassName, jclassName, jclassBytes, outFile) + /* Pipeline that writes classfile representations to disk. */ + private def drainQ3() { + + def sendToDisk(cfr: SubItem3, outFolder: scala.tools.nsc.io.AbstractFile) { + if (cfr != null){ + val SubItem3(jclassName, jclassBytes) = cfr + try { + val outFile = + if (outFolder == null) null + else getFileForClassfile(outFolder, jclassName, ".class") + bytecodeWriter.writeClass(jclassName, jclassName, jclassBytes, outFile) + } + catch { + case e: FileConflictException => + error(s"error writing $jclassName: ${e.getMessage}") + } + } } - catch { - case e: FileConflictException => - error(s"error writing $jclassName: ${e.getMessage}") + + var moreComing = true + // `expected` denotes the arrivalPos whose Item3 should be serialized next + var expected = 0 + + while (moreComing) { + val incoming = q3.poll + moreComing = !incoming.isPoison + if (moreComing) { + val item = incoming + val outFolder = item.outFolder + sendToDisk(item.mirror, outFolder) + sendToDisk(item.plain, outFolder) + sendToDisk(item.bean, outFolder) + expected += 1 + } } + + // we're done + assert(q3.isEmpty, s"Some classfiles weren't written to disk: $q3") + } override def apply(cunit: CompilationUnit): Unit = { -- cgit v1.2.3 From cd1c070ec45cc91a2f3d08c2f190402f19b59f7e Mon Sep 17 00:00:00 2001 From: Miguel Garcia Date: Wed, 21 Aug 2013 17:42:25 +0200 Subject: GenBCode: decouple ClassNode building from encoding as byte array To recap from the previous commit, another baby step towards pipelining of class building and class writing. Implementing similar functionality in GenASM is up for grabs, see https://issues.scala-lang.org/browse/SI-6164 --- .../scala/tools/nsc/backend/jvm/GenBCode.scala | 86 +++++++++++++++++----- 1 file changed, 67 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala b/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala index 6a2ba43f4c..b6ecc66b10 100644 --- a/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala +++ b/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala @@ -23,7 +23,10 @@ import scala.tools.asm * (a) an optional mirror class, * (b) a plain class, and * (c) an optional bean class. - * - each of the ClassNodes above is lowered into a byte-array (ie into a classfile) and serialized. + * - each of the items (a), (b), (c) is placed on queue `q2`. + * Those items will be transferred from that queue to `q3` by Worker2, + * whose job is lowering a ClassNode into a byte-array (ie into a classfile) + * - the classfilse are serialized in `drainQ3()` * * Plain, mirror, and bean classes are built respectively by PlainClassBuilder, JMirrorBuilder, and JBeanInfoBuilder. * @@ -52,6 +55,19 @@ abstract class GenBCode extends BCodeSyncAndTry { private var needsOutFolder = false // whether getOutFolder(claszSymbol) should be invoked for each claszSymbol + /* ---------------- q2 ---------------- */ + + case class Item2(arrivalPos: Int, + mirror: asm.tree.ClassNode, + plain: asm.tree.ClassNode, + bean: asm.tree.ClassNode, + outFolder: scala.tools.nsc.io.AbstractFile) { + def isPoison = { arrivalPos == Int.MaxValue } + } + + private val poison2 = Item2(Int.MaxValue, null, null, null, null) + private val q2 = new _root_.java.util.LinkedList[Item2] + /* ---------------- q3 ---------------- */ /* @@ -88,7 +104,8 @@ abstract class GenBCode extends BCodeSyncAndTry { /* * Checks for duplicate internal names case-insensitively, - * builds ASM ClassNodes for mirror, plain, and bean classes. + * builds ASM ClassNodes for mirror, plain, and bean classes; + * enqueues them in queue-2. * */ def visit(arrivalPos: Int, cd: ClassDef, cunit: CompilationUnit) { @@ -136,31 +153,60 @@ abstract class GenBCode extends BCodeSyncAndTry { // ----------- hand over to next pipeline - addToQ3(arrivalPos, + val item2 = + Item2(arrivalPos, mirrorC, plainC, beanC, outF) - } // end of method visit() + q2 add item2 // at the very end of this method so that no Worker2 thread starts mutating before we're done. - private def addToQ3(arrivalPos: Int, - mirror: asm.tree.ClassNode, - plain: asm.tree.ClassNode, - bean: asm.tree.ClassNode, - outFolder: scala.tools.nsc.io.AbstractFile) { + } // end of method visit() - def getByteArray(cn: asm.tree.ClassNode): Array[Byte] = { - val cw = new CClassWriter(extraProc) - cn.accept(cw) - cw.toByteArray + /* + * Pipeline that takes ClassNodes from queue-2. The unit of work depends on the optimization level: + * + * (a) no optimization involves: + * - converting the plain ClassNode to byte array and placing it on queue-3 + */ + class Worker2 { + + def run() { + while (true) { + val item = q2.poll + if (item.isPoison) { + q3 add poison3 + return + } + else { + try { addToQ3(item) } + catch { + case ex: Throwable => + ex.printStackTrace() + error(s"Error while emitting ${item.plain.name}\n${ex.getMessage}") + } + } + } } - val mirrorC = if (mirror == null) null else SubItem3(mirror.name, getByteArray(mirror)) - val plainC = SubItem3(plain.name, getByteArray(plain)) - val beanC = if (bean == null) null else SubItem3(bean.name, getByteArray(bean)) + private def addToQ3(item: Item2) { - q3 add Item3(arrivalPos, mirrorC, plainC, beanC, outFolder) + def getByteArray(cn: asm.tree.ClassNode): Array[Byte] = { + val cw = new CClassWriter(extraProc) + cn.accept(cw) + cw.toByteArray + } - } + val Item2(arrivalPos, mirror, plain, bean, outFolder) = item + + val mirrorC = if (mirror == null) null else SubItem3(mirror.name, getByteArray(mirror)) + val plainC = SubItem3(plain.name, getByteArray(plain)) + val beanC = if (bean == null) null else SubItem3(bean.name, getByteArray(bean)) + + q3 add Item3(arrivalPos, mirrorC, plainC, beanC, outFolder) + + } + + } // end of class BCodePhase.Worker2 var arrivalPos = 0 @@ -189,7 +235,8 @@ abstract class GenBCode extends BCodeSyncAndTry { needsOutFolder = bytecodeWriter.isInstanceOf[ClassBytecodeWriter] super.run() - q3 add poison3 + q2 add poison2 + (new Worker2).run() drainQ3() // closing output files. @@ -251,6 +298,7 @@ abstract class GenBCode extends BCodeSyncAndTry { } // we're done + assert(q2.isEmpty, s"Some classfiles remained in the second queue: $q2") assert(q3.isEmpty, s"Some classfiles weren't written to disk: $q3") } -- cgit v1.2.3 From 3b3f03749a15afad7b5428cc53bea0a5e759f94d Mon Sep 17 00:00:00 2001 From: Miguel Garcia Date: Wed, 21 Aug 2013 17:32:32 +0200 Subject: GenBCode: decouple ClassDef traversing from ClassNode building In keeping with the baby steps towards pipelining of class building and class writing, this commit gets one step closer to that. Only thing missing: the actual thread-pool. That will be the focus of an upcoming commit. --- .../scala/tools/nsc/backend/jvm/GenBCode.scala | 194 +++++++++++++-------- 1 file changed, 126 insertions(+), 68 deletions(-) (limited to 'src') diff --git a/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala b/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala index b6ecc66b10..193100474c 100644 --- a/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala +++ b/src/compiler/scala/tools/nsc/backend/jvm/GenBCode.scala @@ -17,16 +17,25 @@ import scala.tools.asm /* * Prepare in-memory representations of classfiles using the ASM Tree API, and serialize them to disk. * - * `BCodePhase.apply(CompilationUnit)` is invoked by some external force and that sets in motion: - * - visiting each ClassDef contained in that CompilationUnit - * - lowering the ClassDef into: + * Three pipelines are at work, each taking work items from a queue dedicated to that pipeline: + * + * (There's another pipeline so to speak, the one that populates queue-1 by traversing a CompilationUnit until ClassDefs are found, + * but the "interesting" pipelines are the ones described below) + * + * (1) In the first queue, an item consists of a ClassDef along with its arrival position. + * This position is needed at the time classfiles are serialized to disk, + * so as to emit classfiles in the same order CleanUp handed them over. + * As a result, two runs of the compiler on the same files produce jars that are identical on a byte basis. + * See `ant test.stability` + * + * (2) The second queue contains items where a ClassDef has been lowered into: * (a) an optional mirror class, * (b) a plain class, and * (c) an optional bean class. - * - each of the items (a), (b), (c) is placed on queue `q2`. - * Those items will be transferred from that queue to `q3` by Worker2, - * whose job is lowering a ClassNode into a byte-array (ie into a classfile) - * - the classfilse are serialized in `drainQ3()` + * + * (3) The third queue contains items ready for serialization. + * It's a priority queue that follows the original arrival order, + * so as to emit identical jars on repeated compilation of the same sources. * * Plain, mirror, and bean classes are built respectively by PlainClassBuilder, JMirrorBuilder, and JBeanInfoBuilder. * @@ -53,7 +62,13 @@ abstract class GenBCode extends BCodeSyncAndTry { private var mirrorCodeGen : JMirrorBuilder = null private var beanInfoCodeGen : JBeanInfoBuilder = null - private var needsOutFolder = false // whether getOutFolder(claszSymbol) should be invoked for each claszSymbol + /* ---------------- q1 ---------------- */ + + case class Item1(arrivalPos: Int, cd: ClassDef, cunit: CompilationUnit) { + def isPoison = { arrivalPos == Int.MaxValue } + } + private val poison1 = Item1(Int.MaxValue, null, null) + private val q1 = new java.util.LinkedList[Item1] /* ---------------- q2 ---------------- */ @@ -100,67 +115,93 @@ abstract class GenBCode extends BCodeSyncAndTry { private val poison3 = Item3(Int.MaxValue, null, null, null, null) private val q3 = new java.util.PriorityQueue[Item3](1000, i3comparator) - val caseInsensitively = mutable.Map.empty[String, Symbol] - /* - * Checks for duplicate internal names case-insensitively, - * builds ASM ClassNodes for mirror, plain, and bean classes; - * enqueues them in queue-2. - * + * Pipeline that takes ClassDefs from queue-1, lowers them into an intermediate form, placing them on queue-2 */ - def visit(arrivalPos: Int, cd: ClassDef, cunit: CompilationUnit) { - val claszSymbol = cd.symbol - - // GenASM checks this before classfiles are emitted, https://github.com/scala/scala/commit/e4d1d930693ac75d8eb64c2c3c69f2fc22bec739 - val lowercaseJavaClassName = claszSymbol.javaClassName.toLowerCase - caseInsensitively.get(lowercaseJavaClassName) match { - case None => - caseInsensitively.put(lowercaseJavaClassName, claszSymbol) - case Some(dupClassSym) => - cunit.warning( - claszSymbol.pos, - s"Class ${claszSymbol.javaClassName} differs only in case from ${dupClassSym.javaClassName}. " + - "Such classes will overwrite one another on case-insensitive filesystems." - ) - } + class Worker1(needsOutFolder: Boolean) { + + val caseInsensitively = mutable.Map.empty[String, Symbol] - // -------------- mirror class, if needed -------------- - val mirrorC = - if (isStaticModule(claszSymbol) && isTopLevelModule(claszSymbol)) { - if (claszSymbol.companionClass == NoSymbol) { - mirrorCodeGen.genMirrorClass(claszSymbol, cunit) - } else { - log(s"No mirror class for module with linked class: ${claszSymbol.fullName}"); - null + def run() { + while (true) { + val item = q1.poll + if (item.isPoison) { + q2 add poison2 + return } - } else null + else { + try { visit(item) } + catch { + case ex: Throwable => + ex.printStackTrace() + error(s"Error while emitting ${item.cunit.source}\n${ex.getMessage}") + } + } + } + } - // -------------- "plain" class -------------- - val pcb = new PlainClassBuilder(cunit) - pcb.genPlainClass(cd) - val outF = if (needsOutFolder) getOutFolder(claszSymbol, pcb.thisName, cunit) else null; - val plainC = pcb.cnode + /* + * Checks for duplicate internal names case-insensitively, + * builds ASM ClassNodes for mirror, plain, and bean classes; + * enqueues them in queue-2. + * + */ + def visit(item: Item1) { + val Item1(arrivalPos, cd, cunit) = item + val claszSymbol = cd.symbol + + // GenASM checks this before classfiles are emitted, https://github.com/scala/scala/commit/e4d1d930693ac75d8eb64c2c3c69f2fc22bec739 + val lowercaseJavaClassName = claszSymbol.javaClassName.toLowerCase + caseInsensitively.get(lowercaseJavaClassName) match { + case None => + caseInsensitively.put(lowercaseJavaClassName, claszSymbol) + case Some(dupClassSym) => + item.cunit.warning( + claszSymbol.pos, + s"Class ${claszSymbol.javaClassName} differs only in case from ${dupClassSym.javaClassName}. " + + "Such classes will overwrite one another on case-insensitive filesystems." + ) + } + + // -------------- mirror class, if needed -------------- + val mirrorC = + if (isStaticModule(claszSymbol) && isTopLevelModule(claszSymbol)) { + if (claszSymbol.companionClass == NoSymbol) { + mirrorCodeGen.genMirrorClass(claszSymbol, cunit) + } else { + log(s"No mirror class for module with linked class: ${claszSymbol.fullName}") + null + } + } else null + + // -------------- "plain" class -------------- + val pcb = new PlainClassBuilder(cunit) + pcb.genPlainClass(cd) + val outF = if (needsOutFolder) getOutFolder(claszSymbol, pcb.thisName, cunit) else null; + val plainC = pcb.cnode - // -------------- bean info class, if needed -------------- - val beanC = - if (claszSymbol hasAnnotation BeanInfoAttr) { - beanInfoCodeGen.genBeanInfoClass( - claszSymbol, cunit, - fieldSymbols(claszSymbol), - methodSymbols(cd) - ) - } else null + // -------------- bean info class, if needed -------------- + val beanC = + if (claszSymbol hasAnnotation BeanInfoAttr) { + beanInfoCodeGen.genBeanInfoClass( + claszSymbol, cunit, + fieldSymbols(claszSymbol), + methodSymbols(cd) + ) + } else null - // ----------- hand over to next pipeline + // ----------- hand over to pipeline-2 - val item2 = - Item2(arrivalPos, - mirrorC, plainC, beanC, - outF) + val item2 = + Item2(arrivalPos, + mirrorC, plainC, beanC, + outF) - q2 add item2 // at the very end of this method so that no Worker2 thread starts mutating before we're done. + q2 add item2 // at the very end of this method so that no Worker2 thread starts mutating before we're done. - } // end of method visit() + } // end of method visit(Item1) + + } // end of class BCodePhase.Worker1 /* * Pipeline that takes ClassNodes from queue-2. The unit of work depends on the optimization level: @@ -232,18 +273,12 @@ abstract class GenBCode extends BCodeSyncAndTry { mirrorCodeGen = new JMirrorBuilder beanInfoCodeGen = new JBeanInfoBuilder - needsOutFolder = bytecodeWriter.isInstanceOf[ClassBytecodeWriter] - - super.run() - q2 add poison2 - (new Worker2).run() - drainQ3() + val needsOutfileForSymbol = bytecodeWriter.isInstanceOf[ClassBytecodeWriter] + buildAndSendToDisk(needsOutfileForSymbol) // closing output files. bytecodeWriter.close() - caseInsensitively.clear() - /* TODO Bytecode can be verified (now that all classfiles have been written to disk) * * (1) asm.util.CheckAdapter.verify() @@ -261,6 +296,28 @@ abstract class GenBCode extends BCodeSyncAndTry { clearBCodeTypes() } + /* + * Sequentially: + * (a) place all ClassDefs in queue-1 + * (b) dequeue one at a time from queue-1, convert it to ASM ClassNode, place in queue-2 + * (c) dequeue one at a time from queue-2, convert it to byte-array, place in queue-3 + * (d) serialize to disk by draining queue-3. + */ + private def buildAndSendToDisk(needsOutFolder: Boolean) { + + feedPipeline1() + (new Worker1(needsOutFolder)).run() + (new Worker2).run() + drainQ3() + + } + + /* Feed pipeline-1: place all ClassDefs on q1, recording their arrival position. */ + private def feedPipeline1() { + super.run() + q1 add poison1 + } + /* Pipeline that writes classfile representations to disk. */ private def drainQ3() { @@ -298,6 +355,7 @@ abstract class GenBCode extends BCodeSyncAndTry { } // we're done + assert(q1.isEmpty, s"Some ClassDefs remained in the first queue: $q1") assert(q2.isEmpty, s"Some classfiles remained in the second queue: $q2") assert(q3.isEmpty, s"Some classfiles weren't written to disk: $q3") @@ -310,7 +368,7 @@ abstract class GenBCode extends BCodeSyncAndTry { case EmptyTree => () case PackageDef(_, stats) => stats foreach gen case cd: ClassDef => - visit(arrivalPos, cd, cunit) + q1 add Item1(arrivalPos, cd, cunit) arrivalPos += 1 } } -- cgit v1.2.3