diff options
29 files changed, 1749 insertions, 134 deletions
diff --git a/src/dotty/tools/dotc/config/ScalaSettings.scala b/src/dotty/tools/dotc/config/ScalaSettings.scala index 22e8b99c8..4b583be9d 100644 --- a/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -66,7 +66,6 @@ class ScalaSettings extends Settings.SettingGroup { val logFreeTypes = BooleanSetting("-Xlog-free-types", "Print a message when reification resorts to generating a free type.") val maxClassfileName = IntSetting("-Xmax-classfile-name", "Maximum filename length for generated classes", 255, 72 to 255) val Xmigration28 = BooleanSetting("-Xmigration", "Warn about constructs whose behavior may have changed between 2.7 and 2.8.") - val nouescape = BooleanSetting("-Xno-uescape", "Disable handling of \\u unicode escapes.") val Xnojline = BooleanSetting("-Xnojline", "Do not use JLine for editing.") val Xverify = BooleanSetting("-Xverify", "Verify generic signatures in generated bytecode (asm backend only.)") val plugin = MultiStringSetting("-Xplugin", "file", "Load one or more plugins from files.") diff --git a/src/dotty/tools/dotc/core/Annotations.scala b/src/dotty/tools/dotc/core/Annotations.scala index 98ce4fffd..9086047cd 100644 --- a/src/dotty/tools/dotc/core/Annotations.scala +++ b/src/dotty/tools/dotc/core/Annotations.scala @@ -1,6 +1,7 @@ -package dotty.tools.dotc.core +package dotty.tools.dotc +package core -import Symbols._, Types._, Positions._, Contexts._, Constants._, TypedTrees.tpd._ +import Symbols._, Types._, util.Positions._, Contexts._, Constants._, TypedTrees.tpd._ object Annotations { diff --git a/src/dotty/tools/dotc/core/Contexts.scala b/src/dotty/tools/dotc/core/Contexts.scala index 29da40caf..15f1e80c5 100644 --- a/src/dotty/tools/dotc/core/Contexts.scala +++ b/src/dotty/tools/dotc/core/Contexts.scala @@ -8,8 +8,8 @@ import Names._ import Phases._ import Types._ import Symbols._ -import TypeComparers._, Printers._, NameOps._, SymDenotations._, Positions._ -import TypedTrees.tpd._ +import TypeComparers._, Printers._, NameOps._, SymDenotations._, util.Positions._ +import TypedTrees.tpd._, util.FreshNameCreator import config.Settings._ import config.ScalaSettings import reporting._ @@ -172,7 +172,7 @@ object Contexts { /** The current source file; will be derived from current * compilation unit. */ - def source = io.NoSource // for now + def source = util.NoSource // for now /** Does current phase use an erased types interpretation? */ def erasedTypes: Boolean = phase.erasedTypes @@ -289,6 +289,9 @@ object Contexts { /** The platform */ val platform: Platform = new JavaPlatform + /** The standard fresh name creator */ + val fresh = new FreshNameCreator.Default + /** The loader that loads the members of _root_ */ def rootLoader(root: TermSymbol)(implicit ctx: Context): SymbolLoader = platform.rootLoader(root) diff --git a/src/dotty/tools/dotc/core/Definitions.scala b/src/dotty/tools/dotc/core/Definitions.scala index f62acb015..89d785997 100644 --- a/src/dotty/tools/dotc/core/Definitions.scala +++ b/src/dotty/tools/dotc/core/Definitions.scala @@ -3,7 +3,7 @@ package dotc package core import Types._, Contexts._, Symbols._, Denotations._, SymDenotations._, StdNames._, Names._ -import Flags._, Scopes._, Decorators._, NameOps._, Positions._ +import Flags._, Scopes._, Decorators._, NameOps._, util.Positions._ import scala.annotation.{ switch, meta } import scala.collection.{ mutable, immutable } import PartialFunction._ diff --git a/src/dotty/tools/dotc/core/NameOps.scala b/src/dotty/tools/dotc/core/NameOps.scala index 60ac3fba9..b31cf41d5 100644 --- a/src/dotty/tools/dotc/core/NameOps.scala +++ b/src/dotty/tools/dotc/core/NameOps.scala @@ -2,11 +2,12 @@ package dotty.tools.dotc package core import java.security.MessageDigest -import Chars.isOperatorPart import scala.annotation.switch import scala.io.Codec import Names._, StdNames._, Contexts._, Symbols._, Flags._ import Decorators.StringDecorator +import dotty.tools.dotc.util.Chars +import Chars.isOperatorPart object NameOps { diff --git a/src/dotty/tools/dotc/core/StdNames.scala b/src/dotty/tools/dotc/core/StdNames.scala index f5c3fb845..e4311af6c 100644 --- a/src/dotty/tools/dotc/core/StdNames.scala +++ b/src/dotty/tools/dotc/core/StdNames.scala @@ -240,7 +240,6 @@ object StdNames { val SETTER_SUFFIX: N = encode("_=") val SKOLEM: N = "<skolem>" val SPECIALIZED_INSTANCE: N = "specInstance$" - val STAR: N = "*" val THIS: N = "_$this" val HK_PARAM_PREFIX: N = "_$hk$" diff --git a/src/dotty/tools/dotc/core/SymDenotations.scala b/src/dotty/tools/dotc/core/SymDenotations.scala index a14f28ce4..b4cc9091c 100644 --- a/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/src/dotty/tools/dotc/core/SymDenotations.scala @@ -577,6 +577,12 @@ object SymDenotations { final def symbolicRef(implicit ctx: Context): TypeRef = TypeRef.withSym(owner.thisType, symbol.asType) + /** The termref pointing to this termsymbol + * @throws ClassCastException is this is not a term + */ + def termRef(implicit ctx: Context): TermRef = + TermRef.withSym(owner.thisType, symbol.asTerm) + /** The variance of this type parameter as an Int, with * +1 = Covariant, -1 = Contravariant, 0 = Nonvariant, or not a type parameter */ diff --git a/src/dotty/tools/dotc/core/SymbolLoaders.scala b/src/dotty/tools/dotc/core/SymbolLoaders.scala index e2c5f9774..dc65b77f4 100644 --- a/src/dotty/tools/dotc/core/SymbolLoaders.scala +++ b/src/dotty/tools/dotc/core/SymbolLoaders.scala @@ -10,7 +10,7 @@ package core import java.io.IOException import scala.compat.Platform.currentTime import dotty.tools.io.{ ClassPath, AbstractFile } -import Contexts._, Symbols._, Flags._, SymDenotations._, Types._, Scopes._, Positions._, Names._ +import Contexts._, Symbols._, Flags._, SymDenotations._, Types._, Scopes._, util.Positions._, Names._ import StdNames._ import Decorators.StringDecorator import pickling.ClassfileParser diff --git a/src/dotty/tools/dotc/core/Symbols.scala b/src/dotty/tools/dotc/core/Symbols.scala index dced9823d..c61b7d791 100644 --- a/src/dotty/tools/dotc/core/Symbols.scala +++ b/src/dotty/tools/dotc/core/Symbols.scala @@ -11,7 +11,7 @@ import Decorators._ import Symbols._ import Contexts._ import SymDenotations._, util.Texts._ -import Types._, Annotations._, Positions._, StdNames._, Trees._, NameOps._ +import Types._, Annotations._, util.Positions._, StdNames._, Trees._, NameOps._ import Denotations.{ Denotation, SingleDenotation, MultiDenotation } import collection.mutable import io.AbstractFile diff --git a/src/dotty/tools/dotc/core/Trees.scala b/src/dotty/tools/dotc/core/Trees.scala index 922bf6055..c485eddc1 100644 --- a/src/dotty/tools/dotc/core/Trees.scala +++ b/src/dotty/tools/dotc/core/Trees.scala @@ -1,6 +1,7 @@ -package dotty.tools.dotc.core +package dotty.tools.dotc +package core -import Types._, Names._, Flags._, Positions._, Contexts._, Constants._, SymDenotations._, Symbols._ +import Types._, Names._, Flags._, util.Positions._, Contexts._, Constants._, SymDenotations._, Symbols._ import Denotations._, StdNames._ import annotation.tailrec import language.higherKinds @@ -110,6 +111,7 @@ object Trees { override def getMessage: String = s"type of $tree is not assigned" } + type Untyped = Null type TypedTree = Tree[Type] type UntypedTree = Tree[Nothing] @@ -532,6 +534,9 @@ object Trees { extends NameTree[Nothing] with DefTree[Nothing] { type ThisTree[T] <: NameTree[T] with DefTree[T] with ModuleDef val pos = cpos union impl.pos + def derivedModuleDef(mods: Modifiers[Nothing], name: TermName, impl: Template[Nothing]) = + if (mods == this.mods && name == this.name && (impl eq this.impl)) this + else ModuleDef(mods, name, impl) } /** (vparams) => body */ @@ -541,6 +546,78 @@ object Trees { val pos = unionPos(cpos union body.pos, vparams) } + /** Something in parentheses */ + case class Parens(trees: List[Tree[Nothing]])(implicit cpos: Position) extends Tree[Nothing] { + type ThisType[T] <: Parens + val pos = unionPos(cpos, trees) + } + + // ----- Generic Tree Instances, inherited from `tpt` and `untpd`. + + abstract class Instance[T] { + + type Modifiers = Trees.Modifiers[T] + type Tree = Trees.Tree[T] + type TypTree = Trees.TypTree[T] + type TermTree = Trees.TermTree[T] + type PatternTree = Trees.PatternTree[T] + type DenotingTree = Trees.DenotingTree[T] + type ProxyTree = Trees.ProxyTree[T] + type NameTree = Trees.NameTree[T] + type RefTree = Trees.RefTree[T] + type DefTree = Trees.DefTree[T] + + type TreeCopier = Trees.TreeCopier[T] + type TreeAccumulator[U] = Trees.TreeAccumulator[U, T] + type TreeTransformer = Trees.TreeTransformer[T] + + type Ident = Trees.Ident[T] + type Select = Trees.Select[T] + type This = Trees.This[T] + type Super = Trees.Super[T] + type Apply = Trees.Apply[T] + type TypeApply = Trees.TypeApply[T] + type Literal = Trees.Literal[T] + type New = Trees.New[T] + type Pair = Trees.Pair[T] + type Typed = Trees.Typed[T] + type NamedArg = Trees.NamedArg[T] + type Assign = Trees.Assign[T] + type Block = Trees.Block[T] + type If = Trees.If[T] + type Match = Trees.Match[T] + type CaseDef = Trees.CaseDef[T] + type Return = Trees.Return[T] + type Try = Trees.Try[T] + type Throw = Trees.Throw[T] + type SeqLiteral = Trees.SeqLiteral[T] + type TypeTree = Trees.TypeTree[T] + type SingletonTypeTree = Trees.SingletonTypeTree[T] + type SelectFromTypeTree = Trees.SelectFromTypeTree[T] + type AndTypeTree = Trees.AndTypeTree[T] + type OrTypeTree = Trees.OrTypeTree[T] + type RefineTypeTree = Trees.RefineTypeTree[T] + type AppliedTypeTree = Trees.AppliedTypeTree[T] + type TypeBoundsTree = Trees.TypeBoundsTree[T] + type Bind = Trees.Bind[T] + type Alternative = Trees.Alternative[T] + type UnApply = Trees.UnApply[T] + type ValDef = Trees.ValDef[T] + type DefDef = Trees.DefDef[T] + type TypeDef = Trees.TypeDef[T] + type Template = Trees.Template[T] + type ClassDef = Trees.ClassDef[T] + type Import = Trees.Import[T] + type PackageDef = Trees.PackageDef[T] + type Annotated = Trees.Annotated[T] + type EmptyTree = Trees.EmptyTree[T] + type SharedTree = Trees.SharedTree[T] + + protected implicit def pos(implicit ctx: Context): Position = ctx.position + + def defPos(sym: Symbol)(implicit ctx: Context) = ctx.position union sym.coord.toPosition + } + // ----- Helper functions and classes --------------------------------------- @tailrec final def unionPos(base: Position, trees: List[Tree[_]]): Position = trees match { @@ -712,7 +789,7 @@ object Trees { } } - abstract class TreeTransformer[T, C] { + abstract class FullTreeTransformer[T, C] { var sharedMemo: Map[SharedTree[T], SharedTree[T]] = Map() def transform(tree: Tree[T], c: C): Tree[T] = tree match { @@ -861,6 +938,107 @@ object Trees { def finishSharedTree(tree: Tree[T], old: Tree[T], c: C, plugins: Plugins) = tree } + abstract class TreeTransformer[T] { + var sharedMemo: Map[SharedTree[T], SharedTree[T]] = Map() + + def transform(tree: Tree[T]): Tree[T] = tree match { + case Ident(name) => + tree + case Select(qualifier, name) => + tree.derivedSelect(transform(qualifier), name) + case This(qual) => + tree + case Super(qual, mix) => + tree.derivedSuper(transform(qual), mix) + case Apply(fun, args) => + tree.derivedApply(transform(fun), transform(args)) + case TypeApply(fun, args) => + tree.derivedTypeApply(transform(fun), transform(args)) + case Literal(const) => + tree + case New(tpt) => + tree.derivedNew(transform(tpt)) + case Pair(left, right) => + tree.derivedPair(transform(left), transform(right)) + case Typed(expr, tpt) => + tree.derivedTyped(transform(expr), transform(tpt)) + case NamedArg(name, arg) => + tree.derivedNamedArg(name, transform(arg)) + case Assign(lhs, rhs) => + tree.derivedAssign(transform(lhs), transform(rhs)) + case Block(stats, expr) => + tree.derivedBlock(transform(stats), transform(expr)) + case If(cond, thenp, elsep) => + tree.derivedIf(transform(cond), transform(thenp), transform(elsep)) + case Match(selector, cases) => + tree.derivedMatch(transform(selector), transformSub(cases)) + case CaseDef(pat, guard, body) => + tree.derivedCaseDef(transform(pat), transform(guard), transform(body)) + case Return(expr, from) => + tree.derivedReturn(transform(expr), transformSub(from)) + case Try(block, catches, finalizer) => + tree.derivedTry(transform(block), transformSub(catches), transform(finalizer)) + case Throw(expr) => + tree.derivedThrow(transform(expr)) + case SeqLiteral(elemtpt, elems) => + tree.derivedSeqLiteral(transform(elemtpt), transform(elems)) + case TypeTree(original) => + tree.derivedTypeTree(transform(original)) + case SingletonTypeTree(ref) => + tree.derivedSingletonTypeTree(transform(ref)) + case SelectFromTypeTree(qualifier, name) => + tree.derivedSelectFromTypeTree(transform(qualifier), name) + case AndTypeTree(left, right) => + tree.derivedAndTypeTree(transform(left), transform(right)) + case OrTypeTree(left, right) => + tree.derivedOrTypeTree(transform(left), transform(right)) + case RefineTypeTree(tpt, refinements) => + tree.derivedRefineTypeTree(transform(tpt), transformSub(refinements)) + case AppliedTypeTree(tpt, args) => + tree.derivedAppliedTypeTree(transform(tpt), transform(args)) + case TypeBoundsTree(lo, hi) => + tree.derivedTypeBoundsTree(transform(lo), transform(hi)) + case Bind(name, body) => + tree.derivedBind(name, transform(body)) + case Alternative(trees) => + tree.derivedAlternative(transform(trees)) + case UnApply(fun, args) => + tree.derivedUnApply(transform(fun), transform(args)) + case ValDef(mods, name, tpt, rhs) => + tree.derivedValDef(mods, name, transform(tpt), transform(rhs)) + case DefDef(mods, name, tparams, vparamss, tpt, rhs) => + tree.derivedDefDef(mods, name, transformSub(tparams), vparamss mapConserve (transformSub(_)), transform(tpt), transform(rhs)) + case TypeDef(mods, name, rhs) => + tree.derivedTypeDef(mods, name, transform(rhs)) + case Template(parents, self, body) => + tree.derivedTemplate(transform(parents), transformSub(self), transform(body)) + case ClassDef(mods, name, tparams, impl) => + tree.derivedClassDef(mods, name, transformSub(tparams), transformSub(impl)) + case Import(expr, selectors) => + tree.derivedImport(transform(expr), selectors) + case PackageDef(pid, stats) => + tree.derivedPackageDef(transformSub(pid), transform(stats)) + case Annotated(annot, arg) => + tree.derivedAnnotated(transform(annot), transform(arg)) + case EmptyTree() => + tree + case tree @ SharedTree(shared) => + sharedMemo get tree match { + case Some(tree1) => tree1 + case None => + val tree1 = tree.derivedSharedTree(transform(shared)) + sharedMemo = sharedMemo.updated(tree, tree1) + tree1 + } + } + def transform(trees: List[Tree[T]]): List[Tree[T]] = + trees mapConserve (transform(_)) + def transformSub(tree: Tree[T]): tree.ThisTree[T] = + transform(tree).asInstanceOf[tree.ThisTree[T]] + def transformSub[TT <: Tree[T]](trees: List[TT]): List[TT] = + transform(trees).asInstanceOf[List[TT]] + } + abstract class TreeAccumulator[T, U] extends ((T, Tree[U]) => T) { var sharedMemo: Map[SharedTree[U], T] = Map() def apply(x: T, tree: Tree[U]): T diff --git a/src/dotty/tools/dotc/core/TypedTrees.scala b/src/dotty/tools/dotc/core/TypedTrees.scala index 56ecc7c98..414cd6e2f 100644 --- a/src/dotty/tools/dotc/core/TypedTrees.scala +++ b/src/dotty/tools/dotc/core/TypedTrees.scala @@ -1,73 +1,12 @@ package dotty.tools.dotc package core -import Positions._, Types._, Contexts._, Constants._, Names._, Flags._ +import util.Positions._, Types._, Contexts._, Constants._, Names._, Flags._ import SymDenotations._, Symbols._, StdNames._, Annotations._ object TypedTrees { - object tpd { - - type Modifiers = Trees.Modifiers[Type] - type Tree = Trees.Tree[Type] - type TypTree = Trees.TypTree[Type] - type TermTree = Trees.TermTree[Type] - type PatternTree = Trees.PatternTree[Type] - type DenotingTree = Trees.DenotingTree[Type] - type ProxyTree = Trees.ProxyTree[Type] - type NameTree = Trees.NameTree[Type] - type RefTree = Trees.RefTree[Type] - type DefTree = Trees.DefTree[Type] - - type TreeCopier = Trees.TreeCopier[Type] - type TreeAccumulator[T] = Trees.TreeAccumulator[T, Type] - type TreeTransformer[C] = Trees.TreeTransformer[Type, C] - - type Ident = Trees.Ident[Type] - type Select = Trees.Select[Type] - type This = Trees.This[Type] - type Super = Trees.Super[Type] - type Apply = Trees.Apply[Type] - type TypeApply = Trees.TypeApply[Type] - type Literal = Trees.Literal[Type] - type New = Trees.New[Type] - type Pair = Trees.Pair[Type] - type Typed = Trees.Typed[Type] - type NamedArg = Trees.NamedArg[Type] - type Assign = Trees.Assign[Type] - type Block = Trees.Block[Type] - type If = Trees.If[Type] - type Match = Trees.Match[Type] - type CaseDef = Trees.CaseDef[Type] - type Return = Trees.Return[Type] - type Try = Trees.Try[Type] - type Throw = Trees.Throw[Type] - type SeqLiteral = Trees.SeqLiteral[Type] - type TypeTree = Trees.TypeTree[Type] - type SingletonTypeTree = Trees.SingletonTypeTree[Type] - type SelectFromTypeTree = Trees.SelectFromTypeTree[Type] - type AndTypeTree = Trees.AndTypeTree[Type] - type OrTypeTree = Trees.OrTypeTree[Type] - type RefineTypeTree = Trees.RefineTypeTree[Type] - type AppliedTypeTree = Trees.AppliedTypeTree[Type] - type TypeBoundsTree = Trees.TypeBoundsTree[Type] - type Bind = Trees.Bind[Type] - type Alternative = Trees.Alternative[Type] - type UnApply = Trees.UnApply[Type] - type ValDef = Trees.ValDef[Type] - type DefDef = Trees.DefDef[Type] - type TypeDef = Trees.TypeDef[Type] - type Template = Trees.Template[Type] - type ClassDef = Trees.ClassDef[Type] - type Import = Trees.Import[Type] - type PackageDef = Trees.PackageDef[Type] - type Annotated = Trees.Annotated[Type] - type EmptyTree = Trees.EmptyTree[Type] - type SharedTree = Trees.SharedTree[Type] - - private implicit def pos(implicit ctx: Context): Position = ctx.position - - def defPos(sym: Symbol)(implicit ctx: Context) = ctx.position union sym.coord.toPosition + object tpd extends Trees.Instance[Type] { def Modifiers(sym: Symbol)(implicit ctx: Context): Modifiers = Trees.Modifiers[Type]( sym.flags & ModifierFlags, @@ -664,8 +603,8 @@ object TypedTrees { new TreeMapper(ownerMap = (sym => if (sym == from) to else sym)).apply(tree) } - class TreeMapper(val typeMap: TypeMap = IdentityTypeMap, val ownerMap: Symbol => Symbol = identity)(implicit ctx: Context) extends TreeTransformer[Type, Unit] { - override def transform(tree: tpd.Tree, c: Unit): tpd.Tree = { + class TreeMapper(val typeMap: TypeMap = IdentityTypeMap, val ownerMap: Symbol => Symbol = identity)(implicit ctx: Context) extends TreeTransformer[Type] { + override def transform(tree: tpd.Tree): tpd.Tree = { val tree1 = if (tree.isEmpty) tree else tree.withType(typeMap(tree.tpe)) @@ -681,16 +620,16 @@ object TypedTrees { case _ => tree1 } - super.transform(tree2, c) + super.transform(tree2) } - override def transform(trees: List[tpd.Tree], c: Unit) = { + override def transform(trees: List[tpd.Tree]) = { val locals = localSyms(trees) val mapped = ctx.mapSymbols(locals, typeMap, ownerMap) - if (locals eq mapped) super.transform(trees, c) - else withSubstitution(locals, mapped).transform(trees, c) + if (locals eq mapped) super.transform(trees) + else withSubstitution(locals, mapped).transform(trees) } - def apply[ThisTree <: tpd.Tree](tree: ThisTree): ThisTree = transform(tree, ()).asInstanceOf[ThisTree] + def apply[ThisTree <: tpd.Tree](tree: ThisTree): ThisTree = transform(tree).asInstanceOf[ThisTree] def apply(annot: Annotation): Annotation = { val tree1 = apply(annot.tree) diff --git a/src/dotty/tools/dotc/core/UntypedTrees.scala b/src/dotty/tools/dotc/core/UntypedTrees.scala new file mode 100644 index 000000000..c882b4982 --- /dev/null +++ b/src/dotty/tools/dotc/core/UntypedTrees.scala @@ -0,0 +1,13 @@ +package dotty.tools.dotc +package core + +import util.Positions._, Types._, Contexts._, Constants._, Names._, Flags._ +import SymDenotations._, Symbols._, StdNames._, Annotations._ + +object UntypedTrees { + + object untpd extends Trees.Instance[Nothing] { + } + +} + diff --git a/src/dotty/tools/dotc/core/pickling/ClassfileParser.scala b/src/dotty/tools/dotc/core/pickling/ClassfileParser.scala index 4ba4842e1..02ee9d9cc 100644 --- a/src/dotty/tools/dotc/core/pickling/ClassfileParser.scala +++ b/src/dotty/tools/dotc/core/pickling/ClassfileParser.scala @@ -4,7 +4,7 @@ package core package pickling import Contexts._, Symbols._, Types._, Names._, StdNames._, NameOps._, Scopes._, Decorators._ -import SymDenotations._, UnPickler._, Constants._, Annotations._, Positions._ +import SymDenotations._, UnPickler._, Constants._, Annotations._, util.Positions._ import TypedTrees.tpd._ import java.io.{ File, IOException } import java.lang.Integer.toHexString diff --git a/src/dotty/tools/dotc/core/pickling/UnPickler.scala b/src/dotty/tools/dotc/core/pickling/UnPickler.scala index 3168bb37c..ee8d9de83 100644 --- a/src/dotty/tools/dotc/core/pickling/UnPickler.scala +++ b/src/dotty/tools/dotc/core/pickling/UnPickler.scala @@ -9,7 +9,7 @@ import java.lang.Double.longBitsToDouble import Contexts._, Symbols._, Types._, Scopes._, SymDenotations._, Names._, NameOps._ import StdNames._, Denotations._, NameOps._, Flags._, Constants._, Annotations._ -import Positions._, TypedTrees.tpd._, TypedTrees.TreeOps +import util.Positions._, TypedTrees.tpd._, TypedTrees.TreeOps import util.Texts._ import io.AbstractFile import scala.reflect.internal.pickling.PickleFormat._ diff --git a/src/dotty/tools/dotc/parsing/CharArrayReader.scala b/src/dotty/tools/dotc/parsing/CharArrayReader.scala new file mode 100644 index 000000000..29346b78a --- /dev/null +++ b/src/dotty/tools/dotc/parsing/CharArrayReader.scala @@ -0,0 +1,131 @@ +package dotty.tools +package dotc +package parsing + +import scala.reflect.internal.Chars._ + +abstract class CharArrayReader { self => + + val buf: Array[Char] + + /** Switch whether unicode should be decoded */ + protected def decodeUni: Boolean = true + + /** An error routine to call on bad unicode escapes \\uxxxx. */ + protected def error(msg: String, offset: Int): Unit + + /** the last read character */ + var ch: Char = _ + + /** The offset one past the last read character */ + var charOffset: Int = 0 + + /** The offset before the last read character */ + var lastCharOffset: Int = 0 + + /** The start offset of the current line */ + var lineStartOffset: Int = 0 + + /** The start offset of the line before the current one */ + var lastLineStartOffset: Int = 0 + + private var lastUnicodeOffset = -1 + + /** Is last character a unicode escape \\uxxxx? */ + def isUnicodeEscape = charOffset == lastUnicodeOffset + + /** Advance one character; reducing CR;LF pairs to just LF */ + final def nextChar(): Unit = { + val idx = charOffset + lastCharOffset = idx + if (idx >= buf.length) { + ch = SU + } else { + val c = buf(idx) + ch = c + charOffset = idx + 1 + if (c == '\\') potentialUnicode() + else if (c < ' ') { skipCR(); potentialLineEnd() } + } + } + + def getc() = { nextChar() ; ch } + + /** Advance one character, leaving CR;LF pairs intact. + * This is for use in multi-line strings, so there are no + * "potential line ends" here. + */ + final def nextRawChar(): Unit = { + val idx = charOffset + lastCharOffset = idx + if (idx >= buf.length) { + ch = SU + } else { + val c = buf(charOffset) + ch = c + charOffset = idx + 1 + if (c == '\\') potentialUnicode() + } + } + + /** Interpret \\uxxxx escapes */ + private def potentialUnicode() { + def evenSlashPrefix: Boolean = { + var p = charOffset - 2 + while (p >= 0 && buf(p) == '\\') p -= 1 + (charOffset - p) % 2 == 0 + } + def udigit: Int = { + if (charOffset >= buf.length) { + // Since the positioning code is very insistent about throwing exceptions, + // we have to decrement the position so our error message can be seen, since + // we are one past EOF. This happens with e.g. val x = \ u 1 <EOF> + error("incomplete unicode escape", charOffset - 1) + SU + } + else { + val d = digit2int(buf(charOffset), 16) + if (d >= 0) charOffset += 1 + else error("error in unicode escape", charOffset) + d + } + } + if (charOffset < buf.length && buf(charOffset) == 'u' && decodeUni && evenSlashPrefix) { + do charOffset += 1 + while (charOffset < buf.length && buf(charOffset) == 'u') + val code = udigit << 12 | udigit << 8 | udigit << 4 | udigit + lastUnicodeOffset = charOffset + ch = code.toChar + } + } + + /** replace CR;LF by LF */ + private def skipCR() { + if (ch == CR) + if (charOffset < buf.length && buf(charOffset) == LF) { + charOffset += 1 + ch = LF + } + } + + /** Handle line ends */ + private def potentialLineEnd() { + if (ch == LF || ch == FF) { + lastLineStartOffset = lineStartOffset + lineStartOffset = charOffset + } + } + + def isAtEnd = charOffset >= buf.length + + /** A new reader that takes off at the current character position */ + def lookaheadReader = new CharArrayLookaheadReader + + class CharArrayLookaheadReader extends CharArrayReader { + val buf = self.buf + charOffset = self.charOffset + ch = self.ch + override def decodeUni = self.decodeUni + def error(msg: String, offset: Int) = self.error(msg, offset) + } +} diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala new file mode 100644 index 000000000..2b3ec9bc2 --- /dev/null +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -0,0 +1,958 @@ +package dotty.tools +package dotc +package parsing + +import Tokens._ +import core.Names._, core.Contexts._, core.Decorators._, util.Positions._ +import core.StdNames._ +import util.SourceFile +import java.lang.Character.isDigit +import scala.reflect.internal.Chars._ +import Tokens._ +import scala.annotation.{ switch, tailrec } +import scala.collection.{ mutable, immutable } +import mutable.{ ListBuffer, ArrayBuffer } +import scala.xml.Utility.isNameStart + +object Scanners { + + /** Offset into source character array */ + type Offset = Int + + /** An undefined offset */ + val NoOffset: Offset = -1 + + case class Comment(pos: Position, chrs: String) { + def isDocComment = chrs.startsWith("/**") + } + + type Token = Int + + trait TokenData { + + /** the next token */ + var token: Token = EMPTY + + /** the offset of the first character of the current token */ + var offset: Offset = 0 + + /** the offset of the character following the token preceding this one */ + var lastOffset: Offset = 0 + + /** the name of an identifier */ + var name: TermName = null + + /** the string value of a literal */ + var strVal: String = null + + /** the base of a number */ + var base: Int = 0 + + def copyFrom(td: TokenData) = { + this.token = td.token + this.offset = td.offset + this.lastOffset = td.lastOffset + this.name = td.name + this.strVal = td.strVal + this.base = td.base + } + } + + class Scanner(source: SourceFile)(implicit ctx: Context) extends CharArrayReader with TokenData { + + val buf = source.content + + var keepComments = false + + /** All comments in the reverse order of their position in the source. + * set only when `keepComments` is true. + */ + var revComments: List[Comment] = Nil + + /** the last error offset + */ + var errOffset: Offset = NoOffset + + /** A buffer for comments */ + val commentBuf = new StringBuilder + + /** A character buffer for literals + */ + val litBuf = new StringBuilder + + /** append Unicode character to "litBuf" buffer + */ + protected def putChar(c: Char): Unit = litBuf.append(c) + + /** Clear buffer and set string */ + private def setStrVal() = flushBuf(litBuf) + + private class TokenData0 extends TokenData + + /** we need one token lookahead and one token history + */ + private val next : TokenData = new TokenData0 + private val prev : TokenData = new TokenData0 + + /** a stack of tokens which indicates whether line-ends can be statement separators + * also used for keeping track of nesting levels. + * We keep track of the closing symbol of a region. This can be + * RPAREN if region starts with '(' + * RBRACKET if region starts with '[' + * RBRACE if region starts with '{' + * ARROW if region starts with `case' + * STRINGLIT if region is a string interpolation expression starting with '${' + * (the STRINGLIT appears twice in succession on the stack iff the + * expression is a multiline string literal). + */ + var sepRegions: List[Token] = List() + +// Get next token ------------------------------------------------------------ + + /** Are we directly in a string interpolation expression? + */ + private def inStringInterpolation = + sepRegions.nonEmpty && sepRegions.head == STRINGLIT + + /** Are we directly in a multiline string interpolation expression? + * @pre inStringInterpolation + */ + private def inMultiLineInterpolation = + inStringInterpolation && sepRegions.tail.nonEmpty && sepRegions.tail.head == STRINGPART + + /** read next token and return last offset + */ + def skipToken(): Offset = { + val off = offset + nextToken() + off + } + + def adjustSepRegions(lastToken: Token): Unit = (lastToken: @switch) match { + case LPAREN => + sepRegions = RPAREN :: sepRegions + case LBRACKET => + sepRegions = RBRACKET :: sepRegions + case LBRACE => + sepRegions = RBRACE :: sepRegions + case CASE => + sepRegions = ARROW :: sepRegions + case RBRACE => + while (!sepRegions.isEmpty && sepRegions.head != RBRACE) + sepRegions = sepRegions.tail + if (!sepRegions.isEmpty) sepRegions = sepRegions.tail + case RBRACKET | RPAREN => + if (!sepRegions.isEmpty && sepRegions.head == lastToken) + sepRegions = sepRegions.tail + case ARROW => + if (!sepRegions.isEmpty && sepRegions.head == lastToken) + sepRegions = sepRegions.tail + case STRINGLIT => + if (inMultiLineInterpolation) + sepRegions = sepRegions.tail.tail + else if (inStringInterpolation) + sepRegions = sepRegions.tail + case _ => + } + + /** Produce next token, filling TokenData fields of Scanner. + */ + def nextToken() { + val lastToken = token + adjustSepRegions(lastToken) + + // Read a token or copy it from `next` tokenData + if (next.token == EMPTY) { + lastOffset = lastCharOffset + if (inStringInterpolation) fetchStringPart() + else fetchToken() + if (token == ERROR) adjustSepRegions(STRINGLIT) + } else { + this copyFrom next + next.token = EMPTY + } + + /** Insert NEWLINE or NEWLINES if + * - we are after a newline + * - we are within a { ... } or on toplevel (wrt sepRegions) + * - the current token can start a statement and the one before can end it + * insert NEWLINES if we are past a blank line, NEWLINE otherwise + */ + if (isAfterLineEnd() && + (canEndStatTokens contains lastToken) && + (canStartStatTokens contains token) && + (sepRegions.isEmpty || sepRegions.head == RBRACE)) { + next copyFrom this + offset = lineStartOffset min lastLineStartOffset + token = if (pastBlankLine()) NEWLINES else NEWLINE + } + + postProcessToken() +// print("["+this+"]") + } + + def postProcessToken() = { + // Join CASE + CLASS => CASECLASS, CASE + OBJECT => CASEOBJECT, SEMI + ELSE => ELSE + if (token == CASE) { + prev copyFrom this + val nextLastOffset = lastCharOffset + fetchToken() + def resetOffset() { + offset = prev.offset + lastOffset = prev.lastOffset + } + if (token == CLASS) { + token = CASECLASS + resetOffset() + } else if (token == OBJECT) { + token = CASEOBJECT + resetOffset() + } else { + lastOffset = nextLastOffset + next copyFrom this + this copyFrom prev + } + } else if (token == SEMI) { + prev copyFrom this + fetchToken() + if (token != ELSE) { + next copyFrom this + this copyFrom prev + } + } + } + + /** Is current token first one after a newline? */ + def isAfterLineEnd(): Boolean = + lastOffset < lineStartOffset && + (lineStartOffset <= offset || + lastOffset < lastLineStartOffset && lastLineStartOffset <= offset) + + /** Is there a blank line between the current token and the last one? + * @pre afterLineEnd(). + */ + private def pastBlankLine(): Boolean = { + val end = offset + def recur(idx: Offset, isBlank: Boolean): Boolean = + idx < end && { + val ch = buf(idx) + if (ch == LF || ch == FF) isBlank || recur(idx + 1, true) + else recur(idx + 1, isBlank && ch <= ' ') + } + recur(lastOffset, false) + } + + /** read next token, filling TokenData fields of Scanner. + */ + protected final def fetchToken() { + offset = charOffset - 1 + (ch: @switch) match { + case ' ' | '\t' | CR | LF | FF => + nextChar() + fetchToken() + case 'A' | 'B' | 'C' | 'D' | 'E' | + 'F' | 'G' | 'H' | 'I' | 'J' | + 'K' | 'L' | 'M' | 'N' | 'O' | + 'P' | 'Q' | 'R' | 'S' | 'T' | + 'U' | 'V' | 'W' | 'X' | 'Y' | + 'Z' | '$' | '_' | + 'a' | 'b' | 'c' | 'd' | 'e' | + 'f' | 'g' | 'h' | 'i' | 'j' | + 'k' | 'l' | 'm' | 'n' | 'o' | + 'p' | 'q' | 'r' | 's' | 't' | + 'u' | 'v' | 'w' | 'x' | 'y' | + 'z' => + putChar(ch) + nextChar() + getIdentRest() + if (ch == '"' && token == IDENTIFIER) + token = INTERPOLATIONID + case '<' => // is XMLSTART? + def fetchLT() = { + val last = if (charOffset >= 2) buf(charOffset - 2) else ' ' + nextChar() + last match { + case ' ' | '\t' | '\n' | '{' | '(' | '>' if isNameStart(ch) || ch == '!' || ch == '?' => + token = XMLSTART + case _ => + // Console.println("found '<', but last is '"+in.last+"'"); // DEBUG + putChar('<') + getOperatorRest() + } + } + fetchLT + case '~' | '!' | '@' | '#' | '%' | + '^' | '*' | '+' | '-' | /*'<' | */ + '>' | '?' | ':' | '=' | '&' | + '|' | '\\' => + putChar(ch) + nextChar() + getOperatorRest() + case '/' => + if (skipComment()) { + fetchToken() + } else { + putChar('/') + getOperatorRest() + } + case '0' => + def fetchZero() = { + putChar(ch) + nextChar() + if (ch == 'x' || ch == 'X') { + nextChar() + base = 16 + } else { + /** + * What should leading 0 be in the future? It is potentially dangerous + * to let it be base-10 because of history. Should it be an error? Is + * there a realistic situation where one would need it? + */ + if (isDigit(ch)) + error("Non-zero numbers may not have a leading zero.") + } + getNumber() + } + fetchZero + case '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' => + base = 10 + getNumber() + case '`' => + getBackquotedIdent() + case '\"' => + def fetchDoubleQuote() = { + if (token == INTERPOLATIONID) { + nextRawChar() + if (ch == '\"') { + nextRawChar() + if (ch == '\"') { + nextRawChar() + getStringPart(multiLine = true) + sepRegions = STRINGPART :: sepRegions // indicate string part + sepRegions = STRINGLIT :: sepRegions // once more to indicate multi line string part + } else { + token = STRINGLIT + strVal = "" + } + } else { + getStringPart(multiLine = false) + sepRegions = STRINGLIT :: sepRegions // indicate single line string part + } + } else { + nextChar() + if (ch == '\"') { + nextChar() + if (ch == '\"') { + nextRawChar() + getRawStringLit() + } else { + token = STRINGLIT + strVal = "" + } + } else { + getStringLit() + } + } + } + fetchDoubleQuote + case '\'' => + def fetchSingleQuote() = { + nextChar() + if (isIdentifierStart(ch)) + charLitOr(getIdentRest) + else if (isOperatorPart(ch) && (ch != '\\')) + charLitOr(getOperatorRest) + else { + getLitChar() + if (ch == '\'') { + nextChar() + token = CHARLIT + setStrVal() + } else { + error("unclosed character literal") + } + } + } + fetchSingleQuote + case '.' => + nextChar() + if ('0' <= ch && ch <= '9') { + putChar('.'); getFraction(); setStrVal() + } else { + token = DOT + } + case ';' => + nextChar(); token = SEMI + case ',' => + nextChar(); token = COMMA + case '(' => + nextChar(); token = LPAREN + case '{' => + nextChar(); token = LBRACE + case ')' => + nextChar(); token = RPAREN + case '}' => + nextChar(); token = RBRACE + case '[' => + nextChar(); token = LBRACKET + case ']' => + nextChar(); token = RBRACKET + case SU => + if (isAtEnd) token = EOF + else { + error("illegal character") + nextChar() + } + case _ => + def fetchOther() = { + if (ch == '\u21D2') { + nextChar(); token = ARROW + } else if (ch == '\u2190') { + nextChar(); token = LARROW + } else if (Character.isUnicodeIdentifierStart(ch)) { + putChar(ch) + nextChar() + getIdentRest() + } else if (isSpecial(ch)) { + putChar(ch) + nextChar() + getOperatorRest() + } else { + error(f"illegal character '\\u${ch: Int}%04x'") + nextChar() + } + } + fetchOther + } + } + + private def skipComment(): Boolean = { + def appendToComment(ch: Char) = + if (keepComments) commentBuf.append(ch) + def nextChar() = { + appendToComment(ch) + Scanner.this.nextChar() + } + def skipLine(): Unit = { + nextChar() + if ((ch != CR) && (ch != LF) && (ch != SU)) skipLine() + } + @tailrec + def skipBlock(openComments: Int): Unit = { + val last = ch + nextChar() + if (ch == '/') + if (last == '*') { + if (openComments > 0) skipBlock(openComments - 1) + } else { + nextChar() + if (ch == '*') { nextChar(); skipBlock(openComments + 1) } + else skipBlock(openComments) + } + else if (ch == SU) incompleteInputError("unclosed comment") + else skipBlock(openComments) + } + val start = lastCharOffset + def finishComment(): Boolean = { + if (keepComments) { + val pos = Position(start, charOffset) + nextChar() + revComments = Comment(pos, flushBuf(commentBuf)) :: revComments + } + true + } + nextChar() + if (ch == '/') { skipLine(); finishComment() } + else if (ch == '*') { nextChar(); skipBlock(0); finishComment() } + else false + } + +// Identifiers --------------------------------------------------------------- + + private def getBackquotedIdent() { + nextChar() + getLitChars('`') + if (ch == '`') { + nextChar() + finishNamed(BACKQUOTED_IDENT) + if (name.length == 0) + error("empty quoted identifier") + else if (name == nme.WILDCARD) + error("wildcard invalid as backquoted identifier") + } + else error("unclosed quoted identifier") + } + + private def getIdentRest(): Unit = (ch: @switch) match { + case 'A' | 'B' | 'C' | 'D' | 'E' | + 'F' | 'G' | 'H' | 'I' | 'J' | + 'K' | 'L' | 'M' | 'N' | 'O' | + 'P' | 'Q' | 'R' | 'S' | 'T' | + 'U' | 'V' | 'W' | 'X' | 'Y' | + 'Z' | '$' | + 'a' | 'b' | 'c' | 'd' | 'e' | + 'f' | 'g' | 'h' | 'i' | 'j' | + 'k' | 'l' | 'm' | 'n' | 'o' | + 'p' | 'q' | 'r' | 's' | 't' | + 'u' | 'v' | 'w' | 'x' | 'y' | + 'z' | + '0' | '1' | '2' | '3' | '4' | + '5' | '6' | '7' | '8' | '9' => + putChar(ch) + nextChar() + getIdentRest() + case '_' => + putChar(ch) + nextChar() + getIdentOrOperatorRest() + case SU => // strangely enough, Character.isUnicodeIdentifierPart(SU) returns true! + finishNamed() + case _ => + if (Character.isUnicodeIdentifierPart(ch)) { + putChar(ch) + nextChar() + getIdentRest() + } else { + finishNamed() + } + } + + private def getOperatorRest(): Unit = (ch: @switch) match { + case '~' | '!' | '@' | '#' | '%' | + '^' | '*' | '+' | '-' | '<' | + '>' | '?' | ':' | '=' | '&' | + '|' | '\\' => + putChar(ch); nextChar(); getOperatorRest() + case '/' => + if (skipComment()) finishNamed() + else { putChar('/'); getOperatorRest() } + case _ => + if (isSpecial(ch)) { putChar(ch); nextChar(); getOperatorRest() } + else finishNamed() + } + + private def getIdentOrOperatorRest() { + if (isIdentifierPart(ch)) + getIdentRest() + else ch match { + case '~' | '!' | '@' | '#' | '%' | + '^' | '*' | '+' | '-' | '<' | + '>' | '?' | ':' | '=' | '&' | + '|' | '\\' | '/' => + getOperatorRest() + case _ => + if (isSpecial(ch)) getOperatorRest() + else finishNamed() + } + } + + +// Literals ----------------------------------------------------------------- + + private def getStringLit() = { + getLitChars('"') + if (ch == '"') { + setStrVal() + nextChar() + token = STRINGLIT + } else error("unclosed string literal") + } + + private def getRawStringLit(): Unit = { + if (ch == '\"') { + nextRawChar() + if (isTripleQuote()) { + setStrVal() + token = STRINGLIT + } else + getRawStringLit() + } else if (ch == SU) { + incompleteInputError("unclosed multi-line string literal") + } else { + putChar(ch) + nextRawChar() + getRawStringLit() + } + } + + @annotation.tailrec private def getStringPart(multiLine: Boolean): Unit = { + def finishStringPart() = { + setStrVal() + token = STRINGPART + next.lastOffset = charOffset - 1 + next.offset = charOffset - 1 + } + if (ch == '"') { + if (multiLine) { + nextRawChar() + if (isTripleQuote()) { + setStrVal() + token = STRINGLIT + } else + getStringPart(multiLine) + } else { + nextChar() + setStrVal() + token = STRINGLIT + } + } else if (ch == '$') { + nextRawChar() + if (ch == '$') { + putChar(ch) + nextRawChar() + getStringPart(multiLine) + } else if (ch == '{') { + finishStringPart() + nextRawChar() + next.token = LBRACE + } else if (Character.isUnicodeIdentifierStart(ch)) { + finishStringPart() + do { + putChar(ch) + nextRawChar() + } while (ch != SU && Character.isUnicodeIdentifierPart(ch)) + finishNamed(target = next) + } else { + error("invalid string interpolation: `$$', `$'ident or `$'BlockExpr expected") + } + } else { + val isUnclosedLiteral = !isUnicodeEscape && (ch == SU || (!multiLine && (ch == CR || ch == LF))) + if (isUnclosedLiteral) { + if (multiLine) + incompleteInputError("unclosed multi-line string literal") + else + error("unclosed string literal") + } + else { + putChar(ch) + nextRawChar() + getStringPart(multiLine) + } + } + } + + private def fetchStringPart() = { + offset = charOffset - 1 + getStringPart(multiLine = inMultiLineInterpolation) + } + + private def isTripleQuote(): Boolean = + if (ch == '"') { + nextRawChar() + if (ch == '"') { + nextChar() + while (ch == '"') { + putChar('"') + nextChar() + } + true + } else { + putChar('"') + putChar('"') + false + } + } else { + putChar('"') + false + } + + /** copy current character into litBuf, interpreting any escape sequences, + * and advance to next character. + */ + protected def getLitChar(): Unit = + if (ch == '\\') { + nextChar() + if ('0' <= ch && ch <= '7') { + val leadch: Char = ch + var oct: Int = digit2int(ch, 8) + nextChar() + if ('0' <= ch && ch <= '7') { + oct = oct * 8 + digit2int(ch, 8) + nextChar() + if (leadch <= '3' && '0' <= ch && ch <= '7') { + oct = oct * 8 + digit2int(ch, 8) + nextChar() + } + } + putChar(oct.toChar) + } else { + ch match { + case 'b' => putChar('\b') + case 't' => putChar('\t') + case 'n' => putChar('\n') + case 'f' => putChar('\f') + case 'r' => putChar('\r') + case '\"' => putChar('\"') + case '\'' => putChar('\'') + case '\\' => putChar('\\') + case _ => invalidEscape() + } + nextChar() + } + } else { + putChar(ch) + nextChar() + } + + protected def invalidEscape(): Unit = { + error("invalid escape character", charOffset - 1) + putChar(ch) + } + + private def getLitChars(delimiter: Char) = { + while (ch != delimiter && !isAtEnd && (ch != SU && ch != CR && ch != LF || isUnicodeEscape)) + getLitChar() + } + + /** read fractional part and exponent of floating point number + * if one is present. + */ + protected def getFraction() { + token = DOUBLELIT + while ('0' <= ch && ch <= '9') { + putChar(ch) + nextChar() + } + if (ch == 'e' || ch == 'E') { + val lookahead = lookaheadReader + lookahead.nextChar() + if (lookahead.ch == '+' || lookahead.ch == '-') { + lookahead.nextChar() + } + if ('0' <= lookahead.ch && lookahead.ch <= '9') { + putChar(ch) + nextChar() + if (ch == '+' || ch == '-') { + putChar(ch) + nextChar() + } + while ('0' <= ch && ch <= '9') { + putChar(ch) + nextChar() + } + } + token = DOUBLELIT + } + if (ch == 'd' || ch == 'D') { + putChar(ch) + nextChar() + token = DOUBLELIT + } else if (ch == 'f' || ch == 'F') { + putChar(ch) + nextChar() + token = FLOATLIT + } + checkNoLetter() + } + def checkNoLetter() { + if (isIdentifierPart(ch) && ch >= ' ') + error("Invalid literal number") + } + + /** Read a number into strVal and set base + */ + protected def getNumber() { + while (digit2int(ch, base) >= 0) { + putChar(ch) + nextChar() + } + token = INTLIT + if (base == 10 && ch == '.') { + val isDefinitelyNumber = { + val lookahead = lookaheadReader + val c = lookahead.getc() + (c: @switch) match { + /** Another digit is a giveaway. */ + case '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' => + true + + /** Backquoted idents like 22.`foo`. */ + case '`' => + false + + /** These letters may be part of a literal, or a method invocation on an Int. + */ + case 'd' | 'D' | 'f' | 'F' => + !isIdentifierPart(lookahead.getc()) + + /** A little more special handling for e.g. 5e7 */ + case 'e' | 'E' => + val ch = lookahead.getc() + !isIdentifierPart(ch) || (isDigit(ch) || ch == '+' || ch == '-') + + case x => + !isIdentifierStart(x) + } + } + if (isDefinitelyNumber) { + putChar(ch) + nextChar() + getFraction() + } + } else (ch: @switch) match { + case 'e' | 'E' | 'f' | 'F' | 'd' | 'D' if base == 10 => + getFraction() + case 'l' | 'L' => + nextChar() + token = LONGLIT + case _ => + } + setStrVal() + } + + /** Parse character literal if current character is followed by \', + * or follow with given op and return a symbol literal token + */ + def charLitOr(op: () => Unit) { + putChar(ch) + nextChar() + if (ch == '\'') { + nextChar() + token = CHARLIT + setStrVal() + } else { + op() + token = SYMBOLLIT + strVal = name.toString + } + } + +// Setting token data ---------------------------------------------------- + + /** Clear buffer and set name and token */ + def finishNamed(idtoken: Token = IDENTIFIER, target: TokenData = this): Unit = { + target.name = flushBuf(litBuf).toTermName + target.token = idtoken + if (idtoken == IDENTIFIER) { + val idx = target.name.start + if (idx >= 0 && idx < kwArray.length) target.token = kwArray(idx) + } + } + + /** Return buffer contents and clear */ + def flushBuf(buf: StringBuilder): String = { + val str = buf.toString + buf.clear() + str + } + + /** Convert current strVal to char value + */ + def charVal: Char = if (strVal.length > 0) strVal.charAt(0) else 0 + + /** Convert current strVal, base to long value + * This is tricky because of max negative value. + */ + def intVal(negated: Boolean): Long = { + if (token == CHARLIT && !negated) { + charVal + } else { + var value: Long = 0 + val divider = if (base == 10) 1 else 2 + val limit: Long = + if (token == LONGLIT) Long.MaxValue else Int.MaxValue + var i = 0 + val len = strVal.length + while (i < len) { + val d = digit2int(strVal charAt i, base) + if (d < 0) { + error("malformed integer number") + return 0 + } + if (value < 0 || + limit / (base / divider) < value || + limit - (d / divider) < value * (base / divider) && + !(negated && limit == value * base - 1 + d)) { + error("integer number too large") + return 0 + } + value = value * base + d + i += 1 + } + if (negated) -value else value + } + } + + def intVal: Long = intVal(false) + + /** Convert current strVal, base to double value + */ + def floatVal(negated: Boolean): Double = { + val limit: Double = + if (token == DOUBLELIT) Double.MaxValue else Float.MaxValue + try { + val value: Double = java.lang.Double.valueOf(strVal).doubleValue() + if (value > limit) + error("floating point number too large") + if (negated) -value else value + } catch { + case _: NumberFormatException => + error("malformed floating point number") + 0.0 + } + } + + def floatVal: Double = floatVal(false) + + override def toString = showTokenDetailed(token) + + def show: String = token match { + case IDENTIFIER | BACKQUOTED_IDENT => s"id($name)" + case CHARLIT => s"char($intVal)" + case INTLIT => s"int($intVal)" + case LONGLIT => s"long($intVal)" + case FLOATLIT => s"float($floatVal)" + case DOUBLELIT => s"double($floatVal)" + case STRINGLIT => s"string($strVal)" + case STRINGPART => s"stringpart($strVal)" + case INTERPOLATIONID => s"interpolationid($name)" + case SEMI => ";" + case NEWLINE => ";" + case NEWLINES => ";;" + case COMMA => "," + case _ => showToken(token) + } + +// (does not seem to be needed) def flush = { charOffset = offset; nextChar(); this } + + /* Resume normal scanning after XML */ + def resume(lastToken: Token) = { + token = lastToken + if (next.token != EMPTY && !ctx.reporter.hasErrors) + error("unexpected end of input: possible missing '}' in XML block") + + nextToken() + } + +// Errors ----------------------------------------------------------------- + + /** Generate an error at the given offset */ + def error(msg: String, off: Offset = offset) = { + ctx.error(msg, source atPos Position(off)) + token = ERROR + errOffset = off + } + + /** signal an error where the input ended in the middle of a token */ + def incompleteInputError(msg: String) { + ctx.reporter.incompleteInputError(msg, source atPos Position(offset)) + token = EOF + errOffset = offset + } + + /* Initialization: read first char, then first token */ + nextChar() + nextToken() + } // end Scanner + + // ------------- keyword configuration ----------------------------------- + + private val kwArray: Array[Token] = { + def start(tok: Token) = tok.toString.toTermName.start + val sourceKeywords = keywords.filterNot(_.toString contains " ") + val lastIdx = sourceKeywords.map(start).max + val arr = Array.fill(lastIdx + 1)(IDENTIFIER) + for (kw <- sourceKeywords) arr(start(kw)) = kw + arr + } +} diff --git a/src/dotty/tools/dotc/parsing/Tokens.scala b/src/dotty/tools/dotc/parsing/Tokens.scala new file mode 100644 index 000000000..f573df49d --- /dev/null +++ b/src/dotty/tools/dotc/parsing/Tokens.scala @@ -0,0 +1,171 @@ +package dotty.tools +package dotc +package parsing + +import collection.mutable +import collection.immutable.BitSet +import scala.annotation.switch + +object Tokens { + + final val minToken = EMPTY + final val maxToken = XMLSTART + + type TokenSet = BitSet + + def tokenRange(lo: Int, hi: Int): TokenSet = BitSet(lo to hi: _*) + + def showTokenDetailed(token: Int) = debugString(token) + + def showToken(token: Int) = { + val str = tokenString(token) + if (keywords contains token) s"'$str'" else str + } + + val tokenString, debugString = new Array[String](maxToken + 1) + + def enter(token: Int, str: String, debugStr: String = ""): Unit = { + tokenString(token) = str + debugString(token) = if (debugStr.isEmpty) str else debugStr + } + + /** special tokens */ + final val EMPTY = 0; enter(EMPTY, "<empty>") // a missing token, used in lookahead + final val ERROR = 1; enter(ERROR, "erroneous token") // an erroneous token + final val EOF = 2; enter(EOF, "eof") + + /** literals */ + final val CHARLIT = 3; enter(CHARLIT, "character literal") + final val INTLIT = 4; enter(INTLIT, "integer literal") + final val LONGLIT = 5; enter(LONGLIT, "long literal") + final val FLOATLIT = 6; enter(FLOATLIT, "float literal") + final val DOUBLELIT = 7; enter(DOUBLELIT, "double literal") + final val STRINGLIT = 8; enter(STRINGLIT, "string literal") + final val STRINGPART = 9; enter(STRINGPART, "string literal", "string literal part") + final val INTERPOLATIONID = 10; enter(INTERPOLATIONID, "string interpolator") + final val SYMBOLLIT = 11; enter(SYMBOLLIT, "symbol literal") // TODO: deprecate + + final val literalTokens = tokenRange(CHARLIT, SYMBOLLIT) + + /** identifiers */ + final val IDENTIFIER = 12; enter(IDENTIFIER, "identifier") + final val BACKQUOTED_IDENT = 13; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident") + + final val identifierTokens = BitSet(IDENTIFIER, BACKQUOTED_IDENT) + + def isIdentifier(token : Int) = + token >= IDENTIFIER && token <= BACKQUOTED_IDENT + + /** alphabetic keywords */ + final val IF = 20; enter(IF, "if") + final val FOR = 21; enter(FOR, "for") + final val ELSE = 22; enter(ELSE, "else") + final val THIS = 23; enter(THIS, "this") + final val NULL = 24; enter(NULL, "null") + final val NEW = 25; enter(NEW, "new") + final val WITH = 26; enter(WITH, "with") + final val SUPER = 27; enter(SUPER, "super") + final val CASE = 28; enter(CASE, "case") + final val CASECLASS = 29; enter(CASECLASS, "case class") + final val CASEOBJECT = 30; enter(CASEOBJECT, "case object") + final val VAL = 31; enter(VAL, "val") + final val ABSTRACT = 32; enter(ABSTRACT, "abstract") + final val FINAL = 33; enter(FINAL, "final") + final val PRIVATE = 34; enter(PRIVATE, "private") + final val PROTECTED = 35; enter(PROTECTED, "protected") + final val OVERRIDE = 36; enter(OVERRIDE, "override") + final val IMPLICIT = 37; enter(IMPLICIT, "implicit") + final val VAR = 38; enter(VAR, "var") + final val DEF = 39; enter(DEF, "def") + final val TYPE = 40; enter(TYPE, "type") + final val EXTENDS = 41; enter(EXTENDS, "extends") + final val TRUE = 42; enter(TRUE, "true") + final val FALSE = 43; enter(FALSE, "false") + final val OBJECT = 44; enter(OBJECT, "object") + final val CLASS = 45; enter(CLASS, "class") + final val IMPORT = 46; enter(IMPORT, "import") + final val PACKAGE = 47; enter(PACKAGE, "package") + final val YIELD = 48; enter(YIELD, "yield") + final val DO = 49; enter(DO, "do") + final val TRAIT = 50; enter(TRAIT, "trait") + final val SEALED = 51; enter(SEALED, "sealed") + final val THROW = 52; enter(THROW, "throw") + final val TRY = 53; enter(TRY, "try") + final val CATCH = 54; enter(CATCH, "catch") + final val FINALLY = 55; enter(FINALLY, "finally") + final val WHILE = 56; enter(WHILE, "while") + final val RETURN = 57; enter(RETURN, "return") + final val MATCH = 58; enter(MATCH, "match") + final val FORSOME = 59; enter(FORSOME, "forSome") // TODO: deprecate + final val LAZY = 61; enter(LAZY, "lazy") + final val THEN = 62; enter(THEN, "then") + + final val alphaKeywords = tokenRange(IF, LAZY) + + /** special symbols */ + final val COMMA = 70; enter(COMMA, "','") + final val SEMI = 71; enter(DOT, "'.'") + final val DOT = 72; enter(SEMI, "';'") + final val NEWLINE = 78; enter(NEWLINE, "';'", "new line") + final val NEWLINES = 79; enter(NEWLINES, "';'", "new lines") + + /** special keywords */ + final val USCORE = 73; enter(USCORE, "_") + final val COLON = 74; enter(COLON, ":") + final val EQUALS = 75; enter(EQUALS, "==") + final val LARROW = 76; enter(LARROW, "<-") + final val ARROW = 77; enter(ARROW, "=>") + final val SUBTYPE = 80; enter(SUBTYPE, "<:") + final val SUPERTYPE = 81; enter(SUPERTYPE, ">:") + final val HASH = 82; enter(HASH, "#") + final val AT = 83; enter(AT, "@") + final val VIEWBOUND = 84; enter(VIEWBOUND, "<%") // TODO: deprecate + + final val symbolicKeywords = tokenRange(USCORE, VIEWBOUND) + final val symbolicTokens = tokenRange(COMMA, VIEWBOUND) + final val keywords = alphaKeywords | symbolicKeywords + + /** parentheses */ + final val LPAREN = 90; enter(LPAREN, "'('") + final val RPAREN = 91; enter(RPAREN, "')'") + final val LBRACKET = 92; enter(LBRACKET, "'['") + final val RBRACKET = 93; enter(RBRACKET, "']'") + final val LBRACE = 94; enter(LBRACE, "'{'") + final val RBRACE = 95; enter(RBRACE, "'}'") + + /** XML mode */ + final val XMLSTART = 96; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate + + final val allTokens = tokenRange(minToken, maxToken) + + final val atomicExprTokens = literalTokens | identifierTokens | BitSet( + USCORE, NULL, THIS, SUPER, TRUE, FALSE, RETURN, XMLSTART) + + final val canStartExpressionTokens = atomicExprTokens | BitSet( + LBRACE, LPAREN, IF, DO, WHILE, FOR, NEW, TRY, THROW) + + final val canStartTypeTokens = literalTokens | identifierTokens | BitSet( + THIS, SUPER, USCORE, LPAREN, AT) + + final val templateIntroTokens = BitSet(CLASS, TRAIT, OBJECT, CASECLASS, CASEOBJECT) + + final val dclIntroTokens = BitSet(DEF, VAL, VAR, TYPE) + + final val defIntroTokens = templateIntroTokens | dclIntroTokens + + final val localModifierTokens = BitSet( + ABSTRACT, FINAL, SEALED, IMPLICIT, LAZY) + + final val modifierTokens = localModifierTokens | BitSet( + PRIVATE, PROTECTED, OVERRIDE) + + /** Is token only legal as start of statement (eof also included)? */ + final val mustStartStatTokens = defIntroTokens | modifierTokens | BitSet( + CASE, IMPORT, PACKAGE) + + final val canStartStatTokens = canStartExpressionTokens | mustStartStatTokens | BitSet( + AT) + + final val canEndStatTokens = atomicExprTokens | BitSet( + TYPE, RPAREN, RBRACE, RBRACKET) +} diff --git a/src/dotty/tools/dotc/reporting/ConsoleReporter.scala b/src/dotty/tools/dotc/reporting/ConsoleReporter.scala index 88fdfa386..2372485f4 100644 --- a/src/dotty/tools/dotc/reporting/ConsoleReporter.scala +++ b/src/dotty/tools/dotc/reporting/ConsoleReporter.scala @@ -3,7 +3,7 @@ package dotc package reporting import scala.collection.mutable -import core.Positions.Position +import util.Positions.SourcePosition import core.Contexts._ import Reporter.Severity.{Value => Severity, _} import java.io.{ BufferedReader, IOException, PrintWriter } @@ -24,24 +24,24 @@ class ConsoleReporter( /** maximal number of error messages to be printed */ protected def ErrorLimit = 100 - def formatMessage(msg: String, pos: Position)(implicit ctx: Context) = msg // for now + def formatMessage(msg: String, pos: SourcePosition)(implicit ctx: Context) = msg // for now /** Prints the message. */ def printMessage(msg: String) { writer.print(msg + "\n"); writer.flush() } /** Prints the message with the given position indication. */ - def printMessage(msg: String, pos: Position)(implicit ctx: Context) { + def printMessage(msg: String, pos: SourcePosition)(implicit ctx: Context) { printMessage(formatMessage(msg, pos)) } - def printMessage(msg: String, severity: Severity, pos: Position)(implicit ctx: Context) { + def printMessage(msg: String, severity: Severity, pos: SourcePosition)(implicit ctx: Context) { printMessage(label(severity) + msg, pos) } /** * @param pos ... - def printSourceLine(pos: Position) { + def printSourceLine(pos: SourcePosition) { printMessage(pos.lineContent.stripLineEnd) printColumnMarker(pos) } @@ -50,7 +50,7 @@ class ConsoleReporter( * * @param pos ... */ - def printColumnMarker(pos: Position) = + def printColumnMarker(pos: SourcePosition) = if (pos.isDefined) { printMessage(" " * (pos.column - 1) + "^") } */ @@ -60,7 +60,7 @@ class ConsoleReporter( if ( count(ERROR) > 0) printMessage(countString(ERROR ) + " found") } - override def report(msg: String, severity: Severity, pos: Position)(implicit ctx: Context) { + override def report(msg: String, severity: Severity, pos: SourcePosition)(implicit ctx: Context) { if (severity != ERROR || count(severity) <= ErrorLimit) printMessage(msg, severity, pos) if (ctx.settings.prompt.value) displayPrompt() diff --git a/src/dotty/tools/dotc/reporting/Reporter.scala b/src/dotty/tools/dotc/reporting/Reporter.scala index 2c892938c..3e7eaecde 100644 --- a/src/dotty/tools/dotc/reporting/Reporter.scala +++ b/src/dotty/tools/dotc/reporting/Reporter.scala @@ -3,16 +3,16 @@ package dotc package reporting import core.Contexts._ -import core.Positions._ +import util.Positions._ +import util.{SourceFile, NoSource} import core.Decorators.PhaseListDecorator import collection.mutable -import io.{SourceFile, NoSource} import java.lang.System.currentTimeMillis trait Reporting { this: Context => - def error(msg: String, pos: Position = NoPosition): Unit = reporter.error(msg, pos) - def warning(msg: String, pos: Position = NoPosition): Unit = reporter.warning(msg, pos) - def inform(msg: String, pos: Position = NoPosition): Unit = reporter.info(msg, pos) + def error(msg: String, pos: SourcePosition = NoSourcePosition): Unit = reporter.error(msg, pos) + def warning(msg: String, pos: SourcePosition = NoSourcePosition): Unit = reporter.warning(msg, pos) + def inform(msg: String, pos: SourcePosition = NoSourcePosition): Unit = reporter.info(msg, pos) def log(msg: => String): Unit = if (this.settings.log.value.containsPhase(phase)) @@ -24,6 +24,8 @@ trait Reporting { this: Context => def informTime(msg: => String, start: Long): Unit = informProgress(msg + elapsed(start)) + def deprecationWarning(msg: String, pos: SourcePosition): Unit = ??? + private def elapsed(start: Long) = " in " + (currentTimeMillis - start) + "ms" @@ -78,9 +80,9 @@ abstract class Reporter { import Reporter.Severity.{Value => Severity, _} - protected def report(msg: String, severity: Severity, pos: Position)(implicit ctx: Context): Unit + protected def report(msg: String, severity: Severity, pos: SourcePosition)(implicit ctx: Context): Unit - protected def isHidden(severity: Severity, pos: Position)(implicit ctx: Context) = false + protected def isHidden(severity: Severity, pos: SourcePosition)(implicit ctx: Context) = false val count = new mutable.HashMap[Severity, Int]() { override def default(key: Severity) = 0 @@ -99,7 +101,7 @@ abstract class Reporter { finally _truncationOK = saved } - type ErrorHandler = (String, Position, Context) => Unit + type ErrorHandler = (String, SourcePosition, Context) => Unit private var incompleteHandler: ErrorHandler = error(_, _)(_) def withIncompleteHandler[T](handler: ErrorHandler)(op: => T): T = { val saved = incompleteHandler @@ -112,26 +114,26 @@ abstract class Reporter { def hasWarnings = count(WARNING) > 0 /** For sending messages that are printed only if -verbose is set */ - def info(msg: String, pos: Position = NoPosition)(implicit ctx: Context): Unit = + def info(msg: String, pos: SourcePosition = NoSourcePosition)(implicit ctx: Context): Unit = if (ctx.settings.verbose.value) info0(msg, INFO, pos) /** For sending a message which should not be labeled as a warning/error, * but also shouldn't require -verbose to be visible. */ - def echo(msg: String, pos: Position = NoPosition)(implicit ctx: Context): Unit = + def echo(msg: String, pos: SourcePosition = NoSourcePosition)(implicit ctx: Context): Unit = info0(msg, INFO, pos) - def warning(msg: String, pos: Position = NoPosition)(implicit ctx: Context): Unit = + def warning(msg: String, pos: SourcePosition = NoSourcePosition)(implicit ctx: Context): Unit = if (!ctx.settings.nowarn.value) withoutTruncating(info0(msg, WARNING, pos)) - def error(msg: String, pos: Position = NoPosition)(implicit ctx: Context): Unit = + def error(msg: String, pos: SourcePosition = NoSourcePosition)(implicit ctx: Context): Unit = withoutTruncating(info0(msg, ERROR, pos)) - def incompleteInputError(msg: String, pos: Position = NoPosition)(implicit ctx: Context): Unit = + def incompleteInputError(msg: String, pos: SourcePosition = NoSourcePosition)(implicit ctx: Context): Unit = incompleteHandler(msg, pos, ctx) - private def info0(msg: String, severity: Severity, pos: Position)(implicit ctx: Context): Unit = { + private def info0(msg: String, severity: Severity, pos: SourcePosition)(implicit ctx: Context): Unit = { if (!isHidden(severity, pos)) { count(severity) += 1 report(msg, severity, pos) diff --git a/src/dotty/tools/dotc/reporting/StoreReporter.scala b/src/dotty/tools/dotc/reporting/StoreReporter.scala index 5b9553509..31df36e6f 100644 --- a/src/dotty/tools/dotc/reporting/StoreReporter.scala +++ b/src/dotty/tools/dotc/reporting/StoreReporter.scala @@ -4,7 +4,7 @@ package reporting import core.Contexts.Context import scala.collection.mutable -import core.Positions.Position +import util.Positions.SourcePosition import Reporter.Severity.{Value => Severity} /** @@ -12,12 +12,12 @@ import Reporter.Severity.{Value => Severity} */ class StoreReporter extends Reporter { - class Info(val msg: String, val severity: Severity, val pos: Position) { + class Info(val msg: String, val severity: Severity, val pos: SourcePosition) { override def toString() = "pos: " + pos + " " + msg + " " + severity } val infos = new mutable.LinkedHashSet[Info] - protected def report(msg: String, severity: Severity, pos: Position)(implicit ctx: Context): Unit = { + protected def report(msg: String, severity: Severity, pos: SourcePosition)(implicit ctx: Context): Unit = { infos += new Info(msg, severity, pos) } diff --git a/src/dotty/tools/dotc/reporting/UniqueMessagePositions.scala b/src/dotty/tools/dotc/reporting/UniqueMessagePositions.scala index 5df999aab..704317f23 100644 --- a/src/dotty/tools/dotc/reporting/UniqueMessagePositions.scala +++ b/src/dotty/tools/dotc/reporting/UniqueMessagePositions.scala @@ -3,9 +3,9 @@ package dotc package reporting import scala.collection.mutable -import core.Positions.Position +import util.Positions.SourcePosition +import util.SourceFile import Reporter.Severity.{Value => Severity} -import io.SourceFile import core.Contexts.Context /** @@ -20,7 +20,7 @@ trait UniqueMessagePositions extends Reporter { /** Logs a position and returns true if it was already logged. * @note Two positions are considered identical for logging if they have the same point. */ - override def isHidden(severity: Severity, pos: Position)(implicit ctx: Context): Boolean = + override def isHidden(severity: Severity, pos: SourcePosition)(implicit ctx: Context): Boolean = pos.exists && { positions get (ctx.source, pos.point) match { case Some(level) if level >= severity => true diff --git a/src/dotty/tools/dotc/core/Chars.scala b/src/dotty/tools/dotc/util/Chars.scala index 6b796aa31..99aad620a 100644 --- a/src/dotty/tools/dotc/core/Chars.scala +++ b/src/dotty/tools/dotc/util/Chars.scala @@ -3,11 +3,16 @@ * @author Martin Odersky */ package dotty.tools.dotc -package core +package util -import scala.annotation.{ tailrec, switch } +import scala.annotation.switch import java.lang.{ Character => JCharacter } -import scala.language.postfixOps +import java.lang.{Character => JCharacter} +import java.lang.Character.LETTER_NUMBER +import java.lang.Character.LOWERCASE_LETTER +import java.lang.Character.OTHER_LETTER +import java.lang.Character.TITLECASE_LETTER +import java.lang.Character.UPPERCASE_LETTER /** Contains constants and classifier methods for characters */ object Chars { diff --git a/src/dotty/tools/dotc/util/FreshNameCreator.scala b/src/dotty/tools/dotc/util/FreshNameCreator.scala new file mode 100644 index 000000000..d0c007d94 --- /dev/null +++ b/src/dotty/tools/dotc/util/FreshNameCreator.scala @@ -0,0 +1,38 @@ +package dotty.tools +package dotc +package util + +import scala.collection.mutable + +trait FreshNameCreator { + def newName(): String + def newName(prefix: String): String + + @deprecated("use newName(prefix)", "2.9.0") + def newName(pos: scala.reflect.internal.util.Position, prefix: String): String = newName(prefix) + @deprecated("use newName()", "2.9.0") + def newName(pos: scala.reflect.internal.util.Position): String = newName() +} + +object FreshNameCreator { + class Default extends FreshNameCreator { + protected var counter = 0 + protected val counters = mutable.HashMap[String, Int]() withDefaultValue 0 + + /** + * Create a fresh name with the given prefix. It is guaranteed + * that the returned name has never been returned by a previous + * call to this function (provided the prefix does not end in a digit). + */ + def newName(prefix: String): String = { + val safePrefix = prefix.replaceAll("""[<>]""", """\$""") + counters(safePrefix) += 1 + + safePrefix + counters(safePrefix) + } + def newName(): String = { + counter += 1 + "$" + counter + "$" + } + } +} diff --git a/src/dotty/tools/dotc/core/Positions.scala b/src/dotty/tools/dotc/util/Positions.scala index 5e4673ac4..31eb2ff0d 100644 --- a/src/dotty/tools/dotc/core/Positions.scala +++ b/src/dotty/tools/dotc/util/Positions.scala @@ -1,4 +1,5 @@ -package dotty.tools.dotc.core +package dotty.tools.dotc +package util /** Position format in little endian: * Start: unsigned 26 Bits (works for source files up to 64M) @@ -33,6 +34,16 @@ object Positions { } def exists = this != NoPosition + + def shift(offset: Int) = + if (exists) Position(start + offset, end + offset, point - start) + else this + + def focus = Position(point) + + def withStart(start: Int) = Position(start, this.end, this.point - start) + def withEnd(end: Int) = Position(this.start, end, this.point - this.start) + def withPoint(point: Int) = Position(this.start, this.end, point - this.start) } def Position(start: Int, end: Int, pointOffset: Int = 0): Position = @@ -45,6 +56,15 @@ object Positions { val NoPosition = new Position(-1L) + case class SourcePosition(source: SourceFile, pos: Position) { + def point: Int = pos.point + def start: Int = pos.start + def end: Int = pos.end + def exists = pos.exists + } + + val NoSourcePosition = SourcePosition(NoSource, NoPosition) + /** The coordinate of a symbol. This is either an index or * a point position. */ diff --git a/src/dotty/tools/dotc/util/SourceFile.scala b/src/dotty/tools/dotc/util/SourceFile.scala new file mode 100644 index 000000000..a7a677560 --- /dev/null +++ b/src/dotty/tools/dotc/util/SourceFile.scala @@ -0,0 +1,109 @@ +package dotty.tools +package dotc +package util + +import scala.collection.mutable.ArrayBuffer +import dotty.tools.io._ +import annotation.tailrec +import java.util.regex.Pattern +import java.io.IOException +import Chars._ +import ScriptSourceFile._ +import Positions._ + +object ScriptSourceFile { + private val headerPattern = Pattern.compile("""^(::)?!#.*(\r|\n|\r\n)""", Pattern.MULTILINE) + private val headerStarts = List("#!", "::#!") + + def apply(file: AbstractFile, content: Array[Char]) = { + /** Length of the script header from the given content, if there is one. + * The header begins with "#!" or "::#!" and ends with a line starting + * with "!#" or "::!#". + */ + val headerLength = + if (headerStarts exists (content startsWith _)) { + val matcher = headerPattern matcher content.mkString + if (matcher.find) matcher.end + else throw new IOException("script file does not close its header with !# or ::!#") + } else 0 + new SourceFile(file, content drop headerLength) { + override val underlying = new SourceFile(file, content) + } + } +} + +case class SourceFile(file: AbstractFile, content: Array[Char]) { + + def this(_file: AbstractFile) = this(_file, _file.toCharArray) + def this(sourceName: String, cs: Seq[Char]) = this(new VirtualFile(sourceName), cs.toArray) + def this(file: AbstractFile, cs: Seq[Char]) = this(file, cs.toArray) + + override def equals(that : Any) = that match { + case that : SourceFile => file.path == that.file.path && start == that.start + case _ => false + } + override def hashCode = file.path.## + start.## + + val length = content.length + + /** true for all source files except `NoSource` */ + def exists: Boolean = true + + /** The underlying source file */ + def underlying: SourceFile = this + + /** The start of this file in the underlying source file */ + def start = 0 + + def atPos(pos: Position): SourcePosition = + SourcePosition(underlying, pos) + + def isSelfContained = underlying eq this + + /** Map a position to a position in the underlying source file. + * For regular source files, simply return the argument. + */ + def positionInUltimateSource(position: SourcePosition): SourcePosition = + SourcePosition(underlying, position.pos shift start) + + def isLineBreak(idx: Int) = + if (idx >= length) false else { + val ch = content(idx) + // don't identify the CR in CR LF as a line break, since LF will do. + if (ch == CR) (idx + 1 == length) || (content(idx + 1) != LF) + else isLineBreakChar(ch) + } + + def calculateLineIndices(cs: Array[Char]) = { + val buf = new ArrayBuffer[Int] + buf += 0 + for (i <- 0 until cs.length) if (isLineBreak(i)) buf += i + 1 + buf += cs.length // sentinel, so that findLine below works smoother + buf.toArray + } + private lazy val lineIndices: Array[Int] = calculateLineIndices(content) + + /** Map line to offset of first character in line */ + def lineToOffset(index : Int): Int = lineIndices(index) + + /** A cache to speed up offsetToLine searches to similar lines */ + private var lastLine = 0 + + /** Convert offset to line in this source file + * Lines are numbered from 0 + */ + def offsetToLine(offset: Int): Int = { + val lines = lineIndices + def findLine(lo: Int, hi: Int, mid: Int): Int = + if (offset < lines(mid)) findLine(lo, mid - 1, (lo + mid - 1) / 2) + else if (offset >= lines(mid + 1)) findLine(mid + 1, hi, (mid + 1 + hi) / 2) + else mid + lastLine = findLine(0, lines.length, lastLine) + lastLine + } +} + +object NoSource extends SourceFile("<no source>", Nil) { + override def exists = false +} + diff --git a/src/dotty/tools/io/package.scala b/src/dotty/tools/io/package.scala index 584819c58..2543c38d2 100644 --- a/src/dotty/tools/io/package.scala +++ b/src/dotty/tools/io/package.scala @@ -14,8 +14,6 @@ package object io { // Forwarders from scala.reflect.io type AbstractFile = scala.reflect.io.AbstractFile val AbstractFile = scala.reflect.io.AbstractFile - type SourceFile = AbstractFile // for now - val NoSource = scala.reflect.io.NoAbstractFile // for now type Directory = scala.reflect.io.Directory val Directory = scala.reflect.io.Directory type File = scala.reflect.io.File diff --git a/src/test/loadDefs.scala b/src/test/loadDefs.scala deleted file mode 100644 index 77115d93b..000000000 --- a/src/test/loadDefs.scala +++ /dev/null @@ -1,15 +0,0 @@ -package test - -import dotty.tools.dotc.core._ -import Contexts._ - -object loadDefs { - - def main(args: Array[String]) = { - val base = new ContextBase - implicit val ctx = base.initialCtx - println(ctx.settings) - base.definitions.init() - println("done") - } -}
\ No newline at end of file diff --git a/test/test/ScannerTest.scala b/test/test/ScannerTest.scala new file mode 100644 index 000000000..68886694d --- /dev/null +++ b/test/test/ScannerTest.scala @@ -0,0 +1,51 @@ +package test + +import scala.reflect.io._ +import dotty.tools.dotc.util._ +import dotty.tools.dotc.parsing._ +import Tokens._, Scanners._ +import org.junit.Test + +class ScannerTest extends DottyTest { + + def scan(name: String): Unit = scan(new PlainFile(name)) + + def scan(file: PlainFile): Unit = { + println("***** scanning " + file) + val source = new SourceFile(file) + val scanner = new Scanner(source) + var i = 0 + while (scanner.token != EOF) { +// print("["+scanner.token.show+"]") + scanner.nextToken +// i += 1 +// if (i % 10 == 0) println() + } + } + + def scanDir(path: String): Unit = scanDir(Directory(path)) + + def scanDir(dir: Directory): Unit = { + for (f <- dir.files) + if (f.name.endsWith(".scala")) scan(new PlainFile(f)) + for (d <- dir.dirs) + scanDir(d.path) + } + + @Test + def scanList() = { + println(System.getProperty("user.dir")) + scan("src/dotty/tools/dotc/core/Symbols.scala") + scan("src/dotty/tools/dotc/core/Symbols.scala") + } + + @Test + def scanDotty() = { + scanDir("src") + } + + @Test + def scanScala() = { + scanDir("/Users/odersky/workspace/scala/src") + } +}
\ No newline at end of file diff --git a/test/test/scanPackage.scala b/test/test/scanPackage.scala new file mode 100644 index 000000000..c6a7709b8 --- /dev/null +++ b/test/test/scanPackage.scala @@ -0,0 +1,8 @@ +package test + +object scanPackage extends ScannerTest { + + def main(args: Array[String]): Unit = + scanDir("src") + +}
\ No newline at end of file |