From db544916874accf7d2a90f6f6d95a30cf88588c6 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 16 Nov 2014 19:11:32 -0800 Subject: Added more file-tests, renamed to scalaparser --- .../src/main/scala/scalaparser/ScalaSyntax.scala | 344 ++++++++++++++++++++ .../src/main/scala/scalaparser/syntax/Basic.scala | 36 +++ .../scala/scalaparser/syntax/Identifiers.scala | 24 ++ .../main/scala/scalaparser/syntax/Literals.scala | 56 ++++ .../src/main/scala/scalatex/stages/Parser.scala | 2 +- .../src/main/scala/torimatomeru/ScalaSyntax.scala | 348 --------------------- .../src/main/scala/torimatomeru/syntax/Basic.scala | 36 --- .../scala/torimatomeru/syntax/Identifiers.scala | 24 -- .../main/scala/torimatomeru/syntax/Literals.scala | 56 ---- .../src/test/scala/scalaparser/SyntaxTest.scala | 205 ++++++++++++ .../src/test/scala/scalatex/ParserTests.scala | 2 +- .../src/test/scala/torimatomeru/SyntaxTest.scala | 201 ------------ 12 files changed, 667 insertions(+), 667 deletions(-) create mode 100644 scalatexApi/src/main/scala/scalaparser/ScalaSyntax.scala create mode 100644 scalatexApi/src/main/scala/scalaparser/syntax/Basic.scala create mode 100644 scalatexApi/src/main/scala/scalaparser/syntax/Identifiers.scala create mode 100644 scalatexApi/src/main/scala/scalaparser/syntax/Literals.scala delete mode 100644 scalatexApi/src/main/scala/torimatomeru/ScalaSyntax.scala delete mode 100644 scalatexApi/src/main/scala/torimatomeru/syntax/Basic.scala delete mode 100644 scalatexApi/src/main/scala/torimatomeru/syntax/Identifiers.scala delete mode 100644 scalatexApi/src/main/scala/torimatomeru/syntax/Literals.scala create mode 100644 scalatexApi/src/test/scala/scalaparser/SyntaxTest.scala delete mode 100644 scalatexApi/src/test/scala/torimatomeru/SyntaxTest.scala (limited to 'scalatexApi') diff --git a/scalatexApi/src/main/scala/scalaparser/ScalaSyntax.scala b/scalatexApi/src/main/scala/scalaparser/ScalaSyntax.scala new file mode 100644 index 0000000..0e183a1 --- /dev/null +++ b/scalatexApi/src/main/scala/scalaparser/ScalaSyntax.scala @@ -0,0 +1,344 @@ +package scalaparser +import acyclic.file +import language.implicitConversions +import syntax._ +import org.parboiled2._ + +/** + * Parser for Scala syntax. + * + * The `G` parameter that gets passed in to each rule stands for + * "Greedy", and determines whether or not that rule is to consume + * newlines after the last terminal in that rule. We need to pass it + * everywhere so it can go all the way to the last terminal deep + * inside the parse tree, which can then decide whether or not to + * consume whitespace. + * + * The vast majority of terminals will consume newlines; only rules + * which occur in {} blocks won't have their terminals consume newlines. + * That's why the parser does terminals-consume-newlines-by-default, + * and leaves it up to the dev to thread the `G` variable where-ever + * we want the opposite behavior. + */ +class ScalaSyntax(val input: ParserInput) extends Parser with Basic with Identifiers with Literals { + // Aliases for common things. These things are used in almost every parser + // in the file, so it makes sense to keep them short. + type B = Boolean + val t = true + type R0 = Rule0 + /** + * Parses all whitespace, excluding newlines. This is only + * really useful in e.g. {} blocks, where we want to avoid + * capturing newlines so semicolon-inference would work + */ + def WS = rule { zeroOrMore(Basic.WhitespaceChar | Literals.Comment) } + + /** + * Parses whitespace, including newlines. + * This is the default for most things + */ + def WL = rule{ zeroOrMore(Basic.WhitespaceChar | Literals.Comment | Basic.Newline) } + + /** + * Whitespace which captures or doesn't-capture + * newlines depending on the G that gets passed in + */ + def W(G: B = t) = + if (G) WL + else WS + + /** + * By default, all strings and characters greedily + * capture all whitespace immediately after the token. + */ + implicit private[this] def wspStr(s: String): R0 = rule { str(s) ~ WL } + implicit private[this] def wspChar(s: Char): R0 = rule { ch(s) ~ WL } + + /** + * Occasionally, you want to decide whether or not to + * capture newlines based on the context, so use this + * and pass in G manually. + */ + def StrW(s: String, G: B): R0 = rule { str(s) ~ W(G) } + + + + def pos = cursor -> cursorChar + + /** + * helper printing function + */ + def pr(s: String) = rule { run(println(s"LOGGING $cursor: $s")) } + + def Id(G: B = t) = rule { Identifiers.Id ~ W(G) } + def VarId(G: B = t) = rule { Identifiers.VarId ~ W(G) } + def Literal(G: B = t) = rule { Literals.Literal ~ W(G) } + def Semi = rule { Basic.Semi ~ WL } + def Newline = rule { Basic.Newline ~ WL } + + def QualId(G: B = t) = rule { oneOrMore(Id(false)) separatedBy '.' ~ W(G) } + def Ids = rule { oneOrMore(Id()) separatedBy ',' } + + def Path(G: B = t): R0 = rule { + zeroOrMore(Id(G) ~ '.') ~ "this" ~ zeroOrMore(Id(G)).separatedBy('.') | + StableId(G) + } + def StableId(G: B = t): R0 = rule { + zeroOrMore(Id() ~ '.') ~ ("this" | "super" ~ optional(ClassQualifier)) ~ '.' ~ oneOrMore(Id(G)).separatedBy('.') | + Id(false) ~ zeroOrMore(WL ~ '.' ~ WL ~ Id(false)) ~ W(G) + } + + def ClassQualifier = rule { '[' ~ Id() ~ ']' } + + def Type: R0 = rule { FunctionArgTypes ~ "=>" ~ Type | InfixType ~ optional(ExistentialClause) } + def FunctionArgTypes = rule { InfixType | '(' ~ optional(oneOrMore(ParamType) separatedBy ',') ~ ')' } + + def ExistentialClause = rule { "forSome" ~ '{' ~ oneOrMore(ExistentialDcl).separatedBy(Semi) } + def ExistentialDcl = rule { "type" ~ TypeDcl | "val" ~ ValDcl } + + def InfixType = rule { CompoundType ~ zeroOrMore(Id() ~ optional(Newline) ~ CompoundType) } + def CompoundType = rule { oneOrMore(AnnotType(false)).separatedBy(WL ~ "with") ~ optional(Refinement) } + def AnnotType(G: B = t) = rule { + SimpleType(false) ~ zeroOrMore(WL ~ Annotation) ~ W(G) + } + def SimpleType(G: B = t): R0 = rule { + BasicType(false) ~ + optional(WL ~ + '#' ~ Id(false)) ~ + optional(WL ~ + TypeArgs(false)) ~ + W(G) + } + def BasicType(G: B = t): R0 = rule { + '(' ~ Types ~ ')' | + Path() ~ '.' ~ "type" | + StableId(G) + } + def TypeArgs(G: B = t) = rule { '[' ~ Types ~ StrW("]", G) } + def Types = rule { oneOrMore(Type).separatedBy(',') } + def Refinement = rule { optional(Newline) ~ '{' ~ oneOrMore(RefineStat).separatedBy(Semi) ~ '}' } + def RefineStat = rule { "type" ~ TypeDef | Dcl | MATCH } + def TypePat = rule { Type } + def Ascription = rule { ":" ~ (InfixType | oneOrMore(Annotation) | "_" ~ "*") } + + def ParamType = rule { "=>" ~ Type | Type ~ "*" | Type } + + def Expr(G: B = t): R0 = rule { (Bindings | optional("implicit") ~ Id() | "_") ~ "=>" ~ Expr(G) | Expr1(G) } + def Expr1(G: B = t): R0 = rule { + IfCFlow(G) | + WhileCFlow(G) | + TryCFlow(G) | + DoWhileCFlow(G) | + ForCFlow(G) | + "throw" ~ Expr(G) | + "return" ~ optional(Expr(G)) | + SimpleExpr() ~ ArgumentExprs() ~ '=' ~ Expr(G) | + optional(SimpleExpr() ~ '.') ~ Id() ~ '=' ~ Expr(G) | + PostfixExpr(G) ~ optional("match" ~ '{' ~ CaseClauses ~ '}' | Ascription) + } + + def IfCFlow(G: B = t) = rule { "if" ~ '(' ~ Expr() ~ ')' ~ zeroOrMore(Newline) ~ Expr(G) ~ optional(optional(Semi) ~ "else" ~ Expr(G)) } + def WhileCFlow(G: B = t) = rule { "while" ~ '(' ~ Expr() ~ ')' ~ zeroOrMore(Newline) ~ Expr(G) } + def TryCFlow(G: B = t) = rule { "try" ~ '{' ~ Block ~ StrW("}", G) ~ optional("catch" ~ '{' ~ CaseClauses ~ StrW("}", G)) ~ optional("finally" ~ Expr(G)) } + def DoWhileCFlow(G: B = t) = rule { "do" ~ Expr() ~ optional(Semi) ~ "while" ~ '(' ~ Expr() ~ StrW(")", G) } + def ForCFlow(G: B = t) = rule { "for" ~ ('(' ~ Enumerators ~ ')' | '{' ~ Enumerators ~ '}') ~ zeroOrMore(Newline) ~ optional("yield") ~ Expr(G) } + def PostfixExpr(G: B = t): R0 = rule { InfixExpr(G) ~ optional(Id() ~ optional(Newline)) } + def InfixExpr(G: B = t): R0 = rule { PrefixExpr(G) ~ zeroOrMore(Id() ~ optional(Newline) ~ PrefixExpr(G)) } + def PrefixExpr(G: B = t) = rule { optional(anyOf("-+~!")) ~ SimpleExpr(G) } + + def SimpleExpr(G: B = t): R0 = rule { + SimpleExpr1(false) ~ + zeroOrMore( + WL ~ ('.' ~ Id(false) | TypeArgs(false) | ArgumentExprs(false)) + ) ~ + optional(WL ~ StrW("_", false)) ~ + W(G) + } + + def SimpleExpr1(G: B = t) = rule{ + "new" ~ (ClassTemplate(G) | TemplateBody(G)) | + BlockExpr(G) | + Literal(G) ~ drop[String] | + Path(G) | + '_' | + '(' ~ optional(Exprs) ~ StrW(")", G) + } + + + def Exprs: R0 = rule { oneOrMore(Expr()).separatedBy(',') } + def ArgumentExprs(G: B = t): R0 = rule { + '(' ~ (optional(Exprs ~ ',') ~ PostfixExpr() ~ ':' ~ '_' ~ '*' | optional(Exprs)) ~ StrW(")", G) | + optional(Newline) ~ BlockExpr(G) + } + + def BlockExpr(G: B = t): R0 = rule { '{' ~ (CaseClauses | Block) ~ StrW("}", G) } + def Block: R0 = rule { + zeroOrMore(BlockStat ~ Semi) ~ optional(ResultExpr()) + } + + def BlockStat: R0 = rule { + Semi | + Import(false) | + zeroOrMore(Annotation) ~ (optional("implicit" | "lazy") ~ Def(false) | zeroOrMore(LocalModifier) ~ TmplDef(false)) | + Expr1(false) + } + def ResultExpr(G: B = t): R0 = rule { (Bindings | optional("implicit") ~ Id() | "_") ~ "=>" ~ Block | Expr1(t) } + def Enumerators: R0 = rule { Generator ~ zeroOrMore(Semi ~ Enumerator) } + def Enumerator: R0 = rule { Generator | Guard | Pattern1 ~ '=' ~ Expr() } + def Generator: R0 = rule { Pattern1 ~ "<-" ~ Expr() ~ optional(Guard) } + def CaseClauses: R0 = rule { oneOrMore(CaseClause) } + def CaseClause: R0 = rule { "case" ~ Pattern ~ optional(Guard) ~ "=>" ~ Block } + def Guard: R0 = rule { "if" ~ PostfixExpr() } + def Pattern: R0 = rule { + oneOrMore(Pattern1 ).separatedBy('|') + } + def Pattern1: R0 = rule { '_' ~ ':' ~ TypePat | VarId() ~ ':' ~ TypePat | Pattern2 } + def Pattern2: R0 = rule { VarId() ~ optional("@" ~ Pattern3) | Pattern3 } + def Pattern3: R0 = rule { + SimplePattern ~ zeroOrMore(Id() ~ SimplePattern) + } + def SimplePattern: R0 = rule { + '_' | + Literal() ~ drop[String] | + '(' ~ optional(Patterns) ~ ')' | + StableId() ~ optional('(' ~ (optional(Patterns ~ ',') ~ optional(VarId() ~ '@') ~ '_' ~ '*' | optional(Patterns)) ~ ')') | + VarId() /*| + XmlPattern*/ + } + def Patterns: R0 = rule { '_' ~ '*' | oneOrMore(Pattern).separatedBy(',') } + + def TypeParamClause: R0 = rule { '[' ~ oneOrMore(VariantTypeParam).separatedBy(',') ~ ']' } + def FunTypeParamClause: R0 = rule { '[' ~ oneOrMore(TypeParam).separatedBy(',') ~ ']' } + def VariantTypeParam: R0 = rule { zeroOrMore(Annotation) ~ optional(anyOf("+-")) ~ TypeParam } + def TypeParam: R0 = rule { (Id() | '_') ~ optional(TypeParamClause) ~ optional(">:" ~ Type) ~ optional("<:" ~ Type) ~ zeroOrMore("<%" ~ Type) ~ zeroOrMore(':' ~ Type) } + def ParamClauses: R0 = rule { zeroOrMore(ParamClause) ~ optional(optional(Newline) ~ '(' ~ "implicit" ~ Params ~ ')') } + def ParamClause: R0 = rule { optional(Newline) ~ '(' ~ optional(Params) ~ ')' } + def Params: R0 = rule { zeroOrMore(Param).separatedBy(',') } + def Param: R0 = rule { zeroOrMore(Annotation) ~ Id() ~ optional(':' ~ ParamType) ~ optional('=' ~ Expr()) } + def ClassParamClauses(G: B = t): R0 = rule { zeroOrMore(ClassParamClause(G)) ~ optional(optional(Newline) ~ '(' ~ "implicit" ~ ClassParam ~ StrW(")", G)) } + def ClassParamClause(G: B = t): R0 = rule { optional(Newline) ~ '(' ~ optional(ClassParams) ~ StrW(")", G) } + def ClassParams: R0 = rule { oneOrMore(ClassParam).separatedBy(',') } + def ClassParam: R0 = rule { zeroOrMore(Annotation) ~ optional(zeroOrMore(Modifier) ~ ("val" | "var")) ~ Id() ~ ":" ~ ParamType ~ optional("=" ~ Expr()) } + + def Bindings: R0 = rule { '(' ~ oneOrMore(Binding).separatedBy(',') ~ ')' } + def Binding: R0 = rule { (Id() | '_') ~ optional(':' ~ Type) } + + def Modifier: R0 = rule { LocalModifier | AccessModifier | "override" } + def LocalModifier: R0 = rule { "abstract" | "final" | "sealed" | "implicit" | "lazy" } + def AccessModifier: R0 = rule { ("private" | "protected") ~ optional(AccessQualifier) } + def AccessQualifier: R0 = rule { '[' ~ ("this" | Id()) ~ ']' } + + def Annotation: R0 = rule { '@' ~ SimpleType(false) ~ zeroOrMore(WL ~ ArgumentExprs()) } + def ConstrAnnotation: R0 = rule { '@' ~ SimpleType() ~ ArgumentExprs() } + + def TemplateBody(G: B = t): R0 = rule { + WL ~ + '{' ~ + optional(SelfType) ~ + TemplateStat ~ + zeroOrMore(Semi ~ TemplateStat) ~ + WL ~ + StrW("}", G) + } + def TemplateStat: R0 = rule { + Import(false) | + zeroOrMore(Annotation ~ optional(Newline)) ~ zeroOrMore(Modifier) ~ (Def(false) | Dcl) | + Expr(false) | + MATCH + } + + def SelfType: R0 = rule { "this" ~ ':' ~ Type ~ "=>" | Id() ~ optional(':' ~ Type) ~ "=>" } + + def Import(G: B = t): R0 = rule { "import" ~ oneOrMore(ImportExpr(G)).separatedBy(',') } + + def ImportExpr(G: B = t): R0 = rule { StableId(G) ~ optional('.' ~ (StrW("_", G) | ImportSelectors(G))) } + def ImportSelectors(G: B = t): R0 = rule { '{' ~ zeroOrMore(ImportSelector ~ ',') ~ (ImportSelector | '_') ~ StrW("}", G) } + def ImportSelector: R0 = rule { Id() ~ optional("=>" ~ (Id() | '_')) } + + def Dcl: R0 = rule { + "val" ~ ValDcl | + "var" ~ VarDcl | + "def" ~ FunDcl | + "type" ~ zeroOrMore(Newline) ~ TypeDcl + } + def ValDcl: R0 = rule { Ids ~ ':' ~ Type } + def VarDcl: R0 = rule { Ids ~ ':' ~ Type } + def FunDcl: R0 = rule { FunSig ~ optional(':' ~ Type) } + def FunSig: R0 = rule { Id() ~ optional(FunTypeParamClause) ~ ParamClauses } + def TypeDcl: R0 = rule { Id() ~ optional(TypeParamClause) ~ optional(">:" ~ Type) ~ optional("<:" ~ Type) } + + def PatVarDef(G: B = t): R0 = rule { "val" ~ PatDef(G) | "var" ~ VarDef(G) } + def Def(G: B = t): R0 = rule { "def" ~ FunDef(G) | "type" ~ zeroOrMore(Newline) ~ TypeDef | PatVarDef(G) | TmplDef(G) } + def PatDef(G: B = t): R0 = rule { oneOrMore(Pattern2).separatedBy(',') ~ optional(':' ~ Type) ~ '=' ~ Expr(G) } + def VarDef(G: B = t): R0 = rule { Ids ~ ':' ~ Type ~ '=' ~ '_' | PatDef(G) } + def FunDef(G: B = t): R0 = rule { + "this" ~ ParamClause ~ ParamClauses ~ ('=' ~ ConstrExpr | optional(Newline) ~ ConstrBlock) | + FunSig ~ (optional(':' ~ Type) ~ '=' ~ optional("macro") ~ Expr(G) | optional(Newline) ~ '{' ~ Block ~ '}') + } + def TypeDef: R0 = rule { Id() ~ optional(TypeParamClause) ~ '=' ~ Type } + + def TmplDef(G: B = t): R0 = rule { + "trait" ~ TraitDef(G) | + optional("case") ~ ("class" ~ ClassDef(G) | + "object" ~ ObjectDef(G)) + } + def ClassDef(G: B = t): R0 = rule { + Id() ~ + optional(TypeParamClause) ~ + zeroOrMore(ConstrAnnotation) ~ + optional(AccessModifier) ~ + ClassParamClauses(false) ~ + ClassTemplateOpt(false) ~ + W(G) + + } + def TraitDef(G: B = t): R0 = rule { Id() ~ optional(TypeParamClause) ~ TraitTemplateOpt(G) } + def ObjectDef(G: B = t): R0 = rule { Id() ~ ClassTemplateOpt(G) } + def ClassTemplateOpt(G: B = t): R0 = rule { + WL ~ "extends" ~ ClassTemplate(G) | + optional(WL ~ optional("extends") ~ TemplateBody(G)) + } + def TraitTemplateOpt(G: B = t): R0 = rule { "extends" ~ TraitTemplate(G) | optional(optional("extends") ~ TemplateBody(G)) } + def ClassTemplate(G: B = t): R0 = rule { + optional(EarlyDefs) ~ + ClassParents(false) ~ + optional(WL ~ TemplateBody(false)) ~ + W(G) + } + + def TraitTemplate(G: B = t): R0 = rule { + optional(EarlyDefs) ~ TraitParents(false) ~ optional(TemplateBody(false)) ~ W(G) + } + def ClassParents(G: B = t): R0 = rule { + Constr(false) ~ zeroOrMore(WL ~ "with" ~ AnnotType(G)) ~ W(G) + } + def TraitParents(G: B = t): R0 = rule { + AnnotType(false) ~ zeroOrMore(WL ~ "with" ~ AnnotType(false)) ~ W(G) + } + def Constr(G: B = t): R0 = rule { + AnnotType(false) ~ zeroOrMore(WL ~ ArgumentExprs(false)) ~ + W(G) + } + def EarlyDefs: R0 = rule { + '{' ~ optional(oneOrMore(EarlyDef).separatedBy(Semi)) ~ '}' ~ "with" + } + def EarlyDef: R0 = rule { + zeroOrMore(Annotation ~ optional(Newline)) ~ zeroOrMore(Modifier) ~ PatVarDef(false) + } + def ConstrExpr: R0 = rule { ConstrBlock | SelfInvocation } + def ConstrBlock: R0 = rule { '{' ~ SelfInvocation ~ zeroOrMore(Semi ~ BlockStat) ~ '}' } + def SelfInvocation: R0 = rule { "this" ~ oneOrMore(ArgumentExprs()) } + + def TopStatSeq: R0 = rule { zeroOrMore(TopStat).separatedBy(Semi) } + def TopStat: R0 = rule { Packaging | PackageObject(false) | Import(false) | zeroOrMore(Annotation ~ optional(Newline)) ~ zeroOrMore(Modifier) ~ TmplDef(false) | MATCH } + def Packaging: R0 = rule { "package" ~ QualId() ~ '{' ~ TopStatSeq ~ '}' } + def PackageObject(G: B = t): R0 = rule { "package" ~ "object" ~ ObjectDef(G) } + def CompilationUnit: Rule1[String] = rule { + capture( + zeroOrMore(Semi) ~ + zeroOrMore("package" ~ QualId(false)).separatedBy(Semi) ~ + TopStatSeq ~ + EOI + ) + } +} diff --git a/scalatexApi/src/main/scala/scalaparser/syntax/Basic.scala b/scalatexApi/src/main/scala/scalaparser/syntax/Basic.scala new file mode 100644 index 0000000..bd142a1 --- /dev/null +++ b/scalatexApi/src/main/scala/scalaparser/syntax/Basic.scala @@ -0,0 +1,36 @@ +package scalaparser +package syntax +import acyclic.file +import org.parboiled2._ + +trait Basic { self: Parser => + object Basic{ + def UnicodeExcape = rule { "\\u" ~ 4.times(HexDigit) } + + + //Numbers and digits + def HexDigit = rule { Digit | "a" - "f" | "A" - "Z" } + def Digit = rule { "0" | NonZeroDigit } + def NonZeroDigit = rule { "1" - "9" } + def HexNumeral = rule { "0x" ~ oneOrMore(HexDigit) } + def DecimalNumeral = rule(oneOrMore(Digit)) + def ExponentPart = rule { anyOf("Ee") ~ optional(anyOf("+-")) ~ oneOrMore(Digit) } + def FloatType = rule { anyOf("FfDd") } + + def Parentheses = rule { "(" | ")" | "[" | "]" | "{" | "}" } + def DelimiterChar = rule { "'" | "\"" | "." | ";" | "," } + + def WhitespaceChar = rule { "\u0020" | "\u0009" } + def Newline = rule { "\r\n" | "\n" } + def Semi = rule { ';' | oneOrMore(Newline) } + def OperatorChar = rule { + anyOf("""!#$%&*+-/:<=>?@\^|~""") | + CharPredicate.from(_.getType match { + case Character.OTHER_SYMBOL | Character.MATH_SYMBOL => true; case _ => false + }) + } + def Letter = rule { Upper | Lower | CharPredicate.from(c => c.isLetter | c.isDigit) } + def Lower = rule { "a" - "z" | "$" | "_" | CharPredicate.from(_.isLower) } + def Upper = rule { "A" - "Z" | CharPredicate.from(_.isUpper) } + } +} diff --git a/scalatexApi/src/main/scala/scalaparser/syntax/Identifiers.scala b/scalatexApi/src/main/scala/scalaparser/syntax/Identifiers.scala new file mode 100644 index 0000000..e985f92 --- /dev/null +++ b/scalatexApi/src/main/scala/scalaparser/syntax/Identifiers.scala @@ -0,0 +1,24 @@ +package scalaparser +package syntax +import acyclic.file +import org.parboiled2._ + +trait Identifiers { self: Parser with Basic => + object Identifiers{ + import Basic._ + def Operator = rule(oneOrMore(OperatorChar)) + + def VarId = rule { !(Keywords ~ (WhitespaceChar | Newline | "//" | "/*")) ~ Lower ~ IdRest } + def PlainId = rule { Upper ~ IdRest | VarId | !(Keywords ~ (WhitespaceChar | Newline | "//" | "/*")) ~ Operator } + def Id = rule { PlainId | ("`" ~ oneOrMore(noneOf("`")) ~ "`") } + def IdRest = rule { zeroOrMore(Letter | Digit) ~ optional("_" ~ Operator) } + + + def Keywords = rule { + "abstract" | "case" | "catch" | "class" | "def" | "do" | "else" | "extends" | "false" | "finally" | "final" | "finally" | "forSome" | "for" | "if" | + "implicit" | "import" | "lazy" | "match" | "new" | "null" | "object" | "override" | "package" | "private" | "protected" | "return" | + "sealed" | "super" | "this" | "throw" | "trait" | "try" | "true" | "type" | "val" | "var" | "while" | "with" | "yield" | "_" | + ":" | ";" | "=>" | "=" | "<-" | "<:" | "<%" | ">:" | "#" | "@" | "\u21d2" | "\u2190" + } + } +} diff --git a/scalatexApi/src/main/scala/scalaparser/syntax/Literals.scala b/scalatexApi/src/main/scala/scalaparser/syntax/Literals.scala new file mode 100644 index 0000000..11eb943 --- /dev/null +++ b/scalatexApi/src/main/scala/scalaparser/syntax/Literals.scala @@ -0,0 +1,56 @@ +package scalaparser +package syntax +import acyclic.file +import org.parboiled2._ + +trait Literals { self: Parser with Basic with Identifiers => + object Literals{ + import Basic._ + def FloatingPointLiteral = rule { + capture( + "." ~ oneOrMore(Digit) ~ optional(ExponentPart) ~ optional(FloatType) | + oneOrMore(Digit) ~ ( + "." ~ oneOrMore(Digit) ~ optional(ExponentPart) ~ optional(FloatType) | + ExponentPart ~ optional(FloatType) | + optional(ExponentPart) ~ FloatType)) + } + + def IntegerLiteral = rule { capture((DecimalNumeral | HexNumeral) ~ optional(anyOf("Ll"))) } + + def BooleanLiteral = rule { capture("true" | "false") } + + def MultilineComment: Rule0 = rule { "/*" ~ zeroOrMore(MultilineComment | !"*/" ~ ANY) ~ "*/" } + def Comment: Rule0 = rule { + MultilineComment | + "//" ~ zeroOrMore(!Basic.Newline ~ ANY) ~ (Basic.Newline | EOI) + } + + def Literal = rule { + (capture(optional("-")) ~ (FloatingPointLiteral | IntegerLiteral) ~> ((sign: String, number) => sign + number)) | + BooleanLiteral | + CharacterLiteral | + StringLiteral | + SymbolLiteral | + capture("null") + } + + + def EscapedChars = rule { '\\' ~ anyOf("rnt\\\"") } + + def SymbolLiteral = rule { ''' ~ capture(Identifiers.PlainId) } + + def CharacterLiteral = rule { ''' ~ capture(UnicodeExcape | EscapedChars | !'\\' ~ CharPredicate.from(isPrintableChar)) ~ ''' } + + def MultiLineChars = rule { zeroOrMore(optional('"') ~ optional('"') ~ noneOf("\"")) } + def StringLiteral = rule { + (optional(Identifiers.Id) ~ "\"\"\"" ~ capture(MultiLineChars) ~ capture("\"\"\"" ~ zeroOrMore('"')) ~> ((multilineChars: String, quotes) => multilineChars + quotes.dropRight(3))) | + (optional(Identifiers.Id) ~ '"' ~ capture(zeroOrMore("\\\"" | noneOf("\n\""))) ~ '"') + } + + def isPrintableChar(c: Char): Boolean = { + val block = Character.UnicodeBlock.of(c) + !Character.isISOControl(c) && !Character.isSurrogate(c) && block != null && block != Character.UnicodeBlock.SPECIALS + } + } +} + diff --git a/scalatexApi/src/main/scala/scalatex/stages/Parser.scala b/scalatexApi/src/main/scala/scalatex/stages/Parser.scala index 117806d..403cb72 100644 --- a/scalatexApi/src/main/scala/scalatex/stages/Parser.scala +++ b/scalatexApi/src/main/scala/scalatex/stages/Parser.scala @@ -2,7 +2,7 @@ package scalatex package stages import acyclic.file import org.parboiled2._ -import torimatomeru.ScalaSyntax +import scalaparser.ScalaSyntax /** * Parses the input text into a roughly-structured AST. This AST diff --git a/scalatexApi/src/main/scala/torimatomeru/ScalaSyntax.scala b/scalatexApi/src/main/scala/torimatomeru/ScalaSyntax.scala deleted file mode 100644 index 7538ec6..0000000 --- a/scalatexApi/src/main/scala/torimatomeru/ScalaSyntax.scala +++ /dev/null @@ -1,348 +0,0 @@ -package torimatomeru -import acyclic.file -import language.implicitConversions -import syntax._ -import org.parboiled2._ - -class ScalaSyntax(val input: ParserInput) extends Parser with Basic with Identifiers with Literals { - // Aliases for common things. These things are used in almost every parser - // in the file, so it makes sense to keep them short. - type B = Boolean - val t = true - type R0 = Rule0 - /** - * Parses all whitespace, excluding newlines. This is only - * really useful in e.g. {} blocks, where we want to avoid - * capturing newlines so semicolon-inference would work - */ - def WS = rule { zeroOrMore(Basic.WhitespaceChar | Literals.Comment) } - - /** - * Parses whitespace, including newlines. - * This is the default for most things - */ - def WL = rule{ zeroOrMore(Basic.WhitespaceChar | Literals.Comment | Basic.Newline) } - - /** - * Whitespace which captures or doesn't-capture - * newlines depending on the G that gets passed in - */ - def W(G: B = t) = - if (G) WL - else WS - - /** - * By default, all strings and characters greedily - * capture all whitespace immediately after the token. - */ - implicit private[this] def wspStr(s: String): R0 = rule { str(s) ~ WL } - implicit private[this] def wspChar(s: Char): R0 = rule { ch(s) ~ WL } - - /** - * Occasionally, you want to decide whether or not to - * capture newlines based on the context, so use this - * and pass in G manually. - */ - def StrW(s: String, G: B): R0 = rule { str(s) ~ W(G) } - - - - def pos = cursor -> cursorChar - - /** - * helper printing function - */ - def pr(s: String) = rule { run(println(s"LOGGING $cursor: $s")) } - - ////////////////////////////////////////////////// - // Override rules from dependencies - // in order to handle white spaces - // Note: when you add your AST, make sure to - // only capture super.rule and not the whitespace - ////////////////////////////////////////////////// - - def Id(G: B = t) = rule { Identifiers.Id ~ W(G) } - def VarId(G: B = t) = rule { Identifiers.VarId ~ W(G) } - def Literal(G: B = t) = rule { Literals.Literal ~ W(G) } - def Semi = rule { Basic.Semi ~ WL } - def Newline = rule { Basic.Newline ~ WL } - - /////////////////////////////////////////// - // Qualifiers and Ids - /////////////////////////////////////////// - - def QualId(G: B = t) = rule { oneOrMore(Id(false)) separatedBy '.' ~ W(G) } - def Ids = rule { oneOrMore(Id()) separatedBy ',' } - - def Path(G: B = t): R0 = rule { - zeroOrMore(Id(G) ~ '.') ~ "this" ~ zeroOrMore(Id(G)).separatedBy('.') | - StableId(G) - } - def StableId(G: B = t): R0 = rule { - zeroOrMore(Id() ~ '.') ~ ("this" | "super" ~ optional(ClassQualifier)) ~ '.' ~ oneOrMore(Id(G)).separatedBy('.') | - Id(false) ~ zeroOrMore(WL ~ '.' ~ WL ~ Id(false)) ~ W(G) - } - - def ClassQualifier = rule { '[' ~ Id() ~ ']' } - - /////////////////////////////////////////// - // Types and more Types - /////////////////////////////////////////// - - def Type: R0 = rule { FunctionArgTypes ~ "=>" ~ Type | InfixType ~ optional(ExistentialClause) } - def FunctionArgTypes = rule { InfixType | '(' ~ optional(oneOrMore(ParamType) separatedBy ',') ~ ')' } - - def ExistentialClause = rule { "forSome" ~ '{' ~ oneOrMore(ExistentialDcl).separatedBy(Semi) } - def ExistentialDcl = rule { "type" ~ TypeDcl | "val" ~ ValDcl } - - def InfixType = rule { CompoundType ~ zeroOrMore(Id() ~ optional(Newline) ~ CompoundType) } - def CompoundType = rule { oneOrMore(AnnotType(false)).separatedBy(WL ~ "with") ~ optional(Refinement) } - def AnnotType(G: B = t) = rule { - SimpleType(false) ~ zeroOrMore(WL ~ Annotation) ~ W(G) - } - def SimpleType(G: B = t): R0 = rule { - BasicType(false) ~ - optional(WL ~ - '#' ~ Id(false)) ~ - optional(WL ~ - TypeArgs(false)) ~ - W(G) - } - def BasicType(G: B = t): R0 = rule { - '(' ~ Types ~ ')' | - Path() ~ '.' ~ "type" | - StableId(G) - } - def TypeArgs(G: B = t) = rule { '[' ~ Types ~ StrW("]", G) } - def Types = rule { oneOrMore(Type).separatedBy(',') } - def Refinement = rule { optional(Newline) ~ '{' ~ oneOrMore(RefineStat).separatedBy(Semi) ~ '}' } - def RefineStat = rule { "type" ~ TypeDef | Dcl | MATCH } - def TypePat = rule { Type } - def Ascription = rule { ":" ~ (InfixType | oneOrMore(Annotation) | "_" ~ "*") } - - def ParamType = rule { "=>" ~ Type | Type ~ "*" | Type } - - ///////////////////////////////////////////////// - // Declarations, Expressions and Pattern Matching - ///////////////////////////////////////////////// - - def Expr(G: B = t): R0 = rule { (Bindings | optional("implicit") ~ Id() | "_") ~ "=>" ~ Expr(G) | Expr1(G) } - def Expr1(G: B = t): R0 = rule { - IfCFlow(G) | - WhileCFlow(G) | - TryCFlow(G) | - DoWhileCFlow(G) | - ForCFlow(G) | - "throw" ~ Expr(G) | - "return" ~ optional(Expr(G)) | - SimpleExpr() ~ ArgumentExprs() ~ '=' ~ Expr(G) | - optional(SimpleExpr() ~ '.') ~ Id() ~ '=' ~ Expr(G) | - PostfixExpr(G) ~ optional("match" ~ '{' ~ CaseClauses ~ '}' | Ascription) - } - - def IfCFlow(G: B = t) = rule { "if" ~ '(' ~ Expr() ~ ')' ~ zeroOrMore(Newline) ~ Expr(G) ~ optional(optional(Semi) ~ "else" ~ Expr(G)) } - def WhileCFlow(G: B = t) = rule { "while" ~ '(' ~ Expr() ~ ')' ~ zeroOrMore(Newline) ~ Expr(G) } - def TryCFlow(G: B = t) = rule { "try" ~ '{' ~ Block ~ StrW("}", G) ~ optional("catch" ~ '{' ~ CaseClauses ~ StrW("}", G)) ~ optional("finally" ~ Expr(G)) } - def DoWhileCFlow(G: B = t) = rule { "do" ~ Expr() ~ optional(Semi) ~ "while" ~ '(' ~ Expr() ~ StrW(")", G) } - def ForCFlow(G: B = t) = rule { "for" ~ ('(' ~ Enumerators ~ ')' | '{' ~ Enumerators ~ '}') ~ zeroOrMore(Newline) ~ optional("yield") ~ Expr(G) } - def PostfixExpr(G: B = t): R0 = rule { InfixExpr(G) ~ optional(Id() ~ optional(Newline)) } - def InfixExpr(G: B = t): R0 = rule { PrefixExpr(G) ~ zeroOrMore(Id() ~ optional(Newline) ~ PrefixExpr(G)) } - def PrefixExpr(G: B = t) = rule { optional(anyOf("-+~!")) ~ SimpleExpr(G) } - - def SimpleExpr(G: B = t): R0 = rule { - SimpleExpr1(false) ~ - zeroOrMore( - WL ~ ('.' ~ Id(false) | TypeArgs(false) | ArgumentExprs(false)) - ) ~ - optional(WL ~ StrW("_", false)) ~ - W(G) - } - - def SimpleExpr1(G: B = t) = rule{ - "new" ~ (ClassTemplate(G) | TemplateBody(G)) | - BlockExpr(G) | - Literal(G) ~ drop[String] | - Path(G) | - '_' | - '(' ~ optional(Exprs) ~ StrW(")", G) - } - - - def Exprs: R0 = rule { oneOrMore(Expr()).separatedBy(',') } - def ArgumentExprs(G: B = t): R0 = rule { - '(' ~ (optional(Exprs ~ ',') ~ PostfixExpr() ~ ':' ~ '_' ~ '*' | optional(Exprs)) ~ StrW(")", G) | - optional(Newline) ~ BlockExpr(G) - } - - def BlockExpr(G: B = t): R0 = rule { '{' ~ (CaseClauses | Block) ~ StrW("}", G) } - def Block: R0 = rule { - zeroOrMore(BlockStat ~ Semi) ~ optional(ResultExpr()) - } - - def BlockStat: R0 = rule { - Semi | - Import(false) | - zeroOrMore(Annotation) ~ (optional("implicit" | "lazy") ~ Def(false) | zeroOrMore(LocalModifier) ~ TmplDef(false)) | - Expr1(false) - } - def ResultExpr(G: B = t): R0 = rule { (Bindings | optional("implicit") ~ Id() | "_") ~ "=>" ~ Block | Expr1(t) } - def Enumerators: R0 = rule { Generator ~ zeroOrMore(Semi ~ Enumerator) } - def Enumerator: R0 = rule { Generator | Guard | Pattern1 ~ '=' ~ Expr() } - def Generator: R0 = rule { Pattern1 ~ "<-" ~ Expr() ~ optional(Guard) } - def CaseClauses: R0 = rule { oneOrMore(CaseClause) } - def CaseClause: R0 = rule { "case" ~ Pattern ~ optional(Guard) ~ "=>" ~ Block } - def Guard: R0 = rule { "if" ~ PostfixExpr() } - def Pattern: R0 = rule { - oneOrMore(Pattern1 ).separatedBy('|') - } - def Pattern1: R0 = rule { '_' ~ ':' ~ TypePat | VarId() ~ ':' ~ TypePat | Pattern2 } - def Pattern2: R0 = rule { VarId() ~ optional("@" ~ Pattern3) | Pattern3 } - def Pattern3: R0 = rule { - SimplePattern ~ zeroOrMore(Id() ~ SimplePattern) - } - def SimplePattern: R0 = rule { - '_' | - Literal() ~ drop[String] | - '(' ~ optional(Patterns) ~ ')' | - StableId() ~ optional('(' ~ (optional(Patterns ~ ',') ~ optional(VarId() ~ '@') ~ '_' ~ '*' | optional(Patterns)) ~ ')') | - VarId() /*| - XmlPattern*/ - } - def Patterns: R0 = rule { '_' ~ '*' | oneOrMore(Pattern).separatedBy(',') } - - def TypeParamClause: R0 = rule { '[' ~ oneOrMore(VariantTypeParam).separatedBy(',') ~ ']' } - def FunTypeParamClause: R0 = rule { '[' ~ oneOrMore(TypeParam).separatedBy(',') ~ ']' } - def VariantTypeParam: R0 = rule { zeroOrMore(Annotation) ~ optional(anyOf("+-")) ~ TypeParam } - def TypeParam: R0 = rule { (Id() | '_') ~ optional(TypeParamClause) ~ optional(">:" ~ Type) ~ optional("<:" ~ Type) ~ zeroOrMore("<%" ~ Type) ~ zeroOrMore(':' ~ Type) } - def ParamClauses: R0 = rule { zeroOrMore(ParamClause) ~ optional(optional(Newline) ~ '(' ~ "implicit" ~ Params ~ ')') } - def ParamClause: R0 = rule { optional(Newline) ~ '(' ~ optional(Params) ~ ')' } - def Params: R0 = rule { zeroOrMore(Param).separatedBy(',') } - def Param: R0 = rule { zeroOrMore(Annotation) ~ Id() ~ optional(':' ~ ParamType) ~ optional('=' ~ Expr()) } - def ClassParamClauses(G: B = t): R0 = rule { zeroOrMore(ClassParamClause(G)) ~ optional(optional(Newline) ~ '(' ~ "implicit" ~ ClassParam ~ StrW(")", G)) } - def ClassParamClause(G: B = t): R0 = rule { optional(Newline) ~ '(' ~ optional(ClassParams) ~ StrW(")", G) } - def ClassParams: R0 = rule { oneOrMore(ClassParam).separatedBy(',') } - def ClassParam: R0 = rule { zeroOrMore(Annotation) ~ optional(zeroOrMore(Modifier) ~ ("val" | "var")) ~ Id() ~ ":" ~ ParamType ~ optional("=" ~ Expr()) } - - def Bindings: R0 = rule { '(' ~ oneOrMore(Binding).separatedBy(',') ~ ')' } - def Binding: R0 = rule { (Id() | '_') ~ optional(':' ~ Type) } - - def Modifier: R0 = rule { LocalModifier | AccessModifier | "override" } - def LocalModifier: R0 = rule { "abstract" | "final" | "sealed" | "implicit" | "lazy" } - def AccessModifier: R0 = rule { ("private" | "protected") ~ optional(AccessQualifier) } - def AccessQualifier: R0 = rule { '[' ~ ("this" | Id()) ~ ']' } - - def Annotation: R0 = rule { '@' ~ SimpleType(false) ~ zeroOrMore(WL ~ ArgumentExprs()) } - def ConstrAnnotation: R0 = rule { '@' ~ SimpleType() ~ ArgumentExprs() } - - def TemplateBody(G: B = t): R0 = rule { - WL ~ - '{' ~ - optional(SelfType) ~ - TemplateStat ~ - zeroOrMore(Semi ~ TemplateStat) ~ - WL ~ - StrW("}", G) - } - def TemplateStat: R0 = rule { - Import(false) | - zeroOrMore(Annotation ~ optional(Newline)) ~ zeroOrMore(Modifier) ~ (Def(false) | Dcl) | - Expr(false) | - MATCH - } - - def SelfType: R0 = rule { "this" ~ ':' ~ Type ~ "=>" | Id() ~ optional(':' ~ Type) ~ "=>" } - - def Import(G: B = t): R0 = rule { "import" ~ oneOrMore(ImportExpr(G)).separatedBy(',') } - - //ImportExpr is slightly changed wrt spec because StableId always consumes all the Ids possible, so there is no need to one at the end - def ImportExpr(G: B = t): R0 = rule { StableId(G) ~ optional('.' ~ (StrW("_", G) | ImportSelectors(G))) } - def ImportSelectors(G: B = t): R0 = rule { '{' ~ zeroOrMore(ImportSelector ~ ',') ~ (ImportSelector | '_') ~ StrW("}", G) } - def ImportSelector: R0 = rule { Id() ~ optional("=>" ~ (Id() | '_')) } - - def Dcl: R0 = rule { - "val" ~ ValDcl | - "var" ~ VarDcl | - "def" ~ FunDcl | - "type" ~ zeroOrMore(Newline) ~ TypeDcl - } - def ValDcl: R0 = rule { Ids ~ ':' ~ Type } - def VarDcl: R0 = rule { Ids ~ ':' ~ Type } - def FunDcl: R0 = rule { FunSig ~ optional(':' ~ Type) } - def FunSig: R0 = rule { Id() ~ optional(FunTypeParamClause) ~ ParamClauses } - def TypeDcl: R0 = rule { Id() ~ optional(TypeParamClause) ~ optional(">:" ~ Type) ~ optional("<:" ~ Type) } - - def PatVarDef(G: B = t): R0 = rule { "val" ~ PatDef(G) | "var" ~ VarDef(G) } - def Def(G: B = t): R0 = rule { "def" ~ FunDef(G) | "type" ~ zeroOrMore(Newline) ~ TypeDef | PatVarDef(G) | TmplDef(G) } - def PatDef(G: B = t): R0 = rule { oneOrMore(Pattern2).separatedBy(',') ~ optional(':' ~ Type) ~ '=' ~ Expr(G) } - def VarDef(G: B = t): R0 = rule { Ids ~ ':' ~ Type ~ '=' ~ '_' | PatDef(G) } - def FunDef(G: B = t): R0 = rule { - "this" ~ ParamClause ~ ParamClauses ~ ('=' ~ ConstrExpr | optional(Newline) ~ ConstrBlock) | - FunSig ~ (optional(':' ~ Type) ~ '=' ~ optional("macro") ~ Expr(G) | optional(Newline) ~ '{' ~ Block ~ '}') - } - def TypeDef: R0 = rule { Id() ~ optional(TypeParamClause) ~ '=' ~ Type } - - def TmplDef(G: B = t): R0 = rule { - "trait" ~ TraitDef(G) | - optional("case") ~ ("class" ~ ClassDef(G) | - "object" ~ ObjectDef(G)) - } - def ClassDef(G: B = t): R0 = rule { - Id() ~ - optional(TypeParamClause) ~ - zeroOrMore(ConstrAnnotation) ~ - optional(AccessModifier) ~ - ClassParamClauses(false) ~ - ClassTemplateOpt(false) ~ - W(G) - - } - def TraitDef(G: B = t): R0 = rule { Id() ~ optional(TypeParamClause) ~ TraitTemplateOpt(G) } - def ObjectDef(G: B = t): R0 = rule { Id() ~ ClassTemplateOpt(G) } - def ClassTemplateOpt(G: B = t): R0 = rule { - WL ~ "extends" ~ ClassTemplate(G) | - optional(WL ~ optional("extends") ~ TemplateBody(G)) - } - def TraitTemplateOpt(G: B = t): R0 = rule { "extends" ~ TraitTemplate(G) | optional(optional("extends") ~ TemplateBody(G)) } - def ClassTemplate(G: B = t): R0 = rule { - optional(EarlyDefs) ~ - ClassParents(false) ~ - optional(WL ~ TemplateBody(false)) ~ - W(G) - } - - def TraitTemplate(G: B = t): R0 = rule { - optional(EarlyDefs) ~ TraitParents(false) ~ optional(TemplateBody(false)) ~ W(G) - } - def ClassParents(G: B = t): R0 = rule { - Constr(false) ~ zeroOrMore(WL ~ "with" ~ AnnotType(G)) ~ W(G) - } - def TraitParents(G: B = t): R0 = rule { - AnnotType(false) ~ zeroOrMore(WL ~ "with" ~ AnnotType(false)) ~ W(G) - } - def Constr(G: B = t): R0 = rule { - AnnotType(false) ~ zeroOrMore(WL ~ ArgumentExprs(false)) ~ - W(G) - } - def EarlyDefs: R0 = rule { - '{' ~ optional(oneOrMore(EarlyDef).separatedBy(Semi)) ~ '}' ~ "with" - } - def EarlyDef: R0 = rule { - zeroOrMore(Annotation ~ optional(Newline)) ~ zeroOrMore(Modifier) ~ PatVarDef(false) - } - def ConstrExpr: R0 = rule { ConstrBlock | SelfInvocation } - def ConstrBlock: R0 = rule { '{' ~ SelfInvocation ~ zeroOrMore(Semi ~ BlockStat) ~ '}' } - def SelfInvocation: R0 = rule { "this" ~ oneOrMore(ArgumentExprs()) } - - def TopStatSeq: R0 = rule { zeroOrMore(TopStat).separatedBy(Semi) } - def TopStat: R0 = rule { Packaging | PackageObject(false) | Import(false) | zeroOrMore(Annotation ~ optional(Newline)) ~ zeroOrMore(Modifier) ~ TmplDef(false) | MATCH } - def Packaging: R0 = rule { "package" ~ QualId() ~ '{' ~ TopStatSeq ~ '}' } - def PackageObject(G: B = t): R0 = rule { "package" ~ "object" ~ ObjectDef(G) } - def CompilationUnit: Rule1[String] = rule { - capture( - zeroOrMore(Semi) ~ - zeroOrMore("package" ~ QualId(false)).separatedBy(Semi) ~ - TopStatSeq ~ - EOI - ) - } -} diff --git a/scalatexApi/src/main/scala/torimatomeru/syntax/Basic.scala b/scalatexApi/src/main/scala/torimatomeru/syntax/Basic.scala deleted file mode 100644 index 18cff91..0000000 --- a/scalatexApi/src/main/scala/torimatomeru/syntax/Basic.scala +++ /dev/null @@ -1,36 +0,0 @@ -package torimatomeru -package syntax -import acyclic.file -import org.parboiled2._ - -trait Basic { self: Parser => - object Basic{ - def UnicodeExcape = rule { "\\u" ~ 4.times(HexDigit) } - - - //Numbers and digits - def HexDigit = rule { Digit | "a" - "f" | "A" - "Z" } - def Digit = rule { "0" | NonZeroDigit } - def NonZeroDigit = rule { "1" - "9" } - def HexNumeral = rule { "0x" ~ oneOrMore(HexDigit) } - def DecimalNumeral = rule(oneOrMore(Digit)) - def ExponentPart = rule { anyOf("Ee") ~ optional(anyOf("+-")) ~ oneOrMore(Digit) } - def FloatType = rule { anyOf("FfDd") } - - def Parentheses = rule { "(" | ")" | "[" | "]" | "{" | "}" } - def DelimiterChar = rule { "'" | "\"" | "." | ";" | "," } - - def WhitespaceChar = rule { "\u0020" | "\u0009" } - def Newline = rule { "\r\n" | "\n" } - def Semi = rule { ';' | oneOrMore(Newline) } - def OperatorChar = rule { - anyOf("""!#$%&*+-/:<=>?@\^|~""") | - CharPredicate.from(_.getType match { - case Character.OTHER_SYMBOL | Character.MATH_SYMBOL => true; case _ => false - }) - } - def Letter = rule { Upper | Lower | CharPredicate.from(c => c.isLetter | c.isDigit) } - def Lower = rule { "a" - "z" | "$" | "_" | CharPredicate.from(_.isLower) } - def Upper = rule { "A" - "Z" | CharPredicate.from(_.isUpper) } - } -} diff --git a/scalatexApi/src/main/scala/torimatomeru/syntax/Identifiers.scala b/scalatexApi/src/main/scala/torimatomeru/syntax/Identifiers.scala deleted file mode 100644 index 4be1d4e..0000000 --- a/scalatexApi/src/main/scala/torimatomeru/syntax/Identifiers.scala +++ /dev/null @@ -1,24 +0,0 @@ -package torimatomeru -package syntax -import acyclic.file -import org.parboiled2._ - -trait Identifiers { self: Parser with Basic => - object Identifiers{ - import Basic._ - def Operator = rule(oneOrMore(OperatorChar)) - - def VarId = rule { !(Keywords ~ (WhitespaceChar | Newline | "//" | "/*")) ~ Lower ~ IdRest } - def PlainId = rule { Upper ~ IdRest | VarId | !(Keywords ~ (WhitespaceChar | Newline | "//" | "/*")) ~ Operator } - def Id = rule { PlainId | ("`" ~ oneOrMore(noneOf("`")) ~ "`") } - def IdRest = rule { zeroOrMore(Letter | Digit) ~ optional("_" ~ Operator) } - - - def Keywords = rule { - "abstract" | "case" | "catch" | "class" | "def" | "do" | "else" | "extends" | "false" | "finally" | "final" | "finally" | "forSome" | "for" | "if" | - "implicit" | "import" | "lazy" | "match" | "new" | "null" | "object" | "override" | "package" | "private" | "protected" | "return" | - "sealed" | "super" | "this" | "throw" | "trait" | "try" | "true" | "type" | "val" | "var" | "while" | "with" | "yield" | "_" | - ":" | ";" | "=>" | "=" | "<-" | "<:" | "<%" | ">:" | "#" | "@" | "\u21d2" | "\u2190" - } - } -} diff --git a/scalatexApi/src/main/scala/torimatomeru/syntax/Literals.scala b/scalatexApi/src/main/scala/torimatomeru/syntax/Literals.scala deleted file mode 100644 index b199c30..0000000 --- a/scalatexApi/src/main/scala/torimatomeru/syntax/Literals.scala +++ /dev/null @@ -1,56 +0,0 @@ -package torimatomeru -package syntax -import acyclic.file -import org.parboiled2._ - -trait Literals { self: Parser with Basic with Identifiers => - object Literals{ - import Basic._ - def FloatingPointLiteral = rule { - capture( - "." ~ oneOrMore(Digit) ~ optional(ExponentPart) ~ optional(FloatType) | - oneOrMore(Digit) ~ ( - "." ~ oneOrMore(Digit) ~ optional(ExponentPart) ~ optional(FloatType) | - ExponentPart ~ optional(FloatType) | - optional(ExponentPart) ~ FloatType)) - } - - def IntegerLiteral = rule { capture((DecimalNumeral | HexNumeral) ~ optional(anyOf("Ll"))) } - - def BooleanLiteral = rule { capture("true" | "false") } - - def MultilineComment: Rule0 = rule { "/*" ~ zeroOrMore(MultilineComment | !"*/" ~ ANY) ~ "*/" } - def Comment: Rule0 = rule { - MultilineComment | - "//" ~ zeroOrMore(!Basic.Newline ~ ANY) ~ (Basic.Newline | EOI) - } - - def Literal = rule { - (capture(optional("-")) ~ (FloatingPointLiteral | IntegerLiteral) ~> ((sign: String, number) => sign + number)) | - BooleanLiteral | - CharacterLiteral | - StringLiteral | - SymbolLiteral | - capture("null") - } - - - def EscapedChars = rule { '\\' ~ anyOf("rnt\\\"") } - - def SymbolLiteral = rule { ''' ~ capture(Identifiers.PlainId) } - - def CharacterLiteral = rule { ''' ~ capture(UnicodeExcape | EscapedChars | !'\\' ~ CharPredicate.from(isPrintableChar)) ~ ''' } - - def MultiLineChars = rule { zeroOrMore(optional('"') ~ optional('"') ~ noneOf("\"")) } - def StringLiteral = rule { - ("\"\"\"" ~ capture(MultiLineChars) ~ capture("\"\"\"" ~ zeroOrMore('"')) ~> ((multilineChars: String, quotes) => multilineChars + quotes.dropRight(3))) | - ('"' ~ capture(zeroOrMore("\\\"" | noneOf("\n\""))) ~ '"') - } - - def isPrintableChar(c: Char): Boolean = { - val block = Character.UnicodeBlock.of(c) - !Character.isISOControl(c) && !Character.isSurrogate(c) && block != null && block != Character.UnicodeBlock.SPECIALS - } - } -} - diff --git a/scalatexApi/src/test/scala/scalaparser/SyntaxTest.scala b/scalatexApi/src/test/scala/scalaparser/SyntaxTest.scala new file mode 100644 index 0000000..d9f4820 --- /dev/null +++ b/scalatexApi/src/test/scala/scalaparser/SyntaxTest.scala @@ -0,0 +1,205 @@ +package scalaparser + +import org.parboiled2.ParseError +import utest._ +import utest.framework.Test +import utest.util.Tree + +import scala.util.{Failure, Success} + +object SyntaxTest extends TestSuite{ + def check[T](input: String) = { + new ScalaSyntax(input).CompilationUnit.run() match{ + case Failure(f: ParseError) => + println(f.position) + println(f.formatExpectedAsString) + println(f.formatTraces) + throw new Exception(f.position + "\t" + f.formatTraces) + case Success(parsed) => + assert(parsed == input) + } + } + def tests = TestSuite{ + 'unit { + * - check( + "package torimatomeru" + + ) + * - check( + """ + |package torimatomeru + | + |import org.parboiled2.ParseError + |import utest._ + |import utest.framework.Test + """.stripMargin + + ) + * - check( + """ + |package torimatomeru + | + |import org.parboiled2.ParseError + |import utest._ + |import utest.framework.Test + |import utest.util.Tree + | + |import scala.util.{Failure, Success} + | + |object SyntaxTest extends TestSuite + """.stripMargin + ) + * - check( + """ + |object SyntaxTest extends TestSuite{ + | def check[T](input: String) = { + | + | } + |} + """.stripMargin + ) + * - check( + """ + |object SyntaxTest{ + | a() + | throw 1 + |} + """.stripMargin + ) + * - check( + """ + |object SyntaxTest extends TestSuite{ + | def check[T](input: String) = { + | new ScalaSyntax(input).CompilationUnit.run() match{ + | case Failure(f: ParseError) => + | println(f.position) + | println(f.formatExpectedAsString) + | println(f.formatTraces) + | throw new Exception(f.position + "\t" + f.formatTraces) + | case Success(parsed) => + | assert(parsed == input) + | } + | } + |} + """.stripMargin + ) + * - check( + """package scalatex + | + | + |import org.parboiled2._ + |import torimatomeru.ScalaSyntax + | + |import scalatex.stages.{Trim, Parser, Ast} + |import scalatex.stages.Ast.Block.{IfElse, For, Text} + |import Ast.Chain.Args + | + |object ParserTests extends utest.TestSuite{ + | import Ast._ + | import utest._ + | def check[T](input: String, parse: Parser => scala.util.Try[T], expected: T) = { + | val parsed = parse(new Parser(input)).get + | assert(parsed == expected) + | } + | def tests = TestSuite{} + |} + """.stripMargin + ) + * - check( + """ + |object Moo{ + | a + | .b + | + | c + |} + """.stripMargin + ) + * - check( + """ + |object Moo{ + | filename + | .asInstanceOf[Literal] + |10 + |} + """.stripMargin + ) + * - check( + """ + |object Cow{ + | ().mkString + | + | 1 + |} + """.stripMargin + ) + * - check( + """ + |object O{ + | private[this] val applyMacroFull = 1 + |} + """.stripMargin + ) + * - check( + """ + |object O{ + | private[this] def applyMacroFull(c: Context) + | (expr: c.Expr[String], + | runtimeErrors: Boolean, + | debug: Boolean) + | : c.Expr[Frag] = { + | } + |} + """.stripMargin + ) + * - check( + """ + |object O{ + | class DebugFailure extends Exception + | + | 1 + |} + """.stripMargin + ) + * - check( + """ + |package torimatomeru + | + |package syntax + | + |import org.parboiled2._ + | + """.stripMargin + ) + * - check( + """ + |object Foo{ + | 0 match { + | case A | B => 0 + | } + |} + """.stripMargin + ) + } + 'file{ + def checkFile(path: String) = + check(io.Source.fromFile(path).mkString) + * - checkFile("scalatexApi/src/test/scala/scalaparser/SyntaxTest.scala") + * - checkFile("scalatexApi/src/main/scala/scalaparser/syntax/Basic.scala") + * - checkFile("scalatexApi/src/main/scala/scalaparser/syntax/Identifiers.scala") + * - checkFile("scalatexApi/src/main/scala/scalaparser/syntax/Literals.scala") + // All the commented files seem to make the parser run forever. There's probably + // some exponential performance somewhere in there, but I can't see it =/ + // * - check(io.Source.fromFile("scalatexApi/src/main/scala/scalaparser/ScalaSyntax.scala").mkString) + * - checkFile("scalatexApi/src/test/scala/scalatex/TestUtil.scala") + + + // * - check(io.Source.fromFile("scalatexApi/src/main/scala/scalatex/stages/Compiler.scala").mkString) + * - check(io.Source.fromFile("scalatexApi/src/main/scala/scalatex/stages/Parser.scala").mkString) // * - check(io.Source.fromFile("scalatexApi/src/test/scala/scalatex/ParserTests.scala").mkString) + // * - check(io.Source.fromFile("scalatexApi/src/main/scala/scalatex/stages/Trim.scala").mkString) + // * - check(io.Source.fromFile("scalatexApi/src/main/scala/scalatex/package.scala").mkString) + + } + } + +} diff --git a/scalatexApi/src/test/scala/scalatex/ParserTests.scala b/scalatexApi/src/test/scala/scalatex/ParserTests.scala index be24af9..a4b11b6 100644 --- a/scalatexApi/src/test/scala/scalatex/ParserTests.scala +++ b/scalatexApi/src/test/scala/scalatex/ParserTests.scala @@ -2,7 +2,7 @@ package scalatex import org.parboiled2._ -import torimatomeru.ScalaSyntax +import scalaparser.ScalaSyntax import scalatex.stages.{Trim, Parser, Ast} import scalatex.stages.Ast.Block.{IfElse, For, Text} diff --git a/scalatexApi/src/test/scala/torimatomeru/SyntaxTest.scala b/scalatexApi/src/test/scala/torimatomeru/SyntaxTest.scala deleted file mode 100644 index ed6dcf3..0000000 --- a/scalatexApi/src/test/scala/torimatomeru/SyntaxTest.scala +++ /dev/null @@ -1,201 +0,0 @@ -package torimatomeru - -import org.parboiled2.ParseError -import utest._ -import utest.framework.Test -import utest.util.Tree - -import scala.util.{Failure, Success} - -object SyntaxTest extends TestSuite{ - def check[T](input: String) = { - new ScalaSyntax(input).CompilationUnit.run() match{ - case Failure(f: ParseError) => - println(f.position) - println(f.formatExpectedAsString) - println(f.formatTraces) - throw new Exception(f.position + "\t" + f.formatTraces) - case Success(parsed) => - assert(parsed == input) - } - } - def tests = TestSuite{ - 'unit { - * - check( - "package torimatomeru" - - ) - * - check( - """ - |package torimatomeru - | - |import org.parboiled2.ParseError - |import utest._ - |import utest.framework.Test - """.stripMargin - - ) - * - check( - """ - |package torimatomeru - | - |import org.parboiled2.ParseError - |import utest._ - |import utest.framework.Test - |import utest.util.Tree - | - |import scala.util.{Failure, Success} - | - |object SyntaxTest extends TestSuite - """.stripMargin - ) - * - check( - """ - |object SyntaxTest extends TestSuite{ - | def check[T](input: String) = { - | - | } - |} - """.stripMargin - ) - * - check( - """ - |object SyntaxTest{ - | a() - | throw 1 - |} - """.stripMargin - ) - * - check( - """ - |object SyntaxTest extends TestSuite{ - | def check[T](input: String) = { - | new ScalaSyntax(input).CompilationUnit.run() match{ - | case Failure(f: ParseError) => - | println(f.position) - | println(f.formatExpectedAsString) - | println(f.formatTraces) - | throw new Exception(f.position + "\t" + f.formatTraces) - | case Success(parsed) => - | assert(parsed == input) - | } - | } - |} - """.stripMargin - ) - * - check( - """package scalatex - | - | - |import org.parboiled2._ - |import torimatomeru.ScalaSyntax - | - |import scalatex.stages.{Trim, Parser, Ast} - |import scalatex.stages.Ast.Block.{IfElse, For, Text} - |import Ast.Chain.Args - | - |object ParserTests extends utest.TestSuite{ - | import Ast._ - | import utest._ - | def check[T](input: String, parse: Parser => scala.util.Try[T], expected: T) = { - | val parsed = parse(new Parser(input)).get - | assert(parsed == expected) - | } - | def tests = TestSuite{} - |} - """.stripMargin - ) - * - check( - """ - |object Moo{ - | a - | .b - | - | c - |} - """.stripMargin - ) - * - check( - """ - |object Moo{ - | filename - | .asInstanceOf[Literal] - |10 - |} - """.stripMargin - ) - * - check( - """ - |object Cow{ - | ().mkString - | - | 1 - |} - """.stripMargin - ) - * - check( - """ - |object O{ - | private[this] val applyMacroFull = 1 - |} - """.stripMargin - ) - * - check( - """ - |object O{ - | private[this] def applyMacroFull(c: Context) - | (expr: c.Expr[String], - | runtimeErrors: Boolean, - | debug: Boolean) - | : c.Expr[Frag] = { - | } - |} - """.stripMargin - ) - * - check( - """ - |object O{ - | class DebugFailure extends Exception - | - | 1 - |} - """.stripMargin - ) - * - check( - """ - |package torimatomeru - | - |package syntax - | - |import org.parboiled2._ - | - """.stripMargin - ) - * - check( - """ - |object Foo{ - | 0 match { - | case A | B => 0 - | } - |} - """.stripMargin - ) - } - 'file{ - def checkFile(path: String) = - check(io.Source.fromFile(path).mkString) - * - checkFile("scalatexApi/src/test/scala/torimatomeru/SyntaxTest.scala") - * - checkFile("scalatexApi/src/test/scala/scalatex/TestUtil.scala") - * - checkFile("scalatexApi/src/main/scala/torimatomeru/syntax/Basic.scala") - * - checkFile("scalatexApi/src/main/scala/torimatomeru/syntax/Identifiers.scala") - * - checkFile("scalatexApi/src/main/scala/torimatomeru/syntax/Literals.scala") - -// All the larget files seem to make the parser run forever. There's probably -// some exponential performance somewhere in there, but I can't see it =/ -// * - check(io.Source.fromFile("scalatexApi/src/test/scala/scalatex/ParserTests.scala").mkString) -// * - check(io.Source.fromFile("scalatexApi/src/main/scala/scalatex/package.scala").mkString) -// * - check(io.Source.fromFile("scalatexApi/src/main/scala/torimatomeru/ScalaSyntax.scala").mkString) - } - } - -} -- cgit v1.2.3