From dbbb7a3d9a668bbb8b62bec38f065f2444dacb91 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 1 Feb 2017 14:02:45 +0100 Subject: Refactor templates and pages to deal with `SourceFile` This commit is the first step towards having reportable errors in the template files --- .../tools/dottydoc/staticsite/LiquidTemplate.scala | 34 --------- .../src/dotty/tools/dottydoc/staticsite/Page.scala | 57 ++++++++++----- .../src/dotty/tools/dottydoc/staticsite/Site.scala | 82 ++++++++++++++-------- .../dotty/tools/dottydoc/staticsite/Template.scala | 53 ++++++++++++++ .../src/dotty/tools/dottydoc/staticsite/tags.scala | 6 +- doc-tool/test/SourceFileOps.scala | 19 +++++ .../tools/dottydoc/staticsite/PageTests.scala | 70 +++++++++++------- .../tools/dottydoc/staticsite/SiteTests.scala | 7 +- 8 files changed, 217 insertions(+), 111 deletions(-) delete mode 100644 doc-tool/src/dotty/tools/dottydoc/staticsite/LiquidTemplate.scala create mode 100644 doc-tool/src/dotty/tools/dottydoc/staticsite/Template.scala create mode 100644 doc-tool/test/SourceFileOps.scala (limited to 'doc-tool') diff --git a/doc-tool/src/dotty/tools/dottydoc/staticsite/LiquidTemplate.scala b/doc-tool/src/dotty/tools/dottydoc/staticsite/LiquidTemplate.scala deleted file mode 100644 index e5873d1a9..000000000 --- a/doc-tool/src/dotty/tools/dottydoc/staticsite/LiquidTemplate.scala +++ /dev/null @@ -1,34 +0,0 @@ -package dotty.tools -package dottydoc -package staticsite - -import dotc.config.Printers.dottydoc - -case class LiquidTemplate(contents: String) extends ResourceFinder { - import scala.collection.JavaConverters._ - import liqp.Template - import liqp.filters.Filter - import liqp.parser.Flavor.JEKYLL - import java.util.{ HashMap, Map => JMap } - import filters._ - import tags._ - - /** Register filters to static container */ - Filter.registerFilter(new Reverse) - Filter.registerFilter(new First) - - // For some reason, liqp rejects a straight conversion using `.asJava` - private def toJavaMap(map: Map[String, AnyRef]): HashMap[String, Object] = - map.foldLeft(new HashMap[String, Object]()) { case (map, (k, v)) => - map.put(k, v) - map - } - - def render(params: Map[String, AnyRef], includes: Map[String, String]): String = - Template.parse(contents, JEKYLL) - .`with`(ResourceInclude(params, includes)) - .`with`(RenderReference(params)) - .`with`(RenderTitle(params)) - .`with`(Docstring(params)) - .render(toJavaMap(params)) -} diff --git a/doc-tool/src/dotty/tools/dottydoc/staticsite/Page.scala b/doc-tool/src/dotty/tools/dottydoc/staticsite/Page.scala index f250cbb01..fda41a234 100644 --- a/doc-tool/src/dotty/tools/dottydoc/staticsite/Page.scala +++ b/doc-tool/src/dotty/tools/dottydoc/staticsite/Page.scala @@ -2,35 +2,50 @@ package dotty.tools package dottydoc package staticsite -import dotc.config.Printers.dottydoc +import dotc.util.SourceFile import com.vladsch.flexmark.html.HtmlRenderer import com.vladsch.flexmark.parser.Parser import com.vladsch.flexmark.ext.front.matter.AbstractYamlFrontMatterVisitor +import java.util.{ Map => JMap, List => JList } +import dotc.config.Printers.dottydoc import model.Package -import java.util.{ Map => JMap, List => JList } - +/** When the YAML front matter cannot be parsed, this exception is thrown */ case class IllegalFrontMatter(message: String) extends Exception(message) trait Page { import scala.collection.JavaConverters._ - def includes: Map[String, String] - def pageContent: String + /** Full map of includes, from name to contents */ + def includes: Map[String, Include] + + /** `SourceFile` with contents of page */ + def sourceFile: SourceFile + + /** String containing full unexpanded content of page */ + final lazy val content: String = new String(sourceFile.content) + + /** Parameters to page */ def params: Map[String, AnyRef] + /** Path to template */ + def path: String + + /** YAML front matter from the top of the file */ def yaml: Map[String, AnyRef] = { if (_yaml eq null) initFields() _yaml } + /** HTML generated from page */ def html: String = { if (_html eq null) initFields() _html } + /** First paragraph of page extracted from rendered HTML */ def firstParagraph: String = { if (_html eq null) initFields() @@ -62,7 +77,7 @@ trait Page { protected[this] var _yaml: Map[String, AnyRef /* String | JList[String] */] = _ protected[this] var _html: String = _ protected[this] def initFields() = { - val md = Parser.builder(Site.markdownOptions).build.parse(pageContent) + val md = Parser.builder(Site.markdownOptions).build.parse(content) val yamlCollector = new AbstractYamlFrontMatterVisitor() yamlCollector.visit(md) @@ -82,21 +97,21 @@ trait Page { // YAML must start with "---" and end in either "---" or "..." val withoutYaml = - if (pageContent.startsWith("---\n")) { + if (content.startsWith("---\n")) { val str = - pageContent.lines + content.lines .drop(1) .dropWhile(line => line != "---" && line != "...") .drop(1).mkString("\n") - if (str.isEmpty) throw IllegalFrontMatter(pageContent) + if (str.isEmpty) throw IllegalFrontMatter(content) else str } - else pageContent + else content // make accessible via "{{ page.title }}" in templates val page = Map("page" -> _yaml.asJava) - _html = LiquidTemplate(withoutYaml).render(params ++ page, includes) + _html = LiquidTemplate(path, withoutYaml).render(params ++ page, includes) } /** Takes "page" from `params` map in case this is a second expansion, and @@ -114,12 +129,20 @@ trait Page { .getOrElse(newYaml) } -class HtmlPage(fileContents: => String, val params: Map[String, AnyRef], val includes: Map[String, String]) extends Page { - lazy val pageContent = fileContents -} - -class MarkdownPage(fileContents: => String, val params: Map[String, AnyRef], val includes: Map[String, String], docs: Map[String, Package]) extends Page { - lazy val pageContent = fileContents +class HtmlPage( + val path: String, + val sourceFile: SourceFile, + val params: Map[String, AnyRef], + val includes: Map[String, Include] +) extends Page + +class MarkdownPage( + val path: String, + val sourceFile: SourceFile, + val params: Map[String, AnyRef], + val includes: Map[String, Include], + docs: Map[String, Package] +) extends Page { override protected[this] def initFields() = { super.initFields() diff --git a/doc-tool/src/dotty/tools/dottydoc/staticsite/Site.scala b/doc-tool/src/dotty/tools/dottydoc/staticsite/Site.scala index 4d4dbc75d..05ec113e0 100644 --- a/doc-tool/src/dotty/tools/dottydoc/staticsite/Site.scala +++ b/doc-tool/src/dotty/tools/dottydoc/staticsite/Site.scala @@ -4,7 +4,7 @@ package staticsite import java.nio.file.{ Files, FileSystems } import java.nio.file.StandardCopyOption.REPLACE_EXISTING -import java.io.{ File => JFile } +import java.io.{ File => JFile, OutputStreamWriter, BufferedWriter } import java.util.{ List => JList, Map => JMap, Arrays } import java.nio.file.Path import java.io.ByteArrayInputStream @@ -23,8 +23,10 @@ import com.vladsch.flexmark.util.options.{ DataHolder, MutableDataSet } import dotc.config.Printers.dottydoc import dotc.core.Contexts.Context +import dotc.util.SourceFile import model.Package -import scala.io.Source +import scala.io.{ Codec, Source } +import io.{ AbstractFile, VirtualFile, File } import scala.collection.mutable.ArrayBuffer case class Site(val root: JFile, val projectTitle: String, val documentation: Map[String, Package]) extends ResourceFinder { @@ -69,7 +71,7 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma _blogposts } - + /** TODO */ val sidebar: Sidebar = root .listFiles @@ -83,11 +85,12 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma blogposts .map { file => val BlogPost.extract(year, month, day, name, ext) = file.getName - val fileContents = Source.fromFile(file).mkString + val sourceFile = toSourceFile(file) val params = defaultParams(file, 2).withUrl(s"/blog/$year/$month/$day/$name.html").toMap val page = - if (ext == "md") new MarkdownPage(fileContents, params, includes, documentation) - else new HtmlPage(fileContents, params, includes) + if (ext == "md") + new MarkdownPage(file.getPath, sourceFile, params, includes, documentation) + else new HtmlPage(file.getPath, sourceFile, params, includes) BlogPost(file, page) } .sortBy(_.date) @@ -96,6 +99,15 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma // FileSystem getter private[this] val fs = FileSystems.getDefault + private def stringToSourceFile(name: String, path: String, sourceCode: String): SourceFile = { + val virtualFile = new VirtualFile(name, path) + val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8")) + writer.write(sourceCode) + writer.close() + + new SourceFile(virtualFile, Codec.UTF8) + } + def copyStaticFiles(outDir: JFile = new JFile(root.getAbsolutePath + "/_site")): this.type = { if (!outDir.isDirectory) outDir.mkdirs() if (!outDir.isDirectory) /*dottydoc.*/println(s"couldn't create output folder: $outDir") @@ -156,7 +168,7 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma val target = mkdirs(fs.getPath(outDir.getAbsolutePath + "/api/" + e.path.mkString("/") + suffix)) val params = defaultParams(target.toFile, -1).withPosts(blogInfo).withEntity(e).toMap - val page = new HtmlPage(layouts("api-page"), params, includes) + val page = new HtmlPage("_layouts/api-page.html", layouts("api-page").content, params, includes) val rendered = render(page) val source = new ByteArrayInputStream(rendered.getBytes(StandardCharsets.UTF_8)) @@ -178,11 +190,11 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma createOutput(outDir) { compilableFiles.foreach { asset => val pathFromRoot = stripRoot(asset) - val fileContents = Source.fromFile(asset).mkString + val sourceFile = toSourceFile(asset) val params = defaultParams(asset).withPosts(blogInfo).toMap val page = - if (asset.getName.endsWith(".md")) new MarkdownPage(fileContents, params, includes, documentation) - else new HtmlPage(fileContents, params, includes) + if (asset.getName.endsWith(".md")) new MarkdownPage(pathFromRoot, sourceFile, params, includes, documentation) + else new HtmlPage(pathFromRoot, sourceFile, params, includes) val renderedPage = render(page) val source = new ByteArrayInputStream(renderedPage.getBytes(StandardCharsets.UTF_8)) @@ -196,16 +208,21 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma createOutput(outDir) { blogposts.foreach { file => val BlogPost.extract(year, month, day, name, ext) = file.getName - val fileContents = Source.fromFile(file).mkString + val sourceFile = toSourceFile(file) val date = s"$year-$month-$day 00:00:00" val params = defaultParams(file, 2).withPosts(blogInfo).withDate(date).toMap + + // Output target + val target = mkdirs(fs.getPath(outDir.getAbsolutePath, "blog", year, month, day, name + ".html")) + val page = - if (ext == "md") new MarkdownPage(fileContents, params, includes, documentation) - else new HtmlPage(fileContents, params, includes) + if (ext == "md") + new MarkdownPage(target.toString, sourceFile, params, includes, documentation) + else + new HtmlPage(target.toString, sourceFile, params, includes) val source = new ByteArrayInputStream(render(page).getBytes(StandardCharsets.UTF_8)) - val target = mkdirs(fs.getPath(outDir.getAbsolutePath, "blog", year, month, day, name + ".html")) Files.copy(source, target, REPLACE_EXISTING) } } @@ -283,22 +300,26 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma * If the user supplies a layout that has the same name as one of the * defaults, the user-defined one will take precedence. */ - val layouts: Map[String, String] = { + val layouts: Map[String, Layout] = { val userDefinedLayouts = root .listFiles.find(d => d.getName == "_layouts" && d.isDirectory) .map(collectFiles(_, f => f.endsWith(".md") || f.endsWith(".html"))) - .getOrElse(Map.empty) - .map { case (k, v) => (k.substring(0, k.lastIndexOf('.')), v) } + .getOrElse(Array.empty[JFile]) + .map(f => (f.getName.substring(0, f.getName.lastIndexOf('.')), Layout(f.getPath, toSourceFile(f)))) + .toMap - val defaultLayouts: Map[String, String] = Map( + val defaultLayouts: Map[String, Layout] = Map( "main" -> "/_layouts/main.html", "sidebar" -> "/_layouts/sidebar.html", "doc-page" -> "/_layouts/doc-page.html", "api-page" -> "/_layouts/api-page.html", "blog-page" -> "/_layouts/blog-page.html", "index" -> "/_layouts/index.html" - ).mapValues(getResource) + ).map { + case (name, path) => + (name, Layout(path, stringToSourceFile(name, path, getResource(path)))) + } defaultLayouts ++ userDefinedLayouts } @@ -316,29 +337,34 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma * {% include "some-file" with { key: value } %} * ``` */ - val includes: Map[String, String] = { + val includes: Map[String, Include] = { val userDefinedIncludes = root .listFiles.find(d => d.getName == "_includes" && d.isDirectory) .map(collectFiles(_, f => f.endsWith(".md") || f.endsWith(".html"))) - .getOrElse(Map.empty) + .getOrElse(Array.empty[JFile]) + .map(f => (f.getName, Include(f.getPath, toSourceFile(f)))) + .toMap - val defaultIncludes: Map[String, String] = Map( + val defaultIncludes: Map[String, Include] = Map( "header.html" -> "/_includes/header.html", "scala-logo.svg" -> "/_includes/scala-logo.svg", "toc.html" -> "/_includes/toc.html" - ).mapValues(getResource) - + ).map { + case (name, path) => + (name, Include(path, stringToSourceFile(name, path, getResource(path)))) + } defaultIncludes ++ userDefinedIncludes } - private def collectFiles(dir: JFile, includes: String => Boolean): Map[String, String] = + private def toSourceFile(f: JFile): SourceFile = + SourceFile(AbstractFile.getFile(new File(f)), Source.fromFile(f).toArray) + + private def collectFiles(dir: JFile, includes: String => Boolean): Array[JFile] = dir .listFiles .filter(f => includes(f.getName)) - .map(f => (f.getName, Source.fromFile(f).mkString)) - .toMap /** Render a page to html, the resulting string is the result of the complete * expansion of the template with all its layouts and includes. @@ -350,7 +376,7 @@ case class Site(val root: JFile, val projectTitle: String, val documentation: Ma case Some(layout) => import scala.collection.JavaConverters._ val newParams = page.params ++ params ++ Map("page" -> page.yaml) ++ Map("content" -> page.html) - val expandedTemplate = new HtmlPage(layout, newParams, includes) + val expandedTemplate = new HtmlPage(layout.path, layout.content, newParams, includes) render(expandedTemplate, params) } } diff --git a/doc-tool/src/dotty/tools/dottydoc/staticsite/Template.scala b/doc-tool/src/dotty/tools/dottydoc/staticsite/Template.scala new file mode 100644 index 000000000..00d51f083 --- /dev/null +++ b/doc-tool/src/dotty/tools/dottydoc/staticsite/Template.scala @@ -0,0 +1,53 @@ +package dotty.tools +package dottydoc +package staticsite + +import scala.util.control.NonFatal +import dotc.util.SourceFile + +trait Template { + def path: String + def content: SourceFile + def show: String = new String(content.content) +} + +case class TemplateRenderingError(path: String, ex: Throwable) +extends Exception(s"error rendering $path, $ex") + +case class Layout(path: String, content: SourceFile) extends Template + +case class Include(path: String, content: SourceFile) extends Template + +case class LiquidTemplate(path: String, content: String) extends ResourceFinder { + import scala.collection.JavaConverters._ + import liqp.Template + import liqp.filters.Filter + import liqp.parser.Flavor.JEKYLL + import java.util.{ HashMap, Map => JMap } + import filters._ + import tags._ + + /** Register filters to static container */ + Filter.registerFilter(new Reverse) + Filter.registerFilter(new First) + + // For some reason, liqp rejects a straight conversion using `.asJava` + private def toJavaMap(map: Map[String, AnyRef]): HashMap[String, Object] = + map.foldLeft(new HashMap[String, Object]()) { case (map, (k, v)) => + map.put(k, v) + map + } + + def render(params: Map[String, AnyRef], includes: Map[String, Include]): String = + try { + Template.parse(content, JEKYLL) + .`with`(ResourceInclude(params, includes)) + .`with`(RenderReference(params)) + .`with`(RenderTitle(params)) + .`with`(Docstring(params)) + .render(toJavaMap(params)) + } catch { + case NonFatal(ex) => + throw TemplateRenderingError(path, ex) + } +} diff --git a/doc-tool/src/dotty/tools/dottydoc/staticsite/tags.scala b/doc-tool/src/dotty/tools/dottydoc/staticsite/tags.scala index 7f85846dc..51b3b760f 100644 --- a/doc-tool/src/dotty/tools/dottydoc/staticsite/tags.scala +++ b/doc-tool/src/dotty/tools/dottydoc/staticsite/tags.scala @@ -108,7 +108,7 @@ object tags { } } - case class ResourceInclude(params: Map[String, AnyRef], includes: Map[String, String]) + case class ResourceInclude(params: Map[String, AnyRef], includes: Map[String, Include]) extends Tag("include") { import scala.collection.JavaConverters._ val DefaultExtension = ".html" @@ -124,7 +124,9 @@ object tags { .get(incResource) .map { template => if (nodes.length > 1) ctx.put(origInclude, nodes(1).render(ctx)) - LiquidTemplate(template).render(Map.empty ++ ctx.getVariables.asScala, includes) + + LiquidTemplate(template.path, template.show) + .render(ctx.getVariables.asScala.toMap, includes) } .getOrElse { /*dottydoc.*/println(s"couldn't find include file '$origInclude'") diff --git a/doc-tool/test/SourceFileOps.scala b/doc-tool/test/SourceFileOps.scala new file mode 100644 index 000000000..7b0c2e807 --- /dev/null +++ b/doc-tool/test/SourceFileOps.scala @@ -0,0 +1,19 @@ +package dotty.tools +package dottydoc +package staticsite + +import dotc.util.SourceFile +import java.io.{ BufferedWriter, OutputStreamWriter } +import io.VirtualFile +import scala.io.Codec + +trait SourceFileOps { + def stringToSource(path: String, sourceCode: String): SourceFile = { + val virtualFile = new VirtualFile(path, path) + val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8")) + writer.write(sourceCode) + writer.close() + + new SourceFile(virtualFile, Codec.UTF8) + } +} diff --git a/doc-tool/test/dotty/tools/dottydoc/staticsite/PageTests.scala b/doc-tool/test/dotty/tools/dottydoc/staticsite/PageTests.scala index 14886b681..20a41e70b 100644 --- a/doc-tool/test/dotty/tools/dottydoc/staticsite/PageTests.scala +++ b/doc-tool/test/dotty/tools/dottydoc/staticsite/PageTests.scala @@ -5,19 +5,45 @@ package staticsite import org.junit.Test import org.junit.Assert._ -class PageTests extends DottyDocTest { +import model.Package + +class PageTests extends DottyDocTest with SourceFileOps { import scala.collection.JavaConverters._ + private def markdownPage( + sourceCode: String, + path: String = "test-page", + params: Map[String, AnyRef] = Map.empty, + includes: Map[String, Include] = Map.empty, + docs: Map[String, Package] = Map.empty + ) = new MarkdownPage( + path, + stringToSource(path, sourceCode), + params, + includes, + docs + ) + + private def htmlPage( + sourceCode: String, + path: String = "test-page", + params: Map[String, AnyRef] = Map.empty, + includes: Map[String, Include] = Map.empty, + docs: Map[String, Package] = Map.empty + ) = new HtmlPage( + path, + stringToSource(path, sourceCode), + params, + includes + ) + @Test def mdHas1Key = { - val page = new MarkdownPage( + val page = markdownPage( """|--- |key: |--- | - |great""".stripMargin, - Map.empty, - Map.empty, - Map.empty + |great""".stripMargin ) assert( @@ -29,15 +55,13 @@ class PageTests extends DottyDocTest { } @Test def yamlPreservesLiquidTags = { - val page1 = new MarkdownPage( + val page1 = markdownPage( """|--- |key: |--- | |{{ content }}""".stripMargin, - Map("content" -> "Hello, world!"), - Map.empty, - Map.empty + params = Map("content" -> "Hello, world!") ) assert( @@ -47,11 +71,9 @@ class PageTests extends DottyDocTest { assertEquals("

Hello, world!

\n", page1.html) - val page2 = new MarkdownPage( + val page2 = markdownPage( """|{{ content }}""".stripMargin, - Map("content" -> "hello"), - Map.empty, - Map.empty + params = Map("content" -> "hello") ) assert( page2.yaml == Map(), @@ -59,13 +81,11 @@ class PageTests extends DottyDocTest { ) assertEquals("

hello

\n", page2.html) - val page3 = new MarkdownPage( + val page3 = markdownPage( """|{% if product.title == "Awesome Shoes" %} |These shoes are awesome! |{% endif %}""".stripMargin, - Map("product" -> Map("title" -> "Awesome Shoes").asJava), - Map.empty, - Map.empty + params = Map("product" -> Map("title" -> "Awesome Shoes").asJava) ) assertEquals( @@ -75,20 +95,18 @@ class PageTests extends DottyDocTest { } @Test def simpleHtmlPage = { - val p1 = new HtmlPage("""

{{ "hello, world!" }}

""", Map.empty, Map.empty) + val p1 = htmlPage("""

{{ "hello, world!" }}

""") assert(p1.yaml == Map(), "non-empty yaml found") assertEquals("

hello, world!

", p1.html) } @Test def htmlPageHasNoYaml = { - val page = new HtmlPage( + val page = htmlPage( """|--- |layout: main |--- | - |Hello, world!""".stripMargin, - Map.empty, - Map.empty + |Hello, world!""".stripMargin ) assert(!page.html.contains("---\nlayout: main\n---"), @@ -96,14 +114,12 @@ class PageTests extends DottyDocTest { } @Test def illegalYamlFrontMatter = try { - val page = new HtmlPage( + val page = htmlPage( """|--- |layout: main | | - |Hello, world!""".stripMargin, - Map.empty, - Map.empty + |Hello, world!""".stripMargin ) page.html diff --git a/doc-tool/test/dotty/tools/dottydoc/staticsite/SiteTests.scala b/doc-tool/test/dotty/tools/dottydoc/staticsite/SiteTests.scala index ba431a5c9..77b49700c 100644 --- a/doc-tool/test/dotty/tools/dottydoc/staticsite/SiteTests.scala +++ b/doc-tool/test/dotty/tools/dottydoc/staticsite/SiteTests.scala @@ -5,15 +5,16 @@ package staticsite import org.junit.Test import org.junit.Assert._ -class SiteTests extends DottyDocTest { +class SiteTests extends DottyDocTest with SourceFileOps { import scala.collection.JavaConverters._ val site = new Site(new java.io.File("../doc-tool/resources/"), "test-site", Map.empty) private def html( str: String, + path: String = "test-page", params: Map[String, AnyRef] = Map("docs" -> List.empty.asJava), - includes: Map[String, String] = Map.empty - ) = new HtmlPage(str, params, includes) + includes: Map[String, Include] = Map.empty + ) = new HtmlPage(path, stringToSource(path, str), params, includes) @Test def hasCorrectLayoutFiles = { assert(site.root.exists && site.root.isDirectory, -- cgit v1.2.3