General housekeeping update.

- Bump version to 0.6.0
- Add template functions to merge dictionaries for loading JSON data inside them
- Add extension management separate from MIME type
- Make the `tembed` processor which runs a generic jinja template through embedding in the template

TODO.md

@@ -4,3 +4,5 @@
 * Project global defines, parameters.
 * pre- and post-scripts that will be run from __main__, either some shipped with pixywerk or project-level.
 * Library of template modules? ATOM et al.
+* Some off the shelf website templates and a template manager.
+* Live refreshing server thing which maps a pixywerk tree into a web server's memory and updates on change.

examples/pixywerk.com/publish/atom.xml

@@ -7,7 +7,7 @@
 	<link href="https://pixywerk.com//atom.xml" rel="self" />
 	<link href="https://pixywerk.com/" />
 	<id>urn:uuid:2cbb1961-b1ca-3b73-a6ce-d2e2feae9ab4</id>
-	<updated>2019-05-20T06:32:27.357672+00:00</updated>
+	<updated>2021-02-05T04:45:24.766181+00:00</updated>
 
 	
 	

examples/pixywerk.com/publish/css/main.css

@@ -267,6 +267,11 @@ footer {
     font-variant: small-caps;
 }
 
+pre { line-height: 125%; margin: 0; }
+td.linenos pre { color: #000000; background-color: #f0f0f0; padding: 0 5px 0 5px; }
+span.linenos { color: #000000; background-color: #f0f0f0; padding: 0 5px 0 5px; }
+td.linenos pre.special { color: #000000; background-color: #ffffc0; padding: 0 5px 0 5px; }
+span.linenos.special { color: #000000; background-color: #ffffc0; padding: 0 5px 0 5px; }
 .hll { background-color: #ffffcc }
 .c { color: #008800; font-style: italic } /* Comment */
 .err { border: 1px solid #FF0000 } /* Error */

examples/pixywerk.com/publish/posts/post-2019-04-15.html

@@ -16,7 +16,7 @@
       <h1>My first post</h1>
       <img src="../images/20190415-0.jpg" class="featured">
     </div>
-  <div class="byline">
+  <div class="byline" style="width: 100%">
   <p>Author: Cas Rusnov<br>
     Published: 2019-05-20T01:56:41.720834+00:00
     

examples/pixywerk.com/publish/posts/post-2019-05-16.another-post.html

@@ -16,7 +16,7 @@
       <h1>Another example post!</h1>
       <img src="../images/2019-05-16.png" class="featured">
     </div>
-  <div class="byline">
+  <div class="byline" style="width: 100%">
   <p>Author: Cas Rusnov<br>
     Published: 2019-05-20T02:09:55.805659+00:00
     

examples/pixywerk.com/publish/posts/post-2019-05-19.html

@@ -16,7 +16,7 @@
       <h1>Code Test Post</h1>
       <img src="../images/2019-05-19.png" class="featured">
     </div>
-  <div class="byline">
+  <div class="byline" style="width: 100%">
   <p>Author: Cas Rusnov<br>
     Published: 2019-05-20T06:31:37.352443+00:00
     
@@ -74,7 +74,7 @@ Some python code:
     <span class="c1">#        &#39;cookbook = spicerack.cookbook:main&#39;,</span>
     <span class="c1">#    ],</span>
     <span class="c1"># },</span>
-    <span class="n">include_package_data</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
+    <span class="n">include_package_data</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
     <span class="n">extras_require</span><span class="o">=</span><span class="n">EXTRAS_REQUIRE</span><span class="p">,</span>
     <span class="n">install_requires</span><span class="o">=</span><span class="n">INSTALL_REQUIRES</span><span class="p">,</span>
     <span class="n">keywords</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;cms&quot;</span><span class="p">,</span> <span class="s2">&quot;website&quot;</span><span class="p">,</span> <span class="s2">&quot;compiler&quot;</span><span class="p">],</span>
@@ -84,9 +84,9 @@ Some python code:
     <span class="n">packages</span><span class="o">=</span><span class="n">find_packages</span><span class="p">(</span><span class="n">exclude</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;*.tests&quot;</span><span class="p">,</span> <span class="s2">&quot;*.tests.*&quot;</span><span class="p">]),</span>
     <span class="n">platforms</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;GNU/Linux&quot;</span><span class="p">],</span>
     <span class="n">setup_requires</span><span class="o">=</span><span class="n">SETUP_REQUIRES</span><span class="p">,</span>
-    <span class="n">use_scm_version</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
+    <span class="n">use_scm_version</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
     <span class="n">url</span><span class="o">=</span><span class="s2">&quot;https://git.antpanethon.com/cas/pixywerk2&quot;</span><span class="p">,</span>
-    <span class="n">zip_safe</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+    <span class="n">zip_safe</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
 <span class="p">)</span>
 </pre></div>
 

examples/pixywerk.com/publish/recurse.html

@@ -0,0 +1,134 @@
+<html>
+  <head>
+    <title>Pixywerk2 </title>
+    <link rel="stylesheet" type="text/css" href="./css/main.css">
+  </head>
+  <body>
+    <div class="container">
+      
+	<header><img src="./images/pipe-leak.svg" style="filter: invert(1); height: 2em; float: left; margin: 2px; padding: 0"><h1>Pixywerk2 </h1></header>
+      
+       
+	 <article>
+	   <ul>
+
+<li>{'file_path': 'atom.xml', 'file_name': 'atom.xml', 'mtime': 1558060013.0402112, 'ctime': 1558060013.0482113, 'size': 1209, 'ext': '.xml'}</li>
+
+<li>{'file_path': 'index.thtml', 'file_name': 'index.thtml', 'mtime': 1558317628.2501144, 'ctime': 1558317628.2501144, 'size': 602, 'ext': '.thtml'}</li>
+
+<li>{'file_path': 'setup.py', 'file_name': 'setup.py', 'mtime': 1558332717.6640935, 'ctime': 1558332717.6640935, 'size': 1847, 'ext': '.py'}</li>
+
+<li>{'file_path': 'recurse.thtml', 'file_name': 'recurse.thtml', 'mtime': 1561830991.2152739, 'ctime': 1561830991.223274, 'size': 72, 'ext': '.thtml'}</li>
+
+<li>{'file_path': 'posts/post-2019-04-15.thtml', 'file_name': 'post-2019-04-15.thtml', 'mtime': 1557985040.3647952, 'ctime': 1558317401.7208338, 'size': 106, 'ext': '.thtml'}</li>
+
+<li>{'file_path': 'posts/post-2019-05-16.another-post.thtml', 'file_name': 'post-2019-05-16.another-post.thtml', 'mtime': 1558318195.8056588, 'ctime': 1558318195.8056588, 'size': 79, 'ext': '.thtml'}</li>
+
+<li>{'file_path': 'posts/post-2019-05-19.thtml', 'file_name': 'post-2019-05-19.thtml', 'mtime': 1558333897.3524435, 'ctime': 1558333897.3524435, 'size': 181, 'ext': '.thtml'}</li>
+
+<li>{'file_path': 'css/main.css', 'file_name': 'main.css', 'mtime': 1558333943.9771295, 'ctime': 1558333943.9771295, 'size': 5589, 'ext': '.css'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Bold.woff', 'file_name': 'Teko-Bold.woff', 'mtime': 1557719150.1630452, 'ctime': 1557719150.1630452, 'size': 111224, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Bold.woff2', 'file_name': 'Teko-Bold.woff2', 'mtime': 1557719150.8430567, 'ctime': 1557719150.8430567, 'size': 77468, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Light.woff', 'file_name': 'Teko-Light.woff', 'mtime': 1557719150.863057, 'ctime': 1557719150.863057, 'size': 108568, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Light.woff2', 'file_name': 'Teko-Light.woff2', 'mtime': 1557719151.4950676, 'ctime': 1557719151.4950676, 'size': 75476, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Medium.woff', 'file_name': 'Teko-Medium.woff', 'mtime': 1557719151.5150678, 'ctime': 1557719151.5150678, 'size': 115388, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Medium.woff2', 'file_name': 'Teko-Medium.woff2', 'mtime': 1557719152.1750789, 'ctime': 1557719152.1750789, 'size': 80216, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Regular.woff', 'file_name': 'Teko-Regular.woff', 'mtime': 1557719152.1950793, 'ctime': 1557719152.1950793, 'size': 115048, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-Regular.woff2', 'file_name': 'Teko-Regular.woff2', 'mtime': 1557719152.8550904, 'ctime': 1557719152.8550904, 'size': 80188, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-SemiBold.woff', 'file_name': 'Teko-SemiBold.woff', 'mtime': 1557719152.8750906, 'ctime': 1557719152.8750906, 'size': 117184, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/Teko-SemiBold.woff2', 'file_name': 'Teko-SemiBold.woff2', 'mtime': 1557719153.5591023, 'ctime': 1557719153.5591023, 'size': 81756, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Regular.ttf', 'file_name': 'ExpletusSans-Regular.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6432533, 'size': 55052, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Italic.ttf', 'file_name': 'ExpletusSans-Italic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6472535, 'size': 37344, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Medium.ttf', 'file_name': 'ExpletusSans-Medium.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6472535, 'size': 66676, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-MediumItalic.ttf', 'file_name': 'ExpletusSans-MediumItalic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6512535, 'size': 62900, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-SemiBoldItalic.ttf', 'file_name': 'ExpletusSans-SemiBoldItalic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6552536, 'size': 46812, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-SemiBold.ttf', 'file_name': 'ExpletusSans-SemiBold.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6552536, 'size': 65520, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Bold.ttf', 'file_name': 'ExpletusSans-Bold.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6592536, 'size': 60784, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-BoldItalic.ttf', 'file_name': 'ExpletusSans-BoldItalic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6632538, 'size': 45072, 'ext': '.ttf'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Bold.woff', 'file_name': 'ExpletusSans-Bold.woff', 'mtime': 1557720055.5434017, 'ctime': 1557720055.5434017, 'size': 28176, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Bold.woff2', 'file_name': 'ExpletusSans-Bold.woff2', 'mtime': 1557720055.6754043, 'ctime': 1557720055.6754043, 'size': 20628, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-BoldItalic.woff', 'file_name': 'ExpletusSans-BoldItalic.woff', 'mtime': 1557720055.6794043, 'ctime': 1557720055.6794043, 'size': 21016, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-BoldItalic.woff2', 'file_name': 'ExpletusSans-BoldItalic.woff2', 'mtime': 1557720055.767406, 'ctime': 1557720055.767406, 'size': 15356, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Italic.woff', 'file_name': 'ExpletusSans-Italic.woff', 'mtime': 1557720055.7754061, 'ctime': 1557720055.7754061, 'size': 19132, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Italic.woff2', 'file_name': 'ExpletusSans-Italic.woff2', 'mtime': 1557720055.8514075, 'ctime': 1557720055.8514075, 'size': 14248, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Medium.woff', 'file_name': 'ExpletusSans-Medium.woff', 'mtime': 1557720055.8554077, 'ctime': 1557720055.8554077, 'size': 31592, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Medium.woff2', 'file_name': 'ExpletusSans-Medium.woff2', 'mtime': 1557720056.0074105, 'ctime': 1557720056.0074105, 'size': 22784, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-MediumItalic.woff', 'file_name': 'ExpletusSans-MediumItalic.woff', 'mtime': 1557720056.0154107, 'ctime': 1557720056.0154107, 'size': 27148, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-MediumItalic.woff2', 'file_name': 'ExpletusSans-MediumItalic.woff2', 'mtime': 1557720056.1594133, 'ctime': 1557720056.1594133, 'size': 19972, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Regular.woff', 'file_name': 'ExpletusSans-Regular.woff', 'mtime': 1557720056.1674135, 'ctime': 1557720056.1674135, 'size': 26540, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-Regular.woff2', 'file_name': 'ExpletusSans-Regular.woff2', 'mtime': 1557720056.291416, 'ctime': 1557720056.291416, 'size': 19844, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-SemiBold.woff', 'file_name': 'ExpletusSans-SemiBold.woff', 'mtime': 1557720056.2954159, 'ctime': 1557720056.2954159, 'size': 31536, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-SemiBold.woff2', 'file_name': 'ExpletusSans-SemiBold.woff2', 'mtime': 1557720056.459419, 'ctime': 1557720056.459419, 'size': 22784, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-SemiBoldItalic.woff', 'file_name': 'ExpletusSans-SemiBoldItalic.woff', 'mtime': 1557720056.4634192, 'ctime': 1557720056.4634192, 'size': 22556, 'ext': '.woff'}</li>
+
+<li>{'file_path': 'css/fonts/ExpletusSans-SemiBoldItalic.woff2', 'file_name': 'ExpletusSans-SemiBoldItalic.woff2', 'mtime': 1557720056.563421, 'ctime': 1557720056.563421, 'size': 16484, 'ext': '.woff2'}</li>
+
+<li>{'file_path': 'test/test.thtml', 'file_name': 'test.thtml', 'mtime': 1558064341.65635, 'ctime': 1558064341.65635, 'size': 115, 'ext': '.thtml'}</li>
+
+<li>{'file_path': 'images/placeholder', 'file_name': 'placeholder', 'mtime': 1555378617.4188664, 'ctime': 1555378617.4188664, 'size': 0, 'ext': ''}</li>
+
+<li>{'file_path': 'images/20190415-0.jpg', 'file_name': '20190415-0.jpg', 'mtime': 1555379532.14708, 'ctime': 1555379532.14708, 'size': 73996, 'ext': '.jpg'}</li>
+
+<li>{'file_path': 'images/link-box-variant.svg', 'file_name': 'link-box-variant.svg', 'mtime': 1557815009.004399, 'ctime': 1557815052.4972086, 'size': 1402, 'ext': '.svg'}</li>
+
+<li>{'file_path': 'images/pipe-leak.svg', 'file_name': 'pipe-leak.svg', 'mtime': 1557897588.8718655, 'ctime': 1557897849.3715317, 'size': 448, 'ext': '.svg'}</li>
+
+<li>{'file_path': 'images/2019-05-16.png', 'file_name': '2019-05-16.png', 'mtime': 1558057040.7048073, 'ctime': 1558057040.7048073, 'size': 457198, 'ext': '.png'}</li>
+
+<li>{'file_path': 'images/2019-05-19.png', 'file_name': '2019-05-19.png', 'mtime': 1558331552.2705755, 'ctime': 1558331552.2705755, 'size': 135915, 'ext': '.png'}</li>
+
+<li>{'file_path': 'templates/default.jinja2', 'file_name': 'default.jinja2', 'mtime': 1558317843.4078112, 'ctime': 1558317843.4078112, 'size': 1444, 'ext': '.jinja2'}</li>
+
+<li>{'file_path': 'templates/post.jinja2', 'file_name': 'post.jinja2', 'mtime': 1558498337.0504873, 'ctime': 1558498337.0504873, 'size': 646, 'ext': '.jinja2'}</li>
+
+</ul>
+	 </article>
+       
+       
+	 <nav><ul>
+	     <li>Documentation</li>
+	     <li>Blog</li>
+	     <li><a href="https://git.antpantheon.com/cas/pixywerk2">Gitea<img src="./images/link-box-variant.svg"></a></li>
+	     <li><a href="https://www.github.com/chaomodus/pixywerk2">Github<img src="./images/link-box-variant.svg"></a></li>
+	 </ul></nav>
+       
+       
+	 <footer>Copyright &copy; 2019 by Cas Rusnov <a rel="license" href="http://creativecommons.org/licenses/by/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/80x15.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.
+	 </footer>
+       
+    </div>
+  </body>
+</html>

examples/pixywerk.com/publish/templates/post.jinja2

@@ -8,7 +8,7 @@
       <h1>{{ metadata.title }}</h1>
       <img src="{{metadata.relpath}}/{{metadata.featured}}" class="featured">
     </div>
-  <div class="byline">
+  <div class="byline" style="width: 100%">
   <p>Author: {{ metadata.author }}<br>
     Published: {{ get_time_iso8601(metadata.stat.ctime) }}
     {% if metadata.stat.mtime-metadata.stat.ctime > 512 %}

examples/pixywerk.com/src/recurse.thtml

@@ -0,0 +1,5 @@
+<ul>
+{% for i in get_hier('.', '*') %}
+<li>{{i}}</li>
+{% endfor %}
+</ul>

pixywerk2/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.6.0'

pixywerk2/__main__.py

@@ -11,14 +11,24 @@ import os
 import shutil
 import sys
 import time
-
 from typing import Dict, List, cast
 
+from .metadata import MetaTree
 from .processchain import ProcessorChains
 from .processors.processors import PassthroughException
-from .metadata import MetaTree
-from .template_tools import date_iso8601, file_list, file_name, file_content, file_metadata, time_iso8601, file_raw
 from .pygments import pygments_get_css, pygments_markup_contents_html
+from .template_tools import (
+    date_iso8601,
+    file_content,
+    file_list,
+    file_list_hier,
+    file_json,
+    file_metadata,
+    file_name,
+    file_raw,
+    time_iso8601,
+)
+from .utils import deep_merge_dicts
 
 logger = logging.getLogger()
 
@@ -27,6 +37,12 @@ def setup_logging(verbose: bool = False) -> None:
     pass
 
 
+def parse_var(varspec: str) -> List:
+    if (not ('=' in varspec)):
+        return [varspec, True]
+    return list(varspec.split('=', 2))
+
+
 def get_args(args: List[str]) -> argparse.Namespace:
     parser = argparse.ArgumentParser("Compile a Pixywerk directory into an output directory.")
 
@@ -37,14 +53,14 @@ def get_args(args: List[str]) -> argparse.Namespace:
         "-c", "--clean", help="Remove the target tree before proceeding (by renaming to .bak).", action="store_true"
     )
     parser.add_argument("-s", "--safe", help="Abort if the target directory already exists.", action="store_true")
+    parser.add_argument("-f", "--follow-links", help="Follow symbolic links in the input tree.", action="store_true")
     parser.add_argument("-t", "--template", help="The template directory (default: root/templates)", default=None)
     parser.add_argument("-d", "--dry-run", help="Perform a dry-run.", action="store_true")
     parser.add_argument("-v", "--verbose", help="Output verbosely.", action="store_true")
     parser.add_argument("--processors", help="Specify a path to a processor configuration file.", default=None)
-    # parser.add_argument("--prescript", help="Specify one or more prescripts to run (in order specified) with context of the compile.", default=[], action="append")
-    # parser.add_argument("--postscript", help="Specify one or more postsscripts to run (in order specified) with context of the compile.", default=[], action="append")
+    parser.add_argument(
+        "-D", "--define", help="Add a variable to the metadata.", nargs="+", action="extend", type=parse_var)
     result = parser.parse_args(args)
-
     # validate arguments
     if not os.path.isdir(result.root):
         raise FileNotFoundError("can't find root folder {}".format(result.root))
@@ -82,24 +98,31 @@ def main() -> int:
         "author": "",
         "author_email": "",
     }
+    if args.define:
+        for var in args.define:
+            default_metadata[var[0]] = var[1]
     meta_tree = MetaTree(args.root, default_metadata)
     file_list_cache = cast(Dict, {})
     file_cont_cache = cast(Dict, {})
     file_name_cache = cast(Dict, {})
     file_raw_cache = cast(Dict, {})
+    flist = file_list(args.root, file_list_cache)
     default_metadata["globals"] = {
-        "get_file_list": file_list(args.root, file_list_cache),
+        "get_file_list": flist,
+        "get_hier": file_list_hier(args.root, flist),
         "get_file_name": file_name(args.root, meta_tree, process_chains, file_name_cache),
         "get_file_content": file_content(args.root, meta_tree, process_chains, file_cont_cache),
+        "get_json": file_json(args.root),
         "get_raw": file_raw(args.root, file_raw_cache),
         "get_file_metadata": file_metadata(meta_tree),
         "get_time_iso8601": time_iso8601("UTC"),
         "get_date_iso8601": date_iso8601("UTC"),
         "pygments_get_css": pygments_get_css,
         "pygments_markup_contents_html": pygments_markup_contents_html,
+        "merge_dicts": deep_merge_dicts,
     }
 
-    for root, _, files in os.walk(args.root):
+    for root, _, files in os.walk(args.root, followlinks=args.follow_links):
         workroot = os.path.relpath(root, args.root)
         if workroot == ".":
             workroot = ""
@@ -118,7 +141,7 @@ def main() -> int:
                 continue
             metadata = meta_tree.get_metadata(os.path.join(workroot, f))
             chain = process_chains.get_chain_for_filename(os.path.join(root, f), ctx=metadata)
-            print("process {} -> {}".format(os.path.join(root, f), os.path.join(target_dir, chain.output_filename)))
+            print("process {} -> {} -> {}".format(os.path.join(root, f), repr(chain), os.path.join(target_dir, chain.output_filename)))
             if not args.dry_run:
                 try:
                     with open(os.path.join(target_dir, chain.output_filename), "w") as outfile:

pixywerk2/defaults/chains.yaml

@@ -10,6 +10,13 @@ templatable:
     chain:
       - jinja2
 
+# Any object that needs jinja and to be embedded in a parent template
+tembed:
+  extension: null
+  chain:
+    - jinja2
+    - jinja2_page_embed
+
 # Markdown, BBCode and RST are first run through the templater, and then
 # they are processed into HTML, and finally embedded in a page template.
 markdown:
@@ -62,24 +69,24 @@ template-html:
         - jinja2
         - jinja2_page_embed
 
-# Smart CSS are simply converted to CSS.
-sass:
-    extension:
-        - sass
-        - scss
-    chain:
-        - process_sass
-less:
-    extension:
-        - less
-    chain:
-        - process_less
+# # Smart CSS are simply converted to CSS.
+# sass:
+#     extension:
+#         - sass
+#         - scss
+#     chain:
+#         - process_sass
+# less:
+#     extension:
+#         - less
+#     chain:
+#         - process_less
 
-stylus:
-    extension:
-        - styl
-    chain:
-        - process_styl
+# stylus:
+#     extension:
+#         - styl
+#     chain:
+#         - process_styl
 
 # # Images are processed into thumbnails and sized in addition to being retained as their original
 # FIXME implement split chain processor, implement processor arguments,

pixywerk2/metadata.py

@@ -5,8 +5,7 @@ import logging
 import mimetypes
 import os
 import uuid
-
-from typing import Dict, Optional, Union, List, Tuple, Any, cast
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 import jstyleson
 
@@ -94,7 +93,7 @@ class MetaTree:
         """Retrieve the metadata for a given path
 
         The general procedure is to iterate the tree, at each level
-m        load .meta (JSON formatted dictionary) for that level, and
+        load .meta (JSON formatted dictionary) for that level, and
         then finally load the path.meta, and merge these dictionaries
         in descendant order.
 

pixywerk2/processchain.py

@@ -3,8 +3,7 @@
 import os
 import os.path
 import random
-
-from typing import List, Iterable, Optional, Any, Dict, Type, cast
+from typing import Any, Dict, Iterable, List, Optional, Type, cast
 
 import yaml
 
@@ -91,6 +90,9 @@ class ProcessorChain:
             fname = processor.filename(fname, self._ctx)
         return fname
 
+    def __repr__(self) -> str:
+        return "[" + ",".join([x.__class__.__name__ for x in self._processors]) + "]"
+
 
 class ProcessorChains:
     """Load a configuration for processor chains, and provide ability to process the chains given a particular input

pixywerk2/processors/jinja2.py

@@ -1,6 +1,6 @@
 """Define a Jinja2 Processor which applies programmable templating to the input stream."""
 
-from typing import Iterable, Optional, Dict, cast
+from typing import Dict, Iterable, Optional, cast
 
 from jinja2 import Environment, FileSystemLoader
 
@@ -22,11 +22,10 @@ class Jinja2(PassThrough):
             iterable: The post-processed output stream
         """
         ctx = cast(Dict, ctx)
-        template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=['jinja2.ext.do'])
+        template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=["jinja2.ext.do"])
         template_env.globals.update(ctx["globals"])
         template_env.filters.update(ctx["filters"])
         tmpl = template_env.from_string("".join([x for x in input_file]))
         return tmpl.render(metadata=ctx)
 
-
 processor = Jinja2

pixywerk2/processors/jinja2_page_embed.py

@@ -3,8 +3,7 @@
    the target template is rendered)."""
 
 import os
-
-from typing import Iterable, Optional, Dict, cast
+from typing import Dict, Iterable, Optional, cast
 
 from jinja2 import Environment, FileSystemLoader
 
@@ -25,8 +24,7 @@ class Jinja2PageEmbed(Processor):
             str: the new name for the file
 
         """
-
-        return os.path.splitext(oldname)[0] + ".html"
+        return os.path.splitext(oldname)[0] + "." + self.extension(oldname, ctx)
 
     def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
         """Return the mimetype of the post-processed file.
@@ -39,7 +37,7 @@ class Jinja2PageEmbed(Processor):
             str: the new mimetype of the file after processing
 
         """
-        return "text/html"
+        return ctx.get("mime", "text/html")
 
     def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
         """Return an iterable object of the post-processed file.
@@ -52,7 +50,7 @@ class Jinja2PageEmbed(Processor):
             iterable: The post-processed output stream
         """
         ctx = cast(Dict, ctx)
-        template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=['jinja2.ext.do'])
+        template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=["jinja2.ext.do"])
         template_env.globals.update(ctx["globals"])
         template_env.filters.update(ctx["filters"])
         tmpl = template_env.get_template(ctx["template"])
@@ -70,7 +68,7 @@ class Jinja2PageEmbed(Processor):
             str: the new extension of the file after processing
 
         """
-        return "html"
+        return ctx.get("extension", "html")
 
 
 processor = Jinja2PageEmbed

pixywerk2/processors/passthrough.py

@@ -1,10 +1,10 @@
 """Passthrough progcessor which takes input and returns it."""
 
 import os
+from typing import Dict, Iterable, Optional, cast
 
-from .processors import Processor, PassthroughException
 from ..utils import guess_mime
-from typing import Iterable, Optional, Dict, cast
+from .processors import PassthroughException, Processor
 
 
 class PassThrough(Processor):

pixywerk2/processors/process_md.py

@@ -2,8 +2,7 @@
 
 import io
 import os
-
-from typing import Iterable, Optional, Dict
+from typing import Dict, Iterable, Optional
 
 import markdown
 

pixywerk2/processors/processors.py

@@ -1,6 +1,5 @@
 import abc
-
-from typing import Iterable, Optional, Dict
+from typing import Dict, Iterable, Optional
 
 
 class PassthroughException(Exception):
@@ -65,3 +64,6 @@ class Processor(abc.ABC):  # pragma: no cover
         Returns:
             iterable: The post-processed output stream
         """
+
+    def repr(self) -> str:
+        return self.__class__.__name__

pixywerk2/pygments.py

@@ -4,18 +4,17 @@ from typing import Optional
 import pygments
 import pygments.formatters
 import pygments.lexers
-import pygments.util
 import pygments.styles
+import pygments.util
 
 
-
-def pygments_markup_contents_html(input_text: str, file_type: str, style: Optional[str]=None) -> str:
+def pygments_markup_contents_html(input_text: str, file_type: str, style: Optional[str] = None) -> str:
     """Format input string with Pygments and return HTML."""
 
     if style is None:
-        style = 'default'
+        style = "default"
     style = pygments.styles.get_style_by_name(style)
-    formatter = pygments.formatters.get_formatter_by_name('html', style=style)
+    formatter = pygments.formatters.get_formatter_by_name("html", style=style)
     try:
         lexer = pygments.lexers.get_lexer_for_filename(file_type)
     except pygments.util.ClassNotFound:
@@ -26,11 +25,12 @@ def pygments_markup_contents_html(input_text: str, file_type: str, style: Option
 
     return pygments.highlight(input_text, lexer, formatter)
 
-def pygments_get_css(style: Optional[str]=None) -> str:
+
+def pygments_get_css(style: Optional[str] = None) -> str:
     """Return the CSS styles associated with a particular style definition."""
 
     if style is None:
-        style = 'default'
+        style = "default"
     style = pygments.styles.get_style_by_name(style)
-    formatter = pygments.formatters.get_formatter_by_name('html', style=style)
+    formatter = pygments.formatters.get_formatter_by_name("html", style=style)
     return formatter.get_style_defs()

pixywerk2/template_tools.py

@@ -1,21 +1,34 @@
+import copy
 import datetime
 import glob
 import itertools
 import os
+from typing import Callable, Dict, Iterable, List, Union, cast, Tuple
+
+import jstyleson
+
 import pytz
-from typing import Callable, Dict, List, Iterable, Union, cast
 
 from .metadata import MetaTree
 from .processchain import ProcessorChains
+from .utils import deep_merge_dicts
 
 
 def file_list(root: str, listcache: Dict) -> Callable:
-    def get_file_list(path_glob: str, *, sort_order: str = "ctime", reverse: bool = False, limit: int = 0) -> Iterable:
+    def get_file_list(
+            path_glob: Union[str, List[str], Tuple[str]],
+            *,
+            sort_order: str = "ctime",
+            reverse: bool = False,
+            limit: int = 0) -> Iterable:
         stattable = cast(List, [])
-        if path_glob in listcache:
-            stattable = listcache[path_glob]
+        if isinstance(path_glob, str):
+            path_glob = [path_glob]
+        for pglob in path_glob:
+            if pglob in listcache:
+                stattable.extend(listcache[pglob])
             else:
-            for fil in glob.glob(os.path.join(root, path_glob)):
+                for fil in glob.glob(os.path.join(root, pglob)):
                     if os.path.isdir(fil):
                         continue
                     if fil.endswith(".meta") or fil.endswith("~"):
@@ -31,7 +44,7 @@ def file_list(root: str, listcache: Dict) -> Callable:
                             "ext": os.path.splitext(fil)[1],
                         }
                     )
-            listcache[path_glob] = stattable
+                listcache[pglob] = stattable
             ret = sorted(stattable, key=lambda x: x[sort_order], reverse=reverse)
         if limit > 0:
             return itertools.islice(ret, limit)
@@ -40,6 +53,27 @@ def file_list(root: str, listcache: Dict) -> Callable:
     return get_file_list
 
 
+def file_list_hier(root: str, flist: Callable) -> Callable:
+    """Return a callable which, given a directory, will walk the directory and return the files within
+    it that match the glob passed."""
+
+    def get_file_list_hier(path: str, glob: str, *, sort_order: str = "ctime", reverse: bool = False) -> Iterable:
+        output = []
+
+        for pth in os.walk(os.path.join(root, path)):
+            output.extend(
+                flist(
+                    os.path.join(os.path.relpath(os.path.realpath(pth[0]), root), glob),
+                    sort_order=sort_order,
+                    reverse=reverse,
+                )
+            )
+
+        return output
+
+    return get_file_list_hier
+
+
 def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains, namecache: Dict) -> Callable:
     def get_file_name(file_name: str) -> Dict:
         if file_name in namecache:
@@ -51,15 +85,29 @@ def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains,
 
     return get_file_name
 
+
 def file_raw(root: str, contcache: Dict) -> Callable:
     def get_raw(file_name: str) -> str:
         if file_name in contcache:
             return contcache[file_name]
-        with open(os.path.join(root, file_name), 'r', encoding="utf-8") as f:
+        with open(os.path.join(root, file_name), "r", encoding="utf-8") as f:
             return f.read()
 
     return get_raw
 
+
+def file_json(root: str) -> Callable:
+    def get_json(file_name: str, parent: Dict = None) -> Dict:
+        outd = {}
+        if parent is not None:
+            outd = copy.deepcopy(parent)
+
+        with open(os.path.join(root, file_name), "r", encoding="utf-8") as f:
+            return deep_merge_dicts(outd, jstyleson.load(f))
+
+    return get_json
+
+
 def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChains, contcache: Dict) -> Callable:
     def get_file_content(file_name: str) -> Iterable:
         if file_name in contcache:
@@ -67,7 +115,7 @@ def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChain
         metadata = metatree.get_metadata(file_name)
         chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata)
         contcache[file_name] = chain.output
-        return unicode(chain.output)
+        return str(chain.output)
 
     return get_file_content
 
@@ -87,10 +135,11 @@ def time_iso8601(timezone: str) -> Callable:
 
     return get_time_iso8601
 
+
 def date_iso8601(timezone: str) -> Callable:
     tz = pytz.timezone(timezone)
 
     def get_date_iso8601(time_t: Union[int, float]) -> str:
-        return datetime.datetime.fromtimestamp(time_t, tz).strftime('%Y-%m-%d')
+        return datetime.datetime.fromtimestamp(time_t, tz).strftime("%Y-%m-%d")
 
     return get_date_iso8601

pixywerk2/utils.py

@@ -1,11 +1,11 @@
+from typing import Dict, Optional
+import copy
 import mimetypes
 import os
 
-from typing import Dict, Optional
-
 
 def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict:
-    """Merge two dictionaries.
+    """Merge two dictionaries (shallow).
 
     Arguments:
         dict_a (dict): The dictionary to use as the base.
@@ -20,6 +20,36 @@ def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict:
     return dict_z
 
 
+def deep_merge_dicts(dict_a: Dict, dict_b: Dict, _path=None, cpy=False) -> Dict:
+    """Merge two dictionaries (deep).
+    https://stackoverflow.com/questions/7204805/how-to-merge-dictionaries-of-dictionaries/7205107#7205107
+
+    Arguments:
+        dict_a (dict): The dictionary to use as the base.
+        dict_b (dict): The dictionary to update the values with.
+        _path (list): internal use.
+
+    Returns:
+        dict: A new merged dictionary.
+
+    """
+    if cpy:
+        dict_a = copy.deepcopy(dict_a)
+    if _path is None:
+        _path = []
+    for key in dict_b:
+        if key in dict_a:
+            if isinstance(dict_a[key], dict) and isinstance(dict_b[key], dict):
+                deep_merge_dicts(dict_a[key], dict_b[key], _path + [str(key)])
+            elif dict_a[key] == dict_b[key]:
+                pass # same leaf value
+            else:
+                dict_a[key] = copy.deepcopy(dict_b[key])
+        else:
+            dict_a[key] = dict_b[key]
+    return dict_a
+
+
 def guess_mime(path: str) -> Optional[str]:
     """Guess the mime type for a given path.
 

setup.py

@@ -1,6 +1,8 @@
 """Package configuration."""
 from setuptools import find_packages, setup
 
+from pixywerk2 import __version__
+
 LONG_DESCRIPTION = """Pixywerk 2 is a filesystem based static site generator."""
 
 INSTALL_REQUIRES = ["yaml-1.3", "markdown", "jstyleson", "jinja2", "pygments"]
@@ -56,4 +58,5 @@ setup(
     use_scm_version=True,
     url="https://git.antpanethon.com/cas/pixywerk2",
     zip_safe=False,
+    version=__version__,
 )

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist=py{36,37}-{code-quality, unit} #, py37-sphinx
+envlist=py{36,37,38,39}-{code-quality, unit} #, py37-sphinx
 skipsdist = true
 
 [testenv]
@@ -17,6 +17,8 @@ commands =
 basepython =
     py36: python3.6
     py37: python3.7
+    py38: python3.8
+    py39: python3.9
 
 [flake8]
 max-line-length = 120