Major code import.

6 years ago · 43d40f7fce
parent 13fb5dac1c
commit 43d40f7fce
40 changed files with 1228 additions and 0 deletions
--- a/demo/.meta
+++ b/demo/.meta
@ -0,0 +1,7 @@
 {
 "site_root":"https://example.com",
 "title":"Test Metadata",
 "author": "Test User",
 "author_email": "test_user@example.com",
 "uuid_oid_root": "pixywerk-demo"
 }
--- a/demo/atom.xml
+++ b/demo/atom.xml
@ -0,0 +1,33 @@
 <?xml version="1.0" encoding="utf-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom">
 	<title>{{ metadata.title }}</title>
 	<subtitle>{{ metadata.subtitle }}</subtitle>
 	<link href="{{ metadata.site_root }}/{{ metadata.file_name }}" rel="self" />
 	<link href="{{ metadata.site_root }}" />
 	<id>urn:uuid:{{ metadata.uuid }}</id>
 	<updated>{{ get_time_iso8601(metadata['build-time']) }}</updated>
 	{% set posts = get_file_list('blog_posts/*.cont') %}
 	{% for post in posts %}
 	{% set post_meta = get_file_metadata(post['file_path']) %}
 	<entry>
 		<title>{{ post_meta.title }}</title>
 		<link href="{{ metadata.site_root }}/{{post_meta.file_path}}" />
 		<id>urn:uuid:{{ post_meta.uuid }}</id>
 		<updated>{{ get_time_iso8601(post_meta.stat.mtime) }}</updated>
 		<summary>{{post_meta.summary }}</summary>
 		<!-- this would be the snippet, more than summary chunk -->
 		<!-- <content type="xhtml"> -->
 		<!-- 	<div xmlns="http://www.w3.org/1999/xhtml"> -->
 		<!-- 		<p>{{ post_meta.summary }}</p> -->
 		<!-- 	</div> -->
 		<!-- </content> -->
 		<author>
 			<name>{{ post_meta.author }}</name>
 			<email>{{ post_meta.author_email }}</email>
 		</author>
 	</entry>
 	{% endfor %}
 </feed>
--- a/demo/atom.xml.meta
+++ b/demo/atom.xml.meta
@ -0,0 +1,5 @@
 {
 "type": "templatable",
 "title": "Test RSS Feed",
 "subtitle": "Some Subtitle"
 }
--- a/demo/bar/baz/quux/quuux
+++ b/demo/bar/baz/quux/quuux
--- a/demo/blog_posts/anotherpost.cont
+++ b/demo/blog_posts/anotherpost.cont
@ -0,0 +1,5 @@
 Some more post
 la la la
--- a/demo/blog_posts/anotherpost.cont.meta
+++ b/demo/blog_posts/anotherpost.cont.meta
@ -0,0 +1,4 @@
 {
 "title":"Another Post(tm)",
 "summary":"Yet another post"
 }
--- a/demo/blog_posts/test.cont
+++ b/demo/blog_posts/test.cont
@ -0,0 +1 @@
 Some content.
--- a/demo/blog_posts/test.cont.meta
+++ b/demo/blog_posts/test.cont.meta
@ -0,0 +1,4 @@
 {
 "title":"Test.cont",
 "summary":"Some empty test content"
 }
--- a/demo/foo.cont
+++ b/demo/foo.cont
@ -0,0 +1 @@
 yo fresh
--- a/demo/foo.cont.meta
+++ b/demo/foo.cont.meta
@ -0,0 +1,5 @@
 {
 "foo":"bar",
 "title":"A title",
 "summary":"Just a post."
 }
--- a/demo/index.cont
+++ b/demo/index.cont
@ -0,0 +1,19 @@
 <h1>Index of all content</h1>
 {% for f in get_file_list('*', sort_order='file_name') %}
 <a href="{{ get_file_name(f['file_name']) }}">{{get_file_name(f['file_name'])}}</a>
 {% endfor %}
 <p>Including foo.cont.meta:
 <pre>
 {{ get_file_content('foo.cont.meta') }}
 </pre>
 </p>
 <h1>Metadata</h1>
 <table class="metadata">
 <tr><th>key</th><th>value</th></tr>
 {% set metadata = get_file_metadata('foo.cont') %}
 {% for k in metadata.keys() %}
 <tr><td>{{k}}</td><td>{{metadata[k]}}</td></tr>
 {% endfor %}
 </table>
--- a/demo/mapping.json
+++ b/demo/mapping.json
--- a/demo/passthrough.md
+++ b/demo/passthrough.md
@ -0,0 +1,9 @@
 # README #
 This is a test of the emergency compiled HTML system. This is only a *test*.
 [Foo!](foo.html)
 {% for i in range(100) %}
 * {{ i }}
 {% endfor %}
--- a/demo/passthrough.md.meta
+++ b/demo/passthrough.md.meta
@ -0,0 +1,3 @@
 {
 "pragma":["no-proc"]
 }
--- a/demo/readme.md
+++ b/demo/readme.md
@ -0,0 +1,9 @@
 # README #
 This is a test of the emergency compiled HTML system. This is only a *test*.
 [Foo!](foo.html)
 {% for i in range(100) %}
 * {{ i }}
 {% endfor %}
--- a/demo/readme.md.meta
+++ b/demo/readme.md.meta
@ -0,0 +1,3 @@
 {
 "title":"Yo, markdown"
 }
--- a/demo/templates/debug.jinja2
+++ b/demo/templates/debug.jinja2
@ -0,0 +1,32 @@
 <!DOCTYPE html>
 <head>
 <title>Debug for {{path}}</title>
 <style type="text/css">
 table { border: 1px solid black; }
 div { border: 1px solid black; }
 td { border: 1px solid black; }
 </style>
 </head>
 <body>
 <p>{{path}}</p>
 <h1>Content</h1>
 <div class="content">
 {{content}}
 </div>
 <h1>Environment</h1>
 <table class="environment">
 <tr><th>key</th><th>value</th></tr>
 {% for k in environ.keys() %}
 <tr><td>{{k}}</td><td>{{environ[k]}}</td></tr>
 {% endfor %}
 </table>
 <h1>Metadata</h1>
 <table class="metadata">
 <tr><th>key</th><th>value</th></tr>
 {% for k in metadata.keys() %}
 <tr><td>{{k}}</td><td>{{metadata[k]}}</td></tr>
 {% endfor %}
 </table>
 </body>
--- a/demo/templates/default-fs.jinja2
+++ b/demo/templates/default-fs.jinja2
@ -0,0 +1,6 @@
 <table class="werk-file-list">
 <tr class="werk-file-list-head"><th>file</th><th>type</th><th>size</th><th>last change</th></tr>
 {% for f in files.keys() %}
 <tr class="werk-file-list-item"><td><a href="/{{files[f].relpath}}">{{f}}</a></td><td>{{files[f].type}}</td><td>{{files[f].size}}</td><td>{{files[f].ctime | date}}</td></tr>
 {% endfor %}
 </table>
--- a/demo/templates/default.jinja2
+++ b/demo/templates/default.jinja2
@ -0,0 +1,13 @@
 <!DOCTYPE html>
 <head>
 <title>{{metadata.title}}</title>
 <style type="text/css">
 table { border: 1px solid black; }
 div { border: 1px solid black; }
 td { border: 1px solid black; }
 </style>
 </head>
 <body>
 {{content}}
 </body>
 </html>
--- a/pixywerk2/init.py
+++ b/pixywerk2/init.py
--- a/pixywerk2/main.py
+++ b/pixywerk2/main.py
@ -0,0 +1,119 @@
 # iterate source tree
 #   create directors in target tree
 #   for each item:
 #    run processor(s) on item, each processor could be in a chain or a branch
 #    Processors also provide filename munging
 #    output target based on processor output
 import argparse
 import logging
 import os
 import sys
 import time
 from typing import Dict, List, cast
 from .processchain import ProcessorChains
 from .metadata import MetaTree
 from .template_tools import file_list, file_name, file_content, file_metadata, time_iso8601
 logger = logging.getLogger()
 def setup_logging(verbose: bool = False) -> None:
    pass
 def get_args(args: List[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser("Compile a Pixywerk directory into an output directory.")
    parser.add_argument("root", help="The root of the pixywerk directory to process.")
    parser.add_argument("output", help="The output directory to export post-compiled files to.")
    parser.add_argument(
        "-c", "--clean", help="Remove the target tree before proceeding (by renaming to .bak).", action="store_true"
    )
    parser.add_argument("-s", "--safe", help="Abort if the target directory already exists.", action="store_true")
    parser.add_argument("-t", "--template", help="The template directory (default: root/templates)", default=None)
    parser.add_argument("-d", "--dry-run", help="Perform a dry-run.", action="store_true")
    parser.add_argument("-v", "--verbose", help="Output verbosely.", action="store_true")
    parser.add_argument("--processors", help="Specify a path to a processor configuration file.", default=None)
    result = parser.parse_args(args)
    # validate arguments
    if not os.path.isdir(result.root):
        raise FileNotFoundError("can't find root folder {}".format(result.root))
    if not result.template:
        result.template = os.path.join(result.root, "templates")
        result.excludes = [result.template]
    return result
 def main() -> int:
    try:
        args = get_args(sys.argv[1:])
    except FileNotFoundError as ex:
        print("error finding arguments: {}".format(ex))
        return 1
    setup_logging(args.verbose)
    if os.path.exists(args.output) and args.clean:
        bak = "{}.bak-{}".format(args.output, int(time.time()))
        print("cleaning target {} -> {}".format(args.output, bak))
        os.rename(args.output, bak)
    process_chains = ProcessorChains(args.processors)
    default_metadata = {
        "templates": args.template,
        "template": "default.jinja2",
        "dir-template": "default-dir.jinja2",
        "filters": {},
        "build-time": time.time(),
        "build-datetime": time.ctime(),
        "uuid-oid-root": "pixywerk",
    }
    meta_tree = MetaTree(args.root, default_metadata)
    file_list_cache = cast(Dict, {})
    file_cont_cache = cast(Dict, {})
    file_name_cache = cast(Dict, {})
    default_metadata["globals"] = {
        "get_file_list": file_list(args.root, file_list_cache),
        "get_file_name": file_name(args.root, meta_tree, process_chains, file_name_cache),
        "get_file_content": file_content(args.root, meta_tree, process_chains, file_cont_cache),
        "get_file_metadata": file_metadata(meta_tree),
        "get_time_iso8601": time_iso8601("UTC"),
    }
    for root, _, files in os.walk(args.root):
        workroot = os.path.relpath(root, args.root)
        if workroot == ".":
            workroot = ""
        target_dir = os.path.join(args.output, workroot)
        print("mkdir -> {}".format(target_dir))
        if not args.dry_run:
            try:
                os.mkdir(target_dir)
            except FileExistsError:
                if args.safe:
                    print("error, target directory exists, aborting")
                    return 1
        for f in files:
            # fixme global generic filters
            if f.endswith(".meta") or f.endswith("~"):
                continue
            metadata = meta_tree.get_metadata(os.path.join(workroot, f))
            chain = process_chains.get_chain_for_filename(os.path.join(root, f), ctx=metadata)
            print("process {} -> {}".format(os.path.join(root, f), os.path.join(target_dir, chain.output_filename)))
            if not args.dry_run:
                with open(os.path.join(target_dir, chain.output_filename), "w") as outfile:
                    for line in chain.output:
                        outfile.write(line)
    return 0
 if __name__ == "__main__":
    sys.exit(main())
--- a/pixywerk2/defaults/chains.yaml
+++ b/pixywerk2/defaults/chains.yaml
@ -0,0 +1,104 @@
 # Default: output == input
 default:
    extension: default
    chain:
        - passthrough
 # Any object that needs jinja scripts but no other explicit processing
 templatable:
    extension: null
    chain:
        - jinja2
 # Markdown, BBCode and RST are first run through the templater, and then
 # they are processed into HTML, and finally embedded in a page template.
 markdown:
    extension:
        - md
    chain:
        - jinja2
        - process_md
        - jinja2_page_embed
 bbcode:
    extension:
        - bb
        - pp
    chain:
        - jinja2
        - process_pp
        - jinja2_page_embed
 # FIXME implement RST processor
 # restructured:
 #     extension:
 #         - rst
 #     chain:
 #         - jinja2
 #         - process_rst
 #         - jinja2_page_embed
 # # JSON and YAML are split, passed through a pretty printer, and then output
 # FIXME implement split chain processor, implement processor arguments
 # json:
 #     extension:
 #         - json
 #     chain:
 #         - split (passthrough)
 #         - pp_json
 # yaml:
 #     extension:
 #         - yml
 #         - yaml
 #     chain:
 #         - split (passthrough)
 #         - pp_yaml
 # Template-html is first passed through the templater, and then embedded
 # in a page template
 template-html:
    extension:
        - thtml
        - cont
    chain:
        - jinja2
        - jinja2_page_embed
 # Smart CSS are simply converted to CSS.
 sass:
    extension:
        - sass
        - scss
    chain:
        - process_sass
 less:
    extension:
        - less
    chain:
        - process_less
 stylus:
    extension:
        - styl
    chain:
        - process_styl
 # # Images are processed into thumbnails and sized in addition to being retained as their original
 # FIXME implement split chain processor, implement processor arguments,
 # image:
 #     extension:
 #         - jpg
 #         - jpeg
 #         - png
 #     chain:
 #         - split (image_bigthumb)
 #         - split (image_smallthumb)
 #         - passthrough
 # image_bigthumb:
 #     extension:
 #     chain:
 #         - smart_resize (big)
 # image_smallthumb:
 #     extension:
 #     chain:
 #         - smart_resize (small)
--- a/pixywerk2/metadata.py
+++ b/pixywerk2/metadata.py
@ -0,0 +1,145 @@
 """Constructs a tree-like object containing the metadata for a given path, and caches said metadata."""
 import logging
 import mimetypes
 import os
 import uuid
 from typing import Dict, Optional, Union, List, Tuple, Any, cast
 import jstyleson
 from .utils import guess_mime
 # setup mimetypes with some extra ones
 mimetypes.init()
 mimetypes.add_type("text/html", "thtml")
 mimetypes.add_type("text/html", "cont")
 logger = logging.getLogger(__name__)
 class MetaCacheMiss(Exception):
    """Raised on cache miss."""
 class MetaCache:
    """This class provides an in-memory cache for metadata tree."""
    def __init__(self, max_age: float = 200.0):
        """Initialize the cache.
        Arguments:
            max_age (int): the number of seconds to age-out cache items
        """
        self._max_age = max_age
        self._cache: Dict[str, Tuple[float, Any]] = {}
    def get(self, key: str, new_time_stamp: float) -> Any:
        """Get an item from the cache.
        Arguments:
            key (str): the cache key to retieve
            new_time_stamp (int): The time to use to compare the stored time with
        Returns:
            :obj:misc: The previously stored value.
        Raises:
            MetaCacheMiss: on missing key, or on aged out
        """
        if key not in self._cache:
            raise MetaCacheMiss("no item for key {}".format(key))
        if self._cache[key][0] + self._max_age <= new_time_stamp:
            return self._cache[key][1]
        raise MetaCacheMiss("cache expired for key {}".format(key))
    def put(self, key: str, value: Union[Dict, List, int, str, object], time_stamp: float) -> None:
        """Put an item into the cache.
        Arguments:
            key (str): the key to store the cache item under
            value (:obj:misc): the value to store
            time_stamp (float): the time stamp to store the item under
        """
        self._cache[key] = (time_stamp, value)
 class MetaTree:
    """This provides an interface to loading and caching tree metadata for a given directory tree."""
    def __init__(self, root: str, default_metadata: Optional[Dict] = None):
        """Initialize the metadata tree object.
        Arguments:
            root (str): The path to the root of the file tree to operate on.
            default_metadata (dict, optional): The default metadata to apply to the tree
        """
        self._cache = MetaCache()
        if default_metadata is None:
            default_metadata = {}
        self._default_metadata = default_metadata
        if root[-1] != "/":
            root += "/"
        self._root = root
    def get_metadata(self, rel_path: str) -> Dict:
        """Retrieve the metadata for a given path
        The general procedure is to iterate the tree, at each level
 m        load .meta (JSON formatted dictionary) for that level, and
        then finally load the path.meta, and merge these dictionaries
        in descendant order.
        Arguments:
            rel_path (str): The path to retrieve the metadata for (relative to root)
        Returns:
            dict: A dictionary of metadata for that path tree.
        """
        metablob = dict(self._default_metadata)
        # iterate path components from root to target path
        comps = [self._root] + rel_path.split("/")
        fullpath = ""
        for pth in comps:
            fullpath = os.path.join(fullpath, pth)
            st = os.stat(fullpath)
            cachekey = fullpath + ".meta"
            meta = cast(Dict, {})
            try:
                st_meta = os.stat(cachekey)
                meta = self._cache.get(cachekey, st_meta.st_mtime)
            except FileNotFoundError:
                st_meta = None  # type: ignore
            except MetaCacheMiss:
                meta = {}
            if not meta and st_meta:
                meta = jstyleson.load(open(cachekey, "r"))
                self._cache.put(cachekey, meta, st_meta.st_mtime)
            metablob.update(meta)
        # return final dict
        metablob["dir"], metablob["file_name"] = os.path.split(rel_path)
        metablob["file_path"] = rel_path
        metablob["uuid"] = uuid.uuid3(
            uuid.NAMESPACE_OID, metablob["uuid-oid-root"] + os.path.join(self._root, rel_path)
        )
        metablob["os-path"], _ = os.path.split(fullpath)
        metablob["guessed-type"] = guess_mime(os.path.join(self._root, rel_path))
        if "mime-type" not in metablob:
            metablob["mime-type"] = metablob["guessed-type"]
        metablob["stat"] = {}
        for stk in ("st_mtime", "st_ctime", "st_atime", "st_mode", "st_size", "st_ino"):
            metablob["stat"][stk.replace("st_", "")] = getattr(st, stk)
        return metablob
--- a/pixywerk2/processchain.py
+++ b/pixywerk2/processchain.py
@ -0,0 +1,180 @@
 """Interface for chains of processors"""
 import os
 import os.path
 import random
 from typing import List, Iterable, Optional, Any, Dict, Type, cast
 import yaml
 from .processors.processors import Processor
 class ProcessorChain:
    """This implements a wrapper for an arbitrary set of processors and an associated file stream."""
    def __init__(
        self,
        processors: List[Processor],
        file_name: str,
        file_data: Iterable[str],
        file_type: str,
        ctx: Optional[Dict] = None,
    ):
        """Initialize the processing stream.
        Arguments:
             processors (list): A list of processor objects.
             file_data (Iterable): An iterable from which to retrieve the input
             file_type (str): the specified file type for consumer information.
        """
        self._processors = processors
        self._file_data = file_data
        self._file_type = file_type
        self._file_name = file_name
        self._ctx: Dict = {}
        if ctx is not None:
            self._ctx = cast(Dict, ctx)
    @property
    def output(self) -> Iterable:
        """Return an iterable for the output of the process chain
        Returns:
            :obj:'iterable': the iterable
        """
        prev = self._file_data
        for processor in self._processors:
            if processor:
                prev = processor.process(prev, self._ctx)
        return prev
    @property
    def output_mime(self) -> str:
        """Return the post-processed MIME value from the processing chain
        Returns:
            str: the mime type
        """
        fname = self._file_name
        for processor in self._processors:
            fname = processor.mime_type(fname, self._ctx)
        return fname
    @property
    def output_ext(self) -> str:
        """Return the post-processed extension from the processing chain
        Returns:
            str: the extension
        """
        fname = self._file_name
        for processor in self._processors:
            fname = processor.extension(fname, self._ctx)
        return fname
    @property
    def output_filename(self) -> str:
        """Return the post-processed filename from the processing chain
        Returns:
            str: the new filename
        """
        fname = os.path.basename(self._file_name)
        for processor in self._processors:
            fname = processor.filename(fname, self._ctx)
        return fname
 class ProcessorChains:
    """Load a configuration for processor chains, and provide ability to process the chains given a particular input
    file.
    """
    def __init__(self, config: Optional[str] = None):
        """Initialize, with a specified configuration file
        Arguments:
            config (str, optional): The path to a yaml formatted configuration file.
        """
        if config is None:  # pragma: no coverage
            config = os.path.join(os.path.dirname(__file__), "defaults", "chains.yaml")
        self.chainconfig = yaml.load(open(config, "r"))
        self.extensionmap: Dict[str, Any] = {}
        self.processors: Dict[str, Type[Processor]] = {}
        for ch, conf in self.chainconfig.items():
            if conf["extension"] == "default":
                self.default = ch
            else:
                if conf["extension"]:
                    for ex in conf["extension"]:
                        if ex in self.extensionmap or ex is None:
                            # log an error or except or something we'll just override for now.
                            pass
                        self.extensionmap[ex] = ch
            for pr in conf["chain"]:
                if pr in self.processors:
                    continue
                processor_module = __import__("processors", globals(), locals(), [pr], 1)
                self.processors[pr] = processor_module.__dict__[pr].processor
    def get_chain_for_filename(self, filename: str, ctx: Optional[Dict] = None) -> ProcessorChain:
        """Get the ProcessorChain, as configured for a given file by extension.
        Arguments:
            filename (str): The name of the file to get a chain for.
        Returns:
            ProcessorChain: the constructed processor chain.
        """
        r = filename.rsplit(".", 1)
        ftype = "default"
        if r:
            ftype = r[-1]
        if ctx and "pragma" in ctx:
            if "no-proc" in ctx["pragma"]:
                ftype = "default"
        if ctx and "type" in ctx:
            ftype = ctx["type"]
        return self.get_chain_for_file(open(filename, "r"), ftype, filename, ctx)
    def get_chain_for_file(
        self, file_obj: Iterable, file_ext: str, file_name: Optional[str] = None, ctx: Optional[Dict] = None
    ) -> ProcessorChain:
        """Get the ProcessorChain for a given iterable object based on the specified file type
        Arguments:
            file_obj (:obj:`iterable`): The input file stream
            file_ext (str): The type (extension) of the input stream
        Returns:
            ProcessorChain: the constructed processor chain.
        """
        if file_ext not in self.extensionmap or not self.extensionmap[file_ext]:
            if file_ext in self.chainconfig:
                file_type = file_ext
            else:
                file_type = "default"
        else:
            file_type = self.extensionmap[file_ext]
        if not (bool(file_name)):
            file_name = hex(random.randint(0, 65536))
        return ProcessorChain(
            [self.processors[x]() for x in self.chainconfig[file_type]["chain"]],
            cast(str, file_name),
            file_obj,
            file_type,
            ctx,
        )
--- a/pixywerk2/processors/init.py
+++ b/pixywerk2/processors/init.py
@ -0,0 +1 @@
 # processors metadata here
--- a/pixywerk2/processors/jinja2.py
+++ b/pixywerk2/processors/jinja2.py
@ -0,0 +1,32 @@
 """Define a Jinja2 Processor which applies programmable templating to the input stream."""
 from typing import Iterable, Optional, Dict, cast
 from jinja2 import Environment, FileSystemLoader
 from .passthrough import PassThrough
 class Jinja2(PassThrough):
    """Pass the input stream through Jinja2 for scritable templating."""
    def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
        """Return an iterable object of the post-processed file.
        Arguments:
            input_file (iterable): An input stream
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            iterable: The post-processed output stream
        """
        ctx = cast(Dict, ctx)
        template_env = Environment(loader=FileSystemLoader(ctx["templates"]))
        template_env.globals.update(ctx["globals"])
        template_env.filters.update(ctx["filters"])
        tmpl = template_env.from_string("".join([x for x in input_file]))
        return tmpl.render(metadata=ctx)
 processor = Jinja2
--- a/pixywerk2/processors/jinja2_page_embed.py
+++ b/pixywerk2/processors/jinja2_page_embed.py
@ -0,0 +1,76 @@
 """Define a Jinja2 processor which embeds the (presumably HTML) input stream into a Page Template
   as defined in the ctx metadata (the ``content`` variable is assigned to the input stream and
   the target template is rendered)."""
 import os
 from typing import Iterable, Optional, Dict, cast
 from jinja2 import Environment, FileSystemLoader
 from .processors import Processor
 class Jinja2PageEmbed(Processor):
    """Embed input stream as ``content`` variable in page template defined in context key ``template``."""
    def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the filename of the post-processed file.
        Arguments:
            oldname (str): the previous name for the file.
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new name for the file
        """
        return os.path.splitext(oldname)[0] + ".html"
    def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new mimetype of the file after processing
        """
        return "text/html"
    def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
        """Return an iterable object of the post-processed file.
        Arguments:
            input_file (iterable): An input stream
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            iterable: The post-processed output stream
        """
        ctx = cast(Dict, ctx)
        template_env = Environment(loader=FileSystemLoader(ctx["templates"]))
        template_env.globals.update(ctx["globals"])
        template_env.filters.update(ctx["filters"])
        tmpl = template_env.get_template(ctx["template"])
        content = "".join([x for x in input_file])
        return tmpl.render(content=content, metadata=ctx)
    def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new extension of the file after processing
        """
        return "html"
 processor = Jinja2PageEmbed
--- a/pixywerk2/processors/passthrough.py
+++ b/pixywerk2/processors/passthrough.py
@ -0,0 +1,68 @@
 """Passthrough progcessor which takes input and returns it."""
 import os
 from .processors import Processor
 from ..utils import guess_mime
 from typing import Iterable, Optional, Dict, cast
 class PassThrough(Processor):
    """A simple passthrough processor that takes input and sends it to output."""
    def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the filename of the post-processed file.
        Arguments:
            oldname (str): the previous name for the file.
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new name for the file
        """
        return oldname
    def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new mimetype of the file after processing
        """
        result = cast(str, guess_mime(oldname))
        if result == "directory":
            result = "DIR"
        return result
    def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
        """Return an iterable object of the post-processed file.
        Arguments:
            input_file (iterable): An input stream
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            iterable: The post-processed output stream
        """
        return input_file
    def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new extension of the file after processing
        """
        return os.path.splitext(oldname)[-1]
 processor = PassThrough
--- a/pixywerk2/processors/process_less.py
+++ b/pixywerk2/processors/process_less.py
@ -0,0 +1 @@
 processor = None
--- a/pixywerk2/processors/process_md.py
+++ b/pixywerk2/processors/process_md.py
@ -0,0 +1,69 @@
 """Convert an MD stream into an HTML stream"""
 import io
 import os
 from typing import Iterable, Optional, Dict
 import markdown
 from .processors import Processor
 class MarkdownProcessor(Processor):
    """Convert an MD stream into an HTML stream"""
    def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the filename of the post-processed file.
        Arguments:
            oldname (str): the previous name for the file.
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new name for the file
        """
        return os.path.splitext(oldname)[0] + ".html"
    def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new mimetype of the file after processing
        """
        return "text/html"
    def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new extension of the file after processing
        """
        return "html"
    def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
        """Return an iterable object of the post-processed file.
        Arguments:
            input_file (iterable): An input stream
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            iterable: The post-processed output stream
        """
        md = u"".join([x for x in input_file])
        return io.StringIO(markdown.markdown(md, extensions=["extra", "admonition", "wikilinks"]))
 processor = MarkdownProcessor  # pylint: disable=invalid-name
--- a/pixywerk2/processors/process_pp.py
+++ b/pixywerk2/processors/process_pp.py
@ -0,0 +1 @@
 processor = None
--- a/pixywerk2/processors/process_sass.py
+++ b/pixywerk2/processors/process_sass.py
@ -0,0 +1 @@
 processor = None
--- a/pixywerk2/processors/process_styl.py
+++ b/pixywerk2/processors/process_styl.py
@ -0,0 +1 @@
 processor = None
--- a/pixywerk2/processors/processors.py
+++ b/pixywerk2/processors/processors.py
@ -0,0 +1,63 @@
 import abc
 from typing import Iterable, Optional, Dict
 class ProcessorException(Exception):  # pragma: no cover
    """A base exception class to be used by processor objects."""
 class Processor(abc.ABC):  # pragma: no cover
    def __init__(self, *args, **kwargs):
        """Initialize the class."""
    @abc.abstractmethod
    def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the filename of the post-processed file.
        Arguments:
            oldname (str): the previous name for the file.
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new name for the file
        """
    @abc.abstractmethod
    def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new mimetype of the file after processing
        """
    @abc.abstractmethod
    def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
        """Return the mimetype of the post-processed file.
        Arguments:
            oldname (str): the input filename
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            str: the new extension of the file after processing
        """
    @abc.abstractmethod
    def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
        """Return an iterable object of the post-processed file.
        Arguments:
            input_file (iterable): An input stream
            ctx (dict, optional): A context object generated from the processor configuration
        Returns:
            iterable: The post-processed output stream
        """
--- a/pixywerk2/template_tools.py
+++ b/pixywerk2/template_tools.py
@ -0,0 +1,80 @@
 import datetime
 import glob
 import itertools
 import os
 import pytz
 from typing import Callable, Dict, List, Iterable, Union, cast
 from .metadata import MetaTree
 from .processchain import ProcessorChains
 def file_list(root: str, listcache: Dict) -> Callable:
    def get_file_list(path_glob: str, *, sort_order: str = "ctime", reverse: bool = False, limit: int = 0) -> Iterable:
        stattable = cast(List, [])
        if path_glob in listcache:
            stattable = listcache[path_glob]
        else:
            for fil in glob.glob(os.path.join(root, path_glob)):
                if os.path.isdir(fil):
                    continue
                if fil.endswith(".meta") or fil.endswith("~"):
                    continue
                st = os.stat(fil)
                stattable.append(
                    {
                        "file_path": os.path.relpath(fil, root),
                        "file_name": os.path.split(fil)[-1],
                        "mtime": st.st_mtime,
                        "ctime": st.st_ctime,
                        "size": st.st_size,
                        "ext": os.path.splitext(fil)[1],
                    }
                )
            listcache[path_glob] = stattable
        ret = sorted(stattable, key=lambda x: x[sort_order], reverse=reverse)
        if limit > 0:
            return itertools.islice(ret, limit)
        return ret
    return get_file_list
 def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains, namecache: Dict) -> Callable:
    def get_file_name(file_name: str) -> Dict:
        if file_name in namecache:
            return namecache[file_name]
        metadata = metatree.get_metadata(file_name)
        chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata)
        namecache[file_name] = chain.output_filename
        return namecache[file_name]
    return get_file_name
 def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChains, contcache: Dict) -> Callable:
    def get_file_content(file_name: str) -> Iterable:
        if file_name in contcache:
            return contcache[file_name]
        metadata = metatree.get_metadata(file_name)
        chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata)
        contcache[file_name] = chain.output
        return chain.output
    return get_file_content
 def file_metadata(metatree: MetaTree) -> Callable:
    def get_file_metadata(file_name: str) -> Dict:
        return metatree.get_metadata(file_name)
    return get_file_metadata
 def time_iso8601(timezone: str) -> Callable:
    tz = pytz.timezone(timezone)
    def get_time_iso8601(time_t: Union[int, float]) -> str:
        return datetime.datetime.fromtimestamp(time_t, tz).isoformat("T")
    return get_time_iso8601
--- a/pixywerk2/tests/unit/init.py
+++ b/pixywerk2/tests/unit/init.py
--- a/pixywerk2/tests/unit/test_processchain.py
+++ b/pixywerk2/tests/unit/test_processchain.py
@ -0,0 +1,6 @@
 class TestProcessChain:
    def test_process_chain(self):
        pass
    def test_processor_chain(self):
        pass
--- a/pixywerk2/utils.py
+++ b/pixywerk2/utils.py
@ -0,0 +1,42 @@
 import mimetypes
 import os
 from typing import Dict, Optional
 def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict:
    """Merge two dictionaries.
    Arguments:
        dict_a (dict): The dictionary to use as the base.
        dict_b (dict): The dictionary to update the values with.
    Returns:
        dict: A new merged dictionary.
    """
    dict_z = dict_a.copy()
    dict_z.update(dict_b)
    return dict_z
 def guess_mime(path: str) -> Optional[str]:
    """Guess the mime type for a given path.
    Arguments:
        root (str): the root path of the file tree
        path (str): the sub-path within the file tree
    Returns:
        str: the guessed mime-type
    """
    mtypes = mimetypes.guess_type(path)
    ftype = None
    if os.path.isdir(path):
        ftype = "directory"
    elif os.access(path, os.F_OK) and mtypes[0]:
        ftype = mtypes[0]
    else:
        ftype = "application/octet-stream"
    return ftype
--- a/setup.py
+++ b/setup.py
@ -0,0 +1,58 @@
 """Package configuration."""
 from setuptools import find_packages, setup
 LONG_DESCRIPTION = """Pixywerk 2 is a DWIM filesystem based static site generator."""
 INSTALL_REQUIRES = ["yaml-1.3", "markdown", "jstyleson", "jinja2"]
 # Extra dependencies
 EXTRAS_REQUIRE = {
    # Test dependencies
    "tests": [
        "black",
        "bandit>=1.1.0",
        "flake8>=3.2.1",
        "mypy>=0.470",
        "prospector[with_everything]>=0.12.4",
        "pytest-cov>=1.8.0",
        "pytest-xdist>=1.15.0",
        "pytest>=3.0.3",
        "sphinx_rtd_theme>=0.1.6",
        "sphinx-argparse>=0.1.15",
        "Sphinx>=1.4.9",
    ]
 }
 SETUP_REQUIRES = ["pytest-runner>=2.7.1", "setuptools_scm>=1.15.0"]
 setup(
    author="Cassowary Rusnov",
    author_email="rusnovn@gmail.com",
    classifiers=[
        "Development Status :: 1 - Pre-alpha",
        "Environment :: Console",
        "License :: OSI Approved :: MIT",
        "Operating System :: POSIX :: Linux",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3 :: Only",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    description="A filesystem-based DWIM website generator / CMS",
    # entry_points={
    #    'console_scripts': [
    #        'cookbook = spicerack.cookbook:main',
    #    ],
    # },
    extras_require=EXTRAS_REQUIRE,
    install_requires=INSTALL_REQUIRES,
    keywords=["wmf", "automation", "orchestration"],
    license="MIT",
    long_description=LONG_DESCRIPTION,
    name="pixywerk2",  # Must be the same used for __version__ in __init__.py
    packages=find_packages(exclude=["*.tests", "*.tests.*"]),
    platforms=["GNU/Linux"],
    setup_requires=SETUP_REQUIRES,
    use_scm_version=True,
    url="https://git.antpanethon.com/cas/pixywerk2",
    zip_safe=False,
 )
--- a/tox.ini
+++ b/tox.ini
@ -0,0 +1,22 @@
 [tox]
 envlist=py{36,37}-{code-quality, unit} #, py37-sphinx
 skipsdist = true
 [testenv]
 setenv =
    LANG = en_US.UTF-8
 deps = .[tests]
 commands =
 	 unit: py.test --strict --cov-report=term-missing --cov=pixywerk2 pixywerk2/tests/unit {posargs}
 	 code-quality: flake8 pixywerk2
 	 code-quality: black -l 120 --check pixywerk2
 	 code-quality: - prospector -A
 	 code-quality: - mypy --ignore-missing-imports pixywerk2
 	 # sphinx: python setup.py build_sphinx -b html
   	 # sphinx: python setup.py build_sphinx -b man
 basepython =
    py36: python3.6
    py37: python3.7
 [flake8]
 max-line-length = 120