From 43d40f7fce533f90560f76c19b45861a950a3c8d Mon Sep 17 00:00:00 2001 From: Cas Rusnov Date: Sun, 14 Apr 2019 17:50:56 -0700 Subject: [PATCH] Major code import. --- demo/.meta | 7 + demo/atom.xml | 33 ++++ demo/atom.xml.meta | 5 + demo/bar/baz/quux/quuux | 0 demo/blog_posts/anotherpost.cont | 5 + demo/blog_posts/anotherpost.cont.meta | 4 + demo/blog_posts/test.cont | 1 + demo/blog_posts/test.cont.meta | 4 + demo/foo.cont | 1 + demo/foo.cont.meta | 5 + demo/index.cont | 19 +++ demo/mapping.json | 0 demo/passthrough.md | 9 ++ demo/passthrough.md.meta | 3 + demo/readme.md | 9 ++ demo/readme.md.meta | 3 + demo/templates/debug.jinja2 | 32 ++++ demo/templates/default-fs.jinja2 | 6 + demo/templates/default.jinja2 | 13 ++ pixywerk2/__init__.py | 0 pixywerk2/__main__.py | 119 ++++++++++++++ pixywerk2/defaults/chains.yaml | 104 +++++++++++++ pixywerk2/metadata.py | 145 +++++++++++++++++ pixywerk2/processchain.py | 180 ++++++++++++++++++++++ pixywerk2/processors/__init__.py | 1 + pixywerk2/processors/jinja2.py | 32 ++++ pixywerk2/processors/jinja2_page_embed.py | 76 +++++++++ pixywerk2/processors/passthrough.py | 68 ++++++++ pixywerk2/processors/process_less.py | 1 + pixywerk2/processors/process_md.py | 69 +++++++++ pixywerk2/processors/process_pp.py | 1 + pixywerk2/processors/process_sass.py | 1 + pixywerk2/processors/process_styl.py | 1 + pixywerk2/processors/processors.py | 63 ++++++++ pixywerk2/template_tools.py | 80 ++++++++++ pixywerk2/tests/unit/__init__.py | 0 pixywerk2/tests/unit/test_processchain.py | 6 + pixywerk2/utils.py | 42 +++++ setup.py | 58 +++++++ tox.ini | 22 +++ 40 files changed, 1228 insertions(+) create mode 100755 demo/.meta create mode 100644 demo/atom.xml create mode 100644 demo/atom.xml.meta create mode 100755 demo/bar/baz/quux/quuux create mode 100644 demo/blog_posts/anotherpost.cont create mode 100644 demo/blog_posts/anotherpost.cont.meta create mode 100644 demo/blog_posts/test.cont create mode 100644 demo/blog_posts/test.cont.meta create mode 100755 demo/foo.cont create mode 100755 demo/foo.cont.meta create mode 100644 demo/index.cont create mode 100755 demo/mapping.json create mode 100644 demo/passthrough.md create mode 100644 demo/passthrough.md.meta create mode 100644 demo/readme.md create mode 100644 demo/readme.md.meta create mode 100755 demo/templates/debug.jinja2 create mode 100755 demo/templates/default-fs.jinja2 create mode 100755 demo/templates/default.jinja2 create mode 100644 pixywerk2/__init__.py create mode 100644 pixywerk2/__main__.py create mode 100644 pixywerk2/defaults/chains.yaml create mode 100644 pixywerk2/metadata.py create mode 100644 pixywerk2/processchain.py create mode 100644 pixywerk2/processors/__init__.py create mode 100644 pixywerk2/processors/jinja2.py create mode 100644 pixywerk2/processors/jinja2_page_embed.py create mode 100644 pixywerk2/processors/passthrough.py create mode 100644 pixywerk2/processors/process_less.py create mode 100644 pixywerk2/processors/process_md.py create mode 100644 pixywerk2/processors/process_pp.py create mode 100644 pixywerk2/processors/process_sass.py create mode 100644 pixywerk2/processors/process_styl.py create mode 100644 pixywerk2/processors/processors.py create mode 100644 pixywerk2/template_tools.py create mode 100644 pixywerk2/tests/unit/__init__.py create mode 100644 pixywerk2/tests/unit/test_processchain.py create mode 100644 pixywerk2/utils.py create mode 100644 setup.py create mode 100644 tox.ini diff --git a/demo/.meta b/demo/.meta new file mode 100755 index 0000000..77f863c --- /dev/null +++ b/demo/.meta @@ -0,0 +1,7 @@ +{ +"site_root":"https://example.com", +"title":"Test Metadata", +"author": "Test User", +"author_email": "test_user@example.com", +"uuid_oid_root": "pixywerk-demo" +} diff --git a/demo/atom.xml b/demo/atom.xml new file mode 100644 index 0000000..43512f7 --- /dev/null +++ b/demo/atom.xml @@ -0,0 +1,33 @@ + + + + + {{ metadata.title }} + {{ metadata.subtitle }} + + + urn:uuid:{{ metadata.uuid }} + {{ get_time_iso8601(metadata['build-time']) }} + + {% set posts = get_file_list('blog_posts/*.cont') %} + {% for post in posts %} + {% set post_meta = get_file_metadata(post['file_path']) %} + + {{ post_meta.title }} + + urn:uuid:{{ post_meta.uuid }} + {{ get_time_iso8601(post_meta.stat.mtime) }} + {{post_meta.summary }} + + + + + + + + {{ post_meta.author }} + {{ post_meta.author_email }} + + + {% endfor %} + diff --git a/demo/atom.xml.meta b/demo/atom.xml.meta new file mode 100644 index 0000000..dee5897 --- /dev/null +++ b/demo/atom.xml.meta @@ -0,0 +1,5 @@ +{ +"type": "templatable", +"title": "Test RSS Feed", +"subtitle": "Some Subtitle" +} \ No newline at end of file diff --git a/demo/bar/baz/quux/quuux b/demo/bar/baz/quux/quuux new file mode 100755 index 0000000..e69de29 diff --git a/demo/blog_posts/anotherpost.cont b/demo/blog_posts/anotherpost.cont new file mode 100644 index 0000000..cbd4708 --- /dev/null +++ b/demo/blog_posts/anotherpost.cont @@ -0,0 +1,5 @@ +Some more post + + +la la la + diff --git a/demo/blog_posts/anotherpost.cont.meta b/demo/blog_posts/anotherpost.cont.meta new file mode 100644 index 0000000..b9a84d5 --- /dev/null +++ b/demo/blog_posts/anotherpost.cont.meta @@ -0,0 +1,4 @@ +{ +"title":"Another Post(tm)", +"summary":"Yet another post" +} diff --git a/demo/blog_posts/test.cont b/demo/blog_posts/test.cont new file mode 100644 index 0000000..637d210 --- /dev/null +++ b/demo/blog_posts/test.cont @@ -0,0 +1 @@ +Some content. diff --git a/demo/blog_posts/test.cont.meta b/demo/blog_posts/test.cont.meta new file mode 100644 index 0000000..2bc3818 --- /dev/null +++ b/demo/blog_posts/test.cont.meta @@ -0,0 +1,4 @@ +{ +"title":"Test.cont", +"summary":"Some empty test content" +} diff --git a/demo/foo.cont b/demo/foo.cont new file mode 100755 index 0000000..28a04cb --- /dev/null +++ b/demo/foo.cont @@ -0,0 +1 @@ +yo fresh \ No newline at end of file diff --git a/demo/foo.cont.meta b/demo/foo.cont.meta new file mode 100755 index 0000000..59f3331 --- /dev/null +++ b/demo/foo.cont.meta @@ -0,0 +1,5 @@ +{ +"foo":"bar", +"title":"A title", +"summary":"Just a post." +} diff --git a/demo/index.cont b/demo/index.cont new file mode 100644 index 0000000..0c9ab9e --- /dev/null +++ b/demo/index.cont @@ -0,0 +1,19 @@ +

Index of all content

+{% for f in get_file_list('*', sort_order='file_name') %} +{{get_file_name(f['file_name'])}} +{% endfor %} + +

Including foo.cont.meta: +

+{{ get_file_content('foo.cont.meta') }}
+
+

+ +

Metadata

+ + +{% set metadata = get_file_metadata('foo.cont') %} +{% for k in metadata.keys() %} + +{% endfor %} +
keyvalue
{{k}}{{metadata[k]}}
diff --git a/demo/mapping.json b/demo/mapping.json new file mode 100755 index 0000000..e69de29 diff --git a/demo/passthrough.md b/demo/passthrough.md new file mode 100644 index 0000000..373643d --- /dev/null +++ b/demo/passthrough.md @@ -0,0 +1,9 @@ +# README # + +This is a test of the emergency compiled HTML system. This is only a *test*. + +[Foo!](foo.html) + +{% for i in range(100) %} +* {{ i }} +{% endfor %} diff --git a/demo/passthrough.md.meta b/demo/passthrough.md.meta new file mode 100644 index 0000000..d35f55b --- /dev/null +++ b/demo/passthrough.md.meta @@ -0,0 +1,3 @@ +{ +"pragma":["no-proc"] +} diff --git a/demo/readme.md b/demo/readme.md new file mode 100644 index 0000000..373643d --- /dev/null +++ b/demo/readme.md @@ -0,0 +1,9 @@ +# README # + +This is a test of the emergency compiled HTML system. This is only a *test*. + +[Foo!](foo.html) + +{% for i in range(100) %} +* {{ i }} +{% endfor %} diff --git a/demo/readme.md.meta b/demo/readme.md.meta new file mode 100644 index 0000000..a266664 --- /dev/null +++ b/demo/readme.md.meta @@ -0,0 +1,3 @@ +{ +"title":"Yo, markdown" +} \ No newline at end of file diff --git a/demo/templates/debug.jinja2 b/demo/templates/debug.jinja2 new file mode 100755 index 0000000..bcc22fb --- /dev/null +++ b/demo/templates/debug.jinja2 @@ -0,0 +1,32 @@ + + +Debug for {{path}} + + + +

{{path}}

+

Content

+
+{{content}} +
+ +

Environment

+ + +{% for k in environ.keys() %} + +{% endfor %} +
keyvalue
{{k}}{{environ[k]}}
+ +

Metadata

+ + +{% for k in metadata.keys() %} + +{% endfor %} +
keyvalue
{{k}}{{metadata[k]}}
+ diff --git a/demo/templates/default-fs.jinja2 b/demo/templates/default-fs.jinja2 new file mode 100755 index 0000000..09b2c85 --- /dev/null +++ b/demo/templates/default-fs.jinja2 @@ -0,0 +1,6 @@ + + +{% for f in files.keys() %} + +{% endfor %} +
filetypesizelast change
{{f}}{{files[f].type}}{{files[f].size}}{{files[f].ctime | date}}
diff --git a/demo/templates/default.jinja2 b/demo/templates/default.jinja2 new file mode 100755 index 0000000..64e5dcc --- /dev/null +++ b/demo/templates/default.jinja2 @@ -0,0 +1,13 @@ + + +{{metadata.title}} + + + +{{content}} + + diff --git a/pixywerk2/__init__.py b/pixywerk2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pixywerk2/__main__.py b/pixywerk2/__main__.py new file mode 100644 index 0000000..be9065d --- /dev/null +++ b/pixywerk2/__main__.py @@ -0,0 +1,119 @@ +# iterate source tree +# create directors in target tree +# for each item: +# run processor(s) on item, each processor could be in a chain or a branch +# Processors also provide filename munging +# output target based on processor output + +import argparse +import logging +import os +import sys +import time + +from typing import Dict, List, cast + +from .processchain import ProcessorChains +from .metadata import MetaTree +from .template_tools import file_list, file_name, file_content, file_metadata, time_iso8601 + + +logger = logging.getLogger() + + +def setup_logging(verbose: bool = False) -> None: + pass + + +def get_args(args: List[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser("Compile a Pixywerk directory into an output directory.") + + parser.add_argument("root", help="The root of the pixywerk directory to process.") + parser.add_argument("output", help="The output directory to export post-compiled files to.") + + parser.add_argument( + "-c", "--clean", help="Remove the target tree before proceeding (by renaming to .bak).", action="store_true" + ) + parser.add_argument("-s", "--safe", help="Abort if the target directory already exists.", action="store_true") + parser.add_argument("-t", "--template", help="The template directory (default: root/templates)", default=None) + parser.add_argument("-d", "--dry-run", help="Perform a dry-run.", action="store_true") + parser.add_argument("-v", "--verbose", help="Output verbosely.", action="store_true") + parser.add_argument("--processors", help="Specify a path to a processor configuration file.", default=None) + + result = parser.parse_args(args) + + # validate arguments + if not os.path.isdir(result.root): + raise FileNotFoundError("can't find root folder {}".format(result.root)) + + if not result.template: + result.template = os.path.join(result.root, "templates") + result.excludes = [result.template] + + return result + + +def main() -> int: + try: + args = get_args(sys.argv[1:]) + except FileNotFoundError as ex: + print("error finding arguments: {}".format(ex)) + return 1 + setup_logging(args.verbose) + if os.path.exists(args.output) and args.clean: + bak = "{}.bak-{}".format(args.output, int(time.time())) + print("cleaning target {} -> {}".format(args.output, bak)) + os.rename(args.output, bak) + + process_chains = ProcessorChains(args.processors) + + default_metadata = { + "templates": args.template, + "template": "default.jinja2", + "dir-template": "default-dir.jinja2", + "filters": {}, + "build-time": time.time(), + "build-datetime": time.ctime(), + "uuid-oid-root": "pixywerk", + } + meta_tree = MetaTree(args.root, default_metadata) + file_list_cache = cast(Dict, {}) + file_cont_cache = cast(Dict, {}) + file_name_cache = cast(Dict, {}) + default_metadata["globals"] = { + "get_file_list": file_list(args.root, file_list_cache), + "get_file_name": file_name(args.root, meta_tree, process_chains, file_name_cache), + "get_file_content": file_content(args.root, meta_tree, process_chains, file_cont_cache), + "get_file_metadata": file_metadata(meta_tree), + "get_time_iso8601": time_iso8601("UTC"), + } + + for root, _, files in os.walk(args.root): + workroot = os.path.relpath(root, args.root) + if workroot == ".": + workroot = "" + target_dir = os.path.join(args.output, workroot) + print("mkdir -> {}".format(target_dir)) + if not args.dry_run: + try: + os.mkdir(target_dir) + except FileExistsError: + if args.safe: + print("error, target directory exists, aborting") + return 1 + for f in files: + # fixme global generic filters + if f.endswith(".meta") or f.endswith("~"): + continue + metadata = meta_tree.get_metadata(os.path.join(workroot, f)) + chain = process_chains.get_chain_for_filename(os.path.join(root, f), ctx=metadata) + print("process {} -> {}".format(os.path.join(root, f), os.path.join(target_dir, chain.output_filename))) + if not args.dry_run: + with open(os.path.join(target_dir, chain.output_filename), "w") as outfile: + for line in chain.output: + outfile.write(line) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pixywerk2/defaults/chains.yaml b/pixywerk2/defaults/chains.yaml new file mode 100644 index 0000000..4683860 --- /dev/null +++ b/pixywerk2/defaults/chains.yaml @@ -0,0 +1,104 @@ +# Default: output == input +default: + extension: default + chain: + - passthrough + +# Any object that needs jinja scripts but no other explicit processing +templatable: + extension: null + chain: + - jinja2 + +# Markdown, BBCode and RST are first run through the templater, and then +# they are processed into HTML, and finally embedded in a page template. +markdown: + extension: + - md + chain: + - jinja2 + - process_md + - jinja2_page_embed +bbcode: + extension: + - bb + - pp + chain: + - jinja2 + - process_pp + - jinja2_page_embed +# FIXME implement RST processor +# restructured: +# extension: +# - rst +# chain: +# - jinja2 +# - process_rst +# - jinja2_page_embed + +# # JSON and YAML are split, passed through a pretty printer, and then output +# FIXME implement split chain processor, implement processor arguments +# json: +# extension: +# - json +# chain: +# - split (passthrough) +# - pp_json +# yaml: +# extension: +# - yml +# - yaml +# chain: +# - split (passthrough) +# - pp_yaml + +# Template-html is first passed through the templater, and then embedded +# in a page template +template-html: + extension: + - thtml + - cont + chain: + - jinja2 + - jinja2_page_embed + +# Smart CSS are simply converted to CSS. +sass: + extension: + - sass + - scss + chain: + - process_sass +less: + extension: + - less + chain: + - process_less + +stylus: + extension: + - styl + chain: + - process_styl + +# # Images are processed into thumbnails and sized in addition to being retained as their original +# FIXME implement split chain processor, implement processor arguments, +# image: +# extension: +# - jpg +# - jpeg +# - png +# chain: +# - split (image_bigthumb) +# - split (image_smallthumb) +# - passthrough + +# image_bigthumb: +# extension: +# chain: +# - smart_resize (big) + +# image_smallthumb: +# extension: +# chain: +# - smart_resize (small) diff --git a/pixywerk2/metadata.py b/pixywerk2/metadata.py new file mode 100644 index 0000000..8440866 --- /dev/null +++ b/pixywerk2/metadata.py @@ -0,0 +1,145 @@ +"""Constructs a tree-like object containing the metadata for a given path, and caches said metadata.""" + +import logging +import mimetypes +import os +import uuid + +from typing import Dict, Optional, Union, List, Tuple, Any, cast + +import jstyleson + +from .utils import guess_mime + +# setup mimetypes with some extra ones +mimetypes.init() +mimetypes.add_type("text/html", "thtml") +mimetypes.add_type("text/html", "cont") + +logger = logging.getLogger(__name__) + + +class MetaCacheMiss(Exception): + """Raised on cache miss.""" + + +class MetaCache: + """This class provides an in-memory cache for metadata tree.""" + + def __init__(self, max_age: float = 200.0): + """Initialize the cache. + + Arguments: + max_age (int): the number of seconds to age-out cache items + + """ + self._max_age = max_age + self._cache: Dict[str, Tuple[float, Any]] = {} + + def get(self, key: str, new_time_stamp: float) -> Any: + """Get an item from the cache. + + Arguments: + key (str): the cache key to retieve + new_time_stamp (int): The time to use to compare the stored time with + + Returns: + :obj:misc: The previously stored value. + + Raises: + MetaCacheMiss: on missing key, or on aged out + + """ + if key not in self._cache: + raise MetaCacheMiss("no item for key {}".format(key)) + + if self._cache[key][0] + self._max_age <= new_time_stamp: + return self._cache[key][1] + + raise MetaCacheMiss("cache expired for key {}".format(key)) + + def put(self, key: str, value: Union[Dict, List, int, str, object], time_stamp: float) -> None: + """Put an item into the cache. + + Arguments: + key (str): the key to store the cache item under + value (:obj:misc): the value to store + time_stamp (float): the time stamp to store the item under + + """ + self._cache[key] = (time_stamp, value) + + +class MetaTree: + """This provides an interface to loading and caching tree metadata for a given directory tree.""" + + def __init__(self, root: str, default_metadata: Optional[Dict] = None): + """Initialize the metadata tree object. + + Arguments: + root (str): The path to the root of the file tree to operate on. + default_metadata (dict, optional): The default metadata to apply to the tree + + """ + self._cache = MetaCache() + if default_metadata is None: + default_metadata = {} + self._default_metadata = default_metadata + if root[-1] != "/": + root += "/" + self._root = root + + def get_metadata(self, rel_path: str) -> Dict: + """Retrieve the metadata for a given path + + The general procedure is to iterate the tree, at each level +m load .meta (JSON formatted dictionary) for that level, and + then finally load the path.meta, and merge these dictionaries + in descendant order. + + Arguments: + rel_path (str): The path to retrieve the metadata for (relative to root) + + Returns: + dict: A dictionary of metadata for that path tree. + + """ + metablob = dict(self._default_metadata) + # iterate path components from root to target path + comps = [self._root] + rel_path.split("/") + fullpath = "" + for pth in comps: + fullpath = os.path.join(fullpath, pth) + st = os.stat(fullpath) + + cachekey = fullpath + ".meta" + meta = cast(Dict, {}) + try: + st_meta = os.stat(cachekey) + meta = self._cache.get(cachekey, st_meta.st_mtime) + except FileNotFoundError: + st_meta = None # type: ignore + except MetaCacheMiss: + meta = {} + + if not meta and st_meta: + meta = jstyleson.load(open(cachekey, "r")) + self._cache.put(cachekey, meta, st_meta.st_mtime) + + metablob.update(meta) + + # return final dict + metablob["dir"], metablob["file_name"] = os.path.split(rel_path) + metablob["file_path"] = rel_path + metablob["uuid"] = uuid.uuid3( + uuid.NAMESPACE_OID, metablob["uuid-oid-root"] + os.path.join(self._root, rel_path) + ) + metablob["os-path"], _ = os.path.split(fullpath) + metablob["guessed-type"] = guess_mime(os.path.join(self._root, rel_path)) + if "mime-type" not in metablob: + metablob["mime-type"] = metablob["guessed-type"] + metablob["stat"] = {} + for stk in ("st_mtime", "st_ctime", "st_atime", "st_mode", "st_size", "st_ino"): + metablob["stat"][stk.replace("st_", "")] = getattr(st, stk) + + return metablob diff --git a/pixywerk2/processchain.py b/pixywerk2/processchain.py new file mode 100644 index 0000000..d5952d0 --- /dev/null +++ b/pixywerk2/processchain.py @@ -0,0 +1,180 @@ +"""Interface for chains of processors""" + +import os +import os.path +import random + +from typing import List, Iterable, Optional, Any, Dict, Type, cast + +import yaml + +from .processors.processors import Processor + + +class ProcessorChain: + """This implements a wrapper for an arbitrary set of processors and an associated file stream.""" + + def __init__( + self, + processors: List[Processor], + file_name: str, + file_data: Iterable[str], + file_type: str, + ctx: Optional[Dict] = None, + ): + """Initialize the processing stream. + + Arguments: + processors (list): A list of processor objects. + file_data (Iterable): An iterable from which to retrieve the input + file_type (str): the specified file type for consumer information. + + """ + self._processors = processors + self._file_data = file_data + self._file_type = file_type + self._file_name = file_name + self._ctx: Dict = {} + if ctx is not None: + self._ctx = cast(Dict, ctx) + + @property + def output(self) -> Iterable: + """Return an iterable for the output of the process chain + + Returns: + :obj:'iterable': the iterable + + """ + prev = self._file_data + for processor in self._processors: + if processor: + prev = processor.process(prev, self._ctx) + + return prev + + @property + def output_mime(self) -> str: + """Return the post-processed MIME value from the processing chain + + Returns: + str: the mime type + + """ + fname = self._file_name + for processor in self._processors: + fname = processor.mime_type(fname, self._ctx) + return fname + + @property + def output_ext(self) -> str: + """Return the post-processed extension from the processing chain + + Returns: + str: the extension + """ + fname = self._file_name + for processor in self._processors: + fname = processor.extension(fname, self._ctx) + return fname + + @property + def output_filename(self) -> str: + """Return the post-processed filename from the processing chain + + Returns: + str: the new filename + + """ + fname = os.path.basename(self._file_name) + for processor in self._processors: + fname = processor.filename(fname, self._ctx) + return fname + + +class ProcessorChains: + """Load a configuration for processor chains, and provide ability to process the chains given a particular input + file. + """ + + def __init__(self, config: Optional[str] = None): + """Initialize, with a specified configuration file + + Arguments: + config (str, optional): The path to a yaml formatted configuration file. + + """ + if config is None: # pragma: no coverage + config = os.path.join(os.path.dirname(__file__), "defaults", "chains.yaml") + + self.chainconfig = yaml.load(open(config, "r")) + self.extensionmap: Dict[str, Any] = {} + self.processors: Dict[str, Type[Processor]] = {} + for ch, conf in self.chainconfig.items(): + if conf["extension"] == "default": + self.default = ch + else: + if conf["extension"]: + for ex in conf["extension"]: + if ex in self.extensionmap or ex is None: + # log an error or except or something we'll just override for now. + pass + self.extensionmap[ex] = ch + for pr in conf["chain"]: + if pr in self.processors: + continue + processor_module = __import__("processors", globals(), locals(), [pr], 1) + self.processors[pr] = processor_module.__dict__[pr].processor + + def get_chain_for_filename(self, filename: str, ctx: Optional[Dict] = None) -> ProcessorChain: + """Get the ProcessorChain, as configured for a given file by extension. + + Arguments: + filename (str): The name of the file to get a chain for. + + Returns: + ProcessorChain: the constructed processor chain. + """ + r = filename.rsplit(".", 1) + ftype = "default" + if r: + ftype = r[-1] + if ctx and "pragma" in ctx: + if "no-proc" in ctx["pragma"]: + ftype = "default" + + if ctx and "type" in ctx: + ftype = ctx["type"] + return self.get_chain_for_file(open(filename, "r"), ftype, filename, ctx) + + def get_chain_for_file( + self, file_obj: Iterable, file_ext: str, file_name: Optional[str] = None, ctx: Optional[Dict] = None + ) -> ProcessorChain: + """Get the ProcessorChain for a given iterable object based on the specified file type + + Arguments: + file_obj (:obj:`iterable`): The input file stream + file_ext (str): The type (extension) of the input stream + + Returns: + ProcessorChain: the constructed processor chain. + + """ + if file_ext not in self.extensionmap or not self.extensionmap[file_ext]: + if file_ext in self.chainconfig: + file_type = file_ext + else: + file_type = "default" + else: + file_type = self.extensionmap[file_ext] + + if not (bool(file_name)): + file_name = hex(random.randint(0, 65536)) + + return ProcessorChain( + [self.processors[x]() for x in self.chainconfig[file_type]["chain"]], + cast(str, file_name), + file_obj, + file_type, + ctx, + ) diff --git a/pixywerk2/processors/__init__.py b/pixywerk2/processors/__init__.py new file mode 100644 index 0000000..d058ff6 --- /dev/null +++ b/pixywerk2/processors/__init__.py @@ -0,0 +1 @@ +# processors metadata here diff --git a/pixywerk2/processors/jinja2.py b/pixywerk2/processors/jinja2.py new file mode 100644 index 0000000..a330237 --- /dev/null +++ b/pixywerk2/processors/jinja2.py @@ -0,0 +1,32 @@ +"""Define a Jinja2 Processor which applies programmable templating to the input stream.""" + +from typing import Iterable, Optional, Dict, cast + +from jinja2 import Environment, FileSystemLoader + +from .passthrough import PassThrough + + +class Jinja2(PassThrough): + """Pass the input stream through Jinja2 for scritable templating.""" + + def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: + """Return an iterable object of the post-processed file. + + Arguments: + input_file (iterable): An input stream + ctx (dict, optional): A context object generated from the processor configuration + + + Returns: + iterable: The post-processed output stream + """ + ctx = cast(Dict, ctx) + template_env = Environment(loader=FileSystemLoader(ctx["templates"])) + template_env.globals.update(ctx["globals"]) + template_env.filters.update(ctx["filters"]) + tmpl = template_env.from_string("".join([x for x in input_file])) + return tmpl.render(metadata=ctx) + + +processor = Jinja2 diff --git a/pixywerk2/processors/jinja2_page_embed.py b/pixywerk2/processors/jinja2_page_embed.py new file mode 100644 index 0000000..92310b6 --- /dev/null +++ b/pixywerk2/processors/jinja2_page_embed.py @@ -0,0 +1,76 @@ +"""Define a Jinja2 processor which embeds the (presumably HTML) input stream into a Page Template + as defined in the ctx metadata (the ``content`` variable is assigned to the input stream and + the target template is rendered).""" + +import os + +from typing import Iterable, Optional, Dict, cast + +from jinja2 import Environment, FileSystemLoader + +from .processors import Processor + + +class Jinja2PageEmbed(Processor): + """Embed input stream as ``content`` variable in page template defined in context key ``template``.""" + + def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the filename of the post-processed file. + + Arguments: + oldname (str): the previous name for the file. + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new name for the file + + """ + + return os.path.splitext(oldname)[0] + ".html" + + def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new mimetype of the file after processing + + """ + return "text/html" + + def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: + """Return an iterable object of the post-processed file. + + Arguments: + input_file (iterable): An input stream + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + iterable: The post-processed output stream + """ + ctx = cast(Dict, ctx) + template_env = Environment(loader=FileSystemLoader(ctx["templates"])) + template_env.globals.update(ctx["globals"]) + template_env.filters.update(ctx["filters"]) + tmpl = template_env.get_template(ctx["template"]) + content = "".join([x for x in input_file]) + return tmpl.render(content=content, metadata=ctx) + + def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new extension of the file after processing + + """ + return "html" + + +processor = Jinja2PageEmbed diff --git a/pixywerk2/processors/passthrough.py b/pixywerk2/processors/passthrough.py new file mode 100644 index 0000000..9a0601d --- /dev/null +++ b/pixywerk2/processors/passthrough.py @@ -0,0 +1,68 @@ +"""Passthrough progcessor which takes input and returns it.""" + +import os + +from .processors import Processor +from ..utils import guess_mime +from typing import Iterable, Optional, Dict, cast + + +class PassThrough(Processor): + """A simple passthrough processor that takes input and sends it to output.""" + + def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the filename of the post-processed file. + + Arguments: + oldname (str): the previous name for the file. + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new name for the file + + """ + return oldname + + def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new mimetype of the file after processing + + """ + result = cast(str, guess_mime(oldname)) + if result == "directory": + result = "DIR" + return result + + def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: + """Return an iterable object of the post-processed file. + + Arguments: + input_file (iterable): An input stream + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + iterable: The post-processed output stream + """ + return input_file + + def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new extension of the file after processing + + """ + return os.path.splitext(oldname)[-1] + + +processor = PassThrough diff --git a/pixywerk2/processors/process_less.py b/pixywerk2/processors/process_less.py new file mode 100644 index 0000000..a9f542f --- /dev/null +++ b/pixywerk2/processors/process_less.py @@ -0,0 +1 @@ +processor = None diff --git a/pixywerk2/processors/process_md.py b/pixywerk2/processors/process_md.py new file mode 100644 index 0000000..0687bc6 --- /dev/null +++ b/pixywerk2/processors/process_md.py @@ -0,0 +1,69 @@ +"""Convert an MD stream into an HTML stream""" + +import io +import os + +from typing import Iterable, Optional, Dict + +import markdown + +from .processors import Processor + + +class MarkdownProcessor(Processor): + """Convert an MD stream into an HTML stream""" + + def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the filename of the post-processed file. + + Arguments: + oldname (str): the previous name for the file. + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new name for the file + + """ + return os.path.splitext(oldname)[0] + ".html" + + def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new mimetype of the file after processing + + """ + return "text/html" + + def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new extension of the file after processing + + """ + return "html" + + def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: + """Return an iterable object of the post-processed file. + + Arguments: + input_file (iterable): An input stream + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + iterable: The post-processed output stream + """ + md = u"".join([x for x in input_file]) + return io.StringIO(markdown.markdown(md, extensions=["extra", "admonition", "wikilinks"])) + + +processor = MarkdownProcessor # pylint: disable=invalid-name diff --git a/pixywerk2/processors/process_pp.py b/pixywerk2/processors/process_pp.py new file mode 100644 index 0000000..a9f542f --- /dev/null +++ b/pixywerk2/processors/process_pp.py @@ -0,0 +1 @@ +processor = None diff --git a/pixywerk2/processors/process_sass.py b/pixywerk2/processors/process_sass.py new file mode 100644 index 0000000..a9f542f --- /dev/null +++ b/pixywerk2/processors/process_sass.py @@ -0,0 +1 @@ +processor = None diff --git a/pixywerk2/processors/process_styl.py b/pixywerk2/processors/process_styl.py new file mode 100644 index 0000000..a9f542f --- /dev/null +++ b/pixywerk2/processors/process_styl.py @@ -0,0 +1 @@ +processor = None diff --git a/pixywerk2/processors/processors.py b/pixywerk2/processors/processors.py new file mode 100644 index 0000000..3161654 --- /dev/null +++ b/pixywerk2/processors/processors.py @@ -0,0 +1,63 @@ +import abc + +from typing import Iterable, Optional, Dict + + +class ProcessorException(Exception): # pragma: no cover + """A base exception class to be used by processor objects.""" + + +class Processor(abc.ABC): # pragma: no cover + def __init__(self, *args, **kwargs): + """Initialize the class.""" + + @abc.abstractmethod + def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the filename of the post-processed file. + + Arguments: + oldname (str): the previous name for the file. + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new name for the file + + """ + + @abc.abstractmethod + def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new mimetype of the file after processing + + """ + + @abc.abstractmethod + def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: + """Return the mimetype of the post-processed file. + + Arguments: + oldname (str): the input filename + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + str: the new extension of the file after processing + + """ + + @abc.abstractmethod + def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: + """Return an iterable object of the post-processed file. + + Arguments: + input_file (iterable): An input stream + ctx (dict, optional): A context object generated from the processor configuration + + Returns: + iterable: The post-processed output stream + """ diff --git a/pixywerk2/template_tools.py b/pixywerk2/template_tools.py new file mode 100644 index 0000000..56c71f2 --- /dev/null +++ b/pixywerk2/template_tools.py @@ -0,0 +1,80 @@ +import datetime +import glob +import itertools +import os +import pytz +from typing import Callable, Dict, List, Iterable, Union, cast + +from .metadata import MetaTree +from .processchain import ProcessorChains + + +def file_list(root: str, listcache: Dict) -> Callable: + def get_file_list(path_glob: str, *, sort_order: str = "ctime", reverse: bool = False, limit: int = 0) -> Iterable: + stattable = cast(List, []) + if path_glob in listcache: + stattable = listcache[path_glob] + else: + for fil in glob.glob(os.path.join(root, path_glob)): + if os.path.isdir(fil): + continue + if fil.endswith(".meta") or fil.endswith("~"): + continue + st = os.stat(fil) + stattable.append( + { + "file_path": os.path.relpath(fil, root), + "file_name": os.path.split(fil)[-1], + "mtime": st.st_mtime, + "ctime": st.st_ctime, + "size": st.st_size, + "ext": os.path.splitext(fil)[1], + } + ) + listcache[path_glob] = stattable + ret = sorted(stattable, key=lambda x: x[sort_order], reverse=reverse) + if limit > 0: + return itertools.islice(ret, limit) + return ret + + return get_file_list + + +def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains, namecache: Dict) -> Callable: + def get_file_name(file_name: str) -> Dict: + if file_name in namecache: + return namecache[file_name] + metadata = metatree.get_metadata(file_name) + chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata) + namecache[file_name] = chain.output_filename + return namecache[file_name] + + return get_file_name + + +def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChains, contcache: Dict) -> Callable: + def get_file_content(file_name: str) -> Iterable: + if file_name in contcache: + return contcache[file_name] + metadata = metatree.get_metadata(file_name) + chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata) + contcache[file_name] = chain.output + return chain.output + + return get_file_content + + +def file_metadata(metatree: MetaTree) -> Callable: + def get_file_metadata(file_name: str) -> Dict: + return metatree.get_metadata(file_name) + + return get_file_metadata + + +def time_iso8601(timezone: str) -> Callable: + tz = pytz.timezone(timezone) + + def get_time_iso8601(time_t: Union[int, float]) -> str: + return datetime.datetime.fromtimestamp(time_t, tz).isoformat("T") + + return get_time_iso8601 diff --git a/pixywerk2/tests/unit/__init__.py b/pixywerk2/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pixywerk2/tests/unit/test_processchain.py b/pixywerk2/tests/unit/test_processchain.py new file mode 100644 index 0000000..1342fb4 --- /dev/null +++ b/pixywerk2/tests/unit/test_processchain.py @@ -0,0 +1,6 @@ +class TestProcessChain: + def test_process_chain(self): + pass + + def test_processor_chain(self): + pass diff --git a/pixywerk2/utils.py b/pixywerk2/utils.py new file mode 100644 index 0000000..962c9e2 --- /dev/null +++ b/pixywerk2/utils.py @@ -0,0 +1,42 @@ +import mimetypes +import os + +from typing import Dict, Optional + + +def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict: + """Merge two dictionaries. + + Arguments: + dict_a (dict): The dictionary to use as the base. + dict_b (dict): The dictionary to update the values with. + + Returns: + dict: A new merged dictionary. + + """ + dict_z = dict_a.copy() + dict_z.update(dict_b) + return dict_z + + +def guess_mime(path: str) -> Optional[str]: + """Guess the mime type for a given path. + + Arguments: + root (str): the root path of the file tree + path (str): the sub-path within the file tree + + Returns: + str: the guessed mime-type + + """ + mtypes = mimetypes.guess_type(path) + ftype = None + if os.path.isdir(path): + ftype = "directory" + elif os.access(path, os.F_OK) and mtypes[0]: + ftype = mtypes[0] + else: + ftype = "application/octet-stream" + return ftype diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..84d9008 --- /dev/null +++ b/setup.py @@ -0,0 +1,58 @@ +"""Package configuration.""" +from setuptools import find_packages, setup + +LONG_DESCRIPTION = """Pixywerk 2 is a DWIM filesystem based static site generator.""" + +INSTALL_REQUIRES = ["yaml-1.3", "markdown", "jstyleson", "jinja2"] + +# Extra dependencies +EXTRAS_REQUIRE = { + # Test dependencies + "tests": [ + "black", + "bandit>=1.1.0", + "flake8>=3.2.1", + "mypy>=0.470", + "prospector[with_everything]>=0.12.4", + "pytest-cov>=1.8.0", + "pytest-xdist>=1.15.0", + "pytest>=3.0.3", + "sphinx_rtd_theme>=0.1.6", + "sphinx-argparse>=0.1.15", + "Sphinx>=1.4.9", + ] +} + +SETUP_REQUIRES = ["pytest-runner>=2.7.1", "setuptools_scm>=1.15.0"] +setup( + author="Cassowary Rusnov", + author_email="rusnovn@gmail.com", + classifiers=[ + "Development Status :: 1 - Pre-alpha", + "Environment :: Console", + "License :: OSI Approved :: MIT", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + description="A filesystem-based DWIM website generator / CMS", + # entry_points={ + # 'console_scripts': [ + # 'cookbook = spicerack.cookbook:main', + # ], + # }, + extras_require=EXTRAS_REQUIRE, + install_requires=INSTALL_REQUIRES, + keywords=["wmf", "automation", "orchestration"], + license="MIT", + long_description=LONG_DESCRIPTION, + name="pixywerk2", # Must be the same used for __version__ in __init__.py + packages=find_packages(exclude=["*.tests", "*.tests.*"]), + platforms=["GNU/Linux"], + setup_requires=SETUP_REQUIRES, + use_scm_version=True, + url="https://git.antpanethon.com/cas/pixywerk2", + zip_safe=False, +) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..53ca9b0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist=py{36,37}-{code-quality, unit} #, py37-sphinx +skipsdist = true + +[testenv] +setenv = + LANG = en_US.UTF-8 +deps = .[tests] +commands = + unit: py.test --strict --cov-report=term-missing --cov=pixywerk2 pixywerk2/tests/unit {posargs} + code-quality: flake8 pixywerk2 + code-quality: black -l 120 --check pixywerk2 + code-quality: - prospector -A + code-quality: - mypy --ignore-missing-imports pixywerk2 + # sphinx: python setup.py build_sphinx -b html + # sphinx: python setup.py build_sphinx -b man +basepython = + py36: python3.6 + py37: python3.7 + +[flake8] +max-line-length = 120