diff --git a/TODO.md b/TODO.md index af6c80b..ffb5dde 100644 --- a/TODO.md +++ b/TODO.md @@ -4,3 +4,5 @@ * Project global defines, parameters. * pre- and post-scripts that will be run from __main__, either some shipped with pixywerk or project-level. * Library of template modules? ATOM et al. +* Some off the shelf website templates and a template manager. +* Live refreshing server thing which maps a pixywerk tree into a web server's memory and updates on change. diff --git a/READMES/METADATA.md b/docs/METADATA.md similarity index 100% rename from READMES/METADATA.md rename to docs/METADATA.md diff --git a/READMES/Patterns.md b/docs/Patterns.md similarity index 100% rename from READMES/Patterns.md rename to docs/Patterns.md diff --git a/READMES/TemplateFunctions.md b/docs/TemplateFunctions.md similarity index 100% rename from READMES/TemplateFunctions.md rename to docs/TemplateFunctions.md diff --git a/READMES/project-layout.md b/docs/project-layout.md similarity index 100% rename from READMES/project-layout.md rename to docs/project-layout.md diff --git a/examples/pixywerk.com/publish/atom.xml b/examples/pixywerk.com/publish/atom.xml index ccb139c..c9170f9 100644 --- a/examples/pixywerk.com/publish/atom.xml +++ b/examples/pixywerk.com/publish/atom.xml @@ -7,7 +7,7 @@ urn:uuid:2cbb1961-b1ca-3b73-a6ce-d2e2feae9ab4 - 2019-05-20T06:32:27.357672+00:00 + 2021-02-05T04:45:24.766181+00:00 diff --git a/examples/pixywerk.com/publish/css/main.css b/examples/pixywerk.com/publish/css/main.css index 05b7f22..64a56d0 100644 --- a/examples/pixywerk.com/publish/css/main.css +++ b/examples/pixywerk.com/publish/css/main.css @@ -267,6 +267,11 @@ footer { font-variant: small-caps; } +pre { line-height: 125%; margin: 0; } +td.linenos pre { color: #000000; background-color: #f0f0f0; padding: 0 5px 0 5px; } +span.linenos { color: #000000; background-color: #f0f0f0; padding: 0 5px 0 5px; } +td.linenos pre.special { color: #000000; background-color: #ffffc0; padding: 0 5px 0 5px; } +span.linenos.special { color: #000000; background-color: #ffffc0; padding: 0 5px 0 5px; } .hll { background-color: #ffffcc } .c { color: #008800; font-style: italic } /* Comment */ .err { border: 1px solid #FF0000 } /* Error */ diff --git a/examples/pixywerk.com/publish/posts/post-2019-04-15.html b/examples/pixywerk.com/publish/posts/post-2019-04-15.html index b4349d8..c70b00d 100644 --- a/examples/pixywerk.com/publish/posts/post-2019-04-15.html +++ b/examples/pixywerk.com/publish/posts/post-2019-04-15.html @@ -16,7 +16,7 @@

My first post

-
+

Author: Cas Rusnov
Published: 2019-05-20T01:56:41.720834+00:00 diff --git a/examples/pixywerk.com/publish/posts/post-2019-05-16.another-post.html b/examples/pixywerk.com/publish/posts/post-2019-05-16.another-post.html index 0091fb1..a2448ba 100644 --- a/examples/pixywerk.com/publish/posts/post-2019-05-16.another-post.html +++ b/examples/pixywerk.com/publish/posts/post-2019-05-16.another-post.html @@ -16,7 +16,7 @@

Another example post!

-
+

Author: Cas Rusnov
Published: 2019-05-20T02:09:55.805659+00:00 diff --git a/examples/pixywerk.com/publish/posts/post-2019-05-19.html b/examples/pixywerk.com/publish/posts/post-2019-05-19.html index 79b0af6..f1ad673 100644 --- a/examples/pixywerk.com/publish/posts/post-2019-05-19.html +++ b/examples/pixywerk.com/publish/posts/post-2019-05-19.html @@ -16,7 +16,7 @@

Code Test Post

-
+

Author: Cas Rusnov
Published: 2019-05-20T06:31:37.352443+00:00 @@ -74,7 +74,7 @@ Some python code: # 'cookbook = spicerack.cookbook:main', # ], # }, - include_package_data=True, + include_package_data=True, extras_require=EXTRAS_REQUIRE, install_requires=INSTALL_REQUIRES, keywords=["cms", "website", "compiler"], @@ -84,9 +84,9 @@ Some python code: packages=find_packages(exclude=["*.tests", "*.tests.*"]), platforms=["GNU/Linux"], setup_requires=SETUP_REQUIRES, - use_scm_version=True, + use_scm_version=True, url="https://git.antpanethon.com/cas/pixywerk2", - zip_safe=False, + zip_safe=False, )

diff --git a/examples/pixywerk.com/publish/recurse.html b/examples/pixywerk.com/publish/recurse.html new file mode 100644 index 0000000..62b445b --- /dev/null +++ b/examples/pixywerk.com/publish/recurse.html @@ -0,0 +1,134 @@ + + + Pixywerk2 + + + +
+ +

Pixywerk2

+ + +
+
    + +
  • {'file_path': 'atom.xml', 'file_name': 'atom.xml', 'mtime': 1558060013.0402112, 'ctime': 1558060013.0482113, 'size': 1209, 'ext': '.xml'}
    • + +
  • {'file_path': 'index.thtml', 'file_name': 'index.thtml', 'mtime': 1558317628.2501144, 'ctime': 1558317628.2501144, 'size': 602, 'ext': '.thtml'}
    • + +
  • {'file_path': 'setup.py', 'file_name': 'setup.py', 'mtime': 1558332717.6640935, 'ctime': 1558332717.6640935, 'size': 1847, 'ext': '.py'}
    • + +
  • {'file_path': 'recurse.thtml', 'file_name': 'recurse.thtml', 'mtime': 1561830991.2152739, 'ctime': 1561830991.223274, 'size': 72, 'ext': '.thtml'}
    • + +
  • {'file_path': 'posts/post-2019-04-15.thtml', 'file_name': 'post-2019-04-15.thtml', 'mtime': 1557985040.3647952, 'ctime': 1558317401.7208338, 'size': 106, 'ext': '.thtml'}
    • + +
  • {'file_path': 'posts/post-2019-05-16.another-post.thtml', 'file_name': 'post-2019-05-16.another-post.thtml', 'mtime': 1558318195.8056588, 'ctime': 1558318195.8056588, 'size': 79, 'ext': '.thtml'}
    • + +
  • {'file_path': 'posts/post-2019-05-19.thtml', 'file_name': 'post-2019-05-19.thtml', 'mtime': 1558333897.3524435, 'ctime': 1558333897.3524435, 'size': 181, 'ext': '.thtml'}
    • + +
  • {'file_path': 'css/main.css', 'file_name': 'main.css', 'mtime': 1558333943.9771295, 'ctime': 1558333943.9771295, 'size': 5589, 'ext': '.css'}
    • + +
  • {'file_path': 'css/fonts/Teko-Bold.woff', 'file_name': 'Teko-Bold.woff', 'mtime': 1557719150.1630452, 'ctime': 1557719150.1630452, 'size': 111224, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/Teko-Bold.woff2', 'file_name': 'Teko-Bold.woff2', 'mtime': 1557719150.8430567, 'ctime': 1557719150.8430567, 'size': 77468, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/Teko-Light.woff', 'file_name': 'Teko-Light.woff', 'mtime': 1557719150.863057, 'ctime': 1557719150.863057, 'size': 108568, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/Teko-Light.woff2', 'file_name': 'Teko-Light.woff2', 'mtime': 1557719151.4950676, 'ctime': 1557719151.4950676, 'size': 75476, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/Teko-Medium.woff', 'file_name': 'Teko-Medium.woff', 'mtime': 1557719151.5150678, 'ctime': 1557719151.5150678, 'size': 115388, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/Teko-Medium.woff2', 'file_name': 'Teko-Medium.woff2', 'mtime': 1557719152.1750789, 'ctime': 1557719152.1750789, 'size': 80216, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/Teko-Regular.woff', 'file_name': 'Teko-Regular.woff', 'mtime': 1557719152.1950793, 'ctime': 1557719152.1950793, 'size': 115048, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/Teko-Regular.woff2', 'file_name': 'Teko-Regular.woff2', 'mtime': 1557719152.8550904, 'ctime': 1557719152.8550904, 'size': 80188, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/Teko-SemiBold.woff', 'file_name': 'Teko-SemiBold.woff', 'mtime': 1557719152.8750906, 'ctime': 1557719152.8750906, 'size': 117184, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/Teko-SemiBold.woff2', 'file_name': 'Teko-SemiBold.woff2', 'mtime': 1557719153.5591023, 'ctime': 1557719153.5591023, 'size': 81756, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Regular.ttf', 'file_name': 'ExpletusSans-Regular.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6432533, 'size': 55052, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Italic.ttf', 'file_name': 'ExpletusSans-Italic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6472535, 'size': 37344, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Medium.ttf', 'file_name': 'ExpletusSans-Medium.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6472535, 'size': 66676, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-MediumItalic.ttf', 'file_name': 'ExpletusSans-MediumItalic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6512535, 'size': 62900, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-SemiBoldItalic.ttf', 'file_name': 'ExpletusSans-SemiBoldItalic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6552536, 'size': 46812, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-SemiBold.ttf', 'file_name': 'ExpletusSans-SemiBold.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6552536, 'size': 65520, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Bold.ttf', 'file_name': 'ExpletusSans-Bold.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6592536, 'size': 60784, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-BoldItalic.ttf', 'file_name': 'ExpletusSans-BoldItalic.ttf', 'mtime': 1304492400.0, 'ctime': 1557720047.6632538, 'size': 45072, 'ext': '.ttf'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Bold.woff', 'file_name': 'ExpletusSans-Bold.woff', 'mtime': 1557720055.5434017, 'ctime': 1557720055.5434017, 'size': 28176, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Bold.woff2', 'file_name': 'ExpletusSans-Bold.woff2', 'mtime': 1557720055.6754043, 'ctime': 1557720055.6754043, 'size': 20628, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-BoldItalic.woff', 'file_name': 'ExpletusSans-BoldItalic.woff', 'mtime': 1557720055.6794043, 'ctime': 1557720055.6794043, 'size': 21016, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-BoldItalic.woff2', 'file_name': 'ExpletusSans-BoldItalic.woff2', 'mtime': 1557720055.767406, 'ctime': 1557720055.767406, 'size': 15356, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Italic.woff', 'file_name': 'ExpletusSans-Italic.woff', 'mtime': 1557720055.7754061, 'ctime': 1557720055.7754061, 'size': 19132, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Italic.woff2', 'file_name': 'ExpletusSans-Italic.woff2', 'mtime': 1557720055.8514075, 'ctime': 1557720055.8514075, 'size': 14248, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Medium.woff', 'file_name': 'ExpletusSans-Medium.woff', 'mtime': 1557720055.8554077, 'ctime': 1557720055.8554077, 'size': 31592, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Medium.woff2', 'file_name': 'ExpletusSans-Medium.woff2', 'mtime': 1557720056.0074105, 'ctime': 1557720056.0074105, 'size': 22784, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-MediumItalic.woff', 'file_name': 'ExpletusSans-MediumItalic.woff', 'mtime': 1557720056.0154107, 'ctime': 1557720056.0154107, 'size': 27148, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-MediumItalic.woff2', 'file_name': 'ExpletusSans-MediumItalic.woff2', 'mtime': 1557720056.1594133, 'ctime': 1557720056.1594133, 'size': 19972, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Regular.woff', 'file_name': 'ExpletusSans-Regular.woff', 'mtime': 1557720056.1674135, 'ctime': 1557720056.1674135, 'size': 26540, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-Regular.woff2', 'file_name': 'ExpletusSans-Regular.woff2', 'mtime': 1557720056.291416, 'ctime': 1557720056.291416, 'size': 19844, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-SemiBold.woff', 'file_name': 'ExpletusSans-SemiBold.woff', 'mtime': 1557720056.2954159, 'ctime': 1557720056.2954159, 'size': 31536, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-SemiBold.woff2', 'file_name': 'ExpletusSans-SemiBold.woff2', 'mtime': 1557720056.459419, 'ctime': 1557720056.459419, 'size': 22784, 'ext': '.woff2'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-SemiBoldItalic.woff', 'file_name': 'ExpletusSans-SemiBoldItalic.woff', 'mtime': 1557720056.4634192, 'ctime': 1557720056.4634192, 'size': 22556, 'ext': '.woff'}
    • + +
  • {'file_path': 'css/fonts/ExpletusSans-SemiBoldItalic.woff2', 'file_name': 'ExpletusSans-SemiBoldItalic.woff2', 'mtime': 1557720056.563421, 'ctime': 1557720056.563421, 'size': 16484, 'ext': '.woff2'}
    • + +
  • {'file_path': 'test/test.thtml', 'file_name': 'test.thtml', 'mtime': 1558064341.65635, 'ctime': 1558064341.65635, 'size': 115, 'ext': '.thtml'}
    • + +
  • {'file_path': 'images/placeholder', 'file_name': 'placeholder', 'mtime': 1555378617.4188664, 'ctime': 1555378617.4188664, 'size': 0, 'ext': ''}
    • + +
  • {'file_path': 'images/20190415-0.jpg', 'file_name': '20190415-0.jpg', 'mtime': 1555379532.14708, 'ctime': 1555379532.14708, 'size': 73996, 'ext': '.jpg'}
    • + +
  • {'file_path': 'images/link-box-variant.svg', 'file_name': 'link-box-variant.svg', 'mtime': 1557815009.004399, 'ctime': 1557815052.4972086, 'size': 1402, 'ext': '.svg'}
    • + +
  • {'file_path': 'images/pipe-leak.svg', 'file_name': 'pipe-leak.svg', 'mtime': 1557897588.8718655, 'ctime': 1557897849.3715317, 'size': 448, 'ext': '.svg'}
    • + +
  • {'file_path': 'images/2019-05-16.png', 'file_name': '2019-05-16.png', 'mtime': 1558057040.7048073, 'ctime': 1558057040.7048073, 'size': 457198, 'ext': '.png'}
    • + +
  • {'file_path': 'images/2019-05-19.png', 'file_name': '2019-05-19.png', 'mtime': 1558331552.2705755, 'ctime': 1558331552.2705755, 'size': 135915, 'ext': '.png'}
    • + +
  • {'file_path': 'templates/default.jinja2', 'file_name': 'default.jinja2', 'mtime': 1558317843.4078112, 'ctime': 1558317843.4078112, 'size': 1444, 'ext': '.jinja2'}
    • + +
  • {'file_path': 'templates/post.jinja2', 'file_name': 'post.jinja2', 'mtime': 1558498337.0504873, 'ctime': 1558498337.0504873, 'size': 646, 'ext': '.jinja2'}
    • + +
+
+ + + + + + + +
+ + \ No newline at end of file diff --git a/examples/pixywerk.com/publish/templates/post.jinja2 b/examples/pixywerk.com/publish/templates/post.jinja2 index 936cf10..1bb8d42 100644 --- a/examples/pixywerk.com/publish/templates/post.jinja2 +++ b/examples/pixywerk.com/publish/templates/post.jinja2 @@ -8,7 +8,7 @@

{{ metadata.title }}

-
+

Author: {{ metadata.author }}
Published: {{ get_time_iso8601(metadata.stat.ctime) }} {% if metadata.stat.mtime-metadata.stat.ctime > 512 %} diff --git a/examples/pixywerk.com/src/recurse.thtml b/examples/pixywerk.com/src/recurse.thtml new file mode 100644 index 0000000..44cd412 --- /dev/null +++ b/examples/pixywerk.com/src/recurse.thtml @@ -0,0 +1,5 @@ +

    +{% for i in get_hier('.', '*') %} +
  • {{i}}
    • +{% endfor %} +
\ No newline at end of file diff --git a/pixywerk2/__init__.py b/pixywerk2/__init__.py index e69de29..ef7eb44 100644 --- a/pixywerk2/__init__.py +++ b/pixywerk2/__init__.py @@ -0,0 +1 @@ +__version__ = '0.6.0' diff --git a/pixywerk2/__main__.py b/pixywerk2/__main__.py index ff2e88c..bb5d037 100644 --- a/pixywerk2/__main__.py +++ b/pixywerk2/__main__.py @@ -11,14 +11,24 @@ import os import shutil import sys import time - from typing import Dict, List, cast +from .metadata import MetaTree from .processchain import ProcessorChains from .processors.processors import PassthroughException -from .metadata import MetaTree -from .template_tools import date_iso8601, file_list, file_name, file_content, file_metadata, time_iso8601, file_raw from .pygments import pygments_get_css, pygments_markup_contents_html +from .template_tools import ( + date_iso8601, + file_content, + file_list, + file_list_hier, + file_json, + file_metadata, + file_name, + file_raw, + time_iso8601, +) +from .utils import deep_merge_dicts logger = logging.getLogger() @@ -27,6 +37,12 @@ def setup_logging(verbose: bool = False) -> None: pass +def parse_var(varspec: str) -> List: + if (not ('=' in varspec)): + return [varspec, True] + return list(varspec.split('=', 2)) + + def get_args(args: List[str]) -> argparse.Namespace: parser = argparse.ArgumentParser("Compile a Pixywerk directory into an output directory.") @@ -37,14 +53,14 @@ def get_args(args: List[str]) -> argparse.Namespace: "-c", "--clean", help="Remove the target tree before proceeding (by renaming to .bak).", action="store_true" ) parser.add_argument("-s", "--safe", help="Abort if the target directory already exists.", action="store_true") + parser.add_argument("-f", "--follow-links", help="Follow symbolic links in the input tree.", action="store_true") parser.add_argument("-t", "--template", help="The template directory (default: root/templates)", default=None) parser.add_argument("-d", "--dry-run", help="Perform a dry-run.", action="store_true") parser.add_argument("-v", "--verbose", help="Output verbosely.", action="store_true") parser.add_argument("--processors", help="Specify a path to a processor configuration file.", default=None) - # parser.add_argument("--prescript", help="Specify one or more prescripts to run (in order specified) with context of the compile.", default=[], action="append") - # parser.add_argument("--postscript", help="Specify one or more postsscripts to run (in order specified) with context of the compile.", default=[], action="append") + parser.add_argument( + "-D", "--define", help="Add a variable to the metadata.", nargs="+", action="extend", type=parse_var) result = parser.parse_args(args) - # validate arguments if not os.path.isdir(result.root): raise FileNotFoundError("can't find root folder {}".format(result.root)) @@ -82,24 +98,31 @@ def main() -> int: "author": "", "author_email": "", } + if args.define: + for var in args.define: + default_metadata[var[0]] = var[1] meta_tree = MetaTree(args.root, default_metadata) file_list_cache = cast(Dict, {}) file_cont_cache = cast(Dict, {}) file_name_cache = cast(Dict, {}) file_raw_cache = cast(Dict, {}) + flist = file_list(args.root, file_list_cache) default_metadata["globals"] = { - "get_file_list": file_list(args.root, file_list_cache), + "get_file_list": flist, + "get_hier": file_list_hier(args.root, flist), "get_file_name": file_name(args.root, meta_tree, process_chains, file_name_cache), "get_file_content": file_content(args.root, meta_tree, process_chains, file_cont_cache), + "get_json": file_json(args.root), "get_raw": file_raw(args.root, file_raw_cache), "get_file_metadata": file_metadata(meta_tree), "get_time_iso8601": time_iso8601("UTC"), "get_date_iso8601": date_iso8601("UTC"), "pygments_get_css": pygments_get_css, "pygments_markup_contents_html": pygments_markup_contents_html, + "merge_dicts": deep_merge_dicts, } - for root, _, files in os.walk(args.root): + for root, _, files in os.walk(args.root, followlinks=args.follow_links): workroot = os.path.relpath(root, args.root) if workroot == ".": workroot = "" @@ -118,7 +141,7 @@ def main() -> int: continue metadata = meta_tree.get_metadata(os.path.join(workroot, f)) chain = process_chains.get_chain_for_filename(os.path.join(root, f), ctx=metadata) - print("process {} -> {}".format(os.path.join(root, f), os.path.join(target_dir, chain.output_filename))) + print("process {} -> {} -> {}".format(os.path.join(root, f), repr(chain), os.path.join(target_dir, chain.output_filename))) if not args.dry_run: try: with open(os.path.join(target_dir, chain.output_filename), "w") as outfile: diff --git a/pixywerk2/defaults/chains.yaml b/pixywerk2/defaults/chains.yaml index 4683860..459eae0 100644 --- a/pixywerk2/defaults/chains.yaml +++ b/pixywerk2/defaults/chains.yaml @@ -8,7 +8,14 @@ default: templatable: extension: null chain: - - jinja2 + - jinja2 + +# Any object that needs jinja and to be embedded in a parent template +tembed: + extension: null + chain: + - jinja2 + - jinja2_page_embed # Markdown, BBCode and RST are first run through the templater, and then # they are processed into HTML, and finally embedded in a page template. @@ -62,24 +69,24 @@ template-html: - jinja2 - jinja2_page_embed -# Smart CSS are simply converted to CSS. -sass: - extension: - - sass - - scss - chain: - - process_sass -less: - extension: - - less - chain: - - process_less +# # Smart CSS are simply converted to CSS. +# sass: +# extension: +# - sass +# - scss +# chain: +# - process_sass +# less: +# extension: +# - less +# chain: +# - process_less -stylus: - extension: - - styl - chain: - - process_styl +# stylus: +# extension: +# - styl +# chain: +# - process_styl # # Images are processed into thumbnails and sized in addition to being retained as their original # FIXME implement split chain processor, implement processor arguments, diff --git a/pixywerk2/metadata.py b/pixywerk2/metadata.py index 9e45851..f92e534 100644 --- a/pixywerk2/metadata.py +++ b/pixywerk2/metadata.py @@ -5,8 +5,7 @@ import logging import mimetypes import os import uuid - -from typing import Dict, Optional, Union, List, Tuple, Any, cast +from typing import Any, Dict, List, Optional, Tuple, Union, cast import jstyleson @@ -94,7 +93,7 @@ class MetaTree: """Retrieve the metadata for a given path The general procedure is to iterate the tree, at each level -m load .meta (JSON formatted dictionary) for that level, and + load .meta (JSON formatted dictionary) for that level, and then finally load the path.meta, and merge these dictionaries in descendant order. diff --git a/pixywerk2/processchain.py b/pixywerk2/processchain.py index d5952d0..cc1e98c 100644 --- a/pixywerk2/processchain.py +++ b/pixywerk2/processchain.py @@ -3,8 +3,7 @@ import os import os.path import random - -from typing import List, Iterable, Optional, Any, Dict, Type, cast +from typing import Any, Dict, Iterable, List, Optional, Type, cast import yaml @@ -91,6 +90,9 @@ class ProcessorChain: fname = processor.filename(fname, self._ctx) return fname + def __repr__(self) -> str: + return "[" + ",".join([x.__class__.__name__ for x in self._processors]) + "]" + class ProcessorChains: """Load a configuration for processor chains, and provide ability to process the chains given a particular input diff --git a/pixywerk2/processors/jinja2.py b/pixywerk2/processors/jinja2.py index 80b072b..52535b6 100644 --- a/pixywerk2/processors/jinja2.py +++ b/pixywerk2/processors/jinja2.py @@ -1,6 +1,6 @@ """Define a Jinja2 Processor which applies programmable templating to the input stream.""" -from typing import Iterable, Optional, Dict, cast +from typing import Dict, Iterable, Optional, cast from jinja2 import Environment, FileSystemLoader @@ -22,11 +22,10 @@ class Jinja2(PassThrough): iterable: The post-processed output stream """ ctx = cast(Dict, ctx) - template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=['jinja2.ext.do']) + template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=["jinja2.ext.do"]) template_env.globals.update(ctx["globals"]) template_env.filters.update(ctx["filters"]) tmpl = template_env.from_string("".join([x for x in input_file])) return tmpl.render(metadata=ctx) - processor = Jinja2 diff --git a/pixywerk2/processors/jinja2_page_embed.py b/pixywerk2/processors/jinja2_page_embed.py index 21f6f3a..3be143c 100644 --- a/pixywerk2/processors/jinja2_page_embed.py +++ b/pixywerk2/processors/jinja2_page_embed.py @@ -3,8 +3,7 @@ the target template is rendered).""" import os - -from typing import Iterable, Optional, Dict, cast +from typing import Dict, Iterable, Optional, cast from jinja2 import Environment, FileSystemLoader @@ -25,8 +24,7 @@ class Jinja2PageEmbed(Processor): str: the new name for the file """ - - return os.path.splitext(oldname)[0] + ".html" + return os.path.splitext(oldname)[0] + "." + self.extension(oldname, ctx) def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: """Return the mimetype of the post-processed file. @@ -39,7 +37,7 @@ class Jinja2PageEmbed(Processor): str: the new mimetype of the file after processing """ - return "text/html" + return ctx.get("mime", "text/html") def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: """Return an iterable object of the post-processed file. @@ -52,7 +50,7 @@ class Jinja2PageEmbed(Processor): iterable: The post-processed output stream """ ctx = cast(Dict, ctx) - template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=['jinja2.ext.do']) + template_env = Environment(loader=FileSystemLoader(ctx["templates"]), extensions=["jinja2.ext.do"]) template_env.globals.update(ctx["globals"]) template_env.filters.update(ctx["filters"]) tmpl = template_env.get_template(ctx["template"]) @@ -70,7 +68,7 @@ class Jinja2PageEmbed(Processor): str: the new extension of the file after processing """ - return "html" + return ctx.get("extension", "html") processor = Jinja2PageEmbed diff --git a/pixywerk2/processors/passthrough.py b/pixywerk2/processors/passthrough.py index cc6511b..c3f34ae 100644 --- a/pixywerk2/processors/passthrough.py +++ b/pixywerk2/processors/passthrough.py @@ -1,10 +1,10 @@ """Passthrough progcessor which takes input and returns it.""" import os +from typing import Dict, Iterable, Optional, cast -from .processors import Processor, PassthroughException from ..utils import guess_mime -from typing import Iterable, Optional, Dict, cast +from .processors import PassthroughException, Processor class PassThrough(Processor): diff --git a/pixywerk2/processors/process_md.py b/pixywerk2/processors/process_md.py index 0687bc6..30a1b01 100644 --- a/pixywerk2/processors/process_md.py +++ b/pixywerk2/processors/process_md.py @@ -2,8 +2,7 @@ import io import os - -from typing import Iterable, Optional, Dict +from typing import Dict, Iterable, Optional import markdown diff --git a/pixywerk2/processors/processors.py b/pixywerk2/processors/processors.py index 0ff970e..f3312e7 100644 --- a/pixywerk2/processors/processors.py +++ b/pixywerk2/processors/processors.py @@ -1,6 +1,5 @@ import abc - -from typing import Iterable, Optional, Dict +from typing import Dict, Iterable, Optional class PassthroughException(Exception): @@ -65,3 +64,6 @@ class Processor(abc.ABC): # pragma: no cover Returns: iterable: The post-processed output stream """ + + def repr(self) -> str: + return self.__class__.__name__ diff --git a/pixywerk2/pygments.py b/pixywerk2/pygments.py index c96de3b..90938f6 100644 --- a/pixywerk2/pygments.py +++ b/pixywerk2/pygments.py @@ -4,18 +4,17 @@ from typing import Optional import pygments import pygments.formatters import pygments.lexers -import pygments.util import pygments.styles +import pygments.util - -def pygments_markup_contents_html(input_text: str, file_type: str, style: Optional[str]=None) -> str: +def pygments_markup_contents_html(input_text: str, file_type: str, style: Optional[str] = None) -> str: """Format input string with Pygments and return HTML.""" if style is None: - style = 'default' + style = "default" style = pygments.styles.get_style_by_name(style) - formatter = pygments.formatters.get_formatter_by_name('html', style=style) + formatter = pygments.formatters.get_formatter_by_name("html", style=style) try: lexer = pygments.lexers.get_lexer_for_filename(file_type) except pygments.util.ClassNotFound: @@ -26,11 +25,12 @@ def pygments_markup_contents_html(input_text: str, file_type: str, style: Option return pygments.highlight(input_text, lexer, formatter) -def pygments_get_css(style: Optional[str]=None) -> str: + +def pygments_get_css(style: Optional[str] = None) -> str: """Return the CSS styles associated with a particular style definition.""" if style is None: - style = 'default' + style = "default" style = pygments.styles.get_style_by_name(style) - formatter = pygments.formatters.get_formatter_by_name('html', style=style) + formatter = pygments.formatters.get_formatter_by_name("html", style=style) return formatter.get_style_defs() diff --git a/pixywerk2/template_tools.py b/pixywerk2/template_tools.py index 28e4249..328145e 100644 --- a/pixywerk2/template_tools.py +++ b/pixywerk2/template_tools.py @@ -1,38 +1,51 @@ +import copy import datetime import glob import itertools import os +from typing import Callable, Dict, Iterable, List, Union, cast, Tuple + +import jstyleson + import pytz -from typing import Callable, Dict, List, Iterable, Union, cast from .metadata import MetaTree from .processchain import ProcessorChains +from .utils import deep_merge_dicts def file_list(root: str, listcache: Dict) -> Callable: - def get_file_list(path_glob: str, *, sort_order: str = "ctime", reverse: bool = False, limit: int = 0) -> Iterable: + def get_file_list( + path_glob: Union[str, List[str], Tuple[str]], + *, + sort_order: str = "ctime", + reverse: bool = False, + limit: int = 0) -> Iterable: stattable = cast(List, []) - if path_glob in listcache: - stattable = listcache[path_glob] - else: - for fil in glob.glob(os.path.join(root, path_glob)): - if os.path.isdir(fil): - continue - if fil.endswith(".meta") or fil.endswith("~"): - continue - st = os.stat(fil) - stattable.append( - { - "file_path": os.path.relpath(fil, root), - "file_name": os.path.split(fil)[-1], - "mtime": st.st_mtime, - "ctime": st.st_ctime, - "size": st.st_size, - "ext": os.path.splitext(fil)[1], - } - ) - listcache[path_glob] = stattable - ret = sorted(stattable, key=lambda x: x[sort_order], reverse=reverse) + if isinstance(path_glob, str): + path_glob = [path_glob] + for pglob in path_glob: + if pglob in listcache: + stattable.extend(listcache[pglob]) + else: + for fil in glob.glob(os.path.join(root, pglob)): + if os.path.isdir(fil): + continue + if fil.endswith(".meta") or fil.endswith("~"): + continue + st = os.stat(fil) + stattable.append( + { + "file_path": os.path.relpath(fil, root), + "file_name": os.path.split(fil)[-1], + "mtime": st.st_mtime, + "ctime": st.st_ctime, + "size": st.st_size, + "ext": os.path.splitext(fil)[1], + } + ) + listcache[pglob] = stattable + ret = sorted(stattable, key=lambda x: x[sort_order], reverse=reverse) if limit > 0: return itertools.islice(ret, limit) return ret @@ -40,6 +53,27 @@ def file_list(root: str, listcache: Dict) -> Callable: return get_file_list +def file_list_hier(root: str, flist: Callable) -> Callable: + """Return a callable which, given a directory, will walk the directory and return the files within + it that match the glob passed.""" + + def get_file_list_hier(path: str, glob: str, *, sort_order: str = "ctime", reverse: bool = False) -> Iterable: + output = [] + + for pth in os.walk(os.path.join(root, path)): + output.extend( + flist( + os.path.join(os.path.relpath(os.path.realpath(pth[0]), root), glob), + sort_order=sort_order, + reverse=reverse, + ) + ) + + return output + + return get_file_list_hier + + def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains, namecache: Dict) -> Callable: def get_file_name(file_name: str) -> Dict: if file_name in namecache: @@ -51,15 +85,29 @@ def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains, return get_file_name + def file_raw(root: str, contcache: Dict) -> Callable: def get_raw(file_name: str) -> str: if file_name in contcache: return contcache[file_name] - with open(os.path.join(root, file_name), 'r', encoding="utf-8") as f: + with open(os.path.join(root, file_name), "r", encoding="utf-8") as f: return f.read() return get_raw + +def file_json(root: str) -> Callable: + def get_json(file_name: str, parent: Dict = None) -> Dict: + outd = {} + if parent is not None: + outd = copy.deepcopy(parent) + + with open(os.path.join(root, file_name), "r", encoding="utf-8") as f: + return deep_merge_dicts(outd, jstyleson.load(f)) + + return get_json + + def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChains, contcache: Dict) -> Callable: def get_file_content(file_name: str) -> Iterable: if file_name in contcache: @@ -67,7 +115,7 @@ def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChain metadata = metatree.get_metadata(file_name) chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata) contcache[file_name] = chain.output - return unicode(chain.output) + return str(chain.output) return get_file_content @@ -87,10 +135,11 @@ def time_iso8601(timezone: str) -> Callable: return get_time_iso8601 + def date_iso8601(timezone: str) -> Callable: tz = pytz.timezone(timezone) def get_date_iso8601(time_t: Union[int, float]) -> str: - return datetime.datetime.fromtimestamp(time_t, tz).strftime('%Y-%m-%d') + return datetime.datetime.fromtimestamp(time_t, tz).strftime("%Y-%m-%d") return get_date_iso8601 diff --git a/pixywerk2/utils.py b/pixywerk2/utils.py index 962c9e2..d12c490 100644 --- a/pixywerk2/utils.py +++ b/pixywerk2/utils.py @@ -1,11 +1,11 @@ +from typing import Dict, Optional +import copy import mimetypes import os -from typing import Dict, Optional - def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict: - """Merge two dictionaries. + """Merge two dictionaries (shallow). Arguments: dict_a (dict): The dictionary to use as the base. @@ -20,6 +20,36 @@ def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict: return dict_z +def deep_merge_dicts(dict_a: Dict, dict_b: Dict, _path=None, cpy=False) -> Dict: + """Merge two dictionaries (deep). + https://stackoverflow.com/questions/7204805/how-to-merge-dictionaries-of-dictionaries/7205107#7205107 + + Arguments: + dict_a (dict): The dictionary to use as the base. + dict_b (dict): The dictionary to update the values with. + _path (list): internal use. + + Returns: + dict: A new merged dictionary. + + """ + if cpy: + dict_a = copy.deepcopy(dict_a) + if _path is None: + _path = [] + for key in dict_b: + if key in dict_a: + if isinstance(dict_a[key], dict) and isinstance(dict_b[key], dict): + deep_merge_dicts(dict_a[key], dict_b[key], _path + [str(key)]) + elif dict_a[key] == dict_b[key]: + pass # same leaf value + else: + dict_a[key] = copy.deepcopy(dict_b[key]) + else: + dict_a[key] = dict_b[key] + return dict_a + + def guess_mime(path: str) -> Optional[str]: """Guess the mime type for a given path. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 9e41013..6097c30 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ """Package configuration.""" from setuptools import find_packages, setup +from pixywerk2 import __version__ + LONG_DESCRIPTION = """Pixywerk 2 is a filesystem based static site generator.""" INSTALL_REQUIRES = ["yaml-1.3", "markdown", "jstyleson", "jinja2", "pygments"] @@ -56,4 +58,5 @@ setup( use_scm_version=True, url="https://git.antpanethon.com/cas/pixywerk2", zip_safe=False, + version=__version__, ) diff --git a/tox.ini b/tox.ini index 53ca9b0..d6a9161 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py{36,37}-{code-quality, unit} #, py37-sphinx +envlist=py{36,37,38,39}-{code-quality, unit} #, py37-sphinx skipsdist = true [testenv] @@ -17,6 +17,8 @@ commands = basepython = py36: python3.6 py37: python3.7 + py38: python3.8 + py39: python3.9 [flake8] max-line-length = 120