Cas Rusnov
5 years ago
parent
13fb5dac1c
commit
43d40f7fce
@ -0,0 +1,7 @@ |
||||
{ |
||||
"site_root":"https://example.com", |
||||
"title":"Test Metadata", |
||||
"author": "Test User", |
||||
"author_email": "test_user@example.com", |
||||
"uuid_oid_root": "pixywerk-demo" |
||||
} |
||||
@ -0,0 +1,33 @@ |
||||
<?xml version="1.0" encoding="utf-8"?> |
||||
|
||||
<feed xmlns="http://www.w3.org/2005/Atom"> |
||||
|
||||
<title>{{ metadata.title }}</title> |
||||
<subtitle>{{ metadata.subtitle }}</subtitle> |
||||
<link href="{{ metadata.site_root }}/{{ metadata.file_name }}" rel="self" /> |
||||
<link href="{{ metadata.site_root }}" /> |
||||
<id>urn:uuid:{{ metadata.uuid }}</id> |
||||
<updated>{{ get_time_iso8601(metadata['build-time']) }}</updated> |
||||
|
||||
{% set posts = get_file_list('blog_posts/*.cont') %} |
||||
{% for post in posts %} |
||||
{% set post_meta = get_file_metadata(post['file_path']) %} |
||||
<entry> |
||||
<title>{{ post_meta.title }}</title> |
||||
<link href="{{ metadata.site_root }}/{{post_meta.file_path}}" /> |
||||
<id>urn:uuid:{{ post_meta.uuid }}</id> |
||||
<updated>{{ get_time_iso8601(post_meta.stat.mtime) }}</updated> |
||||
<summary>{{post_meta.summary }}</summary> |
||||
<!-- this would be the snippet, more than summary chunk --> |
||||
<!-- <content type="xhtml"> --> |
||||
<!-- <div xmlns="http://www.w3.org/1999/xhtml"> --> |
||||
<!-- <p>{{ post_meta.summary }}</p> --> |
||||
<!-- </div> --> |
||||
<!-- </content> --> |
||||
<author> |
||||
<name>{{ post_meta.author }}</name> |
||||
<email>{{ post_meta.author_email }}</email> |
||||
</author> |
||||
</entry> |
||||
{% endfor %} |
||||
</feed> |
||||
@ -0,0 +1,5 @@ |
||||
{ |
||||
"type": "templatable", |
||||
"title": "Test RSS Feed", |
||||
"subtitle": "Some Subtitle" |
||||
} |
||||
@ -0,0 +1,5 @@ |
||||
Some more post |
||||
|
||||
|
||||
la la la |
||||
|
||||
@ -0,0 +1,4 @@ |
||||
{ |
||||
"title":"Another Post(tm)", |
||||
"summary":"Yet another post" |
||||
} |
||||
@ -0,0 +1 @@ |
||||
Some content. |
||||
@ -0,0 +1,4 @@ |
||||
{ |
||||
"title":"Test.cont", |
||||
"summary":"Some empty test content" |
||||
} |
||||
@ -0,0 +1 @@ |
||||
yo fresh |
||||
@ -0,0 +1,5 @@ |
||||
{ |
||||
"foo":"bar", |
||||
"title":"A title", |
||||
"summary":"Just a post." |
||||
} |
||||
@ -0,0 +1,19 @@ |
||||
<h1>Index of all content</h1> |
||||
{% for f in get_file_list('*', sort_order='file_name') %} |
||||
<a href="{{ get_file_name(f['file_name']) }}">{{get_file_name(f['file_name'])}}</a> |
||||
{% endfor %} |
||||
|
||||
<p>Including foo.cont.meta: |
||||
<pre> |
||||
{{ get_file_content('foo.cont.meta') }} |
||||
</pre> |
||||
</p> |
||||
|
||||
<h1>Metadata</h1> |
||||
<table class="metadata"> |
||||
<tr><th>key</th><th>value</th></tr> |
||||
{% set metadata = get_file_metadata('foo.cont') %} |
||||
{% for k in metadata.keys() %} |
||||
<tr><td>{{k}}</td><td>{{metadata[k]}}</td></tr> |
||||
{% endfor %} |
||||
</table> |
||||
@ -0,0 +1,9 @@ |
||||
# README # |
||||
|
||||
This is a test of the emergency compiled HTML system. This is only a *test*. |
||||
|
||||
[Foo!](foo.html) |
||||
|
||||
{% for i in range(100) %} |
||||
* {{ i }} |
||||
{% endfor %} |
||||
@ -0,0 +1,3 @@ |
||||
{ |
||||
"pragma":["no-proc"] |
||||
} |
||||
@ -0,0 +1,9 @@ |
||||
# README # |
||||
|
||||
This is a test of the emergency compiled HTML system. This is only a *test*. |
||||
|
||||
[Foo!](foo.html) |
||||
|
||||
{% for i in range(100) %} |
||||
* {{ i }} |
||||
{% endfor %} |
||||
@ -0,0 +1,3 @@ |
||||
{ |
||||
"title":"Yo, markdown" |
||||
} |
||||
@ -0,0 +1,32 @@ |
||||
<!DOCTYPE html> |
||||
<head> |
||||
<title>Debug for {{path}}</title> |
||||
<style type="text/css"> |
||||
table { border: 1px solid black; } |
||||
div { border: 1px solid black; } |
||||
td { border: 1px solid black; } |
||||
</style> |
||||
</head> |
||||
<body> |
||||
<p>{{path}}</p> |
||||
<h1>Content</h1> |
||||
<div class="content"> |
||||
{{content}} |
||||
</div> |
||||
|
||||
<h1>Environment</h1> |
||||
<table class="environment"> |
||||
<tr><th>key</th><th>value</th></tr> |
||||
{% for k in environ.keys() %} |
||||
<tr><td>{{k}}</td><td>{{environ[k]}}</td></tr> |
||||
{% endfor %} |
||||
</table> |
||||
|
||||
<h1>Metadata</h1> |
||||
<table class="metadata"> |
||||
<tr><th>key</th><th>value</th></tr> |
||||
{% for k in metadata.keys() %} |
||||
<tr><td>{{k}}</td><td>{{metadata[k]}}</td></tr> |
||||
{% endfor %} |
||||
</table> |
||||
</body> |
||||
@ -0,0 +1,6 @@ |
||||
<table class="werk-file-list"> |
||||
<tr class="werk-file-list-head"><th>file</th><th>type</th><th>size</th><th>last change</th></tr> |
||||
{% for f in files.keys() %} |
||||
<tr class="werk-file-list-item"><td><a href="/{{files[f].relpath}}">{{f}}</a></td><td>{{files[f].type}}</td><td>{{files[f].size}}</td><td>{{files[f].ctime | date}}</td></tr> |
||||
{% endfor %} |
||||
</table> |
||||
@ -0,0 +1,13 @@ |
||||
<!DOCTYPE html> |
||||
<head> |
||||
<title>{{metadata.title}}</title> |
||||
<style type="text/css"> |
||||
table { border: 1px solid black; } |
||||
div { border: 1px solid black; } |
||||
td { border: 1px solid black; } |
||||
</style> |
||||
</head> |
||||
<body> |
||||
{{content}} |
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,119 @@ |
||||
# iterate source tree |
||||
# create directors in target tree |
||||
# for each item: |
||||
# run processor(s) on item, each processor could be in a chain or a branch |
||||
# Processors also provide filename munging |
||||
# output target based on processor output |
||||
|
||||
import argparse |
||||
import logging |
||||
import os |
||||
import sys |
||||
import time |
||||
|
||||
from typing import Dict, List, cast |
||||
|
||||
from .processchain import ProcessorChains |
||||
from .metadata import MetaTree |
||||
from .template_tools import file_list, file_name, file_content, file_metadata, time_iso8601 |
||||
|
||||
|
||||
logger = logging.getLogger() |
||||
|
||||
|
||||
def setup_logging(verbose: bool = False) -> None: |
||||
pass |
||||
|
||||
|
||||
def get_args(args: List[str]) -> argparse.Namespace: |
||||
parser = argparse.ArgumentParser("Compile a Pixywerk directory into an output directory.") |
||||
|
||||
parser.add_argument("root", help="The root of the pixywerk directory to process.") |
||||
parser.add_argument("output", help="The output directory to export post-compiled files to.") |
||||
|
||||
parser.add_argument( |
||||
"-c", "--clean", help="Remove the target tree before proceeding (by renaming to .bak).", action="store_true" |
||||
) |
||||
parser.add_argument("-s", "--safe", help="Abort if the target directory already exists.", action="store_true") |
||||
parser.add_argument("-t", "--template", help="The template directory (default: root/templates)", default=None) |
||||
parser.add_argument("-d", "--dry-run", help="Perform a dry-run.", action="store_true") |
||||
parser.add_argument("-v", "--verbose", help="Output verbosely.", action="store_true") |
||||
parser.add_argument("--processors", help="Specify a path to a processor configuration file.", default=None) |
||||
|
||||
result = parser.parse_args(args) |
||||
|
||||
# validate arguments |
||||
if not os.path.isdir(result.root): |
||||
raise FileNotFoundError("can't find root folder {}".format(result.root)) |
||||
|
||||
if not result.template: |
||||
result.template = os.path.join(result.root, "templates") |
||||
result.excludes = [result.template] |
||||
|
||||
return result |
||||
|
||||
|
||||
def main() -> int: |
||||
try: |
||||
args = get_args(sys.argv[1:]) |
||||
except FileNotFoundError as ex: |
||||
print("error finding arguments: {}".format(ex)) |
||||
return 1 |
||||
setup_logging(args.verbose) |
||||
if os.path.exists(args.output) and args.clean: |
||||
bak = "{}.bak-{}".format(args.output, int(time.time())) |
||||
print("cleaning target {} -> {}".format(args.output, bak)) |
||||
os.rename(args.output, bak) |
||||
|
||||
process_chains = ProcessorChains(args.processors) |
||||
|
||||
default_metadata = { |
||||
"templates": args.template, |
||||
"template": "default.jinja2", |
||||
"dir-template": "default-dir.jinja2", |
||||
"filters": {}, |
||||
"build-time": time.time(), |
||||
"build-datetime": time.ctime(), |
||||
"uuid-oid-root": "pixywerk", |
||||
} |
||||
meta_tree = MetaTree(args.root, default_metadata) |
||||
file_list_cache = cast(Dict, {}) |
||||
file_cont_cache = cast(Dict, {}) |
||||
file_name_cache = cast(Dict, {}) |
||||
default_metadata["globals"] = { |
||||
"get_file_list": file_list(args.root, file_list_cache), |
||||
"get_file_name": file_name(args.root, meta_tree, process_chains, file_name_cache), |
||||
"get_file_content": file_content(args.root, meta_tree, process_chains, file_cont_cache), |
||||
"get_file_metadata": file_metadata(meta_tree), |
||||
"get_time_iso8601": time_iso8601("UTC"), |
||||
} |
||||
|
||||
for root, _, files in os.walk(args.root): |
||||
workroot = os.path.relpath(root, args.root) |
||||
if workroot == ".": |
||||
workroot = "" |
||||
target_dir = os.path.join(args.output, workroot) |
||||
print("mkdir -> {}".format(target_dir)) |
||||
if not args.dry_run: |
||||
try: |
||||
os.mkdir(target_dir) |
||||
except FileExistsError: |
||||
if args.safe: |
||||
print("error, target directory exists, aborting") |
||||
return 1 |
||||
for f in files: |
||||
# fixme global generic filters |
||||
if f.endswith(".meta") or f.endswith("~"): |
||||
continue |
||||
metadata = meta_tree.get_metadata(os.path.join(workroot, f)) |
||||
chain = process_chains.get_chain_for_filename(os.path.join(root, f), ctx=metadata) |
||||
print("process {} -> {}".format(os.path.join(root, f), os.path.join(target_dir, chain.output_filename))) |
||||
if not args.dry_run: |
||||
with open(os.path.join(target_dir, chain.output_filename), "w") as outfile: |
||||
for line in chain.output: |
||||
outfile.write(line) |
||||
return 0 |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
sys.exit(main()) |
||||
@ -0,0 +1,104 @@ |
||||
# Default: output == input |
||||
default: |
||||
extension: default |
||||
chain: |
||||
- passthrough |
||||
|
||||
# Any object that needs jinja scripts but no other explicit processing |
||||
templatable: |
||||
extension: null |
||||
chain: |
||||
- jinja2 |
||||
|
||||
# Markdown, BBCode and RST are first run through the templater, and then |
||||
# they are processed into HTML, and finally embedded in a page template. |
||||
markdown: |
||||
extension: |
||||
- md |
||||
chain: |
||||
- jinja2 |
||||
- process_md |
||||
- jinja2_page_embed |
||||
bbcode: |
||||
extension: |
||||
- bb |
||||
- pp |
||||
chain: |
||||
- jinja2 |
||||
- process_pp |
||||
- jinja2_page_embed |
||||
# FIXME implement RST processor |
||||
# restructured: |
||||
# extension: |
||||
# - rst |
||||
# chain: |
||||
# - jinja2 |
||||
# - process_rst |
||||
# - jinja2_page_embed |
||||
|
||||
# # JSON and YAML are split, passed through a pretty printer, and then output |
||||
# FIXME implement split chain processor, implement processor arguments |
||||
# json: |
||||
# extension: |
||||
# - json |
||||
# chain: |
||||
# - split (passthrough) |
||||
# - pp_json |
||||
# yaml: |
||||
# extension: |
||||
# - yml |
||||
# - yaml |
||||
# chain: |
||||
# - split (passthrough) |
||||
# - pp_yaml |
||||
|
||||
# Template-html is first passed through the templater, and then embedded |
||||
# in a page template |
||||
template-html: |
||||
extension: |
||||
- thtml |
||||
- cont |
||||
chain: |
||||
- jinja2 |
||||
- jinja2_page_embed |
||||
|
||||
# Smart CSS are simply converted to CSS. |
||||
sass: |
||||
extension: |
||||
- sass |
||||
- scss |
||||
chain: |
||||
- process_sass |
||||
less: |
||||
extension: |
||||
- less |
||||
chain: |
||||
- process_less |
||||
|
||||
stylus: |
||||
extension: |
||||
- styl |
||||
chain: |
||||
- process_styl |
||||
|
||||
# # Images are processed into thumbnails and sized in addition to being retained as their original |
||||
# FIXME implement split chain processor, implement processor arguments, |
||||
# image: |
||||
# extension: |
||||
# - jpg |
||||
# - jpeg |
||||
# - png |
||||
# chain: |
||||
# - split (image_bigthumb) |
||||
# - split (image_smallthumb) |
||||
# - passthrough |
||||
|
||||
# image_bigthumb: |
||||
# extension: |
||||
# chain: |
||||
# - smart_resize (big) |
||||
|
||||
# image_smallthumb: |
||||
# extension: |
||||
# chain: |
||||
# - smart_resize (small) |
||||
@ -0,0 +1,145 @@ |
||||
"""Constructs a tree-like object containing the metadata for a given path, and caches said metadata.""" |
||||
|
||||
import logging |
||||
import mimetypes |
||||
import os |
||||
import uuid |
||||
|
||||
from typing import Dict, Optional, Union, List, Tuple, Any, cast |
||||
|
||||
import jstyleson |
||||
|
||||
from .utils import guess_mime |
||||
|
||||
# setup mimetypes with some extra ones |
||||
mimetypes.init() |
||||
mimetypes.add_type("text/html", "thtml") |
||||
mimetypes.add_type("text/html", "cont") |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
class MetaCacheMiss(Exception): |
||||
"""Raised on cache miss.""" |
||||
|
||||
|
||||
class MetaCache: |
||||
"""This class provides an in-memory cache for metadata tree.""" |
||||
|
||||
def __init__(self, max_age: float = 200.0): |
||||
"""Initialize the cache. |
||||
|
||||
Arguments: |
||||
max_age (int): the number of seconds to age-out cache items |
||||
|
||||
""" |
||||
self._max_age = max_age |
||||
self._cache: Dict[str, Tuple[float, Any]] = {} |
||||
|
||||
def get(self, key: str, new_time_stamp: float) -> Any: |
||||
"""Get an item from the cache. |
||||
|
||||
Arguments: |
||||
key (str): the cache key to retieve |
||||
new_time_stamp (int): The time to use to compare the stored time with |
||||
|
||||
Returns: |
||||
:obj:misc: The previously stored value. |
||||
|
||||
Raises: |
||||
MetaCacheMiss: on missing key, or on aged out |
||||
|
||||
""" |
||||
if key not in self._cache: |
||||
raise MetaCacheMiss("no item for key {}".format(key)) |
||||
|
||||
if self._cache[key][0] + self._max_age <= new_time_stamp: |
||||
return self._cache[key][1] |
||||
|
||||
raise MetaCacheMiss("cache expired for key {}".format(key)) |
||||
|
||||
def put(self, key: str, value: Union[Dict, List, int, str, object], time_stamp: float) -> None: |
||||
"""Put an item into the cache. |
||||
|
||||
Arguments: |
||||
key (str): the key to store the cache item under |
||||
value (:obj:misc): the value to store |
||||
time_stamp (float): the time stamp to store the item under |
||||
|
||||
""" |
||||
self._cache[key] = (time_stamp, value) |
||||
|
||||
|
||||
class MetaTree: |
||||
"""This provides an interface to loading and caching tree metadata for a given directory tree.""" |
||||
|
||||
def __init__(self, root: str, default_metadata: Optional[Dict] = None): |
||||
"""Initialize the metadata tree object. |
||||
|
||||
Arguments: |
||||
root (str): The path to the root of the file tree to operate on. |
||||
default_metadata (dict, optional): The default metadata to apply to the tree |
||||
|
||||
""" |
||||
self._cache = MetaCache() |
||||
if default_metadata is None: |
||||
default_metadata = {} |
||||
self._default_metadata = default_metadata |
||||
if root[-1] != "/": |
||||
root += "/" |
||||
self._root = root |
||||
|
||||
def get_metadata(self, rel_path: str) -> Dict: |
||||
"""Retrieve the metadata for a given path |
||||
|
||||
The general procedure is to iterate the tree, at each level |
||||
m load .meta (JSON formatted dictionary) for that level, and |
||||
then finally load the path.meta, and merge these dictionaries |
||||
in descendant order. |
||||
|
||||
Arguments: |
||||
rel_path (str): The path to retrieve the metadata for (relative to root) |
||||
|
||||
Returns: |
||||
dict: A dictionary of metadata for that path tree. |
||||
|
||||
""" |
||||
metablob = dict(self._default_metadata) |
||||
# iterate path components from root to target path |
||||
comps = [self._root] + rel_path.split("/") |
||||
fullpath = "" |
||||
for pth in comps: |
||||
fullpath = os.path.join(fullpath, pth) |
||||
st = os.stat(fullpath) |
||||
|
||||
cachekey = fullpath + ".meta" |
||||
meta = cast(Dict, {}) |
||||
try: |
||||
st_meta = os.stat(cachekey) |
||||
meta = self._cache.get(cachekey, st_meta.st_mtime) |
||||
except FileNotFoundError: |
||||
st_meta = None # type: ignore |
||||
except MetaCacheMiss: |
||||
meta = {} |
||||
|
||||
if not meta and st_meta: |
||||
meta = jstyleson.load(open(cachekey, "r")) |
||||
self._cache.put(cachekey, meta, st_meta.st_mtime) |
||||
|
||||
metablob.update(meta) |
||||
|
||||
# return final dict |
||||
metablob["dir"], metablob["file_name"] = os.path.split(rel_path) |
||||
metablob["file_path"] = rel_path |
||||
metablob["uuid"] = uuid.uuid3( |
||||
uuid.NAMESPACE_OID, metablob["uuid-oid-root"] + os.path.join(self._root, rel_path) |
||||
) |
||||
metablob["os-path"], _ = os.path.split(fullpath) |
||||
metablob["guessed-type"] = guess_mime(os.path.join(self._root, rel_path)) |
||||
if "mime-type" not in metablob: |
||||
metablob["mime-type"] = metablob["guessed-type"] |
||||
metablob["stat"] = {} |
||||
for stk in ("st_mtime", "st_ctime", "st_atime", "st_mode", "st_size", "st_ino"): |
||||
metablob["stat"][stk.replace("st_", "")] = getattr(st, stk) |
||||
|
||||
return metablob |
||||
@ -0,0 +1,180 @@ |
||||
"""Interface for chains of processors""" |
||||
|
||||
import os |
||||
import os.path |
||||
import random |
||||
|
||||
from typing import List, Iterable, Optional, Any, Dict, Type, cast |
||||
|
||||
import yaml |
||||
|
||||
from .processors.processors import Processor |
||||
|
||||
|
||||
class ProcessorChain: |
||||
"""This implements a wrapper for an arbitrary set of processors and an associated file stream.""" |
||||
|
||||
def __init__( |
||||
self, |
||||
processors: List[Processor], |
||||
file_name: str, |
||||
file_data: Iterable[str], |
||||
file_type: str, |
||||
ctx: Optional[Dict] = None, |
||||
): |
||||
"""Initialize the processing stream. |
||||
|
||||
Arguments: |
||||
processors (list): A list of processor objects. |
||||
file_data (Iterable): An iterable from which to retrieve the input |
||||
file_type (str): the specified file type for consumer information. |
||||
|
||||
""" |
||||
self._processors = processors |
||||
self._file_data = file_data |
||||
self._file_type = file_type |
||||
self._file_name = file_name |
||||
self._ctx: Dict = {} |
||||
if ctx is not None: |
||||
self._ctx = cast(Dict, ctx) |
||||
|
||||
@property |
||||
def output(self) -> Iterable: |
||||
"""Return an iterable for the output of the process chain |
||||
|
||||
Returns: |
||||
:obj:'iterable': the iterable |
||||
|
||||
""" |
||||
prev = self._file_data |
||||
for processor in self._processors: |
||||
if processor: |
||||
prev = processor.process(prev, self._ctx) |
||||
|
||||
return prev |
||||
|
||||
@property |
||||
def output_mime(self) -> str: |
||||
"""Return the post-processed MIME value from the processing chain |
||||
|
||||
Returns: |
||||
str: the mime type |
||||
|
||||
""" |
||||
fname = self._file_name |
||||
for processor in self._processors: |
||||
fname = processor.mime_type(fname, self._ctx) |
||||
return fname |
||||
|
||||
@property |
||||
def output_ext(self) -> str: |
||||
"""Return the post-processed extension from the processing chain |
||||
|
||||
Returns: |
||||
str: the extension |
||||
""" |
||||
fname = self._file_name |
||||
for processor in self._processors: |
||||
fname = processor.extension(fname, self._ctx) |
||||
return fname |
||||
|
||||
@property |
||||
def output_filename(self) -> str: |
||||
"""Return the post-processed filename from the processing chain |
||||
|
||||
Returns: |
||||
str: the new filename |
||||
|
||||
""" |
||||
fname = os.path.basename(self._file_name) |
||||
for processor in self._processors: |
||||
fname = processor.filename(fname, self._ctx) |
||||
return fname |
||||
|
||||
|
||||
class ProcessorChains: |
||||
"""Load a configuration for processor chains, and provide ability to process the chains given a particular input |
||||
file. |
||||
""" |
||||
|
||||
def __init__(self, config: Optional[str] = None): |
||||
"""Initialize, with a specified configuration file |
||||
|
||||
Arguments: |
||||
config (str, optional): The path to a yaml formatted configuration file. |
||||
|
||||
""" |
||||
if config is None: # pragma: no coverage |
||||
config = os.path.join(os.path.dirname(__file__), "defaults", "chains.yaml") |
||||
|
||||
self.chainconfig = yaml.load(open(config, "r")) |
||||
self.extensionmap: Dict[str, Any] = {} |
||||
self.processors: Dict[str, Type[Processor]] = {} |
||||
for ch, conf in self.chainconfig.items(): |
||||
if conf["extension"] == "default": |
||||
self.default = ch |
||||
else: |
||||
if conf["extension"]: |
||||
for ex in conf["extension"]: |
||||
if ex in self.extensionmap or ex is None: |
||||
# log an error or except or something we'll just override for now. |
||||
pass |
||||
self.extensionmap[ex] = ch |
||||
for pr in conf["chain"]: |
||||
if pr in self.processors: |
||||
continue |
||||
processor_module = __import__("processors", globals(), locals(), [pr], 1) |
||||
self.processors[pr] = processor_module.__dict__[pr].processor |
||||
|
||||
def get_chain_for_filename(self, filename: str, ctx: Optional[Dict] = None) -> ProcessorChain: |
||||
"""Get the ProcessorChain, as configured for a given file by extension. |
||||
|
||||
Arguments: |
||||
filename (str): The name of the file to get a chain for. |
||||
|
||||
Returns: |
||||
ProcessorChain: the constructed processor chain. |
||||
""" |
||||
r = filename.rsplit(".", 1) |
||||
ftype = "default" |
||||
if r: |
||||
ftype = r[-1] |
||||
if ctx and "pragma" in ctx: |
||||
if "no-proc" in ctx["pragma"]: |
||||
ftype = "default" |
||||
|
||||
if ctx and "type" in ctx: |
||||
ftype = ctx["type"] |
||||
return self.get_chain_for_file(open(filename, "r"), ftype, filename, ctx) |
||||
|
||||
def get_chain_for_file( |
||||
self, file_obj: Iterable, file_ext: str, file_name: Optional[str] = None, ctx: Optional[Dict] = None |
||||
) -> ProcessorChain: |
||||
"""Get the ProcessorChain for a given iterable object based on the specified file type |
||||
|
||||
Arguments: |
||||
file_obj (:obj:`iterable`): The input file stream |
||||
file_ext (str): The type (extension) of the input stream |
||||
|
||||
Returns: |
||||
ProcessorChain: the constructed processor chain. |
||||
|
||||
""" |
||||
if file_ext not in self.extensionmap or not self.extensionmap[file_ext]: |
||||
if file_ext in self.chainconfig: |
||||
file_type = file_ext |
||||
else: |
||||
file_type = "default" |
||||
else: |
||||
file_type = self.extensionmap[file_ext] |
||||
|
||||
if not (bool(file_name)): |
||||
file_name = hex(random.randint(0, 65536)) |
||||
|
||||
return ProcessorChain( |
||||
[self.processors[x]() for x in self.chainconfig[file_type]["chain"]], |
||||
cast(str, file_name), |
||||
file_obj, |
||||
file_type, |
||||
ctx, |
||||
) |
||||
@ -0,0 +1 @@ |
||||
# processors metadata here |
||||
@ -0,0 +1,32 @@ |
||||
"""Define a Jinja2 Processor which applies programmable templating to the input stream.""" |
||||
|
||||
from typing import Iterable, Optional, Dict, cast |
||||
|
||||
from jinja2 import Environment, FileSystemLoader |
||||
|
||||
from .passthrough import PassThrough |
||||
|
||||
|
||||
class Jinja2(PassThrough): |
||||
"""Pass the input stream through Jinja2 for scritable templating.""" |
||||
|
||||
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: |
||||
"""Return an iterable object of the post-processed file. |
||||
|
||||
Arguments: |
||||
input_file (iterable): An input stream |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
|
||||
Returns: |
||||
iterable: The post-processed output stream |
||||
""" |
||||
ctx = cast(Dict, ctx) |
||||
template_env = Environment(loader=FileSystemLoader(ctx["templates"])) |
||||
template_env.globals.update(ctx["globals"]) |
||||
template_env.filters.update(ctx["filters"]) |
||||
tmpl = template_env.from_string("".join([x for x in input_file])) |
||||
return tmpl.render(metadata=ctx) |
||||
|
||||
|
||||
processor = Jinja2 |
||||
@ -0,0 +1,76 @@ |
||||
"""Define a Jinja2 processor which embeds the (presumably HTML) input stream into a Page Template |
||||
as defined in the ctx metadata (the ``content`` variable is assigned to the input stream and |
||||
the target template is rendered).""" |
||||
|
||||
import os |
||||
|
||||
from typing import Iterable, Optional, Dict, cast |
||||
|
||||
from jinja2 import Environment, FileSystemLoader |
||||
|
||||
from .processors import Processor |
||||
|
||||
|
||||
class Jinja2PageEmbed(Processor): |
||||
"""Embed input stream as ``content`` variable in page template defined in context key ``template``.""" |
||||
|
||||
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the filename of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the previous name for the file. |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new name for the file |
||||
|
||||
""" |
||||
|
||||
return os.path.splitext(oldname)[0] + ".html" |
||||
|
||||
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new mimetype of the file after processing |
||||
|
||||
""" |
||||
return "text/html" |
||||
|
||||
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: |
||||
"""Return an iterable object of the post-processed file. |
||||
|
||||
Arguments: |
||||
input_file (iterable): An input stream |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
iterable: The post-processed output stream |
||||
""" |
||||
ctx = cast(Dict, ctx) |
||||
template_env = Environment(loader=FileSystemLoader(ctx["templates"])) |
||||
template_env.globals.update(ctx["globals"]) |
||||
template_env.filters.update(ctx["filters"]) |
||||
tmpl = template_env.get_template(ctx["template"]) |
||||
content = "".join([x for x in input_file]) |
||||
return tmpl.render(content=content, metadata=ctx) |
||||
|
||||
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new extension of the file after processing |
||||
|
||||
""" |
||||
return "html" |
||||
|
||||
|
||||
processor = Jinja2PageEmbed |
||||
@ -0,0 +1,68 @@ |
||||
"""Passthrough progcessor which takes input and returns it.""" |
||||
|
||||
import os |
||||
|
||||
from .processors import Processor |
||||
from ..utils import guess_mime |
||||
from typing import Iterable, Optional, Dict, cast |
||||
|
||||
|
||||
class PassThrough(Processor): |
||||
"""A simple passthrough processor that takes input and sends it to output.""" |
||||
|
||||
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the filename of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the previous name for the file. |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new name for the file |
||||
|
||||
""" |
||||
return oldname |
||||
|
||||
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new mimetype of the file after processing |
||||
|
||||
""" |
||||
result = cast(str, guess_mime(oldname)) |
||||
if result == "directory": |
||||
result = "DIR" |
||||
return result |
||||
|
||||
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: |
||||
"""Return an iterable object of the post-processed file. |
||||
|
||||
Arguments: |
||||
input_file (iterable): An input stream |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
iterable: The post-processed output stream |
||||
""" |
||||
return input_file |
||||
|
||||
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new extension of the file after processing |
||||
|
||||
""" |
||||
return os.path.splitext(oldname)[-1] |
||||
|
||||
|
||||
processor = PassThrough |
||||
@ -0,0 +1 @@ |
||||
processor = None |
||||
@ -0,0 +1,69 @@ |
||||
"""Convert an MD stream into an HTML stream""" |
||||
|
||||
import io |
||||
import os |
||||
|
||||
from typing import Iterable, Optional, Dict |
||||
|
||||
import markdown |
||||
|
||||
from .processors import Processor |
||||
|
||||
|
||||
class MarkdownProcessor(Processor): |
||||
"""Convert an MD stream into an HTML stream""" |
||||
|
||||
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the filename of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the previous name for the file. |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new name for the file |
||||
|
||||
""" |
||||
return os.path.splitext(oldname)[0] + ".html" |
||||
|
||||
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new mimetype of the file after processing |
||||
|
||||
""" |
||||
return "text/html" |
||||
|
||||
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new extension of the file after processing |
||||
|
||||
""" |
||||
return "html" |
||||
|
||||
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: |
||||
"""Return an iterable object of the post-processed file. |
||||
|
||||
Arguments: |
||||
input_file (iterable): An input stream |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
iterable: The post-processed output stream |
||||
""" |
||||
md = u"".join([x for x in input_file]) |
||||
return io.StringIO(markdown.markdown(md, extensions=["extra", "admonition", "wikilinks"])) |
||||
|
||||
|
||||
processor = MarkdownProcessor # pylint: disable=invalid-name |
||||
@ -0,0 +1 @@ |
||||
processor = None |
||||
@ -0,0 +1 @@ |
||||
processor = None |
||||
@ -0,0 +1 @@ |
||||
processor = None |
||||
@ -0,0 +1,63 @@ |
||||
import abc |
||||
|
||||
from typing import Iterable, Optional, Dict |
||||
|
||||
|
||||
class ProcessorException(Exception): # pragma: no cover |
||||
"""A base exception class to be used by processor objects.""" |
||||
|
||||
|
||||
class Processor(abc.ABC): # pragma: no cover |
||||
def __init__(self, *args, **kwargs): |
||||
"""Initialize the class.""" |
||||
|
||||
@abc.abstractmethod |
||||
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the filename of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the previous name for the file. |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new name for the file |
||||
|
||||
""" |
||||
|
||||
@abc.abstractmethod |
||||
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new mimetype of the file after processing |
||||
|
||||
""" |
||||
|
||||
@abc.abstractmethod |
||||
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str: |
||||
"""Return the mimetype of the post-processed file. |
||||
|
||||
Arguments: |
||||
oldname (str): the input filename |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
str: the new extension of the file after processing |
||||
|
||||
""" |
||||
|
||||
@abc.abstractmethod |
||||
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable: |
||||
"""Return an iterable object of the post-processed file. |
||||
|
||||
Arguments: |
||||
input_file (iterable): An input stream |
||||
ctx (dict, optional): A context object generated from the processor configuration |
||||
|
||||
Returns: |
||||
iterable: The post-processed output stream |
||||
""" |
||||
@ -0,0 +1,80 @@ |
||||
import datetime |
||||
import glob |
||||
import itertools |
||||
import os |
||||
import pytz |
||||
from typing import Callable, Dict, List, Iterable, Union, cast |
||||
|
||||
from .metadata import MetaTree |
||||
from .processchain import ProcessorChains |
||||
|
||||
|
||||
def file_list(root: str, listcache: Dict) -> Callable: |
||||
def get_file_list(path_glob: str, *, sort_order: str = "ctime", reverse: bool = False, limit: int = 0) -> Iterable: |
||||
stattable = cast(List, []) |
||||
if path_glob in listcache: |
||||
stattable = listcache[path_glob] |
||||
else: |
||||
for fil in glob.glob(os.path.join(root, path_glob)): |
||||
if os.path.isdir(fil): |
||||
continue |
||||
if fil.endswith(".meta") or fil.endswith("~"): |
||||
continue |
||||
st = os.stat(fil) |
||||
stattable.append( |
||||
{ |
||||
"file_path": os.path.relpath(fil, root), |
||||
"file_name": os.path.split(fil)[-1], |
||||
"mtime": st.st_mtime, |
||||
"ctime": st.st_ctime, |
||||
"size": st.st_size, |
||||
"ext": os.path.splitext(fil)[1], |
||||
} |
||||
) |
||||
listcache[path_glob] = stattable |
||||
ret = sorted(stattable, key=lambda x: x[sort_order], reverse=reverse) |
||||
if limit > 0: |
||||
return itertools.islice(ret, limit) |
||||
return ret |
||||
|
||||
return get_file_list |
||||
|
||||
|
||||
def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains, namecache: Dict) -> Callable: |
||||
def get_file_name(file_name: str) -> Dict: |
||||
if file_name in namecache: |
||||
return namecache[file_name] |
||||
metadata = metatree.get_metadata(file_name) |
||||
chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata) |
||||
namecache[file_name] = chain.output_filename |
||||
return namecache[file_name] |
||||
|
||||
return get_file_name |
||||
|
||||
|
||||
def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChains, contcache: Dict) -> Callable: |
||||
def get_file_content(file_name: str) -> Iterable: |
||||
if file_name in contcache: |
||||
return contcache[file_name] |
||||
metadata = metatree.get_metadata(file_name) |
||||
chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata) |
||||
contcache[file_name] = chain.output |
||||
return chain.output |
||||
|
||||
return get_file_content |
||||
|
||||
|
||||
def file_metadata(metatree: MetaTree) -> Callable: |
||||
def get_file_metadata(file_name: str) -> Dict: |
||||
return metatree.get_metadata(file_name) |
||||
|
||||
return get_file_metadata |
||||
|
||||
|
||||
def time_iso8601(timezone: str) -> Callable: |
||||
tz = pytz.timezone(timezone) |
||||
|
||||
def get_time_iso8601(time_t: Union[int, float]) -> str: |
||||
return datetime.datetime.fromtimestamp(time_t, tz).isoformat("T") |
||||
|
||||
return get_time_iso8601 |
||||
@ -0,0 +1,6 @@ |
||||
class TestProcessChain: |
||||
def test_process_chain(self): |
||||
pass |
||||
|
||||
def test_processor_chain(self): |
||||
pass |
||||
@ -0,0 +1,42 @@ |
||||
import mimetypes |
||||
import os |
||||
|
||||
from typing import Dict, Optional |
||||
|
||||
|
||||
def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict: |
||||
"""Merge two dictionaries. |
||||
|
||||
Arguments: |
||||
dict_a (dict): The dictionary to use as the base. |
||||
dict_b (dict): The dictionary to update the values with. |
||||
|
||||
Returns: |
||||
dict: A new merged dictionary. |
||||
|
||||
""" |
||||
dict_z = dict_a.copy() |
||||
dict_z.update(dict_b) |
||||
return dict_z |
||||
|
||||
|
||||
def guess_mime(path: str) -> Optional[str]: |
||||
"""Guess the mime type for a given path. |
||||
|
||||
Arguments: |
||||
root (str): the root path of the file tree |
||||
path (str): the sub-path within the file tree |
||||
|
||||
Returns: |
||||
str: the guessed mime-type |
||||
|
||||
""" |
||||
mtypes = mimetypes.guess_type(path) |
||||
ftype = None |
||||
if os.path.isdir(path): |
||||
ftype = "directory" |
||||
elif os.access(path, os.F_OK) and mtypes[0]: |
||||
ftype = mtypes[0] |
||||
else: |
||||
ftype = "application/octet-stream" |
||||
return ftype |
||||
@ -0,0 +1,58 @@ |
||||
"""Package configuration.""" |
||||
from setuptools import find_packages, setup |
||||
|
||||
LONG_DESCRIPTION = """Pixywerk 2 is a DWIM filesystem based static site generator.""" |
||||
|
||||
INSTALL_REQUIRES = ["yaml-1.3", "markdown", "jstyleson", "jinja2"] |
||||
|
||||
# Extra dependencies |
||||
EXTRAS_REQUIRE = { |
||||
# Test dependencies |
||||
"tests": [ |
||||
"black", |
||||
"bandit>=1.1.0", |
||||
"flake8>=3.2.1", |
||||
"mypy>=0.470", |
||||
"prospector[with_everything]>=0.12.4", |
||||
"pytest-cov>=1.8.0", |
||||
"pytest-xdist>=1.15.0", |
||||
"pytest>=3.0.3", |
||||
"sphinx_rtd_theme>=0.1.6", |
||||
"sphinx-argparse>=0.1.15", |
||||
"Sphinx>=1.4.9", |
||||
] |
||||
} |
||||
|
||||
SETUP_REQUIRES = ["pytest-runner>=2.7.1", "setuptools_scm>=1.15.0"] |
||||
setup( |
||||
author="Cassowary Rusnov", |
||||
author_email="rusnovn@gmail.com", |
||||
classifiers=[ |
||||
"Development Status :: 1 - Pre-alpha", |
||||
"Environment :: Console", |
||||
"License :: OSI Approved :: MIT", |
||||
"Operating System :: POSIX :: Linux", |
||||
"Programming Language :: Python :: 3.6", |
||||
"Programming Language :: Python :: 3.7", |
||||
"Programming Language :: Python :: 3 :: Only", |
||||
"Topic :: Software Development :: Libraries :: Python Modules", |
||||
], |
||||
description="A filesystem-based DWIM website generator / CMS", |
||||
# entry_points={ |
||||
# 'console_scripts': [ |
||||
# 'cookbook = spicerack.cookbook:main', |
||||
# ], |
||||
# }, |
||||
extras_require=EXTRAS_REQUIRE, |
||||
install_requires=INSTALL_REQUIRES, |
||||
keywords=["wmf", "automation", "orchestration"], |
||||
license="MIT", |
||||
long_description=LONG_DESCRIPTION, |
||||
name="pixywerk2", # Must be the same used for __version__ in __init__.py |
||||
packages=find_packages(exclude=["*.tests", "*.tests.*"]), |
||||
platforms=["GNU/Linux"], |
||||
setup_requires=SETUP_REQUIRES, |
||||
use_scm_version=True, |
||||
url="https://git.antpanethon.com/cas/pixywerk2", |
||||
zip_safe=False, |
||||
) |
||||
@ -0,0 +1,22 @@ |
||||
[tox] |
||||
envlist=py{36,37}-{code-quality, unit} #, py37-sphinx |
||||
skipsdist = true |
||||
|
||||
[testenv] |
||||
setenv = |
||||
LANG = en_US.UTF-8 |
||||
deps = .[tests] |
||||
commands = |
||||
unit: py.test --strict --cov-report=term-missing --cov=pixywerk2 pixywerk2/tests/unit {posargs} |
||||
code-quality: flake8 pixywerk2 |
||||
code-quality: black -l 120 --check pixywerk2 |
||||
code-quality: - prospector -A |
||||
code-quality: - mypy --ignore-missing-imports pixywerk2 |
||||
# sphinx: python setup.py build_sphinx -b html |
||||
# sphinx: python setup.py build_sphinx -b man |
||||
basepython = |
||||
py36: python3.6 |
||||
py37: python3.7 |
||||
|
||||
[flake8] |
||||
max-line-length = 120 |
||||
Reference in new issue