From 1c4b8ed04c179937a98e213f3bd9240b703d917f Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 4 Jan 2021 16:07:40 +1300 Subject: [PATCH] Create better build script (#128) --- scripts/build.py | 234 +++++++++++++++++++++++++++++++++++++++++++++++ scripts/build.sh | 15 +-- 2 files changed, 236 insertions(+), 13 deletions(-) create mode 100644 scripts/build.py diff --git a/scripts/build.py b/scripts/build.py new file mode 100644 index 0000000..2fc509f --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,234 @@ +# A script to add footers and check for dead links +import yaml +import os +import re +import sys +import pathlib +import difflib +import os.path + + +class LOG: + FATAL = 0 + ERROR = 1 + WARN = 2 + INFO = 3 + DEBUG = 4 + + +COLOR = True +LOGLEVEL = LOG.INFO + + +def log(severity, context, message): + if severity > LOGLEVEL: + return + color = { + LOG.FATAL: '\u001b[31m', + LOG.ERROR: '\u001b[35m', + LOG.WARN: '\u001b[33m', + LOG.INFO: '\u001b[34m', + LOG.DEBUG: '\u001b[37m' + }[severity] + seve = { + LOG.FATAL: 'FATL', + LOG.ERROR: 'EROR', + LOG.WARN: 'WARN', + LOG.INFO: 'INFO', + LOG.DEBUG: 'DBUG' + }[severity] + + if COLOR == False: + print(f"{seve} {context}: {message}") + else: + print(f"\u001b[1m{color}{seve}\u001b[0m {context}: {message}") + + +class ContextFilters: + + def warn_unlinked(self, context): + """Warns about any pages which have been created, but not put into the navigation""" + for file in context['all_files']: + if file in context['nav_files']: + continue + log(LOG.WARN, "warn_unlinked", + f"{file} has been created, but not linked in nav") + + +class Filters: + + def warn_deadlink(self, content, context): + """Checks for dead links between pages""" + links = [link.replace(context['mkdocs']['site_url'], "") for _, link in re.findall( + r"\[([\w\W]+?)\]\(([\w\W]+?)\)", content, re.MULTILINE) if ((link.startswith( + "https://") or link.startswith("http://")) and link.startswith(context['mkdocs']['site_url']) or link.startswith("/") or "://" not in link) and "images" not in link] + for l in links: + link = l + if l == '' or l == '/': + link = "index.md" + link = link.split("#")[0] + + path = context['docs_dir'] + link + if not link.startswith("/"): + path = os.path.join( + context['docs_dir'], os.path.dirname(context['file_name']), link) + + if not path.endswith(".md"): + if path.endswith("/"): + path = path[:-1] + ".md" + else: + path = path + ".md" + + abs_path = pathlib.Path(path).resolve() + + if not abs_path.exists(): + log(LOG.WARN, "warn_deadlink", + f"{context['file_name']} contains a dead link to {link} ({path})") + + return content + + def footer(self, content, context): + """Appends a footer to the end of every manuscript file""" + if not hasattr(self, "footer_content"): + self.footer_path = os.path.join( + os.path.dirname(__file__), "recipe-footer.md") + footer_file = open(self.footer_path) + self.footer_content = footer_file.read() + self.footer_search = self.footer_content.split("\n")[ + 0].replace("#", "") + footer_file.close() + if "index.md" in context['file_name']: + log(LOG.DEBUG, "footer", + f"ignoring {context['file_name']} as it contains index.md") + return content + + if self.footer_search in content: + log(LOG.WARN, "footer", + f"Footer is hard-coded in {context['file_name']}") + return content + return content + "\n" + self.footer_content + + +def flattern(obj): + resp = [] + + if type(obj) == dict: + for key, value in obj.items(): + if type(value) == dict or type(value) == list: + resp.extend(flattern(value)) + else: + resp.append(value) + elif type(obj) == list: + for value in obj: + if type(value) == dict or type(value) == list: + resp.extend(flattern(value)) + else: + resp.append(value) + return resp + + +def execute(dry_run, show_diffs, mkdocs_yaml): + log(LOG.DEBUG, "", "Loading mkdocs.yml") + mkdocs = yaml.load(open(mkdocs_yaml).read(), Loader=yaml.Loader) + log(LOG.DEBUG, "mkdocs", mkdocs) + log(LOG.DEBUG, "", "Loaded mkdocs.yml") + + log(LOG.DEBUG, "", "Registering filters") + filters = [getattr(Filters(), func) for func in dir( + Filters) if callable(getattr(Filters, func)) and not func.endswith("__")] + for filter in filters: + log(LOG.INFO, "Filter Info", f"{filter.__name__}: {filter.__doc__}") + log(LOG.DEBUG, "", "Registered filters") + + log(LOG.DEBUG, "", "Registering context filters") + context_filters = [getattr(ContextFilters(), func) for func in dir( + ContextFilters) if callable(getattr(ContextFilters, func)) and not func.endswith("__")] + + for filter in context_filters: + log(LOG.INFO, "Filter Info", f"{filter.__name__}: {filter.__doc__}") + log(LOG.DEBUG, "", "Registered context filters") + + log(LOG.DEBUG, "", "Walking docs_dir for manuscript files") + docs_dir = os.path.join(os.path.dirname(mkdocs_yaml), mkdocs['docs_dir']) + all_files = flattern([[os.path.join(root, file).replace(docs_dir + "/", "") for file in files if file.endswith(".md")] + for root, dirs, files in os.walk(docs_dir)]) + log(LOG.DEBUG, "all_files", all_files) + log(LOG.DEBUG, "", "Walked docs_dir") + + log(LOG.DEBUG, "", "Loading nav entries") + nav_files = flattern(mkdocs['nav']) + log(LOG.DEBUG, "nav_files", nav_files) + log(LOG.DEBUG, "", "Loaded nav entries") + + log(LOG.DEBUG, "", "Constructing context") + context = { + 'docs_dir': docs_dir, + 'all_files': all_files, + 'nav_files': nav_files, + 'mkdocs': mkdocs + } + log(LOG.DEBUG, "", "Constructed context") + + log(LOG.DEBUG, "", "Running context filters") + for filter in context_filters: + log(LOG.INFO, "Global", f"Running {filter.__name__}") + filter(context) + + log(LOG.DEBUG, "", "Running filters per file") + + for file in all_files: + log(LOG.INFO, "", f"Processing {file}") + path = os.path.join(docs_dir, file) + context['file_name'] = file + file_open = open(path, 'r') + original = file_open.read() + content = original + file_open.close() + for filter in filters: + log(LOG.DEBUG, file, f"Running \"{filter.__name__}\" filter") + content = filter(content, context) + if not dry_run: + fo = open(path, 'w') + fo.write(content) + fo.close() + + if show_diffs: + a = original + b = content + + diffs = [] + diff_count = 0 + matcher = difflib.SequenceMatcher(None, a, b) + for opcode, a0, a1, b0, b1 in matcher.get_opcodes(): + diff_count = diff_count + 1 + if opcode == "equal": + diff_count = diff_count - 1 + diffs.append(a[a0:a1]) + elif opcode == "insert": + diffs.append("\u001b[42m" + b[b0:b1]) + elif opcode == "delete": + diffs.append("\u001b[41m" + a[a0:a1]) + elif opcode == "replace": + diffs.append("\u001b[42m" + b[b0:b1]) + diffs.append("\u001b[41m" + a[a0:a1]) + if diff_count != 0: + log(LOG.INFO, file, "\u001b[0m".join(diffs) + "\u001b[0m") + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Stir up a cookbook") + parser.add_argument( + "mkdocs", help="Path to the mkdocs.yml file", type=str) + parser.add_argument( + "-d", "--dry", help="Run the build in dry mode (does not write changes)", action="store_true") + parser.add_argument( + "-i", "--diffs", help="Show diffs for changed files", action="store_true") + + args = parser.parse_args() + + execute(args.dry, args.diffs, args.mkdocs) + + +if __name__ == "__main__": + main() diff --git a/scripts/build.sh b/scripts/build.sh index 2bce5f8..8f1fa17 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,19 +4,8 @@ # Fetch git history so that we get last-updated timestamps git fetch --unshallow -# Append a common footer to all recipes/swarm docs -for i in `find manuscript/ -name "*.md" | grep -v index.md` -do - # Does this recipe already have a "tip your waiter" section? - grep -q "Tip your waiter" $i - if [ $? -eq 1 ] - then - echo -e "\n" >> $i - cat scripts/recipe-footer.md >> $i - else - echo "WARNING - hard-coded footer exists in $i" - fi -done +# Run python build script +python3 scripts/build.py mkdocs.yml # Now build the docs for publishing mkdocs build -f mkdocs.yml