From 41c8d81992d2443cd5c3418df0f461b0af1a6ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 11 Feb 2022 23:49:35 +0100 Subject: [PATCH] feat: Implement execution of code blocks --- .copier-answers.yml | 20 ++ .github/FUNDING.yml | 7 + .github/ISSUE_TEMPLATE/bug_report.md | 32 ++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++ .github/workflows/ci.yml | 111 +++++++ .gitignore | 16 + .gitpod.dockerfile | 7 + .gitpod.yml | 13 + CHANGELOG.md | 7 + CODE_OF_CONDUCT.md | 74 +++++ CONTRIBUTING.md | 137 +++++++++ LICENSE | 15 + Makefile | 53 ++++ README.md | 171 +++++++++++ config/coverage.ini | 21 ++ config/flake8.ini | 110 +++++++ config/mypy.ini | 5 + config/pytest.ini | 16 + docs/changelog.md | 1 + docs/code_of_conduct.md | 1 + docs/contributing.md | 1 + docs/credits.md | 105 +++++++ docs/css/material.css | 4 + docs/css/mkdocstrings.css | 6 + docs/examples.md | 23 ++ docs/gen_ref_nav.py | 34 +++ docs/index.md | 1 + docs/license.md | 3 + duties.py | 357 ++++++++++++++++++++++ markdown_exec.svg | 63 ++++ mkdocs.yml | 87 ++++++ pyproject.toml | 114 +++++++ scripts/multirun.sh | 17 ++ scripts/setup.sh | 28 ++ src/markdown_exec/__init__.py | 83 +++++ src/markdown_exec/py.typed | 0 src/markdown_exec/python.py | 60 ++++ tests/__init__.py | 7 + tests/conftest.py | 30 ++ tests/test_python.py | 35 +++ tests/test_validator.py | 38 +++ 41 files changed, 1933 insertions(+) create mode 100644 .copier-answers.yml create mode 100644 .github/FUNDING.yml create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .gitpod.dockerfile create mode 100644 .gitpod.yml create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 config/coverage.ini create mode 100644 config/flake8.ini create mode 100644 config/mypy.ini create mode 100644 config/pytest.ini create mode 100644 docs/changelog.md create mode 100644 docs/code_of_conduct.md create mode 100644 docs/contributing.md create mode 100644 docs/credits.md create mode 100644 docs/css/material.css create mode 100644 docs/css/mkdocstrings.css create mode 100644 docs/examples.md create mode 100755 docs/gen_ref_nav.py create mode 100644 docs/index.md create mode 100644 docs/license.md create mode 100644 duties.py create mode 100644 markdown_exec.svg create mode 100644 mkdocs.yml create mode 100644 pyproject.toml create mode 100755 scripts/multirun.sh create mode 100755 scripts/setup.sh create mode 100644 src/markdown_exec/__init__.py create mode 100644 src/markdown_exec/py.typed create mode 100644 src/markdown_exec/python.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_python.py create mode 100644 tests/test_validator.py diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..bb4ec9c --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,20 @@ +# Changes here will be overwritten by Copier +_commit: 0.8.1 +_src_path: gh:pawamoy/copier-pdm +author_email: pawamoy@pm.me +author_fullname: Timothée Mazzucotelli +author_username: pawamoy +copyright_date: '2022' +copyright_holder: Timothée Mazzucotelli +copyright_holder_email: pawamoy@pm.me +copyright_license: ISC License +project_description: Utilities to execute code blocks in Markdown files. +project_name: Markdown Exec +python_package_command_line_name: markdown-exec +python_package_distribution_name: markdown-exec +python_package_import_name: markdown_exec +repository_name: markdown-exec +repository_namespace: pawamoy +repository_provider: github.com +use_precommit: false + diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..c71a8d4 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,7 @@ +github: + - pawamoy +ko_fi: pawamoy +liberapay: pawamoy +patreon: pawamoy +custom: + - https://www.paypal.me/pawamoy diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..f8ec6cc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: unconfirmed +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Run command '...' +3. Scroll down to '...' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**System (please complete the following information):** +- `Markdown Exec` version: [e.g. 0.2.1] +- Python version: [e.g. 3.8] +- OS: [Windows/Linux] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..4fe86d5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a9b12e4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,111 @@ +name: ci + +on: + push: + branches: + - master + pull_request: + branches: + - master + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHONIOENCODING: UTF-8 + +jobs: + + quality: + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up PDM + uses: pdm-project/setup-pdm@v2.5 + with: + python-version: "3.8" + + - name: Set cache variables + id: set_variables + run: | + echo "::set-output name=PIP_CACHE::$(pip cache dir)" + echo "::set-output name=PDM_CACHE::$(pdm config cache_dir)" + + - name: Set up cache + uses: actions/cache@v2 + with: + path: | + ${{ steps.set_variables.outputs.PIP_CACHE }} + ${{ steps.set_variables.outputs.PDM_CACHE }} + key: checks-cache + + - name: Resolving dependencies + run: pdm lock + + - name: Install dependencies + run: pdm install -G duty -G docs -G quality -G typing -G security + + - name: Check if the documentation builds correctly + run: pdm run duty check-docs + + - name: Check the code quality + run: pdm run duty check-quality + + - name: Check if the code is correctly typed + run: pdm run duty check-types + + - name: Check for vulnerabilities in dependencies + run: pdm run duty check-dependencies + + tests: + + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11-dev" + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up PDM + uses: pdm-project/setup-pdm@v2.5 + with: + python-version: ${{ matrix.python-version }} + + - name: Set cache variables + id: set_variables + run: | + echo "::set-output name=PIP_CACHE::$(pip cache dir)" + echo "::set-output name=PDM_CACHE::$(pdm config cache_dir)" + + - name: Set up cache + uses: actions/cache@v2 + with: + path: | + ${{ steps.set_variables.outputs.PIP_CACHE }} + ${{ steps.set_variables.outputs.PDM_CACHE }} + key: tests-cache-${{ runner.os }}-${{ matrix.python-version }} + + - name: Install dependencies + run: pdm install -G duty -G tests + + - name: Run the test suite + run: pdm run duty test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a93f1c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +.idea/ +__pycache__/ +*.py[cod] +dist/ +*.egg-info/ +build/ +htmlcov/ +.coverage* +pip-wheel-metadata/ +.pytest_cache/ +.mypy_cache/ +site/ +pdm.lock +.pdm.toml +__pypackages__/ +.venv/ diff --git a/.gitpod.dockerfile b/.gitpod.dockerfile new file mode 100644 index 0000000..33f285c --- /dev/null +++ b/.gitpod.dockerfile @@ -0,0 +1,7 @@ +FROM gitpod/workspace-full +USER gitpod +ENV PIP_USER=no +ENV PYTHON_VERSIONS= +RUN pip3 install pipx; \ + pipx install pdm; \ + pipx ensurepath diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000..23a3c2b --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,13 @@ +vscode: + extensions: + - ms-python.python + +image: + file: .gitpod.dockerfile + +ports: +- port: 8000 + onOpen: notify + +tasks: +- init: make setup diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..49212ce --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + + diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..35f1f53 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at pawamoy@pm.me. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..7e3e1e9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,137 @@ +# Contributing + +Contributions are welcome, and they are greatly appreciated! +Every little bit helps, and credit will always be given. + +## Environment setup + +Nothing easier! + +Fork and clone the repository, then: + +```bash +cd markdown-exec +make setup +``` + +!!! note + If it fails for some reason, + you'll need to install + [PDM](https://github.com/pdm-project/pdm) + manually. + + You can install it with: + + ```bash + python3 -m pip install --user pipx + pipx install pdm + ``` + + Now you can try running `make setup` again, + or simply `pdm install`. + +You now have the dependencies installed. + +You can run the application with `pdm run markdown-exec [ARGS...]`. + +Run `make help` to see all the available actions! + +## Tasks + +This project uses [duty](https://github.com/pawamoy/duty) to run tasks. +A Makefile is also provided. The Makefile will try to run certain tasks +on multiple Python versions. If for some reason you don't want to run the task +on multiple Python versions, you can do one of the following: + +1. `export PYTHON_VERSIONS= `: this will run the task + with only the current Python version +2. run the task directly with `pdm run duty TASK` + +The Makefile detects if a virtual environment is activated, +so `make` will work the same with the virtualenv activated or not. + +## Development + +As usual: + +1. create a new branch: `git checkout -b feature-or-bugfix-name` +1. edit the code and/or the documentation + +If you updated the documentation or the project dependencies: + +1. run `make docs-regen` +1. run `make docs-serve`, + go to http://localhost:8000 and check that everything looks good + +**Before committing:** + +1. run `make format` to auto-format the code +1. run `make check` to check everything (fix any warning) +1. run `make test` to run the tests (fix any issue) +1. follow our [commit message convention](#commit-message-convention) + +If you are unsure about how to fix or ignore a warning, +just let the continuous integration fail, +and we will help you during review. + +Don't bother updating the changelog, we will take care of this. + +## Commit message convention + +Commits messages must follow the +[Angular style](https://gist.github.com/stephenparish/9941e89d80e2bc58a153#format-of-the-commit-message): + +``` +[(scope)]: Subject + +[Body] +``` + +Scope and body are optional. Type can be: + +- `build`: About packaging, building wheels, etc. +- `chore`: About packaging or repo/files management. +- `ci`: About Continuous Integration. +- `docs`: About documentation. +- `feat`: New feature. +- `fix`: Bug fix. +- `perf`: About performance. +- `refactor`: Changes which are not features nor bug fixes. +- `style`: A change in code style/format. +- `tests`: About tests. + +**Subject (and body) must be valid Markdown.** +If you write a body, please add issues references at the end: + +``` +Body. + +References: #10, #11. +Fixes #15. +``` + +## Pull requests guidelines + +Link to any related issue in the Pull Request message. + +During review, we recommend using fixups: + +```bash +# SHA is the SHA of the commit you want to fix +git commit --fixup=SHA +``` + +Once all the changes are approved, you can squash your commits: + +```bash +git rebase -i --autosquash master +``` + +And force-push: + +```bash +git push -f +``` + +If this seems all too complicated, you can push or force-push each new commit, +and we will squash them ourselves if needed, before merging. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f1ce338 --- /dev/null +++ b/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2022, Timothée Mazzucotelli + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5829157 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +.DEFAULT_GOAL := help +SHELL := bash + +DUTY = $(shell [ -n "${VIRTUAL_ENV}" ] || echo pdm run) duty + +args = $(foreach a,$($(subst -,_,$1)_args),$(if $(value $a),$a="$($a)")) +check_quality_args = files +docs_serve_args = host port +release_args = version +test_args = match + +BASIC_DUTIES = \ + changelog \ + check-dependencies \ + clean \ + coverage \ + docs \ + docs-deploy \ + docs-regen \ + docs-serve \ + format \ + release + +QUALITY_DUTIES = \ + check-quality \ + check-docs \ + check-types \ + test + +.PHONY: help +help: + @$(DUTY) --list + +.PHONY: lock +lock: + @pdm lock + +.PHONY: setup +setup: + @bash scripts/setup.sh + +.PHONY: check +check: + @bash scripts/multirun.sh duty check-quality check-types check-docs + @$(DUTY) check-dependencies + +.PHONY: $(BASIC_DUTIES) +$(BASIC_DUTIES): + @$(DUTY) $@ $(call args,$@) + +.PHONY: $(QUALITY_DUTIES) +$(QUALITY_DUTIES): + @bash scripts/multirun.sh duty $@ $(call args,$@) diff --git a/README.md b/README.md new file mode 100644 index 0000000..bc62555 --- /dev/null +++ b/README.md @@ -0,0 +1,171 @@ +# Markdown Exec + +[![ci](https://github.com/pawamoy/markdown-exec/workflows/ci/badge.svg)](https://github.com/pawamoy/markdown-exec/actions?query=workflow%3Aci) +[![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://pawamoy.github.io/markdown-exec/) +[![pypi version](https://img.shields.io/pypi/v/markdown-exec.svg)](https://pypi.org/project/markdown-exec/) +[![gitpod](https://img.shields.io/badge/gitpod-workspace-blue.svg?style=flat)](https://gitpod.io/#https://github.com/pawamoy/markdown-exec) +[![gitter](https://badges.gitter.im/join%20chat.svg)](https://gitter.im/markdown-exec/community) + +Utilities to execute code blocks in Markdown files. + +For example, you write a Python code block that computes some HTML, +and this HTML is injected in place of the code block. + +## Requirements + +Markdown Exec requires Python 3.7 or above. + +
+To install Python 3.7, I recommend using pyenv. + +```bash +# install pyenv +git clone https://github.com/pyenv/pyenv ~/.pyenv + +# setup pyenv (you should also put these three lines in .bashrc or similar) +export PATH="${HOME}/.pyenv/bin:${PATH}" +export PYENV_ROOT="${HOME}/.pyenv" +eval "$(pyenv init -)" + +# install Python 3.7 +pyenv install 3.7.12 + +# make it available globally +pyenv global system 3.7.12 +``` +
+ +## Installation + +With `pip`: +```bash +pip install markdown-exec +``` + +## Configuration + +This extension relies on the +[SuperFences](https://facelessuser.github.io/pymdown-extensions/extensions/superfences/) +extension of +[PyMdown Extensions](https://facelessuser.github.io/pymdown-extensions/). + +To allow execution of code blocks, +configure a custom fence from Python: + +```python +from markdown import Markdown +from markdown_exec import formatter, validator + +Markdown( + extensions=["pymdownx.superfences"], + extension_configs={ + "pymdownx.superfences": { + "custom_fences": [ + { + "name": "python", + "class": "python", + "validator": validator, + "format": formatter, + } + ] + } + } +) +``` + +...or in MkDocs configuration file: + +```yaml +markdown_extensions: +- pymdownx.superfences: + custom_fences: + - name: python + class: python + validator: !!python/name:markdown_exec.validator + format: !!python/name:markdown_exec.formatter +``` + +## Usage + +You are now able to execute code blocks instead of displaying them: + +````md +```python exec="on" +print("Some Python code") +``` +```` + +The `exec` option will be true for every possible value except `0`, `no`, `off` and `false` (case insensitive). + +The standard output and error of executed Python code blocks is not captured +and will be written to the terminal, as usual. + +If you want to "inject" contents into the page, you can use these two functions +in your code blocks (they are available in the global context of execution): + +- `output_html(text)`: inject the HTML text passed as argument. +- `output_markdown(text)`: convert the text passed as argument to HTML and then inject it. + +WARNING: You can call these functions only once, as they internally raise an exception. + +HTML Example: + +=== "Markdown" + + ````md + System information: + + ```python exec="yes" + import platform + output_html( + f""" + + machine: {platform.machine()} + version: {platform.version()} + platform: {platform.platform()} + system: {platform.system()} + + """ + ) + ``` + ```` + +=== "Rendered" + + System information: + + ``` + machine: x86_64 + version: #1 SMP PREEMPT Tue, 01 Feb 2022 21:42:50 +0000 + platform: Linux-5.16.5-arch1-1-x86_64-with-glibc2.33 + system: Linux + ``` + +Markdown Example: + +=== "Markdown" + + ````md + System information: + + ```python exec="yes" + import platform + output_markdown( + f""" + - machine: `{platform.machine()}` + - version: `{platform.version()}` + - platform: `{platform.platform()}` + - system: `{platform.system()}` + """ + ) + ``` + ```` + +=== "Rendered" + + System information: + + - machine: `x86_64` + - version: `#1 SMP PREEMPT Tue, 01 Feb 2022 21:42:50 +0000` + - platform: `Linux-5.16.5-arch1-1-x86_64-with-glibc2.33` + - system: `Linux` \ No newline at end of file diff --git a/config/coverage.ini b/config/coverage.ini new file mode 100644 index 0000000..b1c19ea --- /dev/null +++ b/config/coverage.ini @@ -0,0 +1,21 @@ +[coverage:run] +branch = true +parallel = true +source = + src/ + tests/ + +[coverage:paths] +equivalent = + src/ + __pypackages__/ + +[coverage:report] +precision = 2 +omit = + src/*/__init__.py + src/*/__main__.py + tests/__init__.py + +[coverage:json] +output = htmlcov/coverage.json diff --git a/config/flake8.ini b/config/flake8.ini new file mode 100644 index 0000000..06eae05 --- /dev/null +++ b/config/flake8.ini @@ -0,0 +1,110 @@ +[flake8] +exclude = fixtures,site +max-line-length = 132 +docstring-convention = google +ban-relative-imports = true +ignore = + # redundant with W0622 (builtin override), which is more precise about line number + A001 + # missing docstring in magic method + D105 + # multi-line docstring summary should start at the first line + D212 + # does not support Parameters sections + D417 + # whitespace before ':' (incompatible with Black) + E203 + # redundant with E0602 (undefined variable) + F821 + # black already deals with quoting + Q000 + # use of assert + S101 + # we are not parsing XML + S405 + # line break before binary operator (incompatible with Black) + W503 + # two-lowercase-letters variable DO conform to snake_case naming style + C0103 + # redundant with D102 (missing docstring) + C0116 + # line too long + C0301 + # too many instance attributes + R0902 + # too few public methods + R0903 + # too many public methods + R0904 + # too many branches + R0912 + # too many methods + R0913 + # too many local variables + R0914 + # too many statements + R0915 + # redundant with F401 (unused import) + W0611 + # lazy formatting for logging calls + W1203 + # short name + VNE001 + # f-strings + WPS305 + # common variable names (too annoying) + WPS110 + # redundant with W0622 (builtin override), which is more precise about line number + WPS125 + # too many imports + WPS201 + # too many module members + WPS202 + # overused expression + WPS204 + # too many local variables + WPS210 + # too many arguments + WPS211 + # too many expressions + WPS213 + # too many methods + WPS214 + # too deep nesting + WPS220 + # high Jones complexity + WPS221 + # too many elif branches + WPS223 + # string over-use: can't disable it per file? + WPS226 + # too many public instance attributes + WPS230 + # too complex f-string + WPS237 + # too cumbersome, asks to write class A(object) + WPS306 + # multi-line parameters (incompatible with Black) + WPS317 + # multi-line strings (incompatible with attributes docstrings) + WPS322 + # implicit string concatenation + WPS326 + # explicit string concatenation + WPS336 + # blank line after multiline string + WPS355 + # noqa overuse + WPS402 + # __init__ modules with logic + WPS412 + # print statements + WPS421 + # statement with no effect (not compatible with attribute docstrings) + WPS428 + # redundant with C0415 (not top-level import) + WPS433 + # multiline string + WPS462 + # implicit dict.get usage (generally false-positive) + WPS529 diff --git a/config/mypy.ini b/config/mypy.ini new file mode 100644 index 0000000..814e2ac --- /dev/null +++ b/config/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +ignore_missing_imports = true +exclude = tests/fixtures/ +warn_unused_ignores = true +show_error_codes = true diff --git a/config/pytest.ini b/config/pytest.ini new file mode 100644 index 0000000..ad72bbe --- /dev/null +++ b/config/pytest.ini @@ -0,0 +1,16 @@ +[pytest] +norecursedirs = + .git + .tox + .env + dist + build +python_files = + test_*.py + *_test.py + tests.py +addopts = + --cov + --cov-config config/coverage.ini +testpaths = + tests diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..786b75d --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1 @@ +--8<-- "CHANGELOG.md" diff --git a/docs/code_of_conduct.md b/docs/code_of_conduct.md new file mode 100644 index 0000000..01f2ea2 --- /dev/null +++ b/docs/code_of_conduct.md @@ -0,0 +1 @@ +--8<-- "CODE_OF_CONDUCT.md" diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..ea38c9b --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1 @@ +--8<-- "CONTRIBUTING.md" diff --git a/docs/credits.md b/docs/credits.md new file mode 100644 index 0000000..8eef77f --- /dev/null +++ b/docs/credits.md @@ -0,0 +1,105 @@ +```python exec="yes" +def get_credits(): + import re + from importlib.metadata import metadata, PackageNotFoundError + from itertools import chain + from pathlib import Path + from textwrap import dedent + + import toml + from jinja2 import StrictUndefined + from jinja2.sandbox import SandboxedEnvironment + + project_dir = Path(".") + pyproject = toml.load(project_dir / "pyproject.toml") + project = pyproject["project"] + pdm = pyproject["tool"]["pdm"] + lock_data = toml.load(project_dir / "pdm.lock") + lock_pkgs = {pkg["name"].lower(): pkg for pkg in lock_data["package"]} + project_name = project["name"] + regex = re.compile(r"(?P[\w.-]+)(?P.*)$") + + def get_license(pkg_name): + try: + data = metadata(pkg_name) + except PackageNotFoundError: + return "?" + license = data.get("License", "").replace("UNKNOWN", "") + if not license: + for header, value in data.items(): + if header == "Classifier" and value.startswith("License ::"): + license = value.rsplit("::", 1)[1] + return license or "?" + + def get_deps(base_deps): + deps = {} + for dep in base_deps: + parsed = regex.match(dep).groupdict() + dep_name = parsed["dist"].lower() + deps[dep_name] = {"license": get_license(dep_name), **parsed, **lock_pkgs[dep_name]} + + again = True + while again: + again = False + for pkg_name in lock_pkgs: + if pkg_name in deps: + for pkg_dependency in lock_pkgs[pkg_name].get("dependencies", []): + parsed = regex.match(pkg_dependency).groupdict() + dep_name = parsed["dist"].lower() + if dep_name not in deps: + deps[dep_name] = {"license": get_license(dep_name), **parsed, **lock_pkgs[dep_name]} + again = True + + return deps + + dev_dependencies = get_deps(chain(*pdm.get("dev-dependencies", {}).values())) + prod_dependencies = get_deps( + chain( + project.get("dependencies", []), + chain(*project.get("optional-dependencies", {}).values()), + ) + ) + + template_data = { + "project_name": project_name, + "prod_dependencies": sorted(prod_dependencies.values(), key=lambda dep: dep["name"]), + "dev_dependencies": sorted(dev_dependencies.values(), key=lambda dep: dep["name"]), + "more_credits": "http://pawamoy.github.io/credits/", + } + template_text = dedent( + """ + These projects were used to build `{{ project_name }}`. **Thank you!** + + [`python`](https://www.python.org/) | + [`pdm`](https://pdm.fming.dev/) | + [`copier-pdm`](https://github.com/pawamoy/copier-pdm) + + {% macro dep_line(dep) -%} + [`{{ dep.name }}`](https://pypi.org/project/{{ dep.name }}/) | {{ dep.summary }} | {{ ("`" ~ dep.spec ~ "`") if dep.spec else "" }} | `{{ dep.version }}` | {{ dep.license }} + {%- endmacro %} + + ### Runtime dependencies + + Project | Summary | Version (accepted) | Version (last resolved) | License + ------- | ------- | ------------------ | ----------------------- | ------- + {% for dep in prod_dependencies -%} + {{ dep_line(dep) }} + {% endfor %} + + ### Development dependencies + + Project | Summary | Version (accepted) | Version (last resolved) | License + ------- | ------- | ------------------ | ----------------------- | ------- + {% for dep in dev_dependencies -%} + {{ dep_line(dep) }} + {% endfor %} + + **[More credits from the author]({{ more_credits }})** + """ + ) + jinja_env = SandboxedEnvironment(undefined=StrictUndefined) + return jinja_env.from_string(template_text).render(**template_data) + + +output_markdown(get_credits()) +``` diff --git a/docs/css/material.css b/docs/css/material.css new file mode 100644 index 0000000..9e8c14a --- /dev/null +++ b/docs/css/material.css @@ -0,0 +1,4 @@ +/* More space at the bottom of the page. */ +.md-main__inner { + margin-bottom: 1.5rem; +} diff --git a/docs/css/mkdocstrings.css b/docs/css/mkdocstrings.css new file mode 100644 index 0000000..42c7741 --- /dev/null +++ b/docs/css/mkdocstrings.css @@ -0,0 +1,6 @@ +/* Indentation. */ +div.doc-contents:not(.first) { + padding-left: 25px; + border-left: 4px solid rgba(230, 230, 230); + margin-bottom: 80px; +} diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..2094de9 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,23 @@ +## Draw a graph of module inter-dependencies with [pydeps](https://github.com/thebjorn/pydeps) + +````md +```python exec="true" +```` +```python +from pydeps import cli, colors, py2depgraph, dot +from pydeps.pydeps import depgraph_to_dotsrc +from pydeps.target import Target +cli.verbose = cli._not_verbose +options = cli.parse_args(["src/markdown_exec", "--noshow"]) +colors.START_COLOR = options["start_color"] +target = Target(options["fname"]) +with target.chdir_work(): + dep_graph = py2depgraph.py2dep(target, **options) +dot_src = depgraph_to_dotsrc(target, dep_graph, **options) +svg = dot.call_graphviz_dot(dot_src, "svg").decode() +svg = svg.replace('fill="white"', 'fill="transparent"') +output_html(f"
{svg}
") +``` +```` +``` +```` diff --git a/docs/gen_ref_nav.py b/docs/gen_ref_nav.py new file mode 100755 index 0000000..a7c694c --- /dev/null +++ b/docs/gen_ref_nav.py @@ -0,0 +1,34 @@ +"""Generate the code reference pages and navigation.""" + +from pathlib import Path + +import mkdocs_gen_files + +nav = mkdocs_gen_files.Nav() + +for path in sorted(Path("src").glob("**/*.py")): + module_path = path.relative_to("src").with_suffix("") + doc_path = path.relative_to("src").with_suffix(".md") + full_doc_path = Path("reference", doc_path) + + parts = list(module_path.parts) + if parts[-1] == "__init__": + parts = parts[:-1] + doc_path = doc_path.with_name("index.md") + full_doc_path = full_doc_path.with_name("index.md") + elif parts[-1] == "__main__": + continue + nav_parts = list(parts) + nav[nav_parts] = doc_path + + with mkdocs_gen_files.open(full_doc_path, "w") as fd: + ident = ".".join(parts) + print("::: " + ident, file=fd) + + mkdocs_gen_files.set_edit_path(full_doc_path, path) + +# add pages manually: +# nav["package", "module"] = "path/to/file.md" + +with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: + nav_file.writelines(nav.build_literate_nav()) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..612c7a5 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +--8<-- "README.md" diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..cdacdfe --- /dev/null +++ b/docs/license.md @@ -0,0 +1,3 @@ +``` +--8<-- "LICENSE" +``` diff --git a/duties.py b/duties.py new file mode 100644 index 0000000..3ad804d --- /dev/null +++ b/duties.py @@ -0,0 +1,357 @@ +"""Development tasks.""" + +import importlib +import os +import re +import sys +import tempfile +from contextlib import suppress +from io import StringIO +from pathlib import Path +from typing import List, Optional, Pattern +from urllib.request import urlopen + +from duty import duty + +PY_SRC_PATHS = (Path(_) for _ in ("src", "tests", "duties.py", "docs")) +PY_SRC_LIST = tuple(str(_) for _ in PY_SRC_PATHS) +PY_SRC = " ".join(PY_SRC_LIST) +TESTING = os.environ.get("TESTING", "0") in {"1", "true"} +CI = os.environ.get("CI", "0") in {"1", "true", "yes", ""} +WINDOWS = os.name == "nt" +PTY = not WINDOWS and not CI + + +def _latest(lines: List[str], regex: Pattern) -> Optional[str]: + for line in lines: + match = regex.search(line) + if match: + return match.groupdict()["version"] + return None + + +def _unreleased(versions, last_release): + for index, version in enumerate(versions): + if version.tag == last_release: + return versions[:index] + return versions + + +def update_changelog( + inplace_file: str, + marker: str, + version_regex: str, + template_url: str, +) -> None: + """ + Update the given changelog file in place. + + Arguments: + inplace_file: The file to update in-place. + marker: The line after which to insert new contents. + version_regex: A regular expression to find currently documented versions in the file. + template_url: The URL to the Jinja template used to render contents. + """ + from git_changelog.build import Changelog + from git_changelog.commit import AngularStyle + from jinja2.sandbox import SandboxedEnvironment + + AngularStyle.DEFAULT_RENDER.insert(0, AngularStyle.TYPES["build"]) + env = SandboxedEnvironment(autoescape=False) + template_text = urlopen(template_url).read().decode("utf8") # noqa: S310 + template = env.from_string(template_text) + changelog = Changelog(".", style="angular") + + if len(changelog.versions_list) == 1: + last_version = changelog.versions_list[0] + if last_version.planned_tag is None: + planned_tag = "0.1.0" + last_version.tag = planned_tag + last_version.url += planned_tag + last_version.compare_url = last_version.compare_url.replace("HEAD", planned_tag) + + with open(inplace_file, "r") as changelog_file: + lines = changelog_file.read().splitlines() + + last_released = _latest(lines, re.compile(version_regex)) + if last_released: + changelog.versions_list = _unreleased(changelog.versions_list, last_released) + rendered = template.render(changelog=changelog, inplace=True) + lines[lines.index(marker)] = rendered + + with open(inplace_file, "w") as changelog_file: # noqa: WPS440 + changelog_file.write("\n".join(lines).rstrip("\n") + "\n") + + +@duty +def changelog(ctx): + """ + Update the changelog in-place with latest commits. + + Arguments: + ctx: The context instance (passed automatically). + """ + commit = "166758a98d5e544aaa94fda698128e00733497f4" + template_url = f"https://raw.githubusercontent.com/pawamoy/jinja-templates/{commit}/keepachangelog.md" + ctx.run( + update_changelog, + kwargs={ + "inplace_file": "CHANGELOG.md", + "marker": "", + "version_regex": r"^## \[v?(?P[^\]]+)", + "template_url": template_url, + }, + title="Updating changelog", + pty=PTY, + ) + + +@duty(pre=["check_quality", "check_types", "check_docs", "check_dependencies"]) +def check(ctx): + """ + Check it all! + + Arguments: + ctx: The context instance (passed automatically). + """ + + +@duty +def check_quality(ctx, files=PY_SRC): + """ + Check the code quality. + + Arguments: + ctx: The context instance (passed automatically). + files: The files to check. + """ + ctx.run(f"flake8 --config=config/flake8.ini {files}", title="Checking code quality", pty=PTY) + + +@duty +def check_dependencies(ctx): + """ + Check for vulnerabilities in dependencies. + + Arguments: + ctx: The context instance (passed automatically). + """ + # undo possible patching + # see https://github.com/pyupio/safety/issues/348 + for module in sys.modules: # noqa: WPS528 + if module.startswith("safety.") or module == "safety": + del sys.modules[module] # noqa: WPS420 + + importlib.invalidate_caches() + + # reload original, unpatched safety + from safety.formatter import report + from safety.safety import check as safety_check + from safety.util import read_requirements + + # retrieve the list of dependencies + requirements = ctx.run( + ["pdm", "export", "-f", "requirements", "--without-hashes"], + title="Exporting dependencies as requirements", + allow_overrides=False, + ) + + # check using safety as a library + def safety(): # noqa: WPS430 + packages = list(read_requirements(StringIO(requirements))) + vulns = safety_check(packages=packages, ignore_ids="", key="", db_mirror="", cached=False, proxy={}) + output_report = report(vulns=vulns, full=True, checked_packages=len(packages)) + if vulns: + print(output_report) + + ctx.run(safety, title="Checking dependencies") + + +@duty +def check_docs(ctx): + """ + Check if the documentation builds correctly. + + Arguments: + ctx: The context instance (passed automatically). + """ + Path("htmlcov").mkdir(parents=True, exist_ok=True) + Path("htmlcov/index.html").touch(exist_ok=True) + ctx.run("mkdocs build -s", title="Building documentation", pty=PTY) + + +@duty # noqa: WPS231 +def check_types(ctx): # noqa: WPS231 + """ + Check that the code is correctly typed. + + Arguments: + ctx: The context instance (passed automatically). + """ + # NOTE: the following code works around this issue: + # https://github.com/python/mypy/issues/10633 + + # compute packages directory path + py = f"{sys.version_info.major}.{sys.version_info.minor}" + pkgs_dir = Path("__pypackages__", py, "lib").resolve() + + # build the list of available packages + packages = {} + for package in pkgs_dir.glob("*"): + if package.suffix not in {".dist-info", ".pth"} and package.name != "__pycache__": + packages[package.name] = package + + # handle .pth files + for pth in pkgs_dir.glob("*.pth"): + with suppress(OSError): + for package in Path(pth.read_text().splitlines()[0]).glob("*"): # noqa: WPS440 + if package.suffix != ".dist-info": + packages[package.name] = package + + # create a temporary directory to assign to MYPYPATH + with tempfile.TemporaryDirectory() as tmpdir: + + # symlink the stubs + ignore = set() + for stubs in (path for name, path in packages.items() if name.endswith("-stubs")): # noqa: WPS335 + Path(tmpdir, stubs.name).symlink_to(stubs, target_is_directory=True) + # try to symlink the corresponding package + # see https://www.python.org/dev/peps/pep-0561/#stub-only-packages + pkg_name = stubs.name.replace("-stubs", "") + if pkg_name in packages: + ignore.add(pkg_name) + Path(tmpdir, pkg_name).symlink_to(packages[pkg_name], target_is_directory=True) + + # create temporary mypy config to ignore stubbed packages + newconfig = Path("config", "mypy.ini").read_text() + newconfig += "\n" + "\n\n".join(f"[mypy-{pkg}.*]\nignore_errors=true" for pkg in ignore) + tmpconfig = Path(tmpdir, "mypy.ini") + tmpconfig.write_text(newconfig) + + # set MYPYPATH and run mypy + os.environ["MYPYPATH"] = tmpdir + ctx.run(f"mypy --config-file {tmpconfig} {PY_SRC}", title="Type-checking", pty=PTY) + + +@duty(silent=True) +def clean(ctx): + """ + Delete temporary files. + + Arguments: + ctx: The context instance (passed automatically). + """ + ctx.run("rm -rf .coverage*") + ctx.run("rm -rf .mypy_cache") + ctx.run("rm -rf .pytest_cache") + ctx.run("rm -rf tests/.pytest_cache") + ctx.run("rm -rf build") + ctx.run("rm -rf dist") + ctx.run("rm -rf htmlcov") + ctx.run("rm -rf pip-wheel-metadata") + ctx.run("rm -rf site") + ctx.run("find . -type d -name __pycache__ | xargs rm -rf") + ctx.run("find . -name '*.rej' -delete") + + +@duty +def docs(ctx): + """ + Build the documentation locally. + + Arguments: + ctx: The context instance (passed automatically). + """ + ctx.run("mkdocs build", title="Building documentation") + + +@duty +def docs_serve(ctx, host="127.0.0.1", port=8000): + """ + Serve the documentation (localhost:8000). + + Arguments: + ctx: The context instance (passed automatically). + host: The host to serve the docs from. + port: The port to serve the docs on. + """ + ctx.run(f"mkdocs serve -a {host}:{port}", title="Serving documentation", capture=False) + + +@duty +def docs_deploy(ctx): + """ + Deploy the documentation on GitHub pages. + + Arguments: + ctx: The context instance (passed automatically). + """ + ctx.run("mkdocs gh-deploy", title="Deploying documentation") + + +@duty +def format(ctx): + """ + Run formatting tools on the code. + + Arguments: + ctx: The context instance (passed automatically). + """ + ctx.run( + f"autoflake -ir --exclude tests/fixtures --remove-all-unused-imports {PY_SRC}", + title="Removing unused imports", + pty=PTY, + ) + ctx.run(f"isort {PY_SRC}", title="Ordering imports", pty=PTY) + ctx.run(f"black {PY_SRC}", title="Formatting code", pty=PTY) + + +@duty +def release(ctx, version): + """ + Release a new Python package. + + Arguments: + ctx: The context instance (passed automatically). + version: The new version number to use. + """ + ctx.run("git add pyproject.toml CHANGELOG.md", title="Staging files", pty=PTY) + ctx.run(["git", "commit", "-m", f"chore: Prepare release {version}"], title="Committing changes", pty=PTY) + ctx.run(f"git tag {version}", title="Tagging commit", pty=PTY) + if not TESTING: + ctx.run("git push", title="Pushing commits", pty=False) + ctx.run("git push --tags", title="Pushing tags", pty=False) + ctx.run("pdm build", title="Building dist/wheel", pty=PTY) + ctx.run("twine upload --skip-existing dist/*", title="Publishing version", pty=PTY) + docs_deploy.run() + + +@duty(silent=True) +def coverage(ctx): + """ + Report coverage as text and HTML. + + Arguments: + ctx: The context instance (passed automatically). + """ + ctx.run("coverage combine", nofail=True) + ctx.run("coverage report --rcfile=config/coverage.ini", capture=False) + ctx.run("coverage html --rcfile=config/coverage.ini") + + +@duty +def test(ctx, match: str = ""): + """ + Run the test suite. + + Arguments: + ctx: The context instance (passed automatically). + match: A pytest expression to filter selected tests. + """ + py_version = f"{sys.version_info.major}{sys.version_info.minor}" + os.environ["COVERAGE_FILE"] = f".coverage.{py_version}" + ctx.run( + ["pytest", "-c", "config/pytest.ini", "-n", "auto", "-k", match, "tests"], + title="Running tests", + pty=PTY, + ) diff --git a/markdown_exec.svg b/markdown_exec.svg new file mode 100644 index 0000000..3165166 --- /dev/null +++ b/markdown_exec.svg @@ -0,0 +1,63 @@ + + + + + + +G + + + +markdown + +markdown + + + +markdown_exec + +markdown_exec + + + +markdown->markdown_exec + + + + + + + +pymdownx_superfences + +pymdownx. +superfences + + + +markdown->pymdownx_superfences + + + + + +pymdownx + +pymdownx + + + +pymdownx->markdown_exec + + + + + +pymdownx_superfences->markdown_exec + + + + diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..dc0647f --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,87 @@ +site_name: "Markdown Exec" +site_description: "Utilities to execute code blocks in Markdown files." +site_url: "https://pawamoy.github.io/markdown-exec" +repo_url: "https://github.com/pawamoy/markdown-exec" +repo_name: "pawamoy/markdown-exec" +site_dir: "site" + +nav: +- Home: + - Overview: index.md + - Examples: examples.md + - Changelog: changelog.md + - Credits: credits.md + - License: license.md +# defer to gen-files + literate-nav +# - Code Reference: reference/ +- Development: + - Contributing: contributing.md + - Code of Conduct: code_of_conduct.md + - Coverage report: coverage.md +- Author's website: https://pawamoy.github.io/ + +theme: + name: material + icon: + logo: material/currency-sign + features: + - navigation.tabs + - navigation.top + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: teal + accent: purple + toggle: + icon: material/weather-sunny + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: lime + toggle: + icon: material/weather-night + name: Switch to light mode + +extra_css: +- css/material.css +- css/mkdocstrings.css + +markdown_extensions: +- callouts: + strip_period: false +- pymdownx.emoji +- pymdownx.magiclink +- pymdownx.snippets: + check_paths: true +- pymdownx.superfences: + custom_fences: + - name: python + class: python + validator: !!python/name:markdown_exec.validator + format: !!python/name:markdown_exec.formatter +- pymdownx.tabbed: + alternate_style: yes +- pymdownx.tasklist +- toc: + permalink: "¤" + +plugins: +- search +# - gen-files: +# scripts: + # - docs/gen_credits.py + # - docs/gen_ref_nav.py +- literate-nav: + nav_file: SUMMARY.md +- coverage +- mkdocstrings: + watch: + - src/markdown_exec + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/pawamoy + - icon: fontawesome/brands/twitter + link: https://twitter.com/pawamoy diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..758c427 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,114 @@ +[build-system] +requires = ["pdm-pep517"] +build-backend = "pdm.pep517.api" + +[project] +name = "markdown-exec" +description = "Utilities to execute code blocks in Markdown files." +authors = [{name = "Timothée Mazzucotelli", email = "pawamoy@pm.me"}] +license = {file = "LICENSE"} +readme = "README.md" +requires-python = ">=3.7" +keywords = [] +dynamic = ["version"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: ISC License (ISCL)", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Documentation", + "Topic :: Software Development", + "Topic :: Software Development :: Documentation", + "Topic :: Utilities", + "Typing :: Typed", +] +dependencies = [ + "pymdown-extensions>=9", +] + +[project.urls] +Homepage = "https://pawamoy.github.io/markdown-exec" +Documentation = "https://pawamoy.github.io/markdown-exec" +Changelog = "https://pawamoy.github.io/markdown-exec/changelog" +Repository = "https://github.com/pawamoy/markdown-exec" +Issues = "https://github.com/pawamoy/markdown-exec/issues" +Discussions = "https://github.com/pawamoy/markdown-exec/discussions" +Gitter = "https://gitter.im/markdown-exec/community" +Funding = "https://github.com/sponsors/pawamoy" + +[tool.pdm] +version = {use_scm = true} +package-dir = "src" + +[tool.pdm.dev-dependencies] +duty = ["duty>=0.7"] +docs = [ + "mkdocs>=1.2", + "mkdocs-coverage>=0.2", + "mkdocs-gen-files>=0.3", + "mkdocs-literate-nav>=0.4", + "mkdocs-material>=7.3", + "mkdocs-section-index>=0.3", + "mkdocstrings>=0.16", + "toml>=0.10", + "pydeps>=1.10.12", + "markdown-callouts>=0.2.0", +] +format = [ + "autoflake>=1.4", + "black>=21.10b0", + "isort>=5.10", +] +maintain = [ + # TODO: remove this section when git-changelog is more powerful + "git-changelog>=0.4", +] +quality = [ + "darglint>=1.8", + "flake8-bandit>=2.1", + "flake8-black>=0.2", + "flake8-bugbear>=21.9", + "flake8-builtins>=1.5", + "flake8-comprehensions>=3.7", + "flake8-docstrings>=1.6", + "flake8-pytest-style>=1.5", + "flake8-string-format>=0.3", + "flake8-tidy-imports>=4.5", + "flake8-variables-names>=0.0", + "pep8-naming>=0.12", + "wps-light>=0.15", +] +tests = [ + "pytest>=6.2", + "pytest-cov>=3.0", + "pytest-randomly>=3.10", + "pytest-sugar>=0.9", + "pytest-xdist>=2.4", +] +typing = [ + "mypy>=0.910", + "types-markdown>=3.3", + "types-toml>=0.10", +] +security = ["safety>=1.10"] + +[tool.black] +line-length = 120 +exclude = "tests/fixtures" + +[tool.isort] +line_length = 120 +not_skip = "__init__.py" +multi_line_output = 3 +force_single_line = false +balanced_wrapping = true +default_section = "THIRDPARTY" +known_first_party = "markdown_exec" +include_trailing_comma = true diff --git a/scripts/multirun.sh b/scripts/multirun.sh new file mode 100755 index 0000000..609dfd4 --- /dev/null +++ b/scripts/multirun.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e + +PYTHON_VERSIONS="${PYTHON_VERSIONS-3.7 3.8 3.9 3.10 3.11}" + +if [ -n "${PYTHON_VERSIONS}" ]; then + for python_version in ${PYTHON_VERSIONS}; do + if pdm use -f "python${python_version}" &>/dev/null; then + echo "> pdm run $@ (Python ${python_version})" + pdm run "$@" + else + echo "> pdm use -f python${python_version}: Python interpreter not available?" >&2 + fi + done +else + pdm run "$@" +fi diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..f0a41cf --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -e + +PYTHON_VERSIONS="${PYTHON_VERSIONS-3.7 3.8 3.9 3.10 3.11}" + +install_with_pipx() { + if ! command -v "$1" &>/dev/null; then + if ! command -v pipx &>/dev/null; then + python3 -m pip install --user pipx + fi + pipx install "$1" + fi +} + +install_with_pipx pdm + +if [ -n "${PYTHON_VERSIONS}" ]; then + for python_version in ${PYTHON_VERSIONS}; do + if pdm use -f "python${python_version}" &>/dev/null; then + echo "> Using Python ${python_version} interpreter" + pdm install + else + echo "> pdm use -f python${python_version}: Python interpreter not available?" >&2 + fi + done +else + pdm install +fi diff --git a/src/markdown_exec/__init__.py b/src/markdown_exec/__init__.py new file mode 100644 index 0000000..817a956 --- /dev/null +++ b/src/markdown_exec/__init__.py @@ -0,0 +1,83 @@ +""" +Markdown Exec package. + +Utilities to execute code blocks in Markdown files. +""" + +# https://facelessuser.github.io/pymdown-extensions/extensions/superfences/#custom-fences +# https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown-extensions/#snippets + +from __future__ import annotations + +from typing import Any + +from markdown import Markdown + +from markdown_exec.python import exec_python + +__all__: list[str] = ["formatter", "validator"] # noqa: WPS410 + + +_formatters = { + "python": exec_python, +} + + +def validator( + language: str, inputs: dict[str, str], options: dict[str, Any], attrs: dict[str, Any], md: Markdown +) -> bool: + """Validate code blocks inputs. + + Parameters: + language: The code language, like python or bash. + inputs: The code block inputs, to be sorted into options and attrs. + options: The container for options. + attrs: The container for attrs: + md: The Markdown instance. + + Returns: + Success or not. + """ + exec_value = _to_bool(inputs.pop("exec", "no")) + if not exec_value: + return False + isolate_value = _to_bool(inputs.pop("isolate", "no")) + options["exec"] = exec_value + options["isolate"] = isolate_value + return True + + +def formatter( + source: str, + language: str, + css_class: str, + options: dict[str, Any], + md: Markdown, + classes: list[str] | None = None, + id_value: str = "", + attrs: dict[str, Any] | None = None, + **kwargs, +) -> str: + """Execute code and return HTML. + + Parameters: + source: The code to execute. + language: The code language, like python or bash. + css_class: The CSS class to add to the HTML element. + options: The container for options. + attrs: The container for attrs: + md: The Markdown instance. + classes: Additional CSS classes. + id_value: An optional HTML id. + attrs: Additional attributes + **kwargs: Additional arguments passed to SuperFences default formatters. + + Returns: + HTML contents. + """ + fmt = _formatters.get(language, lambda source, *args, **kwargs: source) + return fmt(source, md) + + +def _to_bool(value): + return value.lower() not in {"", "no", "off", "false", "0"} diff --git a/src/markdown_exec/py.typed b/src/markdown_exec/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/markdown_exec/python.py b/src/markdown_exec/python.py new file mode 100644 index 0000000..b2e5dfd --- /dev/null +++ b/src/markdown_exec/python.py @@ -0,0 +1,60 @@ +"""Formatter and utils for executing Python code.""" + +from copy import deepcopy +import traceback + +from markdown.core import Markdown +from markupsafe import Markup + + +class MarkdownOutput(Exception): # noqa: N818 + """Exception to return Markdown.""" + + +class HTMLOutput(Exception): # noqa: N818 + """Exception to return HTML.""" + + +def output_markdown(text: str) -> None: + """Output Markdown. + + Parameters: + text: The Markdown to convert and inject back in the page. + + Raises: + MarkdownOutput: Our way of returning without 'return' or 'yield' keywords. + """ + raise MarkdownOutput(text) + + +def output_html(text: str) -> None: + """Output HTML. + + Parameters: + text: The HTML to inject back in the page. + + Raises: + HTMLOutput: Our way of returning without 'return' or 'yield' keywords. + """ + raise HTMLOutput(text) + + +def exec_python(source: str, md: Markdown) -> str: + """Execute code and return HTML. + + Parameters: + source: The code to execute. + md: The Markdown instance. + + Returns: + HTML contents. + """ + try: + exec(source) # noqa: S102 + except MarkdownOutput as output: + return Markup(deepcopy(md).convert(str(output))) + except HTMLOutput as output: + return str(output) + except Exception: + return Markup(deepcopy(md).convert(f"```python\n{traceback.format_exc()}\n```")) + return "" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..96a851c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,7 @@ +"""Tests suite for `markdown_exec`.""" + +from pathlib import Path + +TESTS_DIR = Path(__file__).parent +TMP_DIR = TESTS_DIR / "tmp" +FIXTURES_DIR = TESTS_DIR / "fixtures" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2bc403b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,30 @@ +"""Configuration for the pytest test suite.""" + +import pytest +from markdown import Markdown + +from markdown_exec import formatter, validator + + +@pytest.fixture() +def md() -> Markdown: + """Return a Markdown instance. + + Returns: + Markdown instance. + """ + return Markdown( + extensions=["pymdownx.superfences"], + extension_configs={ + "pymdownx.superfences": { + "custom_fences": [ + { + "name": "python", + "class": "python", + "validator": validator, + "format": formatter, + } + ] + } + }, + ) diff --git a/tests/test_python.py b/tests/test_python.py new file mode 100644 index 0000000..b300ac2 --- /dev/null +++ b/tests/test_python.py @@ -0,0 +1,35 @@ +"""Tests for the `python` module.""" + +from markdown.core import Markdown + + +def test_output_markdown(md: Markdown) -> None: + """Assert Markdown is converted to HTML. + + Parameters: + md: A Markdown instance (fixture). + """ + html = md.convert( + """ + ```python exec="yes" + output_markdown("**Bold!**") + ``` + """ + ) + assert "Bold!" in html + + +def test_output_html(md: Markdown) -> None: + """Assert HTML is injected as is. + + Parameters: + md: A Markdown instance (fixture). + """ + html = md.convert( + """ + ```python exec="yes" + output_html("**Bold!**") + ``` + """ + ) + assert "**Bold!**" in html diff --git a/tests/test_validator.py b/tests/test_validator.py new file mode 100644 index 0000000..245b4c4 --- /dev/null +++ b/tests/test_validator.py @@ -0,0 +1,38 @@ +"""Tests for the `validator` function.""" + +import pytest +from markdown.core import Markdown + +from markdown_exec import validator + + +@pytest.mark.parametrize( + ("exec_value", "expected"), + [ + ("yes", True), + ("YES", True), + ("on", True), + ("ON", True), + ("whynot", True), + ("true", True), + ("TRUE", True), + ("1", True), + ("-1", True), + ("0", False), + ("no", False), + ("NO", False), + ("off", False), + ("OFF", False), + ("false", False), + ("FALSE", False), + ], +) +def test_validate(md: Markdown, exec_value: str, expected: bool) -> None: + """Assert the validator returns True or False given inputs. + + Parameters: + md: A Markdown instance. + exec_value: The exec option value, passed from the code block. + expected: Expected validation result. + """ + assert validator("whatever", inputs={"exec": exec_value}, options={}, attrs={}, md=md) is expected