Skip to content
This repository has been archived by the owner on Dec 16, 2022. It is now read-only.

Commit

Permalink
build and publish models docs (#91)
Browse files Browse the repository at this point in the history 
* build and publish models docs

* fix

* oops, actually fix

* fix PYTHONPATH

* temporarily allow docs to deploy

* revert temp deploy

* fixes

* more formatting fixes

* update CHANGELOG

* fix README
  • Loading branch information
@epwalsh
epwalsh authored Jul 14, 2020
1 parent 4b2178b commit da83a4e
Show file tree
Hide file tree
Showing 62 changed files with 835 additions and 368 deletions.
122 changes: 122 additions & 0 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,3 +127,125 @@ jobs:
- name: Run GPU tests
run: |
make docker-test-run DOCKER_TAG=$DOCKER_TAG ARGS='gpu-test'
# Builds the API documentation and pushes it to the appropriate folder in the
# allennlp-docs repo.
docs:
name: Docs
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v1

- name: Setup SSH Client 🔑
# Only run this on main repo (not forks).
if: github.repository == 'allenai/allennlp-models'
uses: webfactory/[email protected]
with:
ssh-private-key: ${{ secrets.DOCS_DEPLOY_KEY }}

- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.7

- uses: actions/cache@v1
with:
path: ${{ env.pythonLocation }}
key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}

- name: Install requirements
run: |
pip install --upgrade pip setuptools wheel
pip install --upgrade --upgrade-strategy eager -r requirements.txt
pip install --upgrade --upgrade-strategy eager -r dev-requirements.txt
- name: Debug info
run: |
pip freeze
- name: Build docs
run: |
make build-docs
- name: Configure Git
# Only run this on master commits to main repo.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'push'
run: |
git config --global user.email "[email protected]"
git config --global user.name "ai2service"
git config --global push.default simple
- name: Set target folders
# Only run this on master commits to main repo.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'push'
run: |
if [[ $GITHUB_EVENT_NAME == 'release' ]]; then
echo "::set-env name=DOCS_FOLDER::models/${GITHUB_REF#refs/tags/}";
else
echo "::set-env name=DOCS_FOLDER::models/master";
fi
- name: Stage docs
# Only run this on master commits to main repo.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'push'
run: |
echo "Staging docs to $DOCS_FOLDER"
# Checkout allennlp-docs to /allennlp-docs
git clone [email protected]:allenai/allennlp-docs.git ~/allennlp-docs
# Copy the generated docs to the checked out docs repo
rm -rf ~/allennlp-docs/$DOCS_FOLDER/
mkdir -p ~/allennlp-docs/$DOCS_FOLDER
cp -r site/* ~/allennlp-docs/$DOCS_FOLDER
- name: Update shortcuts
# Only run this on master commits to main repo.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'push'
run: |
# Fail immediately if any step fails.
set -e
LATEST=models/$(./scripts/get_version.py latest)
STABLE=models/$(./scripts/get_version.py stable)
cd ~/allennlp-docs/
echo "Updating models/latest/index.html to point to $LATEST"
mkdir -p models/latest
cat >models/latest/index.html << EOL
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Refresh" content="0; url=/${LATEST}/" />
</head>
<body>
<p>Please follow <a href="/${LATEST}/">this link</a>.</p>
</body>
</html>
EOL
echo "Updating models/stable/index.html to point to $STABLE"
mkdir -p models/stable
cat >models/stable/index.html << EOL
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Refresh" content="0; url=/${STABLE}/" />
</head>
<body>
<p>Please follow <a href="/${STABLE}/">this link</a>.</p>
</body>
</html>
EOL
- name: Deploy docs
# Only run this on master commits to main repo.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'push'
run: |
# And push them up to GitHub
cd ~/allennlp-docs/
git add -A
git commit -m "automated update of the models docs"
git push
124 changes: 123 additions & 1 deletion .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,11 +239,133 @@ jobs:
run: |
make docker-test-run DOCKER_TAG=$DOCKER_TAG ARGS='gpu-test'
# Builds the API documentation and pushes it to the appropriate folder in the
# allennlp-docs repo.
docs:
name: Docs
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v1

- name: Setup SSH Client 🔑
# Only run this on main repo (not forks).
if: github.repository == 'allenai/allennlp-models'
uses: webfactory/[email protected]
with:
ssh-private-key: ${{ secrets.DOCS_DEPLOY_KEY }}

- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.7

- uses: actions/cache@v1
with:
path: ${{ env.pythonLocation }}
key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}

- name: Install requirements
run: |
pip install --upgrade pip setuptools wheel
pip install --upgrade --upgrade-strategy eager -r requirements.txt
pip install --upgrade --upgrade-strategy eager -r dev-requirements.txt
- name: Debug info
run: |
pip freeze
- name: Build docs
run: |
make build-docs
- name: Configure Git
# Only run this on releases.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'release'
run: |
git config --global user.email "[email protected]"
git config --global user.name "ai2service"
git config --global push.default simple
- name: Set target folders
# Only run this on releases.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'release'
run: |
if [[ $GITHUB_EVENT_NAME == 'release' ]]; then
echo "::set-env name=DOCS_FOLDER::models/${GITHUB_REF#refs/tags/}";
else
echo "::set-env name=DOCS_FOLDER::models/master";
fi
- name: Stage docs
# Only run this on releases.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'release'
run: |
echo "Staging docs to $DOCS_FOLDER"
# Checkout allennlp-docs to /allennlp-docs
git clone [email protected]:allenai/allennlp-docs.git ~/allennlp-docs
# Copy the generated docs to the checked out docs repo
rm -rf ~/allennlp-docs/$DOCS_FOLDER/
mkdir -p ~/allennlp-docs/$DOCS_FOLDER
cp -r site/* ~/allennlp-docs/$DOCS_FOLDER
- name: Update shortcuts
# Only run this on releases.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'release'
run: |
# Fail immediately if any step fails.
set -e
LATEST=models/$(./scripts/get_version.py latest)
STABLE=models/$(./scripts/get_version.py stable)
cd ~/allennlp-docs/
echo "Updating models/latest/index.html to point to $LATEST"
mkdir -p models/latest
cat >models/latest/index.html << EOL
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Refresh" content="0; url=/${LATEST}/" />
</head>
<body>
<p>Please follow <a href="/${LATEST}/">this link</a>.</p>
</body>
</html>
EOL
echo "Updating models/stable/index.html to point to $STABLE"
mkdir -p models/stable
cat >models/stable/index.html << EOL
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Refresh" content="0; url=/${STABLE}/" />
</head>
<body>
<p>Please follow <a href="/${STABLE}/">this link</a>.</p>
</body>
</html>
EOL
- name: Deploy docs
# Only run this on releases.
if: github.repository == 'allenai/allennlp-models' && github.event_name == 'release'
run: |
# And push them up to GitHub
cd ~/allennlp-docs/
git add -A
git commit -m "automated update of the models docs"
git push
publish:
name: PyPI
# Don't run for forks.
if: github.repository == 'allenai/allennlp-models'
needs: [build, test, docker, pretrained, gpu_checks]
needs: [build, test, docker, docs, pretrained, gpu_checks]
runs-on: ubuntu-latest

steps:
Expand Down
5 changes: 5 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,8 @@ __pycache__
.coverage
.pytest_cache/

# doc stuff
mkdocs.yml
docs/models
docs/*.md
site/
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- A bug with `NextTokenLM` that caused simple gradient interpreters to fail.
- A bug in `training_config` of `qanet` and `bimpm` that used the old version of `regularizer` and `initializer`.
- The fine-grained NER transformer model did not survive an upgrade of the transformers library, but it is now fixed.
- Fixed many minor formatting issues in docstrings. Docs are now published at [https://docs.allennlp.org/models/](https://docs.allennlp.org/models/).

### Changed

Expand Down
59 changes: 59 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,43 @@
VERSION = $(shell python ./scripts/get_version.py current --minimal)

SRC = allennlp_models

MD_DOCS_ROOT = docs/
MD_DOCS_API_ROOT = $(MD_DOCS_ROOT)models/
MD_DOCS_SRC = $(filter-out %/__init__.py $(SRC)/version.py,$(shell find $(SRC) -type f -name '*.py' | grep -v -E 'tests/'))
MD_DOCS = $(subst .py,.md,$(subst $(SRC)/,$(MD_DOCS_API_ROOT),$(MD_DOCS_SRC)))
MD_DOCS_CMD = python scripts/py2md.py
MD_DOCS_CONF = mkdocs.yml
MD_DOCS_CONF_SRC = mkdocs-skeleton.yml
MD_DOCS_TGT = site/
MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md)

DOCKER_TAG = latest
DOCKER_RUN_CMD = docker run --rm \
-v $$HOME/.allennlp:/root/.allennlp \
-v $$HOME/.cache/torch:/root/.cache/torch \
-v $$HOME/nltk_data:/root/nltk_data
ALLENNLP_COMMIT_SHA = $(shell git ls-remote https:/allenai/allennlp master | cut -f 1)

ifeq ($(shell uname),Darwin)
ifeq ($(shell which gsed),)
$(error Please install GNU sed with 'brew install gnu-sed')
else
SED = gsed
endif
else
SED = sed
endif

.PHONY : version
version :
@echo AllenNLP Models $(VERSION)

.PHONY : clean
clean :
rm -rf $(MD_DOCS_TGT)
rm -rf $(MD_DOCS_API_ROOT)
rm -f $(MD_DOCS_ROOT)*.md
rm -rf .pytest_cache/
rm -rf allennlp_models.egg-info/
rm -rf dist/
Expand Down Expand Up @@ -49,6 +75,39 @@ test-with-cov :
test-pretrained :
pytest -v --color=yes -m "pretrained_model_test"

.PHONY : build-all-api-docs
build-all-api-docs : scripts/py2md.py
@PYTHONPATH=./ $(MD_DOCS_CMD) $(subst /,.,$(subst .py,,$(MD_DOCS_SRC))) -o $(MD_DOCS)

.PHONY : build-docs
build-docs : build-all-api-docs $(MD_DOCS_CONF) $(MD_DOCS) $(MD_DOCS_EXTRAS)
mkdocs build

.PHONY : serve-docs
serve-docs : build-all-api-docs $(MD_DOCS_CONF) $(MD_DOCS) $(MD_DOCS_EXTRAS)
mkdocs serve --dirtyreload

.PHONY : update-docs
update-docs : $(MD_DOCS) $(MD_DOCS_EXTRAS)

$(MD_DOCS_ROOT)README.md : README.md
cp $< $@
# Alter the relative path of the README image for the docs.
$(SED) -i '1s/docs/./' $@

$(MD_DOCS_ROOT)%.md : %.md
cp $< $@

scripts/py2md.py :
wget https://raw.githubusercontent.com/allenai/allennlp/master/scripts/py2md.py -O $@

$(MD_DOCS_CONF) : $(MD_DOCS_CONF_SRC) $(MD_DOCS)
@PYTHONPATH=./ python scripts/build_docs_config.py $@ $(MD_DOCS_CONF_SRC) $(MD_DOCS_ROOT) $(MD_DOCS_API_ROOT)

$(MD_DOCS_API_ROOT)%.md : $(SRC)/%.py scripts/py2md.py
mkdir -p $(shell dirname $@)
$(MD_DOCS_CMD) $(subst /,.,$(subst .py,,$<)) --out $@

.PHONY :
docker-image :
docker build \
Expand Down
Loading

0 comments on commit da83a4e

Please sign in to comment.