From f58b612050f29e0e0758bd8c0eb971067b322f15 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Tue, 17 Dec 2019 18:59:06 -0800 Subject: [PATCH] Add gradle release task to remove -SNAPSHOT in examples and jupyter notebooks. Change-Id: I459c1ba848981c8af929db09219ed2a0596ce914 --- build.gradle | 1 + docs/development/README.md | 2 + docs/development/release_process.md | 118 ++++++++++++++++++ jupyter/BERTQA.ipynb | 11 +- jupyter/create_your_first_network.ipynb | 2 +- ...image_classification_with_your_model.ipynb | 11 +- jupyter/load_mxnet_model.ipynb | 11 +- jupyter/object_detection_with_model_zoo.ipynb | 11 +- jupyter/train_your_first_model.ipynb | 11 +- jupyter/transfer_learning_on_cifar10.ipynb | 11 +- model-zoo/README.md | 2 +- tools/gradle/release.gradle | 67 ++++++++++ 12 files changed, 202 insertions(+), 56 deletions(-) create mode 100644 docs/development/release_process.md create mode 100644 tools/gradle/release.gradle diff --git a/build.gradle b/build.gradle index c0d3e446067..a3ba3866fc5 100644 --- a/build.gradle +++ b/build.gradle @@ -72,6 +72,7 @@ configure(javaProjects()) { apply from: file("${rootProject.projectDir}/tools/gradle/publish.gradle") apply from: file("${rootProject.projectDir}/tools/gradle/jacoco.gradle") +apply from: file("${rootProject.projectDir}/tools/gradle/release.gradle") import java.util.regex.Matcher import java.util.regex.Pattern diff --git a/docs/development/README.md b/docs/development/README.md index 029949f79ac..2f4f102a4bf 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -6,5 +6,7 @@ This folder contains documentation and guides for contributors and those working - [Setup development environment](setup.md) - [Development guideline](development_guideline.md) +- [Release process](release_process.md) - [How to add a new model to the main model zoo](add_model_to_model-zoo.md) - [How to add a new model to the MXNet model zoo](add_model_to_mxnet-model-zoo.md) +- [Troubleshooting](troubleshooting.md) diff --git a/docs/development/release_process.md b/docs/development/release_process.md new file mode 100644 index 00000000000..c86dae15031 --- /dev/null +++ b/docs/development/release_process.md @@ -0,0 +1,118 @@ +# Release Process + +## Overview + +This document outlines the procedure to release Deep Java Library (DJL) project to maven central. + +## Step 1: Preparing the Release Candidate + +### Step 1.1 Publish javadoc to S3 bucket + +Make sure you are using correct aws credential and run the following command: + +```shell script +cd djl +./gradlew -Prelease uploadJavadoc +``` + +### Step 1.2: Bump up versions in documents to point to new url + +Edit [README Release Notes section](../../README.md#release-notes) to add link to new release. + +Update build version with the following command: +```shell script +cd djl +./gradlew -PpreviousVersion=X.X.X iFV +``` +Make a commit, get reviewed, and then merge it into github. + +### Step 1.4: Upload javadoc-index.html to S3 bucket + +```shell script +aws s3 cp website/javadoc-index.html s3://javadoc-djl-ai/index.html +``` + +### Step 1.5: Publish mxnet-native library to sonatype staging server + +Run the following command to trigger mxnet-native publishing job: +```shell script +curl -XPOST -u "USERNAME:PERSONAL_TOKEN" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/awslabs/djl/dispatches --data '{"event_type": "mxnet-staging-pub"}' +``` + +### Step 1.6: Remove -SNAPSHOT in examples and jupyter notebooks + +Run the following command with correct version value: +```shell script +cd djl +git clean -xdff +./gradlew release +git commit -a -m 'Remove -SNAPSHOT for release vX.X.X' +git tag -a vX.X.X -m "Releasing version vX.X.X" +git push origin vX.X.X +``` + +### Step 1.7 Create a Release Notes + +Navigate to DJL github site, select "Release" tab and click "Draft a new Release" button. +Select tag that created by previous step. Check "This is a pre-release" checkbox. + +Release notes content should include the following: +- list of new features +- list of bug fixes +- limitations and known issues +- API changes and migration document + +Once "Publish release" button is clicked, a github Action will be triggered, and release packages +will be published sonatype staging server. + +## Step 2: Validate release on staging server + +Login to https://oss.sonatype.org/, and find out staging repo name. + +Run the following command to point maven repository to staging server: +```shell script +cd djl +git checkout vX.X.X +./gradlew -PstagingRepo=aidjl-XXXX staging +``` + +### Validate examples project are working fine + +```shell script +cd examples +./gradlew run +mvn exec:java -Dexec.mainClass="ai.djl.examples.inference.ObjectDetection" +``` + +### Validate jupyter notebooks + +Make sure jupyter notebook and running properly and all javadoc links are accessible. +```shell script +cd jupyter +jupyter notebook +``` + +## Step 3: Validate javadoc url in documents + +Navigate to DJL github site, select tag created by earlier step, open markdown files and +check javadoc links are accessible. + +## Step 4: Publish staging package to maven central + +Login to https://oss.sonatype.org/, close staging packages and publish to maven central. + +## Step 5: Upgrade version for next release + +```shell script +cd djl +./gradlew -PtargetVersion=X.X.X iBV +``` + +Create a PR to get reviewed and merge into github. + +## Step 5: Publish new snapshot to sonatype + +Manually trigger a nightly build with the following command: +```shell script +curl -XPOST -u "USERNAME:PERSONAL_TOKEN" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/awslabs/djl/dispatches --data '{"event_type": "nightly-build"}' +``` diff --git a/jupyter/BERTQA.ipynb b/jupyter/BERTQA.ipynb index 364ec402d86..7fd99602596 100644 --- a/jupyter/BERTQA.ipynb +++ b/jupyter/BERTQA.ipynb @@ -42,15 +42,8 @@ "metadata": {}, "outputs": [], "source": [ - "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/\n", + "\n", "%maven ai.djl:api:0.2.1-SNAPSHOT\n", "%maven ai.djl.mxnet:mxnet-engine:0.2.1-SNAPSHOT\n", "%maven ai.djl:repository:0.2.1-SNAPSHOT\n", diff --git a/jupyter/create_your_first_network.ipynb b/jupyter/create_your_first_network.ipynb index c2a77f54a9b..a68f887ada4 100644 --- a/jupyter/create_your_first_network.ipynb +++ b/jupyter/create_your_first_network.ipynb @@ -28,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "%maven ai.djl:api:0.2.1\n", + "%maven ai.djl:api:0.2.1-SNAPSHOT\n", "%maven org.slf4j:slf4j-api:1.7.26\n", "%maven org.slf4j:slf4j-simple:1.7.26" ] diff --git a/jupyter/image_classification_with_your_model.ipynb b/jupyter/image_classification_with_your_model.ipynb index 8d78f4a3a76..3e0477e87ea 100644 --- a/jupyter/image_classification_with_your_model.ipynb +++ b/jupyter/image_classification_with_your_model.ipynb @@ -21,15 +21,8 @@ "metadata": {}, "outputs": [], "source": [ - "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/\n", + "\n", "%maven ai.djl:api:0.2.1-SNAPSHOT\n", "%maven ai.djl:repository:0.2.1-SNAPSHOT\n", "%maven ai.djl:model-zoo:0.2.1-SNAPSHOT\n", diff --git a/jupyter/load_mxnet_model.ipynb b/jupyter/load_mxnet_model.ipynb index a2131fbb1a6..1c99b0c6f33 100644 --- a/jupyter/load_mxnet_model.ipynb +++ b/jupyter/load_mxnet_model.ipynb @@ -20,15 +20,8 @@ "metadata": {}, "outputs": [], "source": [ - "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/\n", + "\n", "%maven ai.djl:api:0.2.1-SNAPSHOT\n", "%maven ai.djl:repository:0.2.1-SNAPSHOT\n", "%maven ai.djl:model-zoo:0.2.1-SNAPSHOT\n", diff --git a/jupyter/object_detection_with_model_zoo.ipynb b/jupyter/object_detection_with_model_zoo.ipynb index 4300b4cdb6c..189dfa0ecb9 100644 --- a/jupyter/object_detection_with_model_zoo.ipynb +++ b/jupyter/object_detection_with_model_zoo.ipynb @@ -19,15 +19,8 @@ "metadata": {}, "outputs": [], "source": [ - "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/\n", + "\n", "%maven ai.djl:api:0.2.1-SNAPSHOT\n", "%maven ai.djl:repository:0.2.1-SNAPSHOT\n", "%maven ai.djl.mxnet:mxnet-engine:0.2.1-SNAPSHOT\n", diff --git a/jupyter/train_your_first_model.ipynb b/jupyter/train_your_first_model.ipynb index ce12b5f8cde..55e3a594edf 100644 --- a/jupyter/train_your_first_model.ipynb +++ b/jupyter/train_your_first_model.ipynb @@ -20,15 +20,8 @@ "metadata": {}, "outputs": [], "source": [ - "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/\n", + "\n", "%maven ai.djl:api:0.2.1-SNAPSHOT\n", "%maven ai.djl:basicdataset:0.2.1-SNAPSHOT\n", "%maven ai.djl:model-zoo:0.2.1-SNAPSHOT\n", diff --git a/jupyter/transfer_learning_on_cifar10.ipynb b/jupyter/transfer_learning_on_cifar10.ipynb index 196838977d7..5096123f9af 100644 --- a/jupyter/transfer_learning_on_cifar10.ipynb +++ b/jupyter/transfer_learning_on_cifar10.ipynb @@ -33,15 +33,8 @@ "metadata": {}, "outputs": [], "source": [ - "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "%mavenRepo snapshots https://oss.sonatype.org/content/repositories/snapshots/\n", + "\n", "%maven ai.djl:api:0.2.1-SNAPSHOT\n", "%maven ai.djl:basicdataset:0.2.1-SNAPSHOT\n", "%maven ai.djl:model-zoo:0.2.1-SNAPSHOT\n", diff --git a/model-zoo/README.md b/model-zoo/README.md index 5e17c23046f..4499df76f1f 100644 --- a/model-zoo/README.md +++ b/model-zoo/README.md @@ -30,7 +30,7 @@ You can pull the model zoo from the central Maven repository by including the fo ## Pre-trained models -In the 0.2.1 release, you can find the Multilayer Perceptrons (MLP) and Resnet50 pre-trained models in the model zoo. +You can find the Multilayer Perceptrons (MLP) and Resnet50 pre-trained models in the model zoo. ### How to find a pre-trained model in the model zoo diff --git a/tools/gradle/release.gradle b/tools/gradle/release.gradle new file mode 100644 index 00000000000..b55539a9c79 --- /dev/null +++ b/tools/gradle/release.gradle @@ -0,0 +1,67 @@ +task release { + doLast { + def collection = fileTree("jupyter").filter { it.name.endsWith(".ipynb") } + collection += files("examples/build.gradle", "examples/pom.xml") + collection.each { File file -> + file.text = file.text.replaceAll("-SNAPSHOT", "").replaceAll("%mavenRepo", "// %mavenRepo") + } + } +} + +task staging() { + doLast { + if (!project.hasProperty("stagingRepo")) { + throw new GradleException("stagingRepo property is required.") + } + def stagingRepo = project.property("stagingRepo") + + def collection = fileTree("jupyter").filter { it.name.endsWith(".ipynb") } + collection += files("examples/build.gradle", "examples/pom.xml") + collection.each { File file -> + file.text = file.text.replaceAll("https://oss\\.sonatype\\.org/content/repositories/snapshots/", + "https://oss.sonatype.org/service/local/repositories/${stagingRepo}/content/") + .replaceAll("-SNAPSHOT", "") + } + } +} + +task increaseBuildVersion { + doLast { + if (!project.hasProperty("targetVersion")) { + throw new GradleException("targetVersion property is required.") + } + def targetVersion = project.property("targetVersion") + + def collection = fileTree("jupyter").filter { it.name.endsWith(".ipynb") } + collection += files("examples/build.gradle", "examples/pom.xml") + collection.each { File file -> + file.text = file.text.replaceAll("${version}", "${targetVersion}-SNAPSHOT") + } + + File file = new File("build.gradle") + def releaseVersion = version.replaceAll("-SNAPSHOT", "") + file.text = file.text.replaceAll("\"${releaseVersion}\"", "\"${targetVersion}\"") + } +} + +task increaseFinalVersion { + doLast { + if (!project.hasProperty("previousVersion")) { + throw new GradleException("previousVersion property is required.") + } + def previousVersion = project.property("previousVersion") + def releaseVersion = version.replaceAll("-SNAPSHOT", "") + + def collection = fileTree("jupyter").filter { it.name.endsWith(".ipynb") } + collection += fileTree(".").filter { + it.name.endsWith(".md") || it.name.endsWith("overview.html") || it.name.endsWith("javadoc-index.html") + } + + collection.each { File file -> + file.text = file.text.replaceAll("/${previousVersion}/", "/${releaseVersion}/") + .replaceAll(">${previousVersion}<", ">${releaseVersion}<") + .replaceAll(":${previousVersion}", ":${releaseVersion}") + } + } +} +