V0.4

.github/workflows/build-matrix.json

@@ -0,0 +1,55 @@
+[
+ {
+ "OS": "ubuntu",
+ "NAMED_OS": "linux",
+ "RUNNER": "ubuntu-latest",
+ "ARCH": "x64",
+ "TARGET": "x86_64-unknown-linux-gnu",
+ "PACKAGE_MANAGERS": ["pypi", "npm"]
+ },
+ {
+ "OS": "ubuntu",
+ "NAMED_OS": "linux",
+ "RUNNER": ["self-hosted", "Linux", "ARM64"],
+ "ARCH": "arm64",
+ "TARGET": "aarch64-unknown-linux-gnu",
+ "PACKAGE_MANAGERS": ["pypi", "npm"],
+ "CONTAINER": "2_28"
+ },
+ {
+ "OS": "macos",
+ "NAMED_OS": "darwin",
+ "RUNNER": "macos-12",
+ "ARCH": "x64",
+ "TARGET": "x86_64-apple-darwin",
+ "PACKAGE_MANAGERS": ["pypi", "npm"]
+ },
+ {
+ "OS": "macos",
+ "NAMED_OS": "darwin",
+ "RUNNER": "macos-13-xlarge",
+ "ARCH": "arm64",
+ "TARGET": "aarch64-apple-darwin",
+ "PACKAGE_MANAGERS": ["pypi", "npm"]
+ },
+ {
+ "OS": "ubuntu",
+ "NAMED_OS": "linux",
+ "ARCH": "arm64",
+ "TARGET": "aarch64-unknown-linux-musl",
+ "RUNNER": ["self-hosted", "Linux", "ARM64"],
+ "IMAGE": "node:alpine",
+ "CONTAINER_OPTIONS": "--user root --privileged --rm",
+ "PACKAGE_MANAGERS": ["npm"]
+ },
+ {
+ "OS": "ubuntu",
+ "NAMED_OS": "linux",
+ "ARCH": "x64",
+ "TARGET": "x86_64-unknown-linux-musl",
+ "RUNNER": "ubuntu-latest",
+ "IMAGE": "node:alpine",
+ "CONTAINER_OPTIONS": "--user root --privileged",
+ "PACKAGE_MANAGERS": ["npm"]
+ }
+]

.github/workflows/node-create-package-file/action.yml

@@ -69,11 +69,19 @@ runs:
  # set the registry scope
  export registry_scope=`if [ "${{ inputs.npm_scope }}" != '' ]; then echo "${{ inputs.npm_scope }}:"; fi`
  # remove the current name section
- SED_FOR_MACOS=`if [[ "${{ inputs.os }}" =~ .*"macos".* ]]; then echo "''"; fi`
- sed -i $SED_FOR_MACOS '/"name":/d' ./package.json
+ if [[ "${{ inputs.os }}" =~ .*"macos".* ]]; then
+ sed '/"name":/d' ./package.json > temp.json && mv temp.json package.json
+ else
+ sed -i '/"name":/d' ./package.json
+ fi
  # Remove all `///` occurrences to enable the commented out sections
- sed -i -e 's|///||g' package.json
+ if [[ "${{ inputs.os }}" =~ .*"macos".* ]]; then
+ sed 's|///||g' package.json > temp.json && mv temp.json package.json
+ else
+ sed -i 's|///||g' package.json
+ fi
  # generate package.json from the template
  mv package.json package.json.tmpl
  envsubst < package.json.tmpl > "package.json"
  cat package.json
+ echo $(ls *json*) 

.github/workflows/npm-cd.yml

@@ -2,14 +2,6 @@
 
 name: Continuous Deployment
 
-env:
- MATRIX: '[{"OS": "ubuntu","NAMED_OS": "linux","RUNNER": "ubuntu-latest","ARCH": "x64","TARGET": "x86_64-unknown-linux-gnu"},
- {"OS": "ubuntu","NAMED_OS": "linux","RUNNER": ["self-hosted", "Linux", "ARM64"],"ARCH": "arm64","TARGET": "aarch64-unknown-linux-gnu"},
- {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-12","ARCH": "x64","TARGET": "x86_64-apple-darwin"},
- {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-13-xlarge","ARCH": "arm64","TARGET": "aarch64-apple-darwin"},
- {"OS": "ubuntu","NAMED_OS": "linux","ARCH": "arm64","TARGET": "aarch64-unknown-linux-musl","RUNNER": ["self-hosted", "Linux", "ARM64"],"IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged --rm"},
- {"OS": "ubuntu","NAMED_OS": "linux","ARCH": "x64","TARGET": "x86_64-unknown-linux-musl","RUNNER": "ubuntu-latest","IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged"}]'
-
 on:
  pull_request:
  paths:
@@ -29,16 +21,6 @@ permissions:
  id-token: write
 
 jobs:
- output-matrix:
- runs-on: ubuntu-latest
- outputs:
- matrix: ${{ steps.set-matrix.outputs.matrix }}
- steps:
- - name: Set matrix
- id: set-matrix
- run: |
- echo "::set-output name=matrix::${{toJson( env.MATRIX )}}"
-
  start-self-hosted-runner:
  if: github.repository_owner == 'aws'
  runs-on: ubuntu-latest
@@ -56,8 +38,24 @@ jobs:
  aws-region: ${{ secrets.AWS_REGION }}
  ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}
 
+ load-platform-matrix:
+ runs-on: ubuntu-latest
+ outputs:
+ PLATFORM_MATRIX: ${{ steps.load-platform-matrix.outputs.PLATFORM_MATRIX }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: load-platform-matrix
+ id: load-platform-matrix
+ shell: bash
+ run: |
+ # Get the matrix from the matrix.json file, without the object that has the IMAGE key
+ export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["npm"])))' < .github/workflows/build-matrix.json | jq -c .)"
+ echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT
+
  publish-binaries:
- needs: [start-self-hosted-runner, output-matrix]
+ needs: [start-self-hosted-runner, load-platform-matrix]
  if: github.repository_owner == 'aws'
  name: Publish packages to NPM
  runs-on: ${{ matrix.build.RUNNER }}
@@ -67,9 +65,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- build:
- ${{fromJson(needs.output-matrix.outputs.matrix)}}
-
+ build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} 
  steps:
  - name: Setup self-hosted runner access
  if: ${{ contains(matrix.build.RUNNER, 'self-hosted') && matrix.build.TARGET != 'aarch64-unknown-linux-musl' }}
@@ -108,7 +104,7 @@ jobs:
  if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }}
  uses: actions/setup-node@v3
  with:
- node-version: "16"
+ node-version: "20"
  registry-url: "https://registry.npmjs.org"
  architecture: ${{ matrix.build.ARCH }}
  scope: "${{ vars.NPM_SCOPE }}"
@@ -139,15 +135,38 @@ jobs:
  npm_scope: ${{ vars.NPM_SCOPE }}
  publish: "true"
  github-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Check if RC and set a distribution tag for the package
+ shell: bash
+ run: |
+ if [[ "${GITHUB_REF:11}" == *"rc"* ]]
+ then
+ echo "This is a release candidate: ${GITHUB_REF:11}"
+ export npm_tag="next"
+ else
+ echo "This is a stable release: ${GITHUB_REF:11}"
+ export npm_tag="latest"
+ fi
+ echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV
+
+ - name: Check that the release version dont have typo init
+ if: ${{ github.event_name != 'pull_request' && contains(github.ref, '-') && !contains(github.ref, 'rc') }}
+ run: |
+ echo "The release version "${GITHUB_REF:11}" contains a typo, please fix it"
+ echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release."
+ exit 1
 
  - name: Publish to NPM
  if: github.event_name != 'pull_request'
  shell: bash
  working-directory: ./node
  run: |
+ npm pkg fix
  set +e
- # Redirect only stderr
- { npm_publish_err=$(npm publish --access public 2>&1 >&3 3>&-); } 3>&1
+ # 2>&1 1>&3- redirects stderr to stdout and then redirects the original stdout to another file descriptor,
+ # effectively separating stderr and stdout. The 3>&1 at the end redirects the original stdout back to the console.
+ # https:/npm/npm/issues/118#issuecomment-325440 - ignoring notice messages since currentlly they are directed to stderr 
+ { npm_publish_err=$(npm publish --tag ${{ env.NPM_TAG }} --access public 2>&1 1>&3- | grep -v "notice") ;} 3>&1
  if [[ "$npm_publish_err" == *"You cannot publish over the previously published versions"* ]]
  then
  echo "Skipping publishing, package already published"
@@ -159,27 +178,6 @@ jobs:
  env:
  NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
- - name: Pack the Node package
- shell: bash
- working-directory: ./node
- run: |
- # Remove the "cpu" and "os" fileds so the base package would be able to install it on ubuntu
- SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".* ]]; then echo "''"; fi`
- sed -i $SED_FOR_MACOS '/"\/\/\/cpu": \[/,/]/d' ./package.json && sed -i $SED_FOR_MACOS '/"\/\/\/os": \[/,/]/d' ./package.json
- mkdir -p bin
- npm pack --pack-destination ./bin
- ls ./bin
- env:
- NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
-
- - name: Upload the Node package
- if: github.event_name != 'pull_request'
- uses: actions/upload-artifact@v3
- with:
- name: ${{ matrix.build.TARGET }}
- path: ./node/bin
- if-no-files-found: error
-
  # Reset the repository to make sure we get the clean checkout of the action later in other actions.
  # It is not required since in other actions we are cleaning before the action, but it is a good practice to do it here as well.
  - name: Reset repository
@@ -196,14 +194,14 @@ jobs:
  runs-on: ubuntu-latest
  steps:
  - name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: "true"
 
  - name: Install node
  uses: actions/setup-node@v3
  with:
- node-version: "16"
+ node-version: "20"
  registry-url: "https://registry.npmjs.org"
  scope: "${{ vars.NPM_SCOPE }}"
  always-auth: true
@@ -230,33 +228,7 @@ jobs:
  os: ubuntu
  target: "x86_64-unknown-linux-gnu"
  github-token: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Create a directory for the packed packages
- shell: bash
- working-directory: ./node/npm/glide
- run: mkdir packages
-
- - name: Download the packed packages
- id: download
- uses: actions/download-artifact@v3
- with:
- path: ./node/npm/glide/packages
-
- - name: Install the packed packages
- shell: bash
- working-directory: ./node/npm/glide
- run: |
- ls -LR packages/
- packages_list=`find ${{steps.download.outputs.download-path}} -type f -follow -print`
- for package in $packages_list
- do
- if [[ "${package}" == *.tgz ]]
- then
- echo "Installing package $package"
- npm i --no-save "$package"
- fi
- done
-
+
  - name: Check if RC and set a distribution tag for the package
  shell: bash
  run: |
@@ -283,19 +255,18 @@ jobs:
  env:
  NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
- test-release-candidate:
- if: github.event_name != 'pull_request' && contains(github.ref, 'rc')
- name: Test the release candidate
- needs: [publish-base-to-npm, output-matrix]
+ test-release:
+ if: github.event_name != 'pull_request'
+ name: Test the release
+ needs: [publish-base-to-npm, load-platform-matrix]
  runs-on: ${{ matrix.build.RUNNER }}
  container:
  image: ${{ matrix.build.IMAGE || '' }}
  options: ${{ matrix.build.CONTAINER_OPTIONS || 'none'}}
  strategy:
  fail-fast: false
  matrix:
- build:
- ${{fromJson(needs.output-matrix.outputs.matrix)}}
+ build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} 
  steps:
  - name: Setup self-hosted runner access
  if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }}
@@ -306,18 +277,25 @@ jobs:
  run: |
  apk update
  apk add redis git
+ node -v
 
  - name: install Redis and Python for ubuntu
  if: ${{ contains(matrix.build.TARGET, 'linux-gnu') }}
  run: |
  sudo apt-get update
- sudo apt-get install redis-server python3
+ sudo apt-get install redis-server python3 
 
- - name: install Redis and Python for macos
+ - name: install Redis, Python for macos
  if: ${{ contains(matrix.build.RUNNER, 'mac') }}
  run: |
  brew install redis python3
 
+ - name: Checkout
+ if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl'}}
+ uses: actions/checkout@v4
+ with:
+ submodules: "true"
+
  - name: Setup for musl
  if: ${{ contains(matrix.build.TARGET, 'musl') }}
  uses: ./.github/workflows/setup-musl-on-linux
@@ -327,26 +305,44 @@ jobs:
  npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }}
  arch: ${{ matrix.build.ARCH }}
 
- - name: Checkout
- if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl'}}
- uses: actions/checkout@v4
- with:
- submodules: "true"
-
- - name: Install node
- if: ${{ !contains(matrix.build.TARGET, 'musl') }}
+ - name: Setup node
+ if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }}
  uses: actions/setup-node@v3
  with:
  node-version: "16"
  registry-url: "https://registry.npmjs.org"
+ architecture: ${{ matrix.build.ARCH }}
  scope: "${{ vars.NPM_SCOPE }}"
  always-auth: true
+ token: ${{ secrets.NPM_AUTH_TOKEN }}
+
+ - name: Install tsc and compile utils
+ shell: bash
+ working-directory: ./utils
+ run: |
+ npm install
+ npm install -g typescript
+ npx tsc -p ./tsconfig.json
+
+ - name: Check if RC and set a distribution tag for the package
+ shell: bash
+ run: |
+ if [[ "${GITHUB_REF:11}" == *"rc"* ]]
+ then
+ echo "This is a release candidate"
+ export npm_tag="next"
+ else
+ echo "This is a stable release"
+ export npm_tag="latest"
+ fi
+ echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV
 
  - name: Run the tests
  shell: bash
  working-directory: ./utils/release-candidate-testing/node
  run: |
- npm install --no-save @aws/glide-for-redis@next
+ npm install
+ npm install --no-save @aws/glide-for-redis@${{ env.NPM_TAG }}
  npm run test
 
  # Reset the repository to make sure we get the clean checkout of the action later in other actions.