From 495201e69c9f52491453cc4f3541839e33429dbe Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Mon, 24 Jun 2024 13:33:24 +0200 Subject: [PATCH 1/4] PR check generator: add `excludeOsAndVersionCombination` --- .../workflows/__multi-language-autodetect.yml | 6 +++ .github/workflows/__scaling-reserved-ram.yml | 6 +++ .github/workflows/__swift-custom-build.yml | 6 +++ .github/workflows/__unset-environment.yml | 14 ++---- .../checks/multi-language-autodetect.yml | 11 ++++- pr-checks/checks/scaling-reserved-ram.yml | 11 ++++- pr-checks/checks/swift-custom-build.yml | 3 +- pr-checks/checks/unset-environment.yml | 13 +++-- pr-checks/sync.py | 49 ++++++++++++------- 9 files changed, 81 insertions(+), 38 deletions(-) diff --git a/.github/workflows/__multi-language-autodetect.yml b/.github/workflows/__multi-language-autodetect.yml index e2a7cbeb3a..7b917f725c 100644 --- a/.github/workflows/__multi-language-autodetect.yml +++ b/.github/workflows/__multi-language-autodetect.yml @@ -39,10 +39,16 @@ jobs: version: stable-v2.16.6 - os: macos-latest version: default + - os: ubuntu-latest + version: default - os: macos-latest version: linked + - os: ubuntu-latest + version: linked - os: macos-latest version: nightly-latest + - os: ubuntu-latest + version: nightly-latest name: Multi-language repository permissions: contents: read diff --git a/.github/workflows/__scaling-reserved-ram.yml b/.github/workflows/__scaling-reserved-ram.yml index a9cf98c345..add21ad3db 100644 --- a/.github/workflows/__scaling-reserved-ram.yml +++ b/.github/workflows/__scaling-reserved-ram.yml @@ -39,10 +39,16 @@ jobs: version: stable-v2.16.6 - os: macos-latest version: default + - os: ubuntu-latest + version: default - os: macos-latest version: linked + - os: ubuntu-latest + version: linked - os: macos-latest version: nightly-latest + - os: ubuntu-latest + version: nightly-latest name: Scaling reserved RAM permissions: contents: read diff --git a/.github/workflows/__swift-custom-build.yml b/.github/workflows/__swift-custom-build.yml index 765c336670..f298b17e20 100644 --- a/.github/workflows/__swift-custom-build.yml +++ b/.github/workflows/__swift-custom-build.yml @@ -29,10 +29,16 @@ jobs: include: - os: macos-latest version: linked + - os: ubuntu-latest + version: linked - os: macos-latest version: default + - os: ubuntu-latest + version: default - os: macos-latest version: nightly-latest + - os: ubuntu-latest + version: nightly-latest name: Swift analysis using a custom build command permissions: contents: read diff --git a/.github/workflows/__unset-environment.yml b/.github/workflows/__unset-environment.yml index 07275673e8..3f12a3c2b8 100644 --- a/.github/workflows/__unset-environment.yml +++ b/.github/workflows/__unset-environment.yml @@ -27,17 +27,11 @@ jobs: fail-fast: false matrix: include: - - os: macos-12 - version: stable-v2.14.6 - - os: macos-latest - version: stable-v2.15.5 - - os: macos-latest - version: stable-v2.16.6 - - os: macos-latest - version: linked - - os: macos-latest + - os: ubuntu-latest version: default - - os: macos-latest + - os: ubuntu-latest + version: linked + - os: ubuntu-latest version: nightly-latest name: Test unsetting environment variables permissions: diff --git a/pr-checks/checks/multi-language-autodetect.yml b/pr-checks/checks/multi-language-autodetect.yml index 89efa2ca5c..b1558cebc2 100644 --- a/pr-checks/checks/multi-language-autodetect.yml +++ b/pr-checks/checks/multi-language-autodetect.yml @@ -1,7 +1,14 @@ name: "Multi-language repository" description: "An end-to-end integration test of a multi-language repository using automatic language detection" -# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available. -operatingSystems: ["macos"] +operatingSystems: ["macos", "ubuntu"] +excludeOsAndVersionCombination: [ + # Known failure for Swift on Linux before CLI v2.17.4. + [ "ubuntu", "stable-20230403" ], + [ "ubuntu", "stable-v2.13.5" ], + [ "ubuntu", "stable-v2.14.6" ], + [ "ubuntu", "stable-v2.15.5" ], + [ "ubuntu", "stable-v2.16.6" ], +] steps: - uses: actions/setup-go@v5 with: diff --git a/pr-checks/checks/scaling-reserved-ram.yml b/pr-checks/checks/scaling-reserved-ram.yml index acd6c2b24b..8e7143bd49 100644 --- a/pr-checks/checks/scaling-reserved-ram.yml +++ b/pr-checks/checks/scaling-reserved-ram.yml @@ -1,7 +1,14 @@ name: "Scaling reserved RAM" description: "An end-to-end integration test of a multi-language repository with the scaling_reserved_ram feature flag enabled" -# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available. -operatingSystems: ["macos"] +operatingSystems: ["macos", "ubuntu"] +excludeOsAndVersionCombination: [ + # Known failure for Swift on Linux before CLI v2.17.4. + [ "ubuntu", "stable-20230403" ], + [ "ubuntu", "stable-v2.13.5" ], + [ "ubuntu", "stable-v2.14.6" ], + [ "ubuntu", "stable-v2.15.5" ], + [ "ubuntu", "stable-v2.16.6" ], +] env: CODEQL_ACTION_SCALING_RESERVED_RAM: true steps: diff --git a/pr-checks/checks/swift-custom-build.yml b/pr-checks/checks/swift-custom-build.yml index 2984641add..3bdd0748d5 100644 --- a/pr-checks/checks/swift-custom-build.yml +++ b/pr-checks/checks/swift-custom-build.yml @@ -1,8 +1,7 @@ name: "Swift analysis using a custom build command" description: "Tests creation of a Swift database using custom build" versions: ["linked", "default", "nightly-latest"] -# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available. -operatingSystems: ["macos"] +operatingSystems: ["macos", "ubuntu"] env: DOTNET_GENERATE_ASPNET_CERTIFICATE: "false" steps: diff --git a/pr-checks/checks/unset-environment.yml b/pr-checks/checks/unset-environment.yml index d7513114f0..c3803ff1ed 100644 --- a/pr-checks/checks/unset-environment.yml +++ b/pr-checks/checks/unset-environment.yml @@ -1,8 +1,15 @@ name: "Test unsetting environment variables" description: "An end-to-end integration test that unsets some environment variables" -# TODO: Switch back to all versions once CLI v2.17.4 is available and running on ubuntu again. -versions: ["stable-v2.14.6", "stable-v2.15.5", "stable-v2.16.6", "linked", "default", "nightly-latest"] -operatingSystems: ["macos"] # TODO: Switch back to ubuntu for `nightly-latest` and `latest` once CLI v2.17.4 is available. +operatingSystems: ["ubuntu"] +excludeOsAndVersionCombination: [ + # Known failure for Swift on Linux before CLI v2.17.4. + [ "ubuntu", "stable-20230403" ], + [ "ubuntu", "stable-v2.13.5" ], + [ "ubuntu", "stable-v2.14.6" ], + [ "ubuntu", "stable-v2.15.5" ], + [ "ubuntu", "stable-v2.16.6" ], +] + steps: - uses: ./../action/init id: init diff --git a/pr-checks/sync.py b/pr-checks/sync.py index 6d96bd620c..31e818cce3 100755 --- a/pr-checks/sync.py +++ b/pr-checks/sync.py @@ -27,6 +27,12 @@ "nightly-latest" ] +def is_version_and_os_excluded(version, os, exclude_params): + for exclude_param in exclude_params: + if exclude_param[0] == os and exclude_param[1] == version: + return True + return False + # When updating the ruamel.yaml version here, update the PR check in # `.github/workflows/pr-checks.yml` too. header = """# Warning: This file is generated automatically, and should not be modified. @@ -56,27 +62,32 @@ def writeHeader(checkStream): for file in (this_dir / 'checks').glob('*.yml'): with open(file, 'r') as checkStream: checkSpecification = yaml.load(checkStream) - matrix = [] + excludedVersionsAndOses = checkSpecification.get('excludeOsAndVersionCombination', []) for version in checkSpecification.get('versions', defaultTestVersions): - runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"] - if checkSpecification.get('operatingSystems', None): - runnerImages = [image for image in runnerImages for operatingSystem in checkSpecification['operatingSystems'] - if image.startswith(operatingSystem)] - - for runnerImage in runnerImages: - # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer. - # "macos-latest" is now an ARM runner, so we run tests on the old CLIs on Intel runners instead. - if version in ["stable-20230403", "stable-v2.13.4", "stable-v2.13.5", "stable-v2.14.6"] and runnerImage == "macos-latest": - matrix.append({ - 'os': "macos-12", - 'version': version - }) - else: - matrix.append({ - 'os': runnerImage, - 'version': version - }) + runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"] + operatingSystems = checkSpecification.get('operatingSystems', ["ubuntu", "macos", "windows"]) + + for operatingSystem in operatingSystems: + runnerImagesForOs = [image for image in runnerImages if image.startswith(operatingSystem)] + + for runnerImage in runnerImagesForOs: + # Skip appending this combination to the matrix if it is explicitly excluded. + if is_version_and_os_excluded(version, operatingSystem, excludedVersionsAndOses): + continue + + # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer. + # "macos-latest" is now an ARM runner, so we run tests on the old CLIs on Intel runners instead. + if version in ["stable-20230403", "stable-v2.13.4", "stable-v2.13.5", "stable-v2.14.6"] and runnerImage == "macos-latest": + matrix.append({ + 'os': "macos-12", + 'version': version + }) + else: + matrix.append({ + 'os': runnerImage, + 'version': version + }) useAllPlatformBundle = "false" # Default to false if checkSpecification.get('useAllPlatformBundle'): From 462c756cab357d27e4721bea8b772116fceb4945 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Mon, 24 Jun 2024 14:14:23 +0200 Subject: [PATCH 2/4] Switch `debug-artifacts` job back to `ubuntu` Now that CLI v2.17.4+ are available, we can switch this job back to `ubuntu`. As a result, we can also bring back testing on the older CLI versions (which did not work on MacOS). CLI v.2.16.6 has a known failure on Linux so we exclude it from this workflow. This change is orthogonal to the PR check generator change as the check doesn't use the generator. --- .github/workflows/debug-artifacts.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index 0c0182f3d3..17010df62d 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -22,12 +22,10 @@ jobs: fail-fast: false matrix: version: - # TODO: Once CLI v2.17.4 is available and the platform is switched back to ubuntu, - # stable-20230403, stable-v2.13.5, and stable-v2.14.6 can be added back to this matrix, - # and the VERSIONS variable in the bash script below. - # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer. + - stable-20230403 + - stable-v2.13.5 + - stable-v2.14.6 - stable-v2.15.5 - - stable-v2.16.6 - default - linked - nightly-latest @@ -35,7 +33,7 @@ jobs: env: CODEQL_ACTION_TEST_MODE: true timeout-minutes: 45 - runs-on: macos-latest # TODO: Switch back to ubuntu for `nightly-latest` and `linked` once CLI v2.17.4 is available. + runs-on: ubuntu-latest steps: - name: Check out repository uses: actions/checkout@v4 @@ -73,7 +71,7 @@ jobs: - name: Check expected artifacts exist shell: bash run: | - VERSIONS="stable-v2.15.5 stable-v2.16.6 default linked nightly-latest" + VERSIONS="stable-20230403 stable-v2.13.5 stable-v2.14.6 stable-v2.15.5 default linked nightly-latest" LANGUAGES="cpp csharp go java javascript python" for version in $VERSIONS; do pushd "./my-debug-artifacts-${version//./}" From 7c35b4f6b532751db53ff3fab0e00b16f7fa0179 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Mon, 24 Jun 2024 16:33:10 +0200 Subject: [PATCH 3/4] Rename for clarity per PR review --- pr-checks/sync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pr-checks/sync.py b/pr-checks/sync.py index 31e818cce3..08520890b2 100755 --- a/pr-checks/sync.py +++ b/pr-checks/sync.py @@ -27,7 +27,7 @@ "nightly-latest" ] -def is_version_and_os_excluded(version, os, exclude_params): +def is_os_and_version_excluded(version, os, exclude_params): for exclude_param in exclude_params: if exclude_param[0] == os and exclude_param[1] == version: return True @@ -63,7 +63,7 @@ def writeHeader(checkStream): with open(file, 'r') as checkStream: checkSpecification = yaml.load(checkStream) matrix = [] - excludedVersionsAndOses = checkSpecification.get('excludeOsAndVersionCombination', []) + excludedOsesAndVersions = checkSpecification.get('excludeOsAndVersionCombination', []) for version in checkSpecification.get('versions', defaultTestVersions): runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"] operatingSystems = checkSpecification.get('operatingSystems', ["ubuntu", "macos", "windows"]) @@ -73,7 +73,7 @@ def writeHeader(checkStream): for runnerImage in runnerImagesForOs: # Skip appending this combination to the matrix if it is explicitly excluded. - if is_version_and_os_excluded(version, operatingSystem, excludedVersionsAndOses): + if is_os_and_version_excluded(version, operatingSystem, excludedOsesAndVersions): continue # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer. From 5d576606ec0fd17f00d70aadf8737174c148fdda Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Mon, 24 Jun 2024 17:02:29 +0200 Subject: [PATCH 4/4] Switch param order for clarity --- pr-checks/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pr-checks/sync.py b/pr-checks/sync.py index 08520890b2..0c347b10b5 100755 --- a/pr-checks/sync.py +++ b/pr-checks/sync.py @@ -27,7 +27,7 @@ "nightly-latest" ] -def is_os_and_version_excluded(version, os, exclude_params): +def is_os_and_version_excluded(os, version, exclude_params): for exclude_param in exclude_params: if exclude_param[0] == os and exclude_param[1] == version: return True @@ -73,7 +73,7 @@ def writeHeader(checkStream): for runnerImage in runnerImagesForOs: # Skip appending this combination to the matrix if it is explicitly excluded. - if is_os_and_version_excluded(version, operatingSystem, excludedOsesAndVersions): + if is_os_and_version_excluded(operatingSystem, version, excludedOsesAndVersions): continue # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer.