Support Azure universal packages as a binary caching provider (#1491)

* add universal support

* fix

* fix

* review

* review

* fix

include/vcpkg/base/message-data.inc.h

@@ -1490,6 +1490,12 @@ DECLARE_MESSAGE(HelpBinaryCachingAzBlob,
  "**Experimental: will change or be removed without warning**\n"
  "Adds an Azure Blob Storage source. Uses Shared Access Signature validation. <url> should include "
  "the container path. <sas> must be be prefixed with a \"?\".")
+DECLARE_MESSAGE(HelpBinaryCachingAzUpkg,
+ (),
+ "Printed as the 'definition' for 'x-az-universal,<organization>,<project>,<feed>[,<rw>]'.",
+ "**Experimental: will change or be removed without warning**\n"
+ "Adds a Universal Package Azure Artifacts source. Uses the Azure CLI "
+ "(az artifacts) for uploads and downloads.")
 DECLARE_MESSAGE(HelpBinaryCachingCos,
  (),
  "Printed as the 'definition' for 'x-cos,<prefix>[,<rw>]'.",
@@ -1811,6 +1817,10 @@ DECLARE_MESSAGE(InvalidArgumentRequiresBaseUrlAndToken,
  (msg::binary_source),
  "",
  "invalid argument: binary config '{binary_source}' requires at least a base-url and a SAS token")
+DECLARE_MESSAGE(InvalidArgumentRequiresFourOrFiveArguments,
+ (msg::binary_source),
+ "",
+ "invalid argument: binary config '{binary_source}' requires 4 or 5 arguments")
 DECLARE_MESSAGE(InvalidArgumentRequiresNoneArguments,
  (msg::binary_source),
  "",
@@ -2576,6 +2586,11 @@ DECLARE_MESSAGE(RestoredPackagesFromAWS,
  (msg::count, msg::elapsed),
  "",
  "Restored {count} package(s) from AWS in {elapsed}. Use --debug to see more details.")
+DECLARE_MESSAGE(RestoredPackagesFromAZUPKG,
+ (msg::count, msg::elapsed),
+ "",
+ "Restored {count} package(s) from Universal Packages in {elapsed}. "
+ "Use --debug to see more details.")
 DECLARE_MESSAGE(RestoredPackagesFromCOS,
  (msg::count, msg::elapsed),
  "",

include/vcpkg/binarycaching.h

@@ -121,6 +121,13 @@ namespace vcpkg
  std::string commit;
  };
 
+ struct AzureUpkgSource
+ {
+ std::string organization;
+ std::string project;
+ std::string feed;
+ };
+
  struct BinaryConfigParserState
  {
  bool nuget_interactive = false;
@@ -147,6 +154,9 @@ namespace vcpkg
  bool gha_write = false;
  bool gha_read = false;
 
+ std::vector<AzureUpkgSource> upkg_templates_to_get;
+ std::vector<AzureUpkgSource> upkg_templates_to_put;
+
  std::vector<std::string> sources_to_read;
  std::vector<std::string> sources_to_write;
 

include/vcpkg/binarycaching.private.h

@@ -19,19 +19,19 @@ namespace vcpkg
  // - v?<X>.<Y><whatever> -> <X>.<Y>.0-vcpkg<abitag>
  // - v?<X>.<Y>.<Z><whatever> -> <X>.<Y>.<Z>-vcpkg<abitag>
  // - anything else -> 0.0.0-vcpkg<abitag>
- std::string format_version_for_nugetref(StringView version_text, StringView abi_tag);
+ std::string format_version_for_feedref(StringView version_text, StringView abi_tag);
 
- struct NugetReference
+ struct FeedReference
  {
- NugetReference(std::string id, std::string version) : id(std::move(id)), version(std::move(version)) { }
+ FeedReference(std::string id, std::string version) : id(std::move(id)), version(std::move(version)) { }
 
  std::string id;
  std::string version;
 
  std::string nupkg_filename() const { return Strings::concat(id, '.', version, ".nupkg"); }
  };
 
- NugetReference make_nugetref(const InstallPlanAction& action, StringView prefix);
+ FeedReference make_nugetref(const InstallPlanAction& action, StringView prefix);
 
  std::string generate_nuspec(const Path& package_dir,
  const InstallPlanAction& action,

include/vcpkg/metrics.h

@@ -24,6 +24,7 @@ namespace vcpkg
  BinaryCachingHttp,
  BinaryCachingNuget,
  BinaryCachingSource,
+ BinaryCachingUpkg,
  ErrorVersioningDisabled,
  ErrorVersioningNoBaseline,
  GitHubRepository,

include/vcpkg/tools.h

@@ -23,6 +23,7 @@ namespace vcpkg
  static constexpr StringLiteral GIT = "git";
  static constexpr StringLiteral GSUTIL = "gsutil";
  static constexpr StringLiteral AWSCLI = "aws";
+ static constexpr StringLiteral AZCLI = "az";
  static constexpr StringLiteral COSCLI = "coscli";
  static constexpr StringLiteral MONO = "mono";
  static constexpr StringLiteral NINJA = "ninja";
@@ -45,6 +46,11 @@ namespace vcpkg
  virtual const std::string& get_tool_version(StringView tool, MessageSink& status_sink) const = 0;
  };
 
+ ExpectedL<std::string> extract_prefixed_nonquote(StringLiteral prefix,
+ StringLiteral tool_name,
+ std::string&& output,
+ const Path& exe_path);
+
  ExpectedL<std::string> extract_prefixed_nonwhitespace(StringLiteral prefix,
  StringLiteral tool_name,
  std::string&& output,

locales/messages.json

@@ -858,6 +858,8 @@
  "HelpBinaryCachingAwsHeader": "Azure Web Services sources",
  "HelpBinaryCachingAzBlob": "**Experimental: will change or be removed without warning**\nAdds an Azure Blob Storage source. Uses Shared Access Signature validation. <url> should include the container path. <sas> must be be prefixed with a \"?\".",
  "_HelpBinaryCachingAzBlob.comment": "Printed as the 'definition' for 'x-azblob,<url>,<sas>[,<rw>]'.",
+ "HelpBinaryCachingAzUpkg": "**Experimental: will change or be removed without warning**\nAdds a Universal Package Azure Artifacts source. Uses the Azure CLI (az artifacts) for uploads and downloads.",
+ "_HelpBinaryCachingAzUpkg.comment": "Printed as the 'definition' for 'x-az-universal,<organization>,<project>,<feed>[,<rw>]'.",
  "HelpBinaryCachingCos": "**Experimental: will change or be removed without warning**\nAdds an COS source. Uses the cos CLI for uploads and downloads. <prefix> should include the scheme 'cos://' and be suffixed with a \"/\".",
  "_HelpBinaryCachingCos.comment": "Printed as the 'definition' for 'x-cos,<prefix>[,<rw>]'.",
  "HelpBinaryCachingDefaults": "Adds the default file-based location. Based on your system settings, the default path to store binaries is \"{path}\". This consults %LOCALAPPDATA%/%APPDATA% on Windows and $XDG_CACHE_HOME or $HOME on other platforms.",
@@ -997,6 +999,8 @@
  "_InvalidArgumentRequiresBaseUrl.comment": "An example of {base_url} is azblob://. An example of {binary_source} is azblob.",
  "InvalidArgumentRequiresBaseUrlAndToken": "invalid argument: binary config '{binary_source}' requires at least a base-url and a SAS token",
  "_InvalidArgumentRequiresBaseUrlAndToken.comment": "An example of {binary_source} is azblob.",
+ "InvalidArgumentRequiresFourOrFiveArguments": "invalid argument: binary config '{binary_source}' requires 4 or 5 arguments",
+ "_InvalidArgumentRequiresFourOrFiveArguments.comment": "An example of {binary_source} is azblob.",
  "InvalidArgumentRequiresNoWildcards": "cannot fix Windows path case for path containing wildcards: {path}",
  "_InvalidArgumentRequiresNoWildcards.comment": "An example of {path} is /foo/bar.",
  "InvalidArgumentRequiresNoneArguments": "invalid argument: binary config '{binary_source}' does not take arguments",
@@ -1359,6 +1363,8 @@
  "_ResponseFileCode.comment": "Explains to the user that they can use response files on the command line, 'response_file' must have no spaces and be a legal file name.",
  "RestoredPackagesFromAWS": "Restored {count} package(s) from AWS in {elapsed}. Use --debug to see more details.",
  "_RestoredPackagesFromAWS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
+ "RestoredPackagesFromAZUPKG": "Restored {count} package(s) from Universal Packages in {elapsed}. Use --debug to see more details.",
+ "_RestoredPackagesFromAZUPKG.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
  "RestoredPackagesFromCOS": "Restored {count} package(s) from COS in {elapsed}. Use --debug to see more details.",
  "_RestoredPackagesFromCOS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
  "RestoredPackagesFromFiles": "Restored {count} package(s) from {path} in {elapsed}. Use --debug to see more details.",

src/vcpkg-test/binarycaching.cpp

@@ -166,30 +166,30 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]")
  REQUIRE(assignee.is_restored());
 }
 
-TEST_CASE ("format_version_for_nugetref semver-ish", "[format_version_for_nugetref]")
+TEST_CASE ("format_version_for_feedref semver-ish", "[format_version_for_feedref]")
 {
- REQUIRE(format_version_for_nugetref("0.0.0", "abitag") == "0.0.0-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("1.0.1", "abitag") == "1.0.1-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("1.01.000", "abitag") == "1.1.0-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("1.2", "abitag") == "1.2.0-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("v52", "abitag") == "52.0.0-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("v09.01.02", "abitag") == "9.1.2-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("1.1.1q", "abitag") == "1.1.1-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("1", "abitag") == "1.0.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("0.0.0", "abitag") == "0.0.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("1.0.1", "abitag") == "1.0.1-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("1.01.000", "abitag") == "1.1.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("1.2", "abitag") == "1.2.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("v52", "abitag") == "52.0.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("v09.01.02", "abitag") == "9.1.2-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("1.1.1q", "abitag") == "1.1.1-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("1", "abitag") == "1.0.0-vcpkgabitag");
 }
 
-TEST_CASE ("format_version_for_nugetref date", "[format_version_for_nugetref]")
+TEST_CASE ("format_version_for_feedref date", "[format_version_for_feedref]")
 {
- REQUIRE(format_version_for_nugetref("2020-06-26", "abitag") == "2020.6.26-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("20-06-26", "abitag") == "0.0.0-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("2020-06-26-release", "abitag") == "2020.6.26-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("2020-06-26000", "abitag") == "2020.6.26-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("2020-06-26", "abitag") == "2020.6.26-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("20-06-26", "abitag") == "0.0.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("2020-06-26-release", "abitag") == "2020.6.26-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("2020-06-26000", "abitag") == "2020.6.26-vcpkgabitag");
 }
 
-TEST_CASE ("format_version_for_nugetref generic", "[format_version_for_nugetref]")
+TEST_CASE ("format_version_for_feedref generic", "[format_version_for_feedref]")
 {
- REQUIRE(format_version_for_nugetref("apr", "abitag") == "0.0.0-vcpkgabitag");
- REQUIRE(format_version_for_nugetref("", "abitag") == "0.0.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("apr", "abitag") == "0.0.0-vcpkgabitag");
+ REQUIRE(format_version_for_feedref("", "abitag") == "0.0.0-vcpkgabitag");
 }
 
 TEST_CASE ("generate_nuspec", "[generate_nuspec]")
@@ -236,11 +236,11 @@ Build-Depends: bzip
  compiler_info.version = "compilerversion";
  ipa.abi_info.get()->compiler_info = compiler_info;
 
- NugetReference ref2 = make_nugetref(ipa, "prefix_");
+ FeedReference ref2 = make_nugetref(ipa, "prefix_");
 
  REQUIRE(ref2.nupkg_filename() == "prefix_zlib2_x64-windows.1.5.0-vcpkgpackageabi.nupkg");
 
- NugetReference ref = make_nugetref(ipa, "");
+ FeedReference ref = make_nugetref(ipa, "");
 
  REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-vcpkgpackageabi.nupkg");
 

src/vcpkg-test/configparser.cpp

@@ -560,6 +560,42 @@ TEST_CASE ("BinaryConfigParser HTTP provider", "[binaryconfigparser]")
  }
 }
 
+TEST_CASE ("BinaryConfigParser Universal Packages provider", "[binaryconfigparser]")
+{
+ // Scheme: x-az-universal,<organization>,<project>,<feed>[,<readwrite>]
+ {
+ auto parsed =
+ parse_binary_provider_configs("x-az-universal,test_organization,test_project_name,test_feed,read", {});
+ auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
+ REQUIRE(state.upkg_templates_to_get.size() == 1);
+ REQUIRE(state.upkg_templates_to_get[0].feed == "test_feed");
+ REQUIRE(state.upkg_templates_to_get[0].organization == "test_organization");
+ REQUIRE(state.upkg_templates_to_get[0].project == "test_project_name");
+ }
+ {
+ auto parsed =
+ parse_binary_provider_configs("x-az-universal,test_organization,test_project_name,test_feed,readwrite", {});
+ auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
+ REQUIRE(state.upkg_templates_to_get.size() == 1);
+ REQUIRE(state.upkg_templates_to_put.size() == 1);
+ REQUIRE(state.upkg_templates_to_get[0].feed == "test_feed");
+ REQUIRE(state.upkg_templates_to_get[0].organization == "test_organization");
+ REQUIRE(state.upkg_templates_to_get[0].project == "test_project_name");
+ REQUIRE(state.upkg_templates_to_put[0].feed == "test_feed");
+ REQUIRE(state.upkg_templates_to_put[0].organization == "test_organization");
+ REQUIRE(state.upkg_templates_to_put[0].project == "test_project_name");
+ }
+ {
+ auto parsed = parse_binary_provider_configs(
+ "x-az-universal,test_organization,test_project_name,test_feed,extra_argument,readwrite", {});
+ REQUIRE(!parsed.has_value());
+ }
+ {
+ auto parsed = parse_binary_provider_configs("x-az-universal,missing_args,read", {});
+ REQUIRE(!parsed.has_value());
+ }
+}
+
 TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]")
 {
  CHECK(parse_download_configuration({}));

src/vcpkg-test/tools.cpp

@@ -128,3 +128,15 @@ TEST_CASE ("extract_prefixed_nonwhitespace", "[tools]")
  CHECK(error_result.error() == "error: fooutil (fooutil.exe) produced unexpected output when attempting to "
  "determine the version:\nmalformed output");
 }
+
+TEST_CASE ("extract_prefixed_nonquote", "[tools]")
+{
+ CHECK(extract_prefixed_nonquote("fooutil version ", "fooutil", "fooutil version 1.2\"", "fooutil.exe")
+ .value_or_exit(VCPKG_LINE_INFO) == "1.2");
+ CHECK(extract_prefixed_nonquote("fooutil version ", "fooutil", "fooutil version 1.2 \" ", "fooutil.exe")
+ .value_or_exit(VCPKG_LINE_INFO) == "1.2 ");
+ auto error_result = extract_prefixed_nonquote("fooutil version ", "fooutil", "malformed output", "fooutil.exe");
+ CHECK(!error_result.has_value());
+ CHECK(error_result.error() == "error: fooutil (fooutil.exe) produced unexpected output when attempting to "
+ "determine the version:\nmalformed output");
+}