[vcpkg] Added python script to generate all packages file list, added…

… to azur… (#12177) * Added python script to generate all packages file list, added to azure pipeline to build at the end of the run * fixed an issue that causes some leading slashes were being removed * Header Database now includes hpp files for C++ header files * Changed Header Database to include all files under \include\ folder * Apply suggestions from code review, changing condition from eq to ne Co-authored-by: Billy O'Neal <[email protected]> * Update last condition from eq to ne Co-authored-by: Billy O'Neal <[email protected]>
microsoft · Jul 6, 2020 · aa2b9ef · aa2b9ef
1 parent 8974699
commit aa2b9ef
Show file tree

Hide file tree

Showing 5 changed files with 86 additions and 0 deletions.
diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -48,3 +48,15 @@ jobs:
  PathtoPublish: '$(System.ArtifactsDirectory)/failure-logs'
  ArtifactName: 'x64-linux port build failure logs'
  condition: failed()
+ - bash: |
+ python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/
+ displayName: 'Build a file list for all packages'
+ condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+ - task: PublishBuildArtifacts@1
+ displayName: 'Upload file lists for all packages'
+ condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+ inputs:
+ PathtoPublish: scripts/list_files
+ ArtifactName: "x64-linux package file lists"
diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -52,3 +52,15 @@ jobs:
  PathtoPublish: '$(System.ArtifactsDirectory)/failure-logs'
  ArtifactName: 'x64-osx port build failure logs'
  condition: failed()
+ - bash: |
+ python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
+ displayName: 'Build a file list for all packages'
+ condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+ - task: PublishBuildArtifacts@1
+ displayName: 'Upload file lists for all packages'
+ condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+ inputs:
+ PathtoPublish: scripts/list_files
+ ArtifactName: "x64-osx package file lists"
diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -61,3 +61,19 @@ jobs:
  PathtoPublish: '$(System.ArtifactsDirectory)\failure-logs'
  ArtifactName: '${{ parameters.triplet }} port build failure logs'
  condition: failed()
+ - task: PowerShell@2
+ displayName: "Generating all packages files"
+ condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+ inputs:
+ targetType: inline
+ script: |
+ $env:VCPKG_DOWNLOADS = "D:\downloads"
+ ./vcpkg.exe fetch python3
+ & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\
+ - task: PublishBuildArtifacts@1
+ displayName: 'Upload file lists for all packages'
+ condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+ inputs:
+ PathtoPublish: scripts/list_files
+ ArtifactName: "${{ parameters.triplet }} package file lists"
diff --git a/scripts/file_script.py b/scripts/file_script.py
@@ -0,0 +1,39 @@
+import os
+import os.path
+import sys
+
+
+keyword = "include/"
+
+def getFiles(path):
+ files = os.listdir(path)
+ return list(filter(lambda x: x[0] != '.', files))
+
+def gen_all_file_strings(path, files, headers, output):
+ for file in files:
+ package = file[:file.find("_")]
+ f = open(path + file)
+ for line in f:
+ idx = line.strip().find(keyword)
+ if idx >= 0 and line.strip()[-1] != "/":
+ headers.write(package + ":" + line[idx + len(keyword):])
+ output.write(package + ":" + line[idx-1:])
+ elif line.strip()[-1] != "/":
+ output.write(package + ":" + line[line.find("/"):])
+ f.close()
+
+def main(path):
+ try:
+ os.mkdir("scripts/list_files")
+ except FileExistsError:
+ print("Path already exists, continuing...")
+
+ headers = open("scripts/list_files/VCPKGHeadersDatabase.txt", mode='w')
+ output = open("scripts/list_files/VCPKGDatabase.txt", mode='w')
+ gen_all_file_strings(path, getFiles(path), headers, output)
+ headers.close()
+ output.close()
+
+if __name__ == "__main__":
+ main(sys.argv[1])
+
diff --git a/scripts/vcpkgTools.xml b/scripts/vcpkgTools.xml
@@ -1,5 +1,12 @@
 <?xml version="1.0"?>
 <tools version="2">
+ <tool name="python3" os="windows">
+ <version>3.8.3</version>
+ <exeRelativePath>python.exe</exeRelativePath>
+ <url>https://www.python.org/ftp/python/3.8.3/python-3.8.3-embed-win32.zip</url>
+ <sha512>8c9078f55b1b5d694e0e809eee6ccf8a6e15810dd4649e8ae1209bff30e102d49546ce970a5d519349ca7759d93146f459c316dc440737171f018600255dcd0a</sha512>
+ <archiveName>python-3.8.3-embed-win32.zip</archiveName>
+ </tool>
  <tool name="cmake" os="windows">
  <version>3.17.2</version>
  <exeRelativePath>cmake-3.17.2-win32-x86\bin\cmake.exe</exeRelativePath>