Restore file permissions after unzip

aws_lambda_builders/workflows/dotnet_clipackage/actions.py

@@ -108,7 +108,7 @@ def execute(self):
 
  # The dotnet lambda package command outputs a zip file for the package. To make this compatible
  # with the workflow, unzip the zip file into the artifacts directory and then delete the zip archive.
- self.os_utils.expand_zip(zipfullpath, self.artifacts_dir)
+ self.os_utils.unzip(zipfullpath, self.artifacts_dir)
 
  except DotnetCLIExecutionError as ex:
  raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/dotnet_clipackage/utils.py

@@ -1,14 +1,15 @@
 """
 Commonly used utilities
 """
-
+import logging
 import os
 import platform
-import shutil
 import subprocess
 import zipfile
 from aws_lambda_builders.utils import which
 
+LOG = logging.getLogger(__name__)
+
 
 class OSUtils(object):
  """
@@ -25,11 +26,126 @@ def is_windows(self):
  def which(self, executable, executable_search_paths=None):
  return which(executable, executable_search_paths=executable_search_paths)
 
- def expand_zip(self, zipfullpath, destination_dir):
- ziparchive = zipfile.ZipFile(zipfullpath, "r")
- ziparchive.extractall(destination_dir)
- ziparchive.close()
- os.remove(zipfullpath)
+ def unzip(self, zip_file_path, output_dir, permission=None):
+ """
+ This method and dependent methods were copied from SAM CLI, but with the addition of deleting the zip file
+ https:/aws/aws-sam-cli/blob/458076265651237a662a372f54d5b3df49fd6797/samcli/local/lambdafn/zip.py#L81
+
+ Unzip the given file into the given directory while preserving file permissions in the process.
+ Parameters
+ ----------
+ zip_file_path : str
+ Path to the zip file
+ output_dir : str
+ Path to the directory where the it should be unzipped to
+ permission : int
+ Permission to set in an octal int form
+ """
+
+ with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
+
+ # For each item in the zip file, extract the file and set permissions if available
+ for file_info in zip_ref.infolist():
+ extracted_path = self._extract(file_info, output_dir, zip_ref)
+
+ # If the extracted_path is a symlink, do not set the permissions. If the target of the symlink does not
+ # exist, then os.chmod will fail with FileNotFoundError
+ if not os.path.islink(extracted_path):
+ self._set_permissions(file_info, extracted_path)
+ self._override_permissions(extracted_path, permission)
+
+ if not os.path.islink(extracted_path):
+ self._override_permissions(output_dir, permission)
+
+ os.remove(zip_file_path)
+
+ def _is_symlink(self, file_info):
+ """
+ Check the upper 4 bits of the external attribute for a symlink.
+ See: https://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute
+ Parameters
+ ----------
+ file_info : zipfile.ZipInfo
+ The ZipInfo for a ZipFile
+ Returns
+ -------
+ bool
+ A response regarding whether the ZipInfo defines a symlink or not.
+ """
+
+ return (file_info.external_attr >> 28) == 0xA
+
+ def _extract(self, file_info, output_dir, zip_ref):
+ """
+ Unzip the given file into the given directory while preserving file permissions in the process.
+ Parameters
+ ----------
+ file_info : zipfile.ZipInfo
+ The ZipInfo for a ZipFile
+ output_dir : str
+ Path to the directory where the it should be unzipped to
+ zip_ref : zipfile.ZipFile
+ The ZipFile we are working with.
+ Returns
+ -------
+ string
+ Returns the target path the Zip Entry was extracted to.
+ """
+
+ # Handle any regular file/directory entries
+ if not self._is_symlink(file_info):
+ return zip_ref.extract(file_info, output_dir)
+
+ source = zip_ref.read(file_info.filename).decode("utf8")
+ link_name = os.path.normpath(os.path.join(output_dir, file_info.filename))
+
+ # make leading dirs if needed
+ leading_dirs = os.path.dirname(link_name)
+ if not os.path.exists(leading_dirs):
+ os.makedirs(leading_dirs)
+
+ # If the link already exists, delete it or symlink() fails
+ if os.path.lexists(link_name):
+ os.remove(link_name)
+
+ # Create a symbolic link pointing to source named link_name.
+ os.symlink(source, link_name)
+
+ return link_name
+
+ def _override_permissions(self, path, permission):
+ """
+ Forcefully override the permissions on the path
+ Parameters
+ ----------
+ path str
+ Path where the file or directory
+ permission octal int
+ Permission to set
+ """
+ if permission:
+ os.chmod(path, permission)
+
+ def _set_permissions(self, zip_file_info, extracted_path):
+ """
+ Sets permissions on the extracted file by reading the ``external_attr`` property of given file info.
+ Parameters
+ ----------
+ zip_file_info : zipfile.ZipInfo
+ Object containing information about a file within a zip archive
+ extracted_path : str
+ Path where the file has been extracted to
+ """
+
+ # Permission information is stored in first two bytes.
+ permission = zip_file_info.external_attr >> 16
+ if not permission:
+ # Zips created on certain Windows machines, however, might not have any permission information on them.
+ # Skip setting a permission on these files.
+ LOG.debug("File %s in zipfile does not have permission information", zip_file_info.filename)
+ return
+
+ os.chmod(extracted_path, permission)
 
  @property
  def pipe(self):

tests/integration/workflows/dotnet_clipackage/test_dotnet.py

@@ -45,6 +45,10 @@ def verify_architecture(self, deps_file_name, expected_architecture, version=Non
 
  self.assertEqual(target, target_name)
 
+ def verify_execute_permissions(self, entrypoint_file_name):
+ entrypoint_file_path = os.path.join(self.artifacts_dir, entrypoint_file_name)
+ self.assertTrue(os.access(entrypoint_file_path, os.X_OK))
+
 
 class TestDotnet31(TestDotnetBase):
  """
@@ -192,3 +196,29 @@ def test_with_defaults_file_arm64(self):
 
  self.assertEqual(expected_files, output_files)
  self.verify_architecture("WithDefaultsFile.deps.json", "linux-arm64", version="6.0")
+
+ def test_with_custom_runtime(self):
+ source_dir = os.path.join(self.TEST_DATA_FOLDER, "CustomRuntime6")
+
+ self.builder.build(
+ source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=X86_64
+ )
+
+ expected_files = {
+ "Amazon.Lambda.Core.dll",
+ "Amazon.Lambda.RuntimeSupport.dll",
+ "Amazon.Lambda.Serialization.SystemTextJson.dll",
+ "bootstrap",
+ "bootstrap.deps.json",
+ "bootstrap.dll",
+ "bootstrap.pdb",
+ "bootstrap.runtimeconfig.json",
+ }
+
+ output_files = set(os.listdir(self.artifacts_dir))
+
+ self.assertEqual(expected_files, output_files)
+ self.verify_architecture("bootstrap.deps.json", "linux-x64", version="6.0")
+ # Execute permissions are required for custom runtimes which bootstrap themselves, otherwise `sam local invoke`
+ # won't have permission to run the file
+ self.verify_execute_permissions("bootstrap")

tests/integration/workflows/dotnet_clipackage/testdata/CustomRuntime6/CustomRuntime6.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+ <PropertyGroup>
+ <OutputType>Exe</OutputType>
+ <TargetFramework>net6.0</TargetFramework>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <Nullable>enable</Nullable>
+ <AWSProjectType>Lambda</AWSProjectType>
+ <AssemblyName>bootstrap</AssemblyName>
+ <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+ <PublishReadyToRun>true</PublishReadyToRun>
+ </PropertyGroup>
+ <!-- 
+ When publishing Lambda functions for ARM64 to the provided.al2 runtime a newer version of libicu needs to be included
+ in the deployment bundle because .NET requires a newer version of libicu then is preinstalled with Amazon Linux 2.
+ -->
+ <ItemGroup Condition="'$(RuntimeIdentifier)' == 'linux-arm64'">
+ <RuntimeHostConfigurationOption Include="System.Globalization.AppLocalIcu" Value="68.2.0.9" />
+ <PackageReference Include="Microsoft.ICU.ICU4C.Runtime" Version="68.2.0.9" />
+ </ItemGroup>
+ <ItemGroup>
+ <PackageReference Include="Amazon.Lambda.RuntimeSupport" Version="1.8.2" />
+ <PackageReference Include="Amazon.Lambda.Core" Version="2.1.0" />
+ <PackageReference Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.3.0" />
+ </ItemGroup>
+</Project>

tests/integration/workflows/dotnet_clipackage/testdata/CustomRuntime6/Function.cs

@@ -0,0 +1,21 @@
+using Amazon.Lambda.Core;
+using Amazon.Lambda.RuntimeSupport;
+using Amazon.Lambda.Serialization.SystemTextJson;
+
+namespace CustomRuntime6;
+
+public class Function
+{
+ private static async Task Main(string[] args)
+ {
+ Func<string, ILambdaContext, string> handler = FunctionHandler;
+ await LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer())
+ .Build()
+ .RunAsync();
+ }
+
+ public static string FunctionHandler(string input, ILambdaContext context)
+ {
+ return input.ToUpper();
+ }
+}

tests/integration/workflows/dotnet_clipackage/testdata/CustomRuntime6/aws-lambda-tools-defaults.json

@@ -0,0 +1,16 @@
+{
+ "Information": [
+ "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.",
+ "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.",
+ "dotnet lambda help",
+ "All the command line options for the Lambda command can be specified in this file."
+ ],
+ "profile": "",
+ "region": "",
+ "configuration": "Release",
+ "function-runtime": "provided.al2",
+ "function-memory-size": 256,
+ "function-timeout": 30,
+ "function-handler": "bootstrap",
+ "msbuild-parameters": "--self-contained true"
+}

tests/unit/workflows/dotnet_clipackage/test_utils.py

@@ -0,0 +1,34 @@
+import os
+import stat
+import tempfile
+from unittest import TestCase
+from zipfile import ZipFile
+
+from aws_lambda_builders.workflows.dotnet_clipackage.utils import OSUtils
+
+
+class TestDotnetCliPackageWorkflow(TestCase):
+ def test_unzip_keeps_execute_permission_on_linux(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ with tempfile.TemporaryDirectory(dir=temp_dir) as output_dir:
+ test_file_name = "myFileToZip"
+ path_to_file_to_zip = os.path.join(temp_dir, test_file_name)
+ path_to_zip_file = os.path.join(temp_dir, "myZip.zip")
+ expected_output_file = os.path.join(output_dir, test_file_name)
+ with open(path_to_file_to_zip, "a") as the_file:
+ the_file.write("Hello World!")
+
+ # Set execute permissions on the file before zipping (won't do anything on Windows)
+ st = os.stat(path_to_file_to_zip)
+ os.chmod(path_to_file_to_zip, st.st_mode | stat.S_IEXEC | stat.S_IXOTH | stat.S_IXGRP)
+
+ # Zip the file
+ with ZipFile(path_to_zip_file, "w") as myzip:
+ myzip.write(path_to_file_to_zip, test_file_name)
+
+ # Unzip the file
+ OSUtils().unzip(path_to_zip_file, output_dir)
+ self.assertTrue(os.path.exists(expected_output_file))
+
+ # Assert that execute permissions are still applied
+ self.assertTrue(os.access(expected_output_file, os.X_OK))