From 88d774b5a7303c5856c30ed00cebbad987f18151 Mon Sep 17 00:00:00 2001 From: Jawadtp Date: Wed, 18 Oct 2023 14:40:05 +0530 Subject: [PATCH 01/13] Clean functionality works for select metadata types --- .../core/source_transforms/transforms.py | 217 +++++++++++++++++- cumulusci/salesforce_api/package_zip.py | 5 + cumulusci/tasks/salesforce/Deploy.py | 10 +- cumulusci/utils/__init__.py | 13 ++ 4 files changed, 243 insertions(+), 2 deletions(-) diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index ab311b1456..dd8fb6c5df 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -11,7 +11,7 @@ from lxml import etree as ET from pydantic import BaseModel, root_validator - +from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged from cumulusci.core.dependencies.utils import TaskContext from cumulusci.core.enums import StrEnum from cumulusci.core.exceptions import CumulusCIException, TaskOptionsError @@ -23,6 +23,7 @@ temporary_dir, tokenize_namespace, zip_clean_metaxml, + zip_clean_profile_metaxml ) from cumulusci.utils.xml import metadata_tree from cumulusci.utils.ziputils import process_text_in_zipfile @@ -211,6 +212,220 @@ def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: ) return zip_clean_metaxml(zf) +class CleanProfileMetaXMLTransform(SourceTransform): + """Source transform that cleans *-meta.xml files of invalid references.""" + + options_model = None + + identifier = "clean_profiles" + + + def _create_package_xml(self, input_dict: dict, api_version: str): + package_xml = '\n' + package_xml += '\n' + + for name, members in input_dict.items(): + package_xml += " \n" + for member in members: + package_xml += f" {member}\n" + package_xml += f" {name}\n" + package_xml += " \n" + + package_xml+=( + ' \n' + ' *\n' + ' ApexClass\n' + ' \n' + ' \n' + ' *\n' + ' Flow\n' + ' \n' + ' \n' + ' *\n' + ' ApexPage\n' + ' \n' + ' \n' + ' *\n' + ' CustomPermission\n' + ' \n' + ' \n' + ' *\n' + ' CustomTab\n' + ' \n' + ' \n' + ' *\n' + ' CustomApplication\n' + ' \n' + ' \n' + ' *\n' + ' CustomObject\n' + ' \n' + ' \n' + ' TestProfile\n' + ' Profile\n' + ' \n') + + package_xml += f" {api_version}\n" + package_xml += "\n" + + return package_xml + + def strip_namespace(self, element): + for elem in element.iter(): + if '}' in elem.tag: + elem.tag = elem.tag.split('}', 1)[1] + return element + + def fetchObjectAPINamesAndUserPermissionsFromProfileXML(self, docroot): + objectAPINames = set() + userPermissionNames = set() + + docroot = self.strip_namespace(docroot) + + for objectPermission in docroot.findall('.//objectPermissions'): + objectAPINames.add(objectPermission.find('object').text) + + for recordTypeVisibility in docroot.findall('.//recordTypeVisibilities'): + objectAPINames.add(recordTypeVisibility.find('recordType').text.split('.')[0]) + + for fieldPermission in docroot.findall('.//fieldPermissions'): + objectAPINames.add(fieldPermission.find('field').text.split('.')[0]) + + for userPermission in docroot.findall('.//userPermissions'): + userPermissionNames.add(userPermission.find('name').text) + + return (objectAPINames, userPermissionNames) + + def CleanProfileXML(self, docroot, orgMetadataDict): + docroot = self.strip_namespace(docroot) + + for objectPermission in docroot.findall('.//objectPermissions'): + if objectPermission.find('object').text not in orgMetadataDict["objects"]: + docroot.remove(objectPermission) + + for fieldPermission in docroot.findall('.//fieldPermissions'): + if fieldPermission.find('field').text not in orgMetadataDict["fields"]: + docroot.remove(fieldPermission) + + for tabVisibility in docroot.findall('.//tabVisibilities'): + if tabVisibility.find('tab').text not in orgMetadataDict["tabs"]: + docroot.remove(tabVisibility) + + for applicationVisibility in docroot.findall('.//applicationVisibilities'): + if applicationVisibility.find('application').text not in orgMetadataDict["applications"]: + docroot.remove(applicationVisibility) + + for customPermission in docroot.findall('.//customPermissions'): + if customPermission.find('name').text not in orgMetadataDict["customPermissions"]: + docroot.remove(customPermission) + + for pageAccess in docroot.findall('.//pageAccesses'): + if pageAccess.find('apexPage').text not in orgMetadataDict["pages"]: + docroot.remove(pageAccess) + + for classAccess in docroot.findall('.//classAccesses'): + if classAccess.find('apexClass').text not in orgMetadataDict["classes"]: + docroot.remove(classAccess) + + for flowAccess in docroot.findall('.//flowAccesses'): + if flowAccess.find('flow').text not in orgMetadataDict["flows"]: + docroot.remove(flowAccess) + + for recordTypeVisibility in docroot.findall('.//recordTypeVisibilities'): + if recordTypeVisibility.find('recordType').text not in orgMetadataDict["recordTypes"]: + docroot.remove(recordTypeVisibility) + + return docroot + + + def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: + context.logger.info( + "Cleaning profile meta.xml files of invalid references" + ) + + objectAPINames = set() + userPermissionsNames = set() + + for name in zf.namelist(): + if name.endswith('.profile'): + # Open and remove invalid references + f = zf.open(name) + docroot = ET.parse(f).getroot() + objectAPINamesTemp, userPermissionsNamesTemp = self.fetchObjectAPINamesAndUserPermissionsFromProfileXML(docroot=docroot) + objectAPINames.update(objectAPINamesTemp) + userPermissionsNames.update(userPermissionsNamesTemp) + + package_xml = self._create_package_xml(input_dict={'CustomObject': list(objectAPINames)}, api_version='58') + context.logger.info(package_xml) + + api = ApiRetrieveUnpackaged(context, package_xml=package_xml, api_version='58') + + obj_zf = api() + context.logger.info(obj_zf.namelist()) + + # for name in obj_zf.namelist(): + # f = zf.open(name) + # docroot = ET.parse(f).getroot() + # context.logger.info(ET.tostring(docroot)) + + obj_zf.extractall('./unpackaged') + + #orgMetadata = obj_zf.open(name) + + orgMetadataDict = {} + + for name in obj_zf.namelist(): + if name=='package.xml': + continue + + + metadataType = name.split('/')[0] + metadataName = name.split('/')[1].split('.')[0] + orgMetadataDict.setdefault(metadataType, set()).update([metadataName]) + + if name.endswith('.object'): + f = obj_zf.open(name) + docroot = ET.parse(f).getroot() + docroot = self.strip_namespace(docroot) + + for field in docroot.findall('fields'): + orgMetadataDict.setdefault('fields', set()).update([metadataName+'.'+field.find('fullName').text]) + + for recordType in docroot.findall('recordTypes'): + orgMetadataDict.setdefault('recordTypes', set()).update([metadataName+'.'+recordType.find('fullName').text]) + + context.logger.info(orgMetadataDict['fields']) + + zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) + # for name in zf.namelist(): + # f = zf.open(name) + # docroot = ET.parse(f).getroot() + # if name.endswith('.profile'): + # # Open and remove invalid references + # cleaned_doc = self.CleanProfileXML(docroot=docroot, orgMetadataDict=orgMetadataDict) + # docroot = cleaned_doc + + # tree = ET.ElementTree(cleaned_doc) + # tree.write(f'unpackaged/{name}_cleaned.xml', pretty_print=True, xml_declaration=True, encoding="utf-8") + + for name in zf.namelist(): + f = zf.open(name) + + if name.endswith('.profile'): + # If it's a .profile file, clean the XML content + docroot = ET.parse(f).getroot() + cleaned_doc = self.CleanProfileXML(docroot, orgMetadataDict) + tree = ET.ElementTree(cleaned_doc) + cleaned_content = io.BytesIO() + tree.write(f'unpackaged/{name}_cleaned.xml', pretty_print=True, xml_declaration=True, encoding="utf-8") + tree.write(cleaned_content, pretty_print=True, xml_declaration=True, encoding="utf-8") + zip_dest.writestr(name, cleaned_content.getvalue()) + else: + # If it's not a .profile file, add it to the new ZIP archive as is + zip_dest.writestr(name, f.read()) + + return zip_dest + class BundleStaticResourcesOptions(BaseModel): static_resource_path: str diff --git a/cumulusci/salesforce_api/package_zip.py b/cumulusci/salesforce_api/package_zip.py index 5e6fbefdff..75e9fdf6fb 100644 --- a/cumulusci/salesforce_api/package_zip.py +++ b/cumulusci/salesforce_api/package_zip.py @@ -13,6 +13,7 @@ BundleStaticResourcesOptions, BundleStaticResourcesTransform, CleanMetaXMLTransform, + CleanProfileMetaXMLTransform, NamespaceInjectionOptions, NamespaceInjectionTransform, RemoveFeatureParametersTransform, @@ -189,6 +190,10 @@ def _process(self): # -meta.xml cleaning if self.options.get("clean_meta_xml", True): transforms.append(CleanMetaXMLTransform()) + + if self.options.get("clean_profiles", True): + transforms.append(CleanProfileMetaXMLTransform()) + # Static resource bundling relpath = self.options.get("static_resource_path") if relpath and os.path.exists(relpath): diff --git a/cumulusci/tasks/salesforce/Deploy.py b/cumulusci/tasks/salesforce/Deploy.py index 4f42a8a14d..e1a33a9b5f 100644 --- a/cumulusci/tasks/salesforce/Deploy.py +++ b/cumulusci/tasks/salesforce/Deploy.py @@ -56,7 +56,12 @@ class Deploy(BaseSalesforceMetadataApiTask): "transforms": { "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms." }, - "rest_deploy": {"description": "If True, deploy metadata using REST API"}, + "rest_deploy": { + "description": "If True, deploy metadata using REST API" + }, + "clean_profiles": { + "description": "If specified, all profiles are cleaned of invalid references before deployment." + }, } namespaces = {"sf": "http://soap.sforce.com/2006/04/metadata"} @@ -149,6 +154,9 @@ def _get_package_zip(self, path) -> Optional[str]: "clean_meta_xml": process_bool_arg( self.options.get("clean_meta_xml", True) ), + "clean_profiles": process_bool_arg( + self.options.get("clean_meta_xml", True) + ), "namespace_inject": namespace, "unmanaged": not self._has_namespaced_package(namespace), "namespaced_org": self._is_namespaced_org(namespace), diff --git a/cumulusci/utils/__init__.py b/cumulusci/utils/__init__.py index 605818d96c..72de2d98eb 100644 --- a/cumulusci/utils/__init__.py +++ b/cumulusci/utils/__init__.py @@ -348,6 +348,19 @@ def zip_clean_metaxml(zip_src, logger=None): zip_src.close() return zip_dest +def zip_clean_profile_metaxml(zip_src, logger=None): + """Given a zipfile, cleans all ``*-meta.xml`` files in the zip for + deployment by stripping all ```` elements + """ + zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) + changed = [] + for name in zip_src.namelist(): + logger.info( + f'File name: {name}') + + + zip_src.close() + return zip_dest def doc_task(task_name, task_config, project_config=None, org_config=None): """Document a (project specific) task configuration in RST format.""" From b75aae9c4f5d51dc57d1f6262594f44f2e898a22 Mon Sep 17 00:00:00 2001 From: Jawadtp Date: Wed, 18 Oct 2023 17:58:28 +0530 Subject: [PATCH 02/13] Support added for customMetadata and customSettings --- .../core/source_transforms/transforms.py | 173 +++++++++--------- cumulusci/tasks/salesforce/Deploy.py | 11 +- cumulusci/utils/__init__.py | 14 -- 3 files changed, 88 insertions(+), 110 deletions(-) diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index dd8fb6c5df..2f5db7dad1 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -23,7 +23,6 @@ temporary_dir, tokenize_namespace, zip_clean_metaxml, - zip_clean_profile_metaxml ) from cumulusci.utils.xml import metadata_tree from cumulusci.utils.ziputils import process_text_in_zipfile @@ -219,7 +218,6 @@ class CleanProfileMetaXMLTransform(SourceTransform): identifier = "clean_profiles" - def _create_package_xml(self, input_dict: dict, api_version: str): package_xml = '\n' package_xml += '\n' @@ -231,45 +229,25 @@ def _create_package_xml(self, input_dict: dict, api_version: str): package_xml += f" {name}\n" package_xml += " \n" - package_xml+=( - ' \n' - ' *\n' - ' ApexClass\n' - ' \n' - ' \n' - ' *\n' - ' Flow\n' - ' \n' - ' \n' - ' *\n' - ' ApexPage\n' - ' \n' - ' \n' - ' *\n' - ' CustomPermission\n' - ' \n' - ' \n' - ' *\n' - ' CustomTab\n' - ' \n' - ' \n' - ' *\n' - ' CustomApplication\n' - ' \n' - ' \n' - ' *\n' - ' CustomObject\n' - ' \n' - ' \n' - ' TestProfile\n' - ' Profile\n' - ' \n') - + metadataToFetch = [ + "ApexClass", "Flow", "ApexPage", "CustomPermission", + "CustomTab", "CustomApplication", "CustomObject", "CustomMetadataType" + ] + + for item in metadataToFetch: + package_xml+=( + ' \n' + ' *\n' + f' {item}\n' + ' \n') + package_xml += f" {api_version}\n" package_xml += "\n" return package_xml + + def strip_namespace(self, element): for elem in element.iter(): if '}' in elem.tag: @@ -277,6 +255,8 @@ def strip_namespace(self, element): return element def fetchObjectAPINamesAndUserPermissionsFromProfileXML(self, docroot): + # Scans through the Profile.xml file whose docroot is specified to collect object API names and user permissions. + # These are used to build a Profile.xml file to fetch information from the org. objectAPINames = set() userPermissionNames = set() @@ -297,6 +277,7 @@ def fetchObjectAPINamesAndUserPermissionsFromProfileXML(self, docroot): return (objectAPINames, userPermissionNames) def CleanProfileXML(self, docroot, orgMetadataDict): + # Function to clean the profile xml whose docroot is provided of invalid references. docroot = self.strip_namespace(docroot) for objectPermission in docroot.findall('.//objectPermissions'): @@ -335,96 +316,106 @@ def CleanProfileXML(self, docroot, orgMetadataDict): if recordTypeVisibility.find('recordType').text not in orgMetadataDict["recordTypes"]: docroot.remove(recordTypeVisibility) + for customSettingAccess in docroot.findall('.//customSettingAccesses'): + if customSettingAccess.find('name').text not in orgMetadataDict["customSettings"]: + docroot.remove(customSettingAccess) + + for customMetadataTypeAccess in docroot.findall('.//customMetadataTypeAccesses'): + if customMetadataTypeAccess.find('name').text not in orgMetadataDict["customMetadataType"]: + docroot.remove(customMetadataTypeAccess) + return docroot + def cleanProfilesinZipOfInvalidReferences(self, zf: ZipFile, orgMetadataDict: dict): + zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) + + for name in zf.namelist(): + f = zf.open(name) + + if name.endswith('.profile'): + # If it's a .profile file, clean the XML content + docroot = ET.parse(f).getroot() + cleaned_doc = self.CleanProfileXML(docroot, orgMetadataDict) + tree = ET.ElementTree(cleaned_doc) + cleaned_content = io.BytesIO() + #tree.write(f'unpackaged/{name}_cleaned.xml', pretty_print=True, xml_declaration=True, encoding="utf-8") + tree.write(cleaned_content, pretty_print=True, xml_declaration=True, encoding="utf-8") + zip_dest.writestr(name, cleaned_content.getvalue()) + else: + # If it's not a .profile file, add it to the new ZIP archive with no changes. + zip_dest.writestr(name, f.read()) - def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: - context.logger.info( - "Cleaning profile meta.xml files of invalid references" - ) - + return zip_dest + + def collectObjectAPINamesAndUserPermissionNamesFromProfiles(self, zf: ZipFile): objectAPINames = set() userPermissionsNames = set() for name in zf.namelist(): if name.endswith('.profile'): - # Open and remove invalid references f = zf.open(name) docroot = ET.parse(f).getroot() objectAPINamesTemp, userPermissionsNamesTemp = self.fetchObjectAPINamesAndUserPermissionsFromProfileXML(docroot=docroot) objectAPINames.update(objectAPINamesTemp) userPermissionsNames.update(userPermissionsNamesTemp) - package_xml = self._create_package_xml(input_dict={'CustomObject': list(objectAPINames)}, api_version='58') - context.logger.info(package_xml) - - api = ApiRetrieveUnpackaged(context, package_xml=package_xml, api_version='58') - - obj_zf = api() - context.logger.info(obj_zf.namelist()) - - # for name in obj_zf.namelist(): - # f = zf.open(name) - # docroot = ET.parse(f).getroot() - # context.logger.info(ET.tostring(docroot)) - - obj_zf.extractall('./unpackaged') - - #orgMetadata = obj_zf.open(name) - + return (objectAPINames, userPermissionsNames) + + def createMetadataDictFromZipfile(self, zf: ZipFile): orgMetadataDict = {} - for name in obj_zf.namelist(): + for name in zf.namelist(): if name=='package.xml': continue - metadataType = name.split('/')[0] metadataName = name.split('/')[1].split('.')[0] - orgMetadataDict.setdefault(metadataType, set()).update([metadataName]) if name.endswith('.object'): - f = obj_zf.open(name) + f = zf.open(name) docroot = ET.parse(f).getroot() docroot = self.strip_namespace(docroot) + if docroot.xpath( "//customSettingsType"): + orgMetadataDict.setdefault('customSettings', set()).update([metadataName]) + continue + + # It is a custom metadata type object + if metadataName.endswith('__mdt'): + orgMetadataDict.setdefault('customMetadataType', set()).update([metadataName]) + for field in docroot.findall('fields'): orgMetadataDict.setdefault('fields', set()).update([metadataName+'.'+field.find('fullName').text]) for recordType in docroot.findall('recordTypes'): orgMetadataDict.setdefault('recordTypes', set()).update([metadataName+'.'+recordType.find('fullName').text]) - context.logger.info(orgMetadataDict['fields']) + orgMetadataDict.setdefault(metadataType, set()).update([metadataName]) + + return orgMetadataDict + + def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: + context.logger.info( + "Cleaning profile meta.xml files of invalid references" + ) - zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) - # for name in zf.namelist(): - # f = zf.open(name) - # docroot = ET.parse(f).getroot() - # if name.endswith('.profile'): - # # Open and remove invalid references - # cleaned_doc = self.CleanProfileXML(docroot=docroot, orgMetadataDict=orgMetadataDict) - # docroot = cleaned_doc - - # tree = ET.ElementTree(cleaned_doc) - # tree.write(f'unpackaged/{name}_cleaned.xml', pretty_print=True, xml_declaration=True, encoding="utf-8") + objectAPINames, userPermissionsNames = self.collectObjectAPINamesAndUserPermissionNamesFromProfiles(zf=zf) - for name in zf.namelist(): - f = zf.open(name) - - if name.endswith('.profile'): - # If it's a .profile file, clean the XML content - docroot = ET.parse(f).getroot() - cleaned_doc = self.CleanProfileXML(docroot, orgMetadataDict) - tree = ET.ElementTree(cleaned_doc) - cleaned_content = io.BytesIO() - tree.write(f'unpackaged/{name}_cleaned.xml', pretty_print=True, xml_declaration=True, encoding="utf-8") - tree.write(cleaned_content, pretty_print=True, xml_declaration=True, encoding="utf-8") - zip_dest.writestr(name, cleaned_content.getvalue()) - else: - # If it's not a .profile file, add it to the new ZIP archive as is - zip_dest.writestr(name, f.read()) + package_xml = self._create_package_xml(input_dict={'CustomObject': list(objectAPINames)}, api_version='58') + + context.logger.info(package_xml) + + api = ApiRetrieveUnpackaged(context, package_xml=package_xml, api_version='58') - return zip_dest + obj_zf = api() + context.logger.info(obj_zf.namelist()) + + obj_zf.extractall('./unpackaged') + + orgMetadataDict = self.createMetadataDictFromZipfile(zf=obj_zf) + + return self.cleanProfilesinZipOfInvalidReferences(zf=zf, orgMetadataDict=orgMetadataDict) + class BundleStaticResourcesOptions(BaseModel): diff --git a/cumulusci/tasks/salesforce/Deploy.py b/cumulusci/tasks/salesforce/Deploy.py index e1a33a9b5f..70c5475eb2 100644 --- a/cumulusci/tasks/salesforce/Deploy.py +++ b/cumulusci/tasks/salesforce/Deploy.py @@ -53,15 +53,15 @@ class Deploy(BaseSalesforceMetadataApiTask): "clean_meta_xml": { "description": "Defaults to True which strips the element from all meta.xml files. The packageVersion element gets added automatically by the target org and is set to whatever version is installed in the org. To disable this, set this option to False" }, + "clean_profiles": { + "description": "Defaults to True in which case all profiles are cleaned of invalid references before deployment." + }, "transforms": { "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms." }, "rest_deploy": { "description": "If True, deploy metadata using REST API" - }, - "clean_profiles": { - "description": "If specified, all profiles are cleaned of invalid references before deployment." - }, + } } namespaces = {"sf": "http://soap.sforce.com/2006/04/metadata"} @@ -155,13 +155,14 @@ def _get_package_zip(self, path) -> Optional[str]: self.options.get("clean_meta_xml", True) ), "clean_profiles": process_bool_arg( - self.options.get("clean_meta_xml", True) + self.options.get("clean_profiles", True) ), "namespace_inject": namespace, "unmanaged": not self._has_namespaced_package(namespace), "namespaced_org": self._is_namespaced_org(namespace), } package_zip = None + with convert_sfdx_source(path, None, self.logger) as src_path: context = TaskContext(self.org_config, self.project_config, self.logger) package_zip = MetadataPackageZipBuilder( diff --git a/cumulusci/utils/__init__.py b/cumulusci/utils/__init__.py index 72de2d98eb..7bcc67ee30 100644 --- a/cumulusci/utils/__init__.py +++ b/cumulusci/utils/__init__.py @@ -348,20 +348,6 @@ def zip_clean_metaxml(zip_src, logger=None): zip_src.close() return zip_dest -def zip_clean_profile_metaxml(zip_src, logger=None): - """Given a zipfile, cleans all ``*-meta.xml`` files in the zip for - deployment by stripping all ```` elements - """ - zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) - changed = [] - for name in zip_src.namelist(): - logger.info( - f'File name: {name}') - - - zip_src.close() - return zip_dest - def doc_task(task_name, task_config, project_config=None, org_config=None): """Document a (project specific) task configuration in RST format.""" from cumulusci.core.utils import import_global From 20bfc49df7c9958d539d01a1e37251a165a30acb Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Thu, 19 Oct 2023 16:17:54 +0530 Subject: [PATCH 03/13] functionality for user permissions --- .../core/source_transforms/transforms.py | 259 ++++-------------- cumulusci/utils/__init__.py | 13 - cumulusci/utils/clean_before_deploy.py | 229 ++++++++++++++++ 3 files changed, 285 insertions(+), 216 deletions(-) create mode 100644 cumulusci/utils/clean_before_deploy.py diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index dd8fb6c5df..f2b026d6b2 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -11,10 +11,12 @@ from lxml import etree as ET from pydantic import BaseModel, root_validator -from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged + from cumulusci.core.dependencies.utils import TaskContext from cumulusci.core.enums import StrEnum from cumulusci.core.exceptions import CumulusCIException, TaskOptionsError +from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged +from cumulusci.salesforce_api.utils import get_simple_salesforce_connection from cumulusci.tasks.metadata.package import RemoveSourceComponents from cumulusci.utils import ( cd, @@ -23,7 +25,11 @@ temporary_dir, tokenize_namespace, zip_clean_metaxml, - zip_clean_profile_metaxml +) +from cumulusci.utils.clean_before_deploy import ( + get_target_entities_from_zip, + return_package_xml_from_zip, + zip_clean_invalid_references, ) from cumulusci.utils.xml import metadata_tree from cumulusci.utils.ziputils import process_text_in_zipfile @@ -212,220 +218,67 @@ def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: ) return zip_clean_metaxml(zf) + class CleanProfileMetaXMLTransform(SourceTransform): """Source transform that cleans *-meta.xml files of invalid references.""" options_model = None - identifier = "clean_profiles" + api_version = "58.0" + + def entities_from_package(self, zf, context): + package_xml = return_package_xml_from_zip(zf) + api = ApiRetrieveUnpackaged( + context, package_xml=package_xml, api_version=self.api_version + ) + retrieved_zf = api() + return get_target_entities_from_zip(retrieved_zf) + + def ret_sf(self, context): + sf = get_simple_salesforce_connection( + context.project_config, + context.org_config, + api_version=self.api_version, + base_url=None, + ) + return sf + + def entities_user_permission(self, sf): + path = "sobjects/PermissionSet/describe" + urlpath = sf.base_url + path + method = "GET" + response = sf._call_salesforce(method, urlpath) + + fields = [ + f["name"].replace("Permissions", "") + for f in response.json()["fields"] + if f["name"].startswith("Permissions") and f["type"] == "boolean" + ] + return set(fields) + def entities_tabs(self, sf): + soql = "SELECT Name FROM TabDefinition" + result = sf.query(soql)["records"] - def _create_package_xml(self, input_dict: dict, api_version: str): - package_xml = '\n' - package_xml += '\n' - - for name, members in input_dict.items(): - package_xml += " \n" - for member in members: - package_xml += f" {member}\n" - package_xml += f" {name}\n" - package_xml += " \n" - - package_xml+=( - ' \n' - ' *\n' - ' ApexClass\n' - ' \n' - ' \n' - ' *\n' - ' Flow\n' - ' \n' - ' \n' - ' *\n' - ' ApexPage\n' - ' \n' - ' \n' - ' *\n' - ' CustomPermission\n' - ' \n' - ' \n' - ' *\n' - ' CustomTab\n' - ' \n' - ' \n' - ' *\n' - ' CustomApplication\n' - ' \n' - ' \n' - ' *\n' - ' CustomObject\n' - ' \n' - ' \n' - ' TestProfile\n' - ' Profile\n' - ' \n') - - package_xml += f" {api_version}\n" - package_xml += "\n" - - return package_xml - - def strip_namespace(self, element): - for elem in element.iter(): - if '}' in elem.tag: - elem.tag = elem.tag.split('}', 1)[1] - return element - - def fetchObjectAPINamesAndUserPermissionsFromProfileXML(self, docroot): - objectAPINames = set() - userPermissionNames = set() - - docroot = self.strip_namespace(docroot) - - for objectPermission in docroot.findall('.//objectPermissions'): - objectAPINames.add(objectPermission.find('object').text) - - for recordTypeVisibility in docroot.findall('.//recordTypeVisibilities'): - objectAPINames.add(recordTypeVisibility.find('recordType').text.split('.')[0]) - - for fieldPermission in docroot.findall('.//fieldPermissions'): - objectAPINames.add(fieldPermission.find('field').text.split('.')[0]) - - for userPermission in docroot.findall('.//userPermissions'): - userPermissionNames.add(userPermission.find('name').text) - - return (objectAPINames, userPermissionNames) - - def CleanProfileXML(self, docroot, orgMetadataDict): - docroot = self.strip_namespace(docroot) - - for objectPermission in docroot.findall('.//objectPermissions'): - if objectPermission.find('object').text not in orgMetadataDict["objects"]: - docroot.remove(objectPermission) - - for fieldPermission in docroot.findall('.//fieldPermissions'): - if fieldPermission.find('field').text not in orgMetadataDict["fields"]: - docroot.remove(fieldPermission) - - for tabVisibility in docroot.findall('.//tabVisibilities'): - if tabVisibility.find('tab').text not in orgMetadataDict["tabs"]: - docroot.remove(tabVisibility) - - for applicationVisibility in docroot.findall('.//applicationVisibilities'): - if applicationVisibility.find('application').text not in orgMetadataDict["applications"]: - docroot.remove(applicationVisibility) - - for customPermission in docroot.findall('.//customPermissions'): - if customPermission.find('name').text not in orgMetadataDict["customPermissions"]: - docroot.remove(customPermission) - - for pageAccess in docroot.findall('.//pageAccesses'): - if pageAccess.find('apexPage').text not in orgMetadataDict["pages"]: - docroot.remove(pageAccess) - - for classAccess in docroot.findall('.//classAccesses'): - if classAccess.find('apexClass').text not in orgMetadataDict["classes"]: - docroot.remove(classAccess) - - for flowAccess in docroot.findall('.//flowAccesses'): - if flowAccess.find('flow').text not in orgMetadataDict["flows"]: - docroot.remove(flowAccess) - - for recordTypeVisibility in docroot.findall('.//recordTypeVisibilities'): - if recordTypeVisibility.find('recordType').text not in orgMetadataDict["recordTypes"]: - docroot.remove(recordTypeVisibility) - - return docroot - + tabs = [record["Name"] for record in result] + return set(tabs) def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: - context.logger.info( - "Cleaning profile meta.xml files of invalid references" - ) + context.logger.info("Cleaning profile meta.xml files of invalid references") - objectAPINames = set() - userPermissionsNames = set() + sf = self.ret_sf(context) - for name in zf.namelist(): - if name.endswith('.profile'): - # Open and remove invalid references - f = zf.open(name) - docroot = ET.parse(f).getroot() - objectAPINamesTemp, userPermissionsNamesTemp = self.fetchObjectAPINamesAndUserPermissionsFromProfileXML(docroot=docroot) - objectAPINames.update(objectAPINamesTemp) - userPermissionsNames.update(userPermissionsNamesTemp) - - package_xml = self._create_package_xml(input_dict={'CustomObject': list(objectAPINames)}, api_version='58') - context.logger.info(package_xml) - - api = ApiRetrieveUnpackaged(context, package_xml=package_xml, api_version='58') - - obj_zf = api() - context.logger.info(obj_zf.namelist()) - - # for name in obj_zf.namelist(): - # f = zf.open(name) - # docroot = ET.parse(f).getroot() - # context.logger.info(ET.tostring(docroot)) - - obj_zf.extractall('./unpackaged') - - #orgMetadata = obj_zf.open(name) - - orgMetadataDict = {} - - for name in obj_zf.namelist(): - if name=='package.xml': - continue - - - metadataType = name.split('/')[0] - metadataName = name.split('/')[1].split('.')[0] - orgMetadataDict.setdefault(metadataType, set()).update([metadataName]) - - if name.endswith('.object'): - f = obj_zf.open(name) - docroot = ET.parse(f).getroot() - docroot = self.strip_namespace(docroot) - - for field in docroot.findall('fields'): - orgMetadataDict.setdefault('fields', set()).update([metadataName+'.'+field.find('fullName').text]) - - for recordType in docroot.findall('recordTypes'): - orgMetadataDict.setdefault('recordTypes', set()).update([metadataName+'.'+recordType.find('fullName').text]) - - context.logger.info(orgMetadataDict['fields']) + target_entites = {} + target_entites.update(self.entities_from_package(zf, context)) + target_entites.setdefault("userPermissions", set()).update( + self.entities_user_permission(sf) + ) + target_entites.setdefault("tabs", set()).update(self.entities_tabs(sf)) - zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) - # for name in zf.namelist(): - # f = zf.open(name) - # docroot = ET.parse(f).getroot() - # if name.endswith('.profile'): - # # Open and remove invalid references - # cleaned_doc = self.CleanProfileXML(docroot=docroot, orgMetadataDict=orgMetadataDict) - # docroot = cleaned_doc - - # tree = ET.ElementTree(cleaned_doc) - # tree.write(f'unpackaged/{name}_cleaned.xml', pretty_print=True, xml_declaration=True, encoding="utf-8") - - for name in zf.namelist(): - f = zf.open(name) - - if name.endswith('.profile'): - # If it's a .profile file, clean the XML content - docroot = ET.parse(f).getroot() - cleaned_doc = self.CleanProfileXML(docroot, orgMetadataDict) - tree = ET.ElementTree(cleaned_doc) - cleaned_content = io.BytesIO() - tree.write(f'unpackaged/{name}_cleaned.xml', pretty_print=True, xml_declaration=True, encoding="utf-8") - tree.write(cleaned_content, pretty_print=True, xml_declaration=True, encoding="utf-8") - zip_dest.writestr(name, cleaned_content.getvalue()) - else: - # If it's not a .profile file, add it to the new ZIP archive as is - zip_dest.writestr(name, f.read()) + print(target_entites["tabs"]) + + return zip_clean_invalid_references(zf, target_entites) - return zip_dest - class BundleStaticResourcesOptions(BaseModel): static_resource_path: str diff --git a/cumulusci/utils/__init__.py b/cumulusci/utils/__init__.py index 72de2d98eb..605818d96c 100644 --- a/cumulusci/utils/__init__.py +++ b/cumulusci/utils/__init__.py @@ -348,19 +348,6 @@ def zip_clean_metaxml(zip_src, logger=None): zip_src.close() return zip_dest -def zip_clean_profile_metaxml(zip_src, logger=None): - """Given a zipfile, cleans all ``*-meta.xml`` files in the zip for - deployment by stripping all ```` elements - """ - zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) - changed = [] - for name in zip_src.namelist(): - logger.info( - f'File name: {name}') - - - zip_src.close() - return zip_dest def doc_task(task_name, task_config, project_config=None, org_config=None): """Document a (project specific) task configuration in RST format.""" diff --git a/cumulusci/utils/clean_before_deploy.py b/cumulusci/utils/clean_before_deploy.py new file mode 100644 index 0000000000..7ed19620c8 --- /dev/null +++ b/cumulusci/utils/clean_before_deploy.py @@ -0,0 +1,229 @@ +import io +import zipfile + +from lxml import etree as ET + + +class FileName: + folder_name: str + extension: str + + def __init__(self, folder_name, extension): + self.folder_name = folder_name + self.extension = extension + + +PROFILE_FILE = FileName("profiles/", ".profile-meta.xml") +PERMISSIONSET_FILE = FileName("permissionsets/", ".permissionset") +FILES_TO_BE_CLEANED = [PROFILE_FILE, PERMISSIONSET_FILE] + + +class PermissionElementXPath: + permission_xpath: str + name_xpath: str + + def __init__(self, permission_xpath, name_xpath): + self.permission_xpath = permission_xpath + self.name_xpath = name_xpath + + def return_parent(self, element: ET.Element): + return element.find(self.name_xpath).text.split(".")[0] + + def return_name(self, element: ET.Element): + return element.find(self.name_xpath).text + + +OBJECT_PERMISSION = PermissionElementXPath(".//objectPermissions", "object") +RECORDTYPE_PERMISSION = PermissionElementXPath( + ".//recordTypeVisibilities", "recordType" +) +FIELD_PERMISSION = PermissionElementXPath(".//fieldPermissions", "field") +USER_PERMISSION = PermissionElementXPath(".//userPermissions", "name") +TAB_PERMISSION = PermissionElementXPath(".//tabVisibilities", "tab") +APP_PERMISSION = PermissionElementXPath(".//applicationVisibilities", "application") +APEXCLASS_PERMISSION = PermissionElementXPath(".//classAccesses", "apexClass") +APEXPAGE_PERMISSION = PermissionElementXPath(".//pageAccesses", "apexPage") +FLOW_PERMISSION = PermissionElementXPath(".//flowAccesses", "flow") +CUSTOMPERM_PERMISSION = PermissionElementXPath(".//customPermissions", "name") +CUSTOMSETTING_PERMISSION = PermissionElementXPath(".//customSettingAccesses", "name") +CUSTOMMETADATA_PERMISSION = PermissionElementXPath( + ".//customMetadataTypeAccesses", "name" +) + +PACKAGEXML_DICT = { + # PERMISSION : is_child + "CustomObject": { + OBJECT_PERMISSION: False, + CUSTOMSETTING_PERMISSION: False, + CUSTOMMETADATA_PERMISSION: False, + RECORDTYPE_PERMISSION: True, + FIELD_PERMISSION: True, + }, + "ApexClass": APEXCLASS_PERMISSION, + "ApexPage": APEXPAGE_PERMISSION, + "Flow": FLOW_PERMISSION, + "CustomTab": TAB_PERMISSION, + "CustomApplication": APP_PERMISSION, + "CustomPermission": CUSTOMPERM_PERMISSION, +} + +FOLDER_PERM_DICT = { + "objects": [OBJECT_PERMISSION, CUSTOMSETTING_PERMISSION, CUSTOMMETADATA_PERMISSION], + "fields": [FIELD_PERMISSION], + "tabs": [TAB_PERMISSION], + "applications": [APP_PERMISSION], + "customPermissions": [CUSTOMPERM_PERMISSION], + "pages": [APEXPAGE_PERMISSION], + "classes": [APEXCLASS_PERMISSION], + "flows": [FLOW_PERMISSION], + "recordTypes": [RECORDTYPE_PERMISSION], + "userPermissions": [USER_PERMISSION], +} + + +def return_package_xml_from_zip(zip_src, api_version: str = "58.0"): + # Iterate through the zip file to generate the package.xml + package_xml_input = {} + for name in zip_src.namelist(): + if any( + name.endswith(item.extension) and name.startswith(item.folder_name) + for item in FILES_TO_BE_CLEANED + ): + file = zip_src.open(name) + root = ET.parse(file).getroot() + root = strip_namespace(root) + for key, value in PACKAGEXML_DICT.items(): + if isinstance(value, dict): + for perm, parent in value.items(): + package_xml_input.setdefault(key, set()).update( + fetch_permissionable_entity_names(root, perm, parent) + ) + else: + package_xml_input.setdefault(key, set()).update( + fetch_permissionable_entity_names(root, value) + ) + package_xml = create_package_xml( + input_dict=package_xml_input, api_version=api_version + ) + return package_xml + + +def create_package_xml(input_dict: dict, api_version: str): + package_xml = '\n' + package_xml += '\n' + + for name, members in input_dict.items(): + package_xml += " \n" + for member in members: + package_xml += f" {member}\n" + package_xml += f" {name}\n" + package_xml += " \n" + + package_xml += f" {api_version}\n" + package_xml += "\n" + return package_xml + + +def get_fields_and_recordtypes(root, objectName): + fields = set() + recordTypes = set() + + for field in root.findall("fields"): + fields.update([objectName + "." + field.find("fullName").text]) + + for recordType in root.findall("recordTypes"): + recordTypes.update([objectName + "." + recordType.find("fullName").text]) + + return fields, recordTypes + + +def get_tabs_from_app(root): + tabs = set() + for tab in root.findall("tabs"): + tabs.update([tab.text]) + return tabs + + +def get_target_entities_from_zip(zip_src): + zip_src.extractall("./unpackaged") + target_entities = {} + for name in zip_src.namelist(): + if name == "package.xml": + continue + metadataType = name.split("/")[0] + metadataName = name.split("/")[1].split(".")[0] + target_entities.setdefault(metadataType, set()).update([metadataName]) + + # If object, fetch all the fields and record types present inside the file + if name.endswith(".object"): + file = zip_src.open(name) + root = ET.parse(file).getroot() + root = strip_namespace(root) + fields, recordTypes = get_fields_and_recordtypes(root, metadataName) + target_entities.setdefault("fields", set()).update(fields) + target_entities.setdefault("recordTypes", set()).update(recordTypes) + + # To handle tabs which are not part of TabDefinition Table + if name.endswith(".app"): + file = zip_src.open(name) + root = ET.parse(file).getroot() + root = strip_namespace(root) + target_entities.setdefault("tabs", set()).update(get_tabs_from_app(root)) + + return target_entities + + +def zip_clean_invalid_references(zip_src, target_entities): + zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) + for name in zip_src.namelist(): + file = zip_src.open(name) + if any( + name.endswith(item.extension) and name.startswith(item.folder_name) + for item in FILES_TO_BE_CLEANED + ): + root = ET.parse(file).getroot() + cleaned_root = CleanXML(root, target_entities) + tree = ET.ElementTree(cleaned_root) + cleaned_content = io.BytesIO() + tree.write( + cleaned_content, + pretty_print=True, + xml_declaration=True, + encoding="utf-8", + ) + zip_dest.writestr(name, cleaned_content.getvalue()) + else: + zip_dest.writestr(name, file.read()) + + return zip_dest + + +def strip_namespace(element): + for elem in element.iter(): + if "}" in elem.tag: + elem.tag = elem.tag.split("}", 1)[1] + return element + + +def fetch_permissionable_entity_names( + root, perm_entity: PermissionElementXPath, parent: bool = False +): + entity_names = set() + for element in root.findall(perm_entity.permission_xpath): + entity_names.add( + perm_entity.return_parent(element) + if parent + else perm_entity.return_name(element) + ) + return entity_names + + +def CleanXML(root, target_entities): + root = strip_namespace(root) + for key, perm_entities in FOLDER_PERM_DICT.items(): + for perm_entity in perm_entities: + for element in root.findall(perm_entity.permission_xpath): + if perm_entity.return_name(element) not in target_entities[key]: + print(f"{key}: {perm_entity.return_name(element)}") + root.remove(element) + return root From 6f7c22a7d27675fe9586911717d7bc3e495eba64 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Thu, 19 Oct 2023 16:19:32 +0530 Subject: [PATCH 04/13] removed print --- cumulusci/core/source_transforms/transforms.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index f2b026d6b2..3d9f97b2a8 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -274,9 +274,6 @@ def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: self.entities_user_permission(sf) ) target_entites.setdefault("tabs", set()).update(self.entities_tabs(sf)) - - print(target_entites["tabs"]) - return zip_clean_invalid_references(zf, target_entites) From 7a5ce956368c909cdfd7663b350d3785b90e0166 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 20 Oct 2023 09:33:09 +0530 Subject: [PATCH 05/13] slight renaming --- cumulusci/core/source_transforms/transforms.py | 6 +++--- cumulusci/salesforce_api/package_zip.py | 8 ++++---- cumulusci/tasks/salesforce/Deploy.py | 8 +++----- ...clean_before_deploy.py => clean_invalid_references.py} | 0 4 files changed, 10 insertions(+), 12 deletions(-) rename cumulusci/utils/{clean_before_deploy.py => clean_invalid_references.py} (100%) diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index 3d9f97b2a8..963aabee45 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -26,7 +26,7 @@ tokenize_namespace, zip_clean_metaxml, ) -from cumulusci.utils.clean_before_deploy import ( +from cumulusci.utils.clean_invalid_references import ( get_target_entities_from_zip, return_package_xml_from_zip, zip_clean_invalid_references, @@ -219,11 +219,11 @@ def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: return zip_clean_metaxml(zf) -class CleanProfileMetaXMLTransform(SourceTransform): +class CleanInvalidReferencesMetaXMLTransform(SourceTransform): """Source transform that cleans *-meta.xml files of invalid references.""" options_model = None - identifier = "clean_profiles" + identifier = "clean_invalid_ref" api_version = "58.0" def entities_from_package(self, zf, context): diff --git a/cumulusci/salesforce_api/package_zip.py b/cumulusci/salesforce_api/package_zip.py index 75e9fdf6fb..ad27e51f19 100644 --- a/cumulusci/salesforce_api/package_zip.py +++ b/cumulusci/salesforce_api/package_zip.py @@ -12,8 +12,8 @@ from cumulusci.core.source_transforms.transforms import ( BundleStaticResourcesOptions, BundleStaticResourcesTransform, + CleanInvalidReferencesMetaXMLTransform, CleanMetaXMLTransform, - CleanProfileMetaXMLTransform, NamespaceInjectionOptions, NamespaceInjectionTransform, RemoveFeatureParametersTransform, @@ -190,9 +190,9 @@ def _process(self): # -meta.xml cleaning if self.options.get("clean_meta_xml", True): transforms.append(CleanMetaXMLTransform()) - - if self.options.get("clean_profiles", True): - transforms.append(CleanProfileMetaXMLTransform()) + + if self.options.get("clean_invalid_ref", True): + transforms.append(CleanInvalidReferencesMetaXMLTransform()) # Static resource bundling relpath = self.options.get("static_resource_path") diff --git a/cumulusci/tasks/salesforce/Deploy.py b/cumulusci/tasks/salesforce/Deploy.py index e1a33a9b5f..2e50bc311f 100644 --- a/cumulusci/tasks/salesforce/Deploy.py +++ b/cumulusci/tasks/salesforce/Deploy.py @@ -56,11 +56,9 @@ class Deploy(BaseSalesforceMetadataApiTask): "transforms": { "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms." }, - "rest_deploy": { - "description": "If True, deploy metadata using REST API" - }, - "clean_profiles": { - "description": "If specified, all profiles are cleaned of invalid references before deployment." + "rest_deploy": {"description": "If True, deploy metadata using REST API"}, + "clean_invalid_ref": { + "description": "If specified, all profiles and permission sets are cleaned of invalid references before deployment." }, } diff --git a/cumulusci/utils/clean_before_deploy.py b/cumulusci/utils/clean_invalid_references.py similarity index 100% rename from cumulusci/utils/clean_before_deploy.py rename to cumulusci/utils/clean_invalid_references.py From d712a1197cd8c1191df14bfc1e7ec3ea2de7b576 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Fri, 20 Oct 2023 12:42:53 +0530 Subject: [PATCH 06/13] resolved issue where source format was failing --- cumulusci/core/source_transforms/transforms.py | 2 +- cumulusci/utils/clean_invalid_references.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index 963aabee45..2d7cefa564 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -250,7 +250,7 @@ def entities_user_permission(self, sf): response = sf._call_salesforce(method, urlpath) fields = [ - f["name"].replace("Permissions", "") + f["name"][len("Permissions") :] for f in response.json()["fields"] if f["name"].startswith("Permissions") and f["type"] == "boolean" ] diff --git a/cumulusci/utils/clean_invalid_references.py b/cumulusci/utils/clean_invalid_references.py index 7ed19620c8..beddd41a71 100644 --- a/cumulusci/utils/clean_invalid_references.py +++ b/cumulusci/utils/clean_invalid_references.py @@ -13,7 +13,7 @@ def __init__(self, folder_name, extension): self.extension = extension -PROFILE_FILE = FileName("profiles/", ".profile-meta.xml") +PROFILE_FILE = FileName("profiles/", ".profile") PERMISSIONSET_FILE = FileName("permissionsets/", ".permissionset") FILES_TO_BE_CLEANED = [PROFILE_FILE, PERMISSIONSET_FILE] @@ -86,7 +86,7 @@ def return_package_xml_from_zip(zip_src, api_version: str = "58.0"): package_xml_input = {} for name in zip_src.namelist(): if any( - name.endswith(item.extension) and name.startswith(item.folder_name) + item.extension in name and name.startswith(item.folder_name) for item in FILES_TO_BE_CLEANED ): file = zip_src.open(name) @@ -102,6 +102,10 @@ def return_package_xml_from_zip(zip_src, api_version: str = "58.0"): package_xml_input.setdefault(key, set()).update( fetch_permissionable_entity_names(root, value) ) + # Remove any entities with no entries + package_xml_input = { + key: value for key, value in package_xml_input.items() if value + } package_xml = create_package_xml( input_dict=package_xml_input, api_version=api_version ) @@ -178,7 +182,7 @@ def zip_clean_invalid_references(zip_src, target_entities): for name in zip_src.namelist(): file = zip_src.open(name) if any( - name.endswith(item.extension) and name.startswith(item.folder_name) + item.extension in name and name.startswith(item.folder_name) for item in FILES_TO_BE_CLEANED ): root = ET.parse(file).getroot() From 98a7ff0e7b63e5a81c5eb08597173b32f1831037 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 25 Oct 2023 11:19:27 +0530 Subject: [PATCH 07/13] Added tests for clean_invalid_references.py --- cumulusci/salesforce_api/package_zip.py | 2 +- .../tests/test_clean_invalid_references.py | 263 ++++++++++++++++++ 2 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 cumulusci/utils/tests/test_clean_invalid_references.py diff --git a/cumulusci/salesforce_api/package_zip.py b/cumulusci/salesforce_api/package_zip.py index ad27e51f19..49c9a89112 100644 --- a/cumulusci/salesforce_api/package_zip.py +++ b/cumulusci/salesforce_api/package_zip.py @@ -191,7 +191,7 @@ def _process(self): if self.options.get("clean_meta_xml", True): transforms.append(CleanMetaXMLTransform()) - if self.options.get("clean_invalid_ref", True): + if self.options.get("clean_invalid_ref", False): transforms.append(CleanInvalidReferencesMetaXMLTransform()) # Static resource bundling diff --git a/cumulusci/utils/tests/test_clean_invalid_references.py b/cumulusci/utils/tests/test_clean_invalid_references.py new file mode 100644 index 0000000000..3b33f662d7 --- /dev/null +++ b/cumulusci/utils/tests/test_clean_invalid_references.py @@ -0,0 +1,263 @@ +import io +import zipfile +from unittest.mock import patch + +import pytest +from lxml import etree as ET + +from cumulusci.utils.clean_invalid_references import ( + CleanXML, + PermissionElementXPath, + create_package_xml, + fetch_permissionable_entity_names, + get_fields_and_recordtypes, + get_tabs_from_app, + get_target_entities_from_zip, + return_package_xml_from_zip, + zip_clean_invalid_references, +) + +EXPECTED_PACKAGEXML = ( + '\n' + '\n' + " \n" + " Account\n" + " Contact\n" + " CustomObject\n" + " \n" + " \n" + " MyClass\n" + " ApexClass\n" + " \n" + " 58.0\n" + "" +) + +SAMPLE_XML = ( + "\n" + " \n" + " Account\n" + " \n" + " \n" + " Contact\n" + " \n" + " \n" + " CustomPermission1\n" + " \n" + "\n" +) + +EXPECTED_XML = ( + "\n" + "\n" + " \n" + " Account\n" + " \n" + "\n" +) + +SAMPLE_OBJ_XML = ( + "\n" + " \n" + " Field2\n" + " \n" + " \n" + " Field1\n" + " \n" + " \n" + " RecordType1\n" + " \n" + " \n" + " RecordType2\n" + " \n" + "" +) + +SAMPLE_APP_XML = "\n" " Tab1\n" " Tab2\n" "" + +PROFILE = ("Profile", "profiles", ".profile-meta.xml") +PERMISSIONSET = ("PermissionSet", "permissionsets", ".permissionset") + + +@pytest.fixture +def sample_root(sample_xml): + xml = sample_xml + return ET.fromstring(xml) + + +@pytest.mark.parametrize("sample_xml", [SAMPLE_XML]) +def test_fetch_permissionable_entity_names(sample_root): + permission_element = PermissionElementXPath(".//objectPermissions", "object") + entity_names = fetch_permissionable_entity_names(sample_root, permission_element) + expected_result = {"Account", "Contact"} + assert entity_names == expected_result + + +@pytest.mark.parametrize("sample_xml", [SAMPLE_OBJ_XML]) +def test_get_fields_and_recordtypes(sample_root): + fields, recordTypes = get_fields_and_recordtypes(sample_root, "SampleObject") + expected_fields = {"SampleObject.Field1", "SampleObject.Field2"} + expected_recordTypes = {"SampleObject.RecordType1", "SampleObject.RecordType2"} + assert fields == expected_fields + assert recordTypes == expected_recordTypes + + +@pytest.mark.parametrize("sample_xml", [SAMPLE_APP_XML]) +def test_get_tabs_from_app(sample_root): + tabs = get_tabs_from_app(sample_root) + expected_tabs = {"Tab1", "Tab2"} + assert tabs == expected_tabs + + +@pytest.mark.parametrize("sample_xml", [SAMPLE_XML]) +def test_CleanXML(sample_root): + target_entities = { + "fields": {"Field1", "Field2"}, + "recordTypes": {"RecordType1", "RecordType2"}, + "tabs": {"Tab1", "Tab2"}, + "objects": {"Account"}, + "customPermissions": {"CustomPermission1"}, + } + + cleaned_root = CleanXML(sample_root, target_entities) + for element in cleaned_root.findall(".//objectPermissions"): + assert element.find("object").text in target_entities["objects"] + for element in cleaned_root.findall(".//customPermissions"): + assert element.find("name").text in target_entities["customPermissions"] + + for element in cleaned_root.findall(".//objectPermissions"): + assert "Contact" not in element.find("object").text + + +@pytest.fixture +def sample_zip(elements): + zip_file = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) + for folder, extension, sample_xml in elements: + zip_file.writestr(f"{folder}/sample{extension}", sample_xml) + + return zip_file + + +@pytest.mark.parametrize( + "elements", + [ + [ + ("objects", ".object", SAMPLE_OBJ_XML), + ("applications", ".app", SAMPLE_APP_XML), + ] + ], +) +def test_get_target_entities_from_zip(sample_zip): + with patch( + "cumulusci.utils.clean_invalid_references.get_fields_and_recordtypes" + ) as mock_fields, patch( + "cumulusci.utils.clean_invalid_references.get_tabs_from_app" + ) as mock_tabs: + mock_fields.return_value = ( + {"sample.Field1", "sample.Field2"}, + {"sample.RecordType1", "sample.RecordType2"}, + ) + mock_tabs.return_value = {"Tab1", "Tab2"} + + target_entities = get_target_entities_from_zip(sample_zip) + + expected_target_entities = { + "objects": {"sample"}, + "fields": {"sample.Field1", "sample.Field2"}, + "recordTypes": {"sample.RecordType1", "sample.RecordType2"}, + "applications": {"sample"}, + "tabs": {"Tab1", "Tab2"}, + } + print(target_entities) + print(expected_target_entities) + assert target_entities == expected_target_entities + + +@patch("cumulusci.utils.clean_invalid_references.fetch_permissionable_entity_names") +@pytest.mark.parametrize( + "elements", + [ + [ + ("profiles", ".profile", SAMPLE_XML), + ("permissionsets", ".permissionset", SAMPLE_XML), + ] + ], +) +def test_return_package_xml_from_zip(mock_fetch_entities, sample_zip): + + # Mock Fetch Entities + def fetch_entities(root, perm, parent=False): + entity_names = set() + for element in root.findall(perm.permission_xpath): + if parent: + entity_names.add(element.find(perm.name_xpath).text.split(".")[0]) + else: + entity_names.add(element.find(perm.name_xpath).text) + return entity_names + + mock_fetch_entities.side_effect = fetch_entities + + with patch( + "cumulusci.utils.clean_invalid_references.create_package_xml" + ) as mock_create_package_xml: + return_package_xml_from_zip(sample_zip, "58.0") + + expected_input_dict = { + "CustomObject": {"Account", "Contact"}, + "CustomPermission": {"CustomPermission1"}, + } + mock_create_package_xml.assert_called_with( + input_dict=expected_input_dict, api_version="58.0" + ) + + +@patch("cumulusci.utils.clean_invalid_references.CleanXML") +@pytest.mark.parametrize( + "elements", + [ + [ + ("profiles", ".profile", SAMPLE_XML), + ("permissionsets", ".permissionset", SAMPLE_XML), + ] + ], +) +def test_zip_clean_invalid_references(mock_clean_xml, sample_zip): + # Mock CleanXML + def clean_xml(root, target_entities): + for element in root.findall(".//objectPermissions"): + if element.find("object").text == "Contact": + root.remove(element) + + for element in root.findall(".//customPermissions"): + root.remove(element) + return root + + mock_clean_xml.side_effect = clean_xml + + # Call the function + target_entities = {"something": {"something"}} + zip_dest = zip_clean_invalid_references(sample_zip, target_entities) + + name_list = ["profiles/sample.profile", "permissionsets/sample.permissionset"] + for name in zip_dest.namelist(): + assert name in name_list + result_root = ET.parse(zip_dest.open(name)).getroot() + expected_root = ET.fromstring(EXPECTED_XML.encode("utf-8")) + assert ET.iselement(result_root) == ET.iselement(expected_root) + + +def test_create_package_xml(): + input_dict = { + "CustomObject": {"Account", "Contact"}, + "ApexClass": {"MyClass"}, + } + api_version = "58.0" + result = create_package_xml(input_dict, api_version) + + result_root = ET.fromstring(result.encode("utf-8")) + expected_root = ET.fromstring(EXPECTED_PACKAGEXML.encode("utf-8")) + assert ET.iselement(result_root) == ET.iselement(expected_root) + + +if __name__ == "__main": + pytest.main() From 6d4b4dc34f2297f6740878e6e9488764a98bc8a7 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 25 Oct 2023 11:21:25 +0530 Subject: [PATCH 08/13] changed usage of api_version --- cumulusci/core/source_transforms/transforms.py | 4 ++-- cumulusci/utils/clean_invalid_references.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index 2d7cefa564..94453a86dd 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -224,10 +224,9 @@ class CleanInvalidReferencesMetaXMLTransform(SourceTransform): options_model = None identifier = "clean_invalid_ref" - api_version = "58.0" def entities_from_package(self, zf, context): - package_xml = return_package_xml_from_zip(zf) + package_xml = return_package_xml_from_zip(zf, self.api_version) api = ApiRetrieveUnpackaged( context, package_xml=package_xml, api_version=self.api_version ) @@ -266,6 +265,7 @@ def entities_tabs(self, sf): def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: context.logger.info("Cleaning profile meta.xml files of invalid references") + self.api_version = context.org_config.latest_api_version sf = self.ret_sf(context) target_entites = {} diff --git a/cumulusci/utils/clean_invalid_references.py b/cumulusci/utils/clean_invalid_references.py index beddd41a71..d227ad576c 100644 --- a/cumulusci/utils/clean_invalid_references.py +++ b/cumulusci/utils/clean_invalid_references.py @@ -81,7 +81,7 @@ def return_name(self, element: ET.Element): } -def return_package_xml_from_zip(zip_src, api_version: str = "58.0"): +def return_package_xml_from_zip(zip_src, api_version: str): # Iterate through the zip file to generate the package.xml package_xml_input = {} for name in zip_src.namelist(): @@ -149,7 +149,6 @@ def get_tabs_from_app(root): def get_target_entities_from_zip(zip_src): - zip_src.extractall("./unpackaged") target_entities = {} for name in zip_src.namelist(): if name == "package.xml": From 781c96e2382c9d003e7cb19fb204b1733bde0b71 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 25 Oct 2023 11:53:11 +0530 Subject: [PATCH 09/13] improved coverage --- .../tests/test_clean_invalid_references.py | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/cumulusci/utils/tests/test_clean_invalid_references.py b/cumulusci/utils/tests/test_clean_invalid_references.py index 3b33f662d7..ee338689e9 100644 --- a/cumulusci/utils/tests/test_clean_invalid_references.py +++ b/cumulusci/utils/tests/test_clean_invalid_references.py @@ -14,6 +14,7 @@ get_tabs_from_app, get_target_entities_from_zip, return_package_xml_from_zip, + strip_namespace, zip_clean_invalid_references, ) @@ -41,6 +42,9 @@ " \n" " Contact\n" " \n" + " \n" + " Opportunity.SomeField\n" + " \n" " \n" " CustomPermission1\n" " \n" @@ -53,6 +57,9 @@ " \n" " Account\n" " \n" + " \n" + " Opportunity.SomeField\n" + " \n" "\n" ) @@ -92,6 +99,13 @@ def test_fetch_permissionable_entity_names(sample_root): expected_result = {"Account", "Contact"} assert entity_names == expected_result + permission_element = PermissionElementXPath(".//fieldPermissions", "field") + entity_names = fetch_permissionable_entity_names( + sample_root, permission_element, parent=True + ) + expected_result = {"Opportunity"} + assert entity_names == expected_result + @pytest.mark.parametrize("sample_xml", [SAMPLE_OBJ_XML]) def test_get_fields_and_recordtypes(sample_root): @@ -127,13 +141,18 @@ def test_CleanXML(sample_root): for element in cleaned_root.findall(".//objectPermissions"): assert "Contact" not in element.find("object").text + for element in cleaned_root.findall(".//fieldPermissions"): + assert "SomeField" not in element.find("field").text @pytest.fixture def sample_zip(elements): zip_file = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) - for folder, extension, sample_xml in elements: - zip_file.writestr(f"{folder}/sample{extension}", sample_xml) + for folder, name, extension, sample_xml in elements: + if folder is not None: + zip_file.writestr(f"{folder}/{name}{extension}", sample_xml) + else: + zip_file.writestr(f"{name}{extension}", sample_xml) return zip_file @@ -142,8 +161,9 @@ def sample_zip(elements): "elements", [ [ - ("objects", ".object", SAMPLE_OBJ_XML), - ("applications", ".app", SAMPLE_APP_XML), + ("objects", "sample", ".object", SAMPLE_OBJ_XML), + ("applications", "sample", ".app", SAMPLE_APP_XML), + (None, "package", ".xml", EXPECTED_PACKAGEXML), ] ], ) @@ -168,8 +188,6 @@ def test_get_target_entities_from_zip(sample_zip): "applications": {"sample"}, "tabs": {"Tab1", "Tab2"}, } - print(target_entities) - print(expected_target_entities) assert target_entities == expected_target_entities @@ -178,8 +196,8 @@ def test_get_target_entities_from_zip(sample_zip): "elements", [ [ - ("profiles", ".profile", SAMPLE_XML), - ("permissionsets", ".permissionset", SAMPLE_XML), + ("profiles", "sample", ".profile", SAMPLE_XML), + ("permissionsets", "sample", ".permissionset", SAMPLE_XML), ] ], ) @@ -203,7 +221,7 @@ def fetch_entities(root, perm, parent=False): return_package_xml_from_zip(sample_zip, "58.0") expected_input_dict = { - "CustomObject": {"Account", "Contact"}, + "CustomObject": {"Account", "Contact", "Opportunity"}, "CustomPermission": {"CustomPermission1"}, } mock_create_package_xml.assert_called_with( @@ -216,8 +234,9 @@ def fetch_entities(root, perm, parent=False): "elements", [ [ - ("profiles", ".profile", SAMPLE_XML), - ("permissionsets", ".permissionset", SAMPLE_XML), + ("profiles", "sample", ".profile", SAMPLE_XML), + ("permissionsets", "sample", ".permissionset", SAMPLE_XML), + ("some_folder", "sample", ".some_extension", SAMPLE_XML), ] ], ) @@ -238,7 +257,11 @@ def clean_xml(root, target_entities): target_entities = {"something": {"something"}} zip_dest = zip_clean_invalid_references(sample_zip, target_entities) - name_list = ["profiles/sample.profile", "permissionsets/sample.permissionset"] + name_list = [ + "profiles/sample.profile", + "permissionsets/sample.permissionset", + "some_folder/sample.some_extension", + ] for name in zip_dest.namelist(): assert name in name_list result_root = ET.parse(zip_dest.open(name)).getroot() @@ -259,5 +282,22 @@ def test_create_package_xml(): assert ET.iselement(result_root) == ET.iselement(expected_root) +def test_strip_namespace(): + xml_string = """ + + Value 1 + Value 2 + + """ + root = ET.fromstring(xml_string) + + root = strip_namespace(root) + element1 = root.find("element1") + element2 = root.find("element2") + + assert element1.text == "Value 1" + assert element2.text == "Value 2" + + if __name__ == "__main": pytest.main() From de442fe0f0a52bd912b4b0244d97ef40527872fa Mon Sep 17 00:00:00 2001 From: Jawadtp Date: Wed, 25 Oct 2023 16:53:21 +0530 Subject: [PATCH 10/13] Tests written for transforms.py --- .../tests/test_transforms.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/cumulusci/core/source_transforms/tests/test_transforms.py b/cumulusci/core/source_transforms/tests/test_transforms.py index 99230f2fe2..9a2ccdd032 100644 --- a/cumulusci/core/source_transforms/tests/test_transforms.py +++ b/cumulusci/core/source_transforms/tests/test_transforms.py @@ -4,6 +4,9 @@ import zipfile from pathlib import Path, PurePosixPath from unittest import mock +from unittest.mock import patch + + from zipfile import ZipFile import pytest @@ -22,6 +25,8 @@ SourceTransformSpec, StripUnwantedComponentsOptions, StripUnwantedComponentTransform, + CleanMetaXMLTransform, + CleanInvalidReferencesMetaXMLTransform ) from cumulusci.salesforce_api.package_zip import MetadataPackageZipBuilder from cumulusci.utils import temporary_dir @@ -1089,3 +1094,81 @@ def test_strip_unwanted_files(task_context): ) == builder.zf ) + + + +@pytest.fixture +def sf(): + client = mock.Mock() + client.query.return_value = {"records": []} + return client + +@pytest.fixture +def sample_zip(elements): + zip_file = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) + for folder, extension, sample_xml in elements: + zip_file.writestr(f"{folder}/sample{extension}", sample_xml) + + return zip_file + +# Test data to test entities_tabs function of CleanInvalidReferencesMetaXMLTransform class. +entities_tabs_test_data = [ + ({"records": [{"Name": "Standard-Account"}, {"Name": "Standard-Asset"}, {"Name": "Standard-Contact"}]}, {"Standard-Account", "Standard-Asset", "Standard-Contact"}), + ({"records": []}, set()), + ({"records": [{"Name": "Standard-Event"}, {"Name": "Standard-Feed"}]}, {"Standard-Event", "Standard-Feed"}), +] + +@pytest.mark.parametrize("query_result, expected_result", entities_tabs_test_data) +def test_entities_tabs(sf, query_result, expected_result): + transform = CleanInvalidReferencesMetaXMLTransform() + transform.sf = sf + sf.query.return_value = query_result + result = transform.entities_tabs(sf) + assert result == expected_result + sf.query.assert_called_once_with("SELECT Name FROM TabDefinition") + +sample_response = { + "fields": [ + {'name': 'PermissionsUP1', 'type': 'boolean'}, + {'name': 'PermissionsUP2', 'type': 'notboolean'}, + {'name': 'PermissionsManagePermissions', 'type': 'boolean'}, + {'name': 'SomeOtherValue', 'type': 'boolean'} + ] +} + +class TestResponse: + @staticmethod + def json(): + return {"fields":[{'name': 'PermissionsUP1', 'type': 'boolean'}, {'name': 'PermissionsUP2', 'type': 'notboolean'}, {'name': 'PermissionsManagePermissions', 'type': 'boolean'}, {'name': 'SomeOtherValue', 'type': 'boolean'}]} + + +def test_entities_user_permission(): + transform = CleanInvalidReferencesMetaXMLTransform() + mock_sf = mock.Mock() + transform.sf = mock_sf + transform.sf.base_url = 'test' + + with patch.object(transform.sf, '_call_salesforce', return_value= TestResponse): + result = transform.entities_user_permission(transform.sf) + + assert result == {'UP1', 'ManagePermissions'} + + +@pytest.mark.parametrize( + "elements", + [[('folder', '.extension', 'sampleText')]] +) +def test_process(sample_zip, task_context): + with patch('cumulusci.salesforce_api.utils.get_simple_salesforce_connection'),\ + patch('cumulusci.utils.clean_invalid_references.return_package_xml_from_zip'),\ + patch('cumulusci.salesforce_api.metadata.ApiRetrieveUnpackaged'),\ + patch('cumulusci.utils.clean_invalid_references.get_target_entities_from_zip' ),\ + patch('cumulusci.core.source_transforms.transforms.CleanInvalidReferencesMetaXMLTransform.entities_user_permission'),\ + patch('cumulusci.core.source_transforms.transforms.CleanInvalidReferencesMetaXMLTransform.entities_tabs'): + transform = CleanInvalidReferencesMetaXMLTransform() + transform.process(sample_zip, task_context) + + + + + From fd4eb78ca2c597481d2f8c03683549cd597eaeff Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 25 Oct 2023 19:12:42 +0530 Subject: [PATCH 11/13] Added tests for transforms.py and added logs --- .../tests/test_transforms.py | 154 ++++++++++-------- .../core/source_transforms/transforms.py | 14 +- cumulusci/utils/clean_invalid_references.py | 11 +- .../tests/test_clean_invalid_references.py | 5 + 4 files changed, 108 insertions(+), 76 deletions(-) diff --git a/cumulusci/core/source_transforms/tests/test_transforms.py b/cumulusci/core/source_transforms/tests/test_transforms.py index 9a2ccdd032..bc318fb30a 100644 --- a/cumulusci/core/source_transforms/tests/test_transforms.py +++ b/cumulusci/core/source_transforms/tests/test_transforms.py @@ -5,13 +5,12 @@ from pathlib import Path, PurePosixPath from unittest import mock from unittest.mock import patch - - from zipfile import ZipFile import pytest from lxml import etree as ET from pydantic import ValidationError +from simple_salesforce import Salesforce from cumulusci.core.exceptions import CumulusCIException, TaskOptionsError from cumulusci.core.source_transforms.transforms import ( @@ -25,9 +24,8 @@ SourceTransformSpec, StripUnwantedComponentsOptions, StripUnwantedComponentTransform, - CleanMetaXMLTransform, - CleanInvalidReferencesMetaXMLTransform ) +from cumulusci.salesforce_api.metadata import BaseMetadataApiCall from cumulusci.salesforce_api.package_zip import MetadataPackageZipBuilder from cumulusci.utils import temporary_dir @@ -1096,79 +1094,101 @@ def test_strip_unwanted_files(task_context): ) - -@pytest.fixture -def sf(): - client = mock.Mock() - client.query.return_value = {"records": []} - return client - -@pytest.fixture -def sample_zip(elements): - zip_file = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) - for folder, extension, sample_xml in elements: - zip_file.writestr(f"{folder}/sample{extension}", sample_xml) - - return zip_file - -# Test data to test entities_tabs function of CleanInvalidReferencesMetaXMLTransform class. -entities_tabs_test_data = [ - ({"records": [{"Name": "Standard-Account"}, {"Name": "Standard-Asset"}, {"Name": "Standard-Contact"}]}, {"Standard-Account", "Standard-Asset", "Standard-Contact"}), - ({"records": []}, set()), - ({"records": [{"Name": "Standard-Event"}, {"Name": "Standard-Feed"}]}, {"Standard-Event", "Standard-Feed"}), -] - -@pytest.mark.parametrize("query_result, expected_result", entities_tabs_test_data) -def test_entities_tabs(sf, query_result, expected_result): - transform = CleanInvalidReferencesMetaXMLTransform() - transform.sf = sf - sf.query.return_value = query_result - result = transform.entities_tabs(sf) - assert result == expected_result - sf.query.assert_called_once_with("SELECT Name FROM TabDefinition") - +# Tests for CleanInvalidReferencesMetaXMLTransform sample_response = { "fields": [ - {'name': 'PermissionsUP1', 'type': 'boolean'}, - {'name': 'PermissionsUP2', 'type': 'notboolean'}, - {'name': 'PermissionsManagePermissions', 'type': 'boolean'}, - {'name': 'SomeOtherValue', 'type': 'boolean'} + {"name": "PermissionsUP1", "type": "boolean"}, + {"name": "PermissionsUP2", "type": "notboolean"}, + {"name": "PermissionsManagePermissions", "type": "boolean"}, + {"name": "SomeOtherValue", "type": "boolean"}, ] } + class TestResponse: @staticmethod def json(): - return {"fields":[{'name': 'PermissionsUP1', 'type': 'boolean'}, {'name': 'PermissionsUP2', 'type': 'notboolean'}, {'name': 'PermissionsManagePermissions', 'type': 'boolean'}, {'name': 'SomeOtherValue', 'type': 'boolean'}]} - - -def test_entities_user_permission(): - transform = CleanInvalidReferencesMetaXMLTransform() - mock_sf = mock.Mock() - transform.sf = mock_sf - transform.sf.base_url = 'test' - - with patch.object(transform.sf, '_call_salesforce', return_value= TestResponse): - result = transform.entities_user_permission(transform.sf) - - assert result == {'UP1', 'ManagePermissions'} - - -@pytest.mark.parametrize( - "elements", - [[('folder', '.extension', 'sampleText')]] -) -def test_process(sample_zip, task_context): - with patch('cumulusci.salesforce_api.utils.get_simple_salesforce_connection'),\ - patch('cumulusci.utils.clean_invalid_references.return_package_xml_from_zip'),\ - patch('cumulusci.salesforce_api.metadata.ApiRetrieveUnpackaged'),\ - patch('cumulusci.utils.clean_invalid_references.get_target_entities_from_zip' ),\ - patch('cumulusci.core.source_transforms.transforms.CleanInvalidReferencesMetaXMLTransform.entities_user_permission'),\ - patch('cumulusci.core.source_transforms.transforms.CleanInvalidReferencesMetaXMLTransform.entities_tabs'): - transform = CleanInvalidReferencesMetaXMLTransform() - transform.process(sample_zip, task_context) + return sample_response + + +def test_clean_invalid_references(task_context): + xml_data = ( + "\n" + " \n" + " Account\n" + " \n" + " \n" + " Contact\n" + " \n" + " \n" + " Opportunity.SomeField\n" + " \n" + " \n" + " CustomPermission1\n" + " \n" + " \n" + " Standard-Account\n" + " \n" + " \n" + " Standard-Fake\n" + " \n" + " \n" + " ManagePermissions\n" + " \n" + "\n" + ) + xml_data_clean = ( + "\n" + "\n" + " \n" + " Account\n" + " \n" + " \n" + " Opportunity.SomeField\n" + " \n" + " \n" + " Standard-Account\n" + " \n" + " \n" + " ManagePermissions\n" + " \n" + "\n" + ) + obj_xml = ( + "\n" + " \n" + " SomeField\n" + " \n" + " \n" + " RecordType1\n" + " \n" + "" + ) + output_zip = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) + output_zip.writestr("objects/Opportunity.object", obj_xml) + output_zip.writestr("objects/Account.object", obj_xml) + query_result = { + "records": [ + {"Name": "Standard-Account"}, + {"Name": "Standard-Asset"}, + {"Name": "Standard-Contact"}, + ] + } + with patch.object(Salesforce, "query", return_value=query_result), patch.object( + Salesforce, "_call_salesforce", return_value=TestResponse + ), patch.object(BaseMetadataApiCall, "__call__", return_value=output_zip): + builder = MetadataPackageZipBuilder.from_zipfile( + ZipFileSpec({Path("profiles/Foo.profile"): xml_data}).as_zipfile(), + options={"clean_invalid_ref": True}, + context=task_context, + ) + for name in builder.zf.namelist(): + assert name == "profiles/Foo.profile" + result_root = ET.parse(builder.zf.open(name)).getroot() + expected_root = ET.fromstring(xml_data_clean.encode("utf-8")) + assert ET.iselement(result_root) == ET.iselement(expected_root) diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index 94453a86dd..df8b033560 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -216,7 +216,9 @@ def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: context.logger.info( "Cleaning meta.xml files of packageVersion elements for deploy" ) - return zip_clean_metaxml(zf) + zip_dest = zip_clean_metaxml(zf) + context.logger.info("[Done]\n") + return zip_dest class CleanInvalidReferencesMetaXMLTransform(SourceTransform): @@ -230,6 +232,7 @@ def entities_from_package(self, zf, context): api = ApiRetrieveUnpackaged( context, package_xml=package_xml, api_version=self.api_version ) + context.logger.info("Retrieving entities from package.xml") retrieved_zf = api() return get_target_entities_from_zip(retrieved_zf) @@ -263,7 +266,9 @@ def entities_tabs(self, sf): return set(tabs) def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: - context.logger.info("Cleaning profile meta.xml files of invalid references") + context.logger.info( + "Cleaning profiles and permission sets meta.xml files of invalid references" + ) self.api_version = context.org_config.latest_api_version sf = self.ret_sf(context) @@ -274,7 +279,10 @@ def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: self.entities_user_permission(sf) ) target_entites.setdefault("tabs", set()).update(self.entities_tabs(sf)) - return zip_clean_invalid_references(zf, target_entites) + zip_dest = zip_clean_invalid_references(zf, target_entites) + + context.logger.info("Done cleaning profiles and permission sets\n") + return zip_dest class BundleStaticResourcesOptions(BaseModel): diff --git a/cumulusci/utils/clean_invalid_references.py b/cumulusci/utils/clean_invalid_references.py index d227ad576c..05f593fe2b 100644 --- a/cumulusci/utils/clean_invalid_references.py +++ b/cumulusci/utils/clean_invalid_references.py @@ -149,13 +149,13 @@ def get_tabs_from_app(root): def get_target_entities_from_zip(zip_src): - target_entities = {} + target_entities = {key: set() for key in FOLDER_PERM_DICT.keys()} for name in zip_src.namelist(): if name == "package.xml": continue metadataType = name.split("/")[0] metadataName = name.split("/")[1].split(".")[0] - target_entities.setdefault(metadataType, set()).update([metadataName]) + target_entities[metadataType].update([metadataName]) # If object, fetch all the fields and record types present inside the file if name.endswith(".object"): @@ -163,15 +163,15 @@ def get_target_entities_from_zip(zip_src): root = ET.parse(file).getroot() root = strip_namespace(root) fields, recordTypes = get_fields_and_recordtypes(root, metadataName) - target_entities.setdefault("fields", set()).update(fields) - target_entities.setdefault("recordTypes", set()).update(recordTypes) + target_entities["fields"].update(fields) + target_entities["recordTypes"].update(recordTypes) # To handle tabs which are not part of TabDefinition Table if name.endswith(".app"): file = zip_src.open(name) root = ET.parse(file).getroot() root = strip_namespace(root) - target_entities.setdefault("tabs", set()).update(get_tabs_from_app(root)) + target_entities["tabs"].update(get_tabs_from_app(root)) return target_entities @@ -227,6 +227,5 @@ def CleanXML(root, target_entities): for perm_entity in perm_entities: for element in root.findall(perm_entity.permission_xpath): if perm_entity.return_name(element) not in target_entities[key]: - print(f"{key}: {perm_entity.return_name(element)}") root.remove(element) return root diff --git a/cumulusci/utils/tests/test_clean_invalid_references.py b/cumulusci/utils/tests/test_clean_invalid_references.py index ee338689e9..cc206cbcd7 100644 --- a/cumulusci/utils/tests/test_clean_invalid_references.py +++ b/cumulusci/utils/tests/test_clean_invalid_references.py @@ -187,6 +187,11 @@ def test_get_target_entities_from_zip(sample_zip): "recordTypes": {"sample.RecordType1", "sample.RecordType2"}, "applications": {"sample"}, "tabs": {"Tab1", "Tab2"}, + "classes": set(), + "customPermissions": set(), + "flows": set(), + "pages": set(), + "userPermissions": set(), } assert target_entities == expected_target_entities From b179b01bcf499758a264295d22468e9ecccf4901 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 1 Nov 2023 16:59:29 +0530 Subject: [PATCH 12/13] Handled corner cases --- .../tests/test_transforms.py | 58 +++-- .../core/source_transforms/transforms.py | 60 +---- cumulusci/salesforce_api/package_zip.py | 3 +- cumulusci/utils/clean_invalid_references.py | 152 ++++++++++++- .../tests/test_clean_invalid_references.py | 215 +++++++++++++++++- 5 files changed, 399 insertions(+), 89 deletions(-) diff --git a/cumulusci/core/source_transforms/tests/test_transforms.py b/cumulusci/core/source_transforms/tests/test_transforms.py index bc318fb30a..e13ea2dae9 100644 --- a/cumulusci/core/source_transforms/tests/test_transforms.py +++ b/cumulusci/core/source_transforms/tests/test_transforms.py @@ -1095,20 +1095,48 @@ def test_strip_unwanted_files(task_context): # Tests for CleanInvalidReferencesMetaXMLTransform -sample_response = { +up_result_dict = { "fields": [ - {"name": "PermissionsUP1", "type": "boolean"}, - {"name": "PermissionsUP2", "type": "notboolean"}, - {"name": "PermissionsManagePermissions", "type": "boolean"}, - {"name": "SomeOtherValue", "type": "boolean"}, + {"name": "PermissionsEdit", "type": "boolean"}, + {"name": "PermissionsView", "type": "boolean"}, ] } +field_result_dict = { + "fields": [ + { + "name": "Field", + "picklistValues": [{"value": "Option1"}, {"value": "Option2"}], + } + ], +} +tabs_result_dict = { + "records": [ + {"Name": "Tab1"}, + {"Name": "Tab2"}, + ], +} +objects_result_dict = { + "fields": [ + { + "name": "SobjectType", + "picklistValues": [{"value": "Object1"}, {"value": "Object2"}], + }, + ], +} + +def return_response(method, urlpath): + response = mock.Mock() + if "sobjects/PermissionSet/describe" in urlpath: + response.json.return_value = up_result_dict + elif "sobjects/FieldPermissions/describe" in urlpath: + response.json.return_value = field_result_dict + elif "sobjects/ObjectPermissions/describe" in urlpath: + response.json.return_value = objects_result_dict + elif "query/?q=SELECT+Name+FROM+PermissionSetTabSetting+GROUP+BY+Name" in urlpath: + response.json.return_value = tabs_result_dict -class TestResponse: - @staticmethod - def json(): - return sample_response + return response def test_clean_invalid_references(task_context): @@ -1171,16 +1199,8 @@ def test_clean_invalid_references(task_context): output_zip.writestr("objects/Opportunity.object", obj_xml) output_zip.writestr("objects/Account.object", obj_xml) - query_result = { - "records": [ - {"Name": "Standard-Account"}, - {"Name": "Standard-Asset"}, - {"Name": "Standard-Contact"}, - ] - } - - with patch.object(Salesforce, "query", return_value=query_result), patch.object( - Salesforce, "_call_salesforce", return_value=TestResponse + with patch.object( + Salesforce, "_call_salesforce", side_effect=return_response ), patch.object(BaseMetadataApiCall, "__call__", return_value=output_zip): builder = MetadataPackageZipBuilder.from_zipfile( ZipFileSpec({Path("profiles/Foo.profile"): xml_data}).as_zipfile(), diff --git a/cumulusci/core/source_transforms/transforms.py b/cumulusci/core/source_transforms/transforms.py index df8b033560..fa2179980c 100644 --- a/cumulusci/core/source_transforms/transforms.py +++ b/cumulusci/core/source_transforms/transforms.py @@ -15,8 +15,6 @@ from cumulusci.core.dependencies.utils import TaskContext from cumulusci.core.enums import StrEnum from cumulusci.core.exceptions import CumulusCIException, TaskOptionsError -from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged -from cumulusci.salesforce_api.utils import get_simple_salesforce_connection from cumulusci.tasks.metadata.package import RemoveSourceComponents from cumulusci.utils import ( cd, @@ -26,11 +24,7 @@ tokenize_namespace, zip_clean_metaxml, ) -from cumulusci.utils.clean_invalid_references import ( - get_target_entities_from_zip, - return_package_xml_from_zip, - zip_clean_invalid_references, -) +from cumulusci.utils.clean_invalid_references import zip_clean_invalid_references from cumulusci.utils.xml import metadata_tree from cumulusci.utils.ziputils import process_text_in_zipfile @@ -225,62 +219,14 @@ class CleanInvalidReferencesMetaXMLTransform(SourceTransform): """Source transform that cleans *-meta.xml files of invalid references.""" options_model = None - identifier = "clean_invalid_ref" - - def entities_from_package(self, zf, context): - package_xml = return_package_xml_from_zip(zf, self.api_version) - api = ApiRetrieveUnpackaged( - context, package_xml=package_xml, api_version=self.api_version - ) - context.logger.info("Retrieving entities from package.xml") - retrieved_zf = api() - return get_target_entities_from_zip(retrieved_zf) - - def ret_sf(self, context): - sf = get_simple_salesforce_connection( - context.project_config, - context.org_config, - api_version=self.api_version, - base_url=None, - ) - return sf - - def entities_user_permission(self, sf): - path = "sobjects/PermissionSet/describe" - urlpath = sf.base_url + path - method = "GET" - response = sf._call_salesforce(method, urlpath) - - fields = [ - f["name"][len("Permissions") :] - for f in response.json()["fields"] - if f["name"].startswith("Permissions") and f["type"] == "boolean" - ] - return set(fields) - def entities_tabs(self, sf): - soql = "SELECT Name FROM TabDefinition" - result = sf.query(soql)["records"] - - tabs = [record["Name"] for record in result] - return set(tabs) + identifier = "clean_invalid_ref" def process(self, zf: ZipFile, context: TaskContext) -> ZipFile: context.logger.info( "Cleaning profiles and permission sets meta.xml files of invalid references" ) - - self.api_version = context.org_config.latest_api_version - sf = self.ret_sf(context) - - target_entites = {} - target_entites.update(self.entities_from_package(zf, context)) - target_entites.setdefault("userPermissions", set()).update( - self.entities_user_permission(sf) - ) - target_entites.setdefault("tabs", set()).update(self.entities_tabs(sf)) - zip_dest = zip_clean_invalid_references(zf, target_entites) - + zip_dest = zip_clean_invalid_references(zf, context) context.logger.info("Done cleaning profiles and permission sets\n") return zip_dest diff --git a/cumulusci/salesforce_api/package_zip.py b/cumulusci/salesforce_api/package_zip.py index 49c9a89112..e8bea53327 100644 --- a/cumulusci/salesforce_api/package_zip.py +++ b/cumulusci/salesforce_api/package_zip.py @@ -19,6 +19,7 @@ RemoveFeatureParametersTransform, SourceTransform, ) +from cumulusci.core.utils import process_bool_arg from cumulusci.utils.ziputils import hash_zipfile_contents INSTALLED_PACKAGE_PACKAGE_XML = """ @@ -191,7 +192,7 @@ def _process(self): if self.options.get("clean_meta_xml", True): transforms.append(CleanMetaXMLTransform()) - if self.options.get("clean_invalid_ref", False): + if process_bool_arg(self.options.get("clean_invalid_ref") or False): transforms.append(CleanInvalidReferencesMetaXMLTransform()) # Static resource bundling diff --git a/cumulusci/utils/clean_invalid_references.py b/cumulusci/utils/clean_invalid_references.py index 05f593fe2b..b79f900c76 100644 --- a/cumulusci/utils/clean_invalid_references.py +++ b/cumulusci/utils/clean_invalid_references.py @@ -1,7 +1,14 @@ import io import zipfile +from concurrent.futures import ThreadPoolExecutor +from typing import Callable, Dict from lxml import etree as ET +from simple_salesforce.api import Salesforce + +from cumulusci.core.dependencies.utils import TaskContext +from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged +from cumulusci.salesforce_api.utils import get_simple_salesforce_connection class FileName: @@ -81,7 +88,94 @@ def return_name(self, element: ET.Element): } -def return_package_xml_from_zip(zip_src, api_version: str): +def run_queries_in_parallel( + queries: Dict[str, str], run_query: Callable[[str], dict], num_threads: int = 4 +): + """Accepts a set of queries structured as {'query_name': 'query'} + and a run_query function that runs a particular query. Runs queries in parallel and returns the queries""" + results_dict = {} + + with ThreadPoolExecutor(max_workers=num_threads) as executor: + futures = { + query_name: executor.submit(run_query, query) + for query_name, query in queries.items() + } + + for query_name, future in futures.items(): + try: + query_result = future.result() + results_dict[query_name] = query_result + except Exception as e: + raise Exception(f"Error executing query '{query_name}': {type(e)}: {e}") + else: + queries.pop(query_name, None) + + return results_dict + + +def entities_from_query(sf: Salesforce): + # Define the query + def run_query(path: str): + urlpath = sf.base_url + path + method = "GET" + return sf._call_salesforce(method, urlpath).json() + + # Queries + queries = {} + queries["userPermissions"] = "sobjects/PermissionSet/describe" + queries["fields"] = "sobjects/FieldPermissions/describe" + queries["objects"] = "sobjects/ObjectPermissions/describe" + queries["tabs"] = "query/?q=SELECT+Name+FROM+PermissionSetTabSetting+GROUP+BY+Name" + + # Run all queries + result = run_queries_in_parallel(queries, run_query) + + # Process the results + user_permissions = process_user_permissions(result["userPermissions"]) + fields = process_fields(result["fields"]) + tabs = process_tabs(result["tabs"]) + objects = process_objects(result["objects"]) + + return user_permissions, fields, tabs, objects + + +def process_user_permissions(result_dict: dict) -> set: + permissions = [ + f["name"][len("Permissions") :] + for f in result_dict["fields"] + if f["name"].startswith("Permissions") and f["type"] == "boolean" + ] + return set(permissions) + + +def process_fields(result_dict: dict) -> set: + field_entities = [] + for field in result_dict["fields"]: + if field.get("name") == "Field": + field_entities.extend( + picklistValue["value"] + for picklistValue in field.get("picklistValues", []) + ) + return set(field_entities) + + +def process_tabs(result_dict: dict) -> set: + tabs = [tab["Name"] for tab in result_dict["records"]] + return set(tabs) + + +def process_objects(result_dict: dict) -> set: + objects = [] + for obj in result_dict["fields"]: + if obj.get("name") == "SobjectType": + objects.extend( + picklistValue["value"] + for picklistValue in obj.get("picklistValues", []) + ) + return set(objects) + + +def return_package_xml_from_zip(zip_src: zipfile.ZipFile, api_version: str): # Iterate through the zip file to generate the package.xml package_xml_input = {} for name in zip_src.namelist(): @@ -128,7 +222,7 @@ def create_package_xml(input_dict: dict, api_version: str): return package_xml -def get_fields_and_recordtypes(root, objectName): +def get_fields_and_recordtypes(root: ET.Element, objectName): fields = set() recordTypes = set() @@ -141,14 +235,14 @@ def get_fields_and_recordtypes(root, objectName): return fields, recordTypes -def get_tabs_from_app(root): +def get_tabs_from_app(root: ET.Element): tabs = set() for tab in root.findall("tabs"): tabs.update([tab.text]) return tabs -def get_target_entities_from_zip(zip_src): +def get_target_entities_from_zip(zip_src: zipfile.ZipFile): target_entities = {key: set() for key in FOLDER_PERM_DICT.keys()} for name in zip_src.namelist(): if name == "package.xml": @@ -176,7 +270,7 @@ def get_target_entities_from_zip(zip_src): return target_entities -def zip_clean_invalid_references(zip_src, target_entities): +def clean_zip_file(zip_src: zipfile.ZipFile, target_entities: Dict[str, set]): zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) for name in zip_src.namelist(): file = zip_src.open(name) @@ -201,7 +295,7 @@ def zip_clean_invalid_references(zip_src, target_entities): return zip_dest -def strip_namespace(element): +def strip_namespace(element: ET.Element): for elem in element.iter(): if "}" in elem.tag: elem.tag = elem.tag.split("}", 1)[1] @@ -209,7 +303,7 @@ def strip_namespace(element): def fetch_permissionable_entity_names( - root, perm_entity: PermissionElementXPath, parent: bool = False + root: ET.Element, perm_entity: PermissionElementXPath, parent: bool = False ): entity_names = set() for element in root.findall(perm_entity.permission_xpath): @@ -221,11 +315,51 @@ def fetch_permissionable_entity_names( return entity_names -def CleanXML(root, target_entities): +def CleanXML(root: ET.Element, target_entities: Dict[str, set]): root = strip_namespace(root) for key, perm_entities in FOLDER_PERM_DICT.items(): for perm_entity in perm_entities: for element in root.findall(perm_entity.permission_xpath): - if perm_entity.return_name(element) not in target_entities[key]: + if perm_entity.return_name(element) not in target_entities.get(key, []): root.remove(element) return root + + +def entities_from_package(zf: zipfile.ZipFile, context: TaskContext, api_version: str): + package_xml = return_package_xml_from_zip(zf, api_version) + api = ApiRetrieveUnpackaged( + context, package_xml=package_xml, api_version=api_version + ) + context.logger.info("Retrieving entities from package.xml") + retrieved_zf = api() + return get_target_entities_from_zip(retrieved_zf) + + +def ret_sf(context: TaskContext, api_version: str): + sf = get_simple_salesforce_connection( + context.project_config, + context.org_config, + api_version=api_version, + base_url=None, + ) + return sf + + +def zip_clean_invalid_references(zf: zipfile.ZipFile, context: TaskContext): + # Set API version + api_version = context.org_config.latest_api_version + + # Query and get entities + sf = ret_sf(context, api_version) + userPermissions, fields, tabs, objects = entities_from_query(sf) + + # Update the target entites + target_entites = {} + target_entites.update(entities_from_package(zf, context, api_version)) + target_entites.setdefault("userPermissions", set()).update(userPermissions) + target_entites.setdefault("tabs", set()).update(tabs) + target_entites.setdefault("fields", set()).update(fields) + target_entites.setdefault("objects", set()).update(objects) + + # Clean the zip file + return clean_zip_file(zf, target_entites) diff --git a/cumulusci/utils/tests/test_clean_invalid_references.py b/cumulusci/utils/tests/test_clean_invalid_references.py index cc206cbcd7..b9cceb1370 100644 --- a/cumulusci/utils/tests/test_clean_invalid_references.py +++ b/cumulusci/utils/tests/test_clean_invalid_references.py @@ -1,19 +1,30 @@ import io import zipfile -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest from lxml import etree as ET +from simple_salesforce.api import Salesforce +from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged from cumulusci.utils.clean_invalid_references import ( CleanXML, PermissionElementXPath, + clean_zip_file, create_package_xml, + entities_from_package, + entities_from_query, fetch_permissionable_entity_names, get_fields_and_recordtypes, get_tabs_from_app, get_target_entities_from_zip, + process_fields, + process_objects, + process_tabs, + process_user_permissions, + ret_sf, return_package_xml_from_zip, + run_queries_in_parallel, strip_namespace, zip_clean_invalid_references, ) @@ -245,7 +256,7 @@ def fetch_entities(root, perm, parent=False): ] ], ) -def test_zip_clean_invalid_references(mock_clean_xml, sample_zip): +def test_clean_zip_file(mock_clean_xml, sample_zip): # Mock CleanXML def clean_xml(root, target_entities): for element in root.findall(".//objectPermissions"): @@ -260,7 +271,7 @@ def clean_xml(root, target_entities): # Call the function target_entities = {"something": {"something"}} - zip_dest = zip_clean_invalid_references(sample_zip, target_entities) + zip_dest = clean_zip_file(sample_zip, target_entities) name_list = [ "profiles/sample.profile", @@ -304,5 +315,203 @@ def test_strip_namespace(): assert element2.text == "Value 2" +up_result_dict = { + "fields": [ + {"name": "PermissionsEdit", "type": "boolean"}, + {"name": "PermissionsView", "type": "boolean"}, + ] +} +field_result_dict = { + "fields": [ + { + "name": "Field", + "picklistValues": [{"value": "Option1"}, {"value": "Option2"}], + } + ], +} +tabs_result_dict = { + "records": [ + {"Name": "Tab1"}, + {"Name": "Tab2"}, + ], +} +objects_result_dict = { + "fields": [ + { + "name": "SobjectType", + "picklistValues": [{"value": "Object1"}, {"value": "Object2"}], + }, + ], +} + + +def test_process_user_permissions(): + result = process_user_permissions(up_result_dict) + expected = {"Edit", "View"} + assert result == expected + + +def test_process_fields(): + result = process_fields(field_result_dict) + expected = {"Option1", "Option2"} + assert result == expected + + +def test_process_tabs(): + result = process_tabs(tabs_result_dict) + expected = {"Tab1", "Tab2"} + assert result == expected + + +def test_process_objects(): + result = process_objects(objects_result_dict) + expected = {"Object1", "Object2"} + assert result == expected + + +def sample_run_query(query): + if query == "query1": + return ["Query 1 result"] + elif query == "query2": + return ["Query 2 result"] + elif query == "query_data_error": + raise ValueError("Some error occurred.") + + +def test_run_queries_in_parallel(): + queries = { + "query1": "query1", + "query2": "query2", + } + + results = run_queries_in_parallel(queries, sample_run_query, num_threads=2) + + assert "query1" in results + assert "query2" in results + assert results["query1"] == ["Query 1 result"] + assert results["query2"] == ["Query 2 result"] + + +def test_run_queries_in_parallel_exception_handling(): + queries = { + "query_that_raises": "query_data_error", + } + + with pytest.raises(Exception) as exc_info: + run_queries_in_parallel(queries, sample_run_query) + assert "Error executing query 'query_that_raises':" in str(exc_info.value) + + +class SampleResponse: + def __init__(self, result_list): + self.result_list = result_list + + def json(self): + return self.result_list + + +def call_salesforce(method, urlpath): + result = { + "123" + "sobjects/PermissionSet/describe": SampleResponse(up_result_dict), + "123" + "sobjects/FieldPermissions/describe": SampleResponse(field_result_dict), + "123" + + "sobjects/ObjectPermissions/describe": SampleResponse(objects_result_dict), + "123" + + "query/?q=SELECT+Name+FROM+PermissionSetTabSetting+GROUP+BY+Name": SampleResponse( + tabs_result_dict + ), + } + return result[urlpath] + + +def test_entities_from_query(task_context): + sf = Mock() + sf.base_url = "123" + sf._call_salesforce = call_salesforce + + result_userpermissions = {"Edit", "View"} + result_fields = {"Option1", "Option2"} + result_tabs = {"Tab1", "Tab2"} + result_objects = {"Object1", "Object2"} + + with patch( + "cumulusci.utils.clean_invalid_references.process_user_permissions", + return_value=result_userpermissions, + ) as mock_process_user_permissions, patch( + "cumulusci.utils.clean_invalid_references.process_fields", + return_value=result_fields, + ) as mock_process_fields, patch( + "cumulusci.utils.clean_invalid_references.process_tabs", + return_value=result_tabs, + ) as mock_process_tabs, patch( + "cumulusci.utils.clean_invalid_references.process_objects", + return_value=result_objects, + ) as mock_process_objects: + userpermissions, fields, tabs, objects = entities_from_query(sf) + assert userpermissions == result_userpermissions + assert fields == result_fields + assert tabs == result_tabs + assert objects == result_objects + + mock_process_user_permissions.assert_called_with(up_result_dict) + mock_process_fields.assert_called_with(field_result_dict) + mock_process_tabs.assert_called_with(tabs_result_dict) + mock_process_objects.assert_called_with(objects_result_dict) + + +def test_ret_sf(task_context): + sf = ret_sf(task_context, "58.0") + assert isinstance(sf, Salesforce) + + +@pytest.mark.parametrize( + "elements", + [ + [ + ("sample", "sample", ".sample", SAMPLE_XML), + ] + ], +) +def test_entities_from_package(task_context, sample_zip): + zf = Mock() + with patch( + "cumulusci.utils.clean_invalid_references.return_package_xml_from_zip", + return_value="package_xml", + ), patch.object(ApiRetrieveUnpackaged, "__call__", return_value=sample_zip), patch( + "cumulusci.utils.clean_invalid_references.get_target_entities_from_zip" + ) as mock_get_target: + entities_from_package(zf, task_context, "58.0") + mock_get_target.assert_called_once_with(sample_zip) + + +def test_zip_clean_invalid_references(task_context): + userPermissions = {"UserPermissions"} + fields = {"fields"} + tabs = {"tabs"} + objects = {"objects"} + entities = {"applications": {"applications"}} + + expected_target_entites = {} + expected_target_entites.setdefault("userPermissions", set()).update(userPermissions) + expected_target_entites.setdefault("tabs", set()).update(tabs) + expected_target_entites.setdefault("fields", set()).update(fields) + expected_target_entites.setdefault("objects", set()).update(objects) + expected_target_entites.update(entities) + + zf = Mock() + + with patch("cumulusci.utils.clean_invalid_references.ret_sf"), patch( + "cumulusci.utils.clean_invalid_references.entities_from_query", + return_value=(userPermissions, fields, tabs, objects), + ), patch( + "cumulusci.utils.clean_invalid_references.entities_from_package", + return_value=entities, + ), patch( + "cumulusci.utils.clean_invalid_references.clean_zip_file" + ) as mock_clean_zip_file: + zip_clean_invalid_references(zf, task_context) + mock_clean_zip_file.assert_called_once_with(zf, expected_target_entites) + + if __name__ == "__main": pytest.main() From 1d2a52cc2e9aa53f4dae03ce97b1db290b202f40 Mon Sep 17 00:00:00 2001 From: aditya-balachander Date: Wed, 8 Nov 2023 21:46:59 +0530 Subject: [PATCH 13/13] Made requested changes --- .../tests/test_transforms.py | 36 +++- cumulusci/salesforce_api/package_zip.py | 1 + cumulusci/tasks/salesforce/Deploy.py | 8 +- cumulusci/utils/clean_invalid_references.py | 187 ++++++++++++------ .../tests/test_clean_invalid_references.py | 134 +++++++++---- 5 files changed, 253 insertions(+), 113 deletions(-) diff --git a/cumulusci/core/source_transforms/tests/test_transforms.py b/cumulusci/core/source_transforms/tests/test_transforms.py index e13ea2dae9..9e994d73e1 100644 --- a/cumulusci/core/source_transforms/tests/test_transforms.py +++ b/cumulusci/core/source_transforms/tests/test_transforms.py @@ -1099,6 +1099,16 @@ def test_strip_unwanted_files(task_context): "fields": [ {"name": "PermissionsEdit", "type": "boolean"}, {"name": "PermissionsView", "type": "boolean"}, + {"name": "PermissionsViewPackage", "type": "boolean"}, + ] +} +pd_result_dict = { + "records": [ + {"Permission": "ViewSetup", "RequiredPermission": "View"}, + {"Permission": "EditSetup", "RequiredPermission": "Edit"}, + {"Permission": "EditSetup", "RequiredPermission": "View"}, + {"Permission": "ManageSetup", "RequiredPermission": "Manage"}, + {"Permission": "ViewPackage", "RequiredPermission": "View"}, ] } field_result_dict = { @@ -1135,6 +1145,11 @@ def return_response(method, urlpath): response.json.return_value = objects_result_dict elif "query/?q=SELECT+Name+FROM+PermissionSetTabSetting+GROUP+BY+Name" in urlpath: response.json.return_value = tabs_result_dict + elif ( + "tooling/query/?q=SELECT+Permission,+RequiredPermission+FROM+PermissionDependency+WHERE+PermissionType+=+'User Permission'+AND+RequiredPermissionType+=+'User Permission'" + in urlpath + ): + response.json.return_value = pd_result_dict return response @@ -1161,7 +1176,7 @@ def test_clean_invalid_references(task_context): " Standard-Fake\n" " \n" " \n" - " ManagePermissions\n" + " ViewPackage\n" " \n" "\n" ) @@ -1179,7 +1194,7 @@ def test_clean_invalid_references(task_context): " Standard-Account\n" " \n" " \n" - " ManagePermissions\n" + " ViewPackage\n" " \n" "\n" ) @@ -1195,6 +1210,8 @@ def test_clean_invalid_references(task_context): "" ) + package_xml = "\n" " 58.0\n" "" + output_zip = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) output_zip.writestr("objects/Opportunity.object", obj_xml) output_zip.writestr("objects/Account.object", obj_xml) @@ -1203,12 +1220,17 @@ def test_clean_invalid_references(task_context): Salesforce, "_call_salesforce", side_effect=return_response ), patch.object(BaseMetadataApiCall, "__call__", return_value=output_zip): builder = MetadataPackageZipBuilder.from_zipfile( - ZipFileSpec({Path("profiles/Foo.profile"): xml_data}).as_zipfile(), + ZipFileSpec( + { + Path("profiles/Foo.profile"): xml_data, + Path("package.xml"): package_xml, + } + ).as_zipfile(), options={"clean_invalid_ref": True}, context=task_context, ) for name in builder.zf.namelist(): - assert name == "profiles/Foo.profile" - result_root = ET.parse(builder.zf.open(name)).getroot() - expected_root = ET.fromstring(xml_data_clean.encode("utf-8")) - assert ET.iselement(result_root) == ET.iselement(expected_root) + if name == "profiles/Foo.profile": + result_root = ET.parse(builder.zf.open(name)).getroot() + expected_root = ET.fromstring(xml_data_clean.encode("utf-8")) + assert ET.iselement(result_root) == ET.iselement(expected_root) diff --git a/cumulusci/salesforce_api/package_zip.py b/cumulusci/salesforce_api/package_zip.py index e8bea53327..2ee6710030 100644 --- a/cumulusci/salesforce_api/package_zip.py +++ b/cumulusci/salesforce_api/package_zip.py @@ -192,6 +192,7 @@ def _process(self): if self.options.get("clean_meta_xml", True): transforms.append(CleanMetaXMLTransform()) + # To clean profiles and permissionsets of invalid references if process_bool_arg(self.options.get("clean_invalid_ref") or False): transforms.append(CleanInvalidReferencesMetaXMLTransform()) diff --git a/cumulusci/tasks/salesforce/Deploy.py b/cumulusci/tasks/salesforce/Deploy.py index bc75290366..3a41ae24ff 100644 --- a/cumulusci/tasks/salesforce/Deploy.py +++ b/cumulusci/tasks/salesforce/Deploy.py @@ -60,15 +60,12 @@ class Deploy(BaseSalesforceMetadataApiTask): "clean_meta_xml": { "description": "Defaults to True which strips the element from all meta.xml files. The packageVersion element gets added automatically by the target org and is set to whatever version is installed in the org. To disable this, set this option to False" }, - "clean_profiles": { - "description": "Defaults to True in which case all profiles are cleaned of invalid references before deployment." - }, "transforms": { "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms." }, "rest_deploy": {"description": "If True, deploy metadata using REST API"}, "clean_invalid_ref": { - "description": "If specified, all profiles and permission sets are cleaned of invalid references before deployment." + "description": "If True, all profiles and permission sets are cleaned of invalid references before deployment." }, } @@ -226,9 +223,6 @@ def _get_package_zip(self, path) -> Union[str, dict, None]: "clean_meta_xml": process_bool_arg( self.options.get("clean_meta_xml", True) ), - "clean_profiles": process_bool_arg( - self.options.get("clean_profiles", True) - ), "namespace_inject": namespace, "unmanaged": not self._has_namespaced_package(namespace), "namespaced_org": self._is_namespaced_org(namespace), diff --git a/cumulusci/utils/clean_invalid_references.py b/cumulusci/utils/clean_invalid_references.py index b79f900c76..ba18ddc87b 100644 --- a/cumulusci/utils/clean_invalid_references.py +++ b/cumulusci/utils/clean_invalid_references.py @@ -1,16 +1,18 @@ import io import zipfile from concurrent.futures import ThreadPoolExecutor -from typing import Callable, Dict +from typing import Callable, Dict, Union from lxml import etree as ET from simple_salesforce.api import Salesforce from cumulusci.core.dependencies.utils import TaskContext +from cumulusci.core.exceptions import CumulusCIUsageError from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged from cumulusci.salesforce_api.utils import get_simple_salesforce_connection +# Class for folder name and extension of the entities needed to be cleaned class FileName: folder_name: str extension: str @@ -20,11 +22,13 @@ def __init__(self, folder_name, extension): self.extension = extension +# Add here any other entity that you want cleaned in a similar fashion PROFILE_FILE = FileName("profiles/", ".profile") PERMISSIONSET_FILE = FileName("permissionsets/", ".permissionset") FILES_TO_BE_CLEANED = [PROFILE_FILE, PERMISSIONSET_FILE] +# Class for defining the xpath of different entities class PermissionElementXPath: permission_xpath: str name_xpath: str @@ -40,6 +44,7 @@ def return_name(self, element: ET.Element): return element.find(self.name_xpath).text +# The entities and their xpath OBJECT_PERMISSION = PermissionElementXPath(".//objectPermissions", "object") RECORDTYPE_PERMISSION = PermissionElementXPath( ".//recordTypeVisibilities", "recordType" @@ -57,6 +62,8 @@ def return_name(self, element: ET.Element): ".//customMetadataTypeAccesses", "name" ) +# Contains the relation between the field (key) inside a package.xml file +# And the field entities and their xpath (value) PACKAGEXML_DICT = { # PERMISSION : is_child "CustomObject": { @@ -74,6 +81,9 @@ def return_name(self, element: ET.Element): "CustomPermission": CUSTOMPERM_PERMISSION, } +# Contains the relation between the folder names (key) present inside the zip file from +# ApiRetrievedUnpackaged and the entities and their xpath (value) which are present inside +# those folders FOLDER_PERM_DICT = { "objects": [OBJECT_PERMISSION, CUSTOMSETTING_PERMISSION, CUSTOMMETADATA_PERMISSION], "fields": [FIELD_PERMISSION], @@ -88,50 +98,57 @@ def return_name(self, element: ET.Element): } -def run_queries_in_parallel( - queries: Dict[str, str], run_query: Callable[[str], dict], num_threads: int = 4 +def run_calls_in_parallel( + queries: Dict[str, str], run_call: Callable[[str], dict], num_threads: int = 4 ): - """Accepts a set of queries structured as {'query_name': 'query'} - and a run_query function that runs a particular query. Runs queries in parallel and returns the queries""" + """Accepts a set of calls structured as {'call_name': 'call'} + and a run_call function that runs a particular call. Runs calls in parallel and returns the resuts""" results_dict = {} with ThreadPoolExecutor(max_workers=num_threads) as executor: futures = { - query_name: executor.submit(run_query, query) - for query_name, query in queries.items() + call_name: executor.submit(run_call, call) + for call_name, call in queries.items() } - for query_name, future in futures.items(): + for call_name, future in futures.items(): try: - query_result = future.result() - results_dict[query_name] = query_result + call_result = future.result() + results_dict[call_name] = call_result except Exception as e: - raise Exception(f"Error executing query '{query_name}': {type(e)}: {e}") - else: - queries.pop(query_name, None) + raise Exception(f"Error executing call '{call_name}': {type(e)}: {e}") return results_dict -def entities_from_query(sf: Salesforce): - # Define the query - def run_query(path: str): +def entities_from_api_calls(sf: Salesforce): + """Retrieves the set of UserPermissions, Fields, Objects and Tabs from the target org + by using API calls and returns them""" + # Define the run_call method + def run_call(path: str): urlpath = sf.base_url + path method = "GET" return sf._call_salesforce(method, urlpath).json() # Queries - queries = {} - queries["userPermissions"] = "sobjects/PermissionSet/describe" - queries["fields"] = "sobjects/FieldPermissions/describe" - queries["objects"] = "sobjects/ObjectPermissions/describe" - queries["tabs"] = "query/?q=SELECT+Name+FROM+PermissionSetTabSetting+GROUP+BY+Name" - - # Run all queries - result = run_queries_in_parallel(queries, run_query) + api_calls = {} + api_calls["userPermissions"] = "tooling/sobjects/PermissionSet/describe" + api_calls[ + "permissionDependency" + ] = "tooling/query/?q=SELECT+Permission,+RequiredPermission+FROM+PermissionDependency+WHERE+PermissionType+=+'User Permission'+AND+RequiredPermissionType+=+'User Permission'" + api_calls["fields"] = "sobjects/FieldPermissions/describe" + api_calls["objects"] = "sobjects/ObjectPermissions/describe" + api_calls[ + "tabs" + ] = "query/?q=SELECT+Name+FROM+PermissionSetTabSetting+GROUP+BY+Name" + + # Run all api_calls + result = run_calls_in_parallel(api_calls, run_call) # Process the results - user_permissions = process_user_permissions(result["userPermissions"]) + user_permissions = process_user_permissions( + result["userPermissions"], result["permissionDependency"] + ) fields = process_fields(result["fields"]) tabs = process_tabs(result["tabs"]) objects = process_objects(result["objects"]) @@ -139,16 +156,25 @@ def run_query(path: str): return user_permissions, fields, tabs, objects -def process_user_permissions(result_dict: dict) -> set: - permissions = [ - f["name"][len("Permissions") :] - for f in result_dict["fields"] - if f["name"].startswith("Permissions") and f["type"] == "boolean" - ] - return set(permissions) +def process_user_permissions( + result_dict_user_permission: dict, result_dict_permission_dependency: dict +) -> dict: + """Process the result of the API Call and return UserPermissions""" + user_permissions = {} + for field in result_dict_user_permission["fields"]: + if field["name"].startswith("Permissions") and field["type"] == "boolean": + user_permissions.setdefault(field["name"][len("Permissions") :], set()) + + for record in result_dict_permission_dependency["records"]: + user_permissions.setdefault(record["Permission"], set()).update( + [record["RequiredPermission"]] + ) + + return user_permissions def process_fields(result_dict: dict) -> set: + """Process the result of the API Call and return Fields""" field_entities = [] for field in result_dict["fields"]: if field.get("name") == "Field": @@ -160,11 +186,12 @@ def process_fields(result_dict: dict) -> set: def process_tabs(result_dict: dict) -> set: - tabs = [tab["Name"] for tab in result_dict["records"]] - return set(tabs) + """Process the result of the API Call and return Tabs""" + return {tab["Name"] for tab in result_dict["records"]} def process_objects(result_dict: dict) -> set: + """Process the result of the API Call and return Objects""" objects = [] for obj in result_dict["fields"]: if obj.get("name") == "SobjectType": @@ -176,7 +203,9 @@ def process_objects(result_dict: dict) -> set: def return_package_xml_from_zip(zip_src: zipfile.ZipFile, api_version: str): - # Iterate through the zip file to generate the package.xml + """Iterates through the zip file, searches for profile/permissionset files, + extracts all permissionable entities, creates a package.xml file containing them, + and returns the resulting package.xml file""" package_xml_input = {} for name in zip_src.namelist(): if any( @@ -207,6 +236,7 @@ def return_package_xml_from_zip(zip_src: zipfile.ZipFile, api_version: str): def create_package_xml(input_dict: dict, api_version: str): + """Given the entities, create the package.xml""" package_xml = '\n' package_xml += '\n' @@ -223,6 +253,7 @@ def create_package_xml(input_dict: dict, api_version: str): def get_fields_and_recordtypes(root: ET.Element, objectName): + """Get the fields and record types that are present inside the .object file""" fields = set() recordTypes = set() @@ -235,14 +266,9 @@ def get_fields_and_recordtypes(root: ET.Element, objectName): return fields, recordTypes -def get_tabs_from_app(root: ET.Element): - tabs = set() - for tab in root.findall("tabs"): - tabs.update([tab.text]) - return tabs - - def get_target_entities_from_zip(zip_src: zipfile.ZipFile): + """Accepts the resulting zip file from ApiRetrievedUnpackaged + and returns all the entities which are present in the target org""" target_entities = {key: set() for key in FOLDER_PERM_DICT.keys()} for name in zip_src.namelist(): if name == "package.xml": @@ -260,17 +286,14 @@ def get_target_entities_from_zip(zip_src: zipfile.ZipFile): target_entities["fields"].update(fields) target_entities["recordTypes"].update(recordTypes) - # To handle tabs which are not part of TabDefinition Table - if name.endswith(".app"): - file = zip_src.open(name) - root = ET.parse(file).getroot() - root = strip_namespace(root) - target_entities["tabs"].update(get_tabs_from_app(root)) - return target_entities -def clean_zip_file(zip_src: zipfile.ZipFile, target_entities: Dict[str, set]): +def clean_zip_file( + zip_src: zipfile.ZipFile, target_entities: Dict[str, Union[set, dict]] +): + """Parses through the zip file and removes any of the permissionable entities + which are not present in the target_entities""" zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED) for name in zip_src.namelist(): file = zip_src.open(name) @@ -279,7 +302,7 @@ def clean_zip_file(zip_src: zipfile.ZipFile, target_entities: Dict[str, set]): for item in FILES_TO_BE_CLEANED ): root = ET.parse(file).getroot() - cleaned_root = CleanXML(root, target_entities) + cleaned_root = clean_xml(root, target_entities) tree = ET.ElementTree(cleaned_root) cleaned_content = io.BytesIO() tree.write( @@ -296,6 +319,7 @@ def clean_zip_file(zip_src: zipfile.ZipFile, target_entities: Dict[str, set]): def strip_namespace(element: ET.Element): + """Remove the namespace in the xml tree""" for elem in element.iter(): if "}" in elem.tag: elem.tag = elem.tag.split("}", 1)[1] @@ -305,6 +329,8 @@ def strip_namespace(element: ET.Element): def fetch_permissionable_entity_names( root: ET.Element, perm_entity: PermissionElementXPath, parent: bool = False ): + """Return the name of the entity given the xpath. + If parent is set to True, the entity's parent name is fetched""" entity_names = set() for element in root.findall(perm_entity.permission_xpath): entity_names.add( @@ -315,17 +341,39 @@ def fetch_permissionable_entity_names( return entity_names -def CleanXML(root: ET.Element, target_entities: Dict[str, set]): +def clean_xml(root: ET.Element, target_entities: Dict[str, Union[set, dict]]): + """Given the xml tree, remove entities not present in target_entities""" root = strip_namespace(root) + user_permissions_source_org = {} for key, perm_entities in FOLDER_PERM_DICT.items(): for perm_entity in perm_entities: for element in root.findall(perm_entity.permission_xpath): - if perm_entity.return_name(element) not in target_entities.get(key, []): + entity_name = perm_entity.return_name(element) + # If entity is a user permission, store it + if key == "userPermissions": + user_permissions_source_org[entity_name] = element + if entity_name not in target_entities.get(key, []): root.remove(element) + + # Remove those UserPermissions which dont have required dependencies + # Find elements that should be removed based on dependencies + elements_to_remove = set() + for permission, dependencies_list in target_entities["userPermissions"].items(): + if permission in user_permissions_source_org and not all( + dependency in user_permissions_source_org + for dependency in dependencies_list + ): + elements_to_remove.add(user_permissions_source_org[permission]) + # Remove elements + for element in elements_to_remove: + root.remove(element) return root def entities_from_package(zf: zipfile.ZipFile, context: TaskContext, api_version: str): + """Creates package.xml with all permissionable entities from the profiles and permissionsets. + Uses ApiRetrieveUnpackaged to retieve all these entities from the target org and generate a + list of entities present in the target org""" package_xml = return_package_xml_from_zip(zf, api_version) api = ApiRetrieveUnpackaged( context, package_xml=package_xml, api_version=api_version @@ -345,21 +393,34 @@ def ret_sf(context: TaskContext, api_version: str): return sf +def ret_api_version(zf: zipfile.ZipFile): + with zf.open("package.xml") as package_xml_file: + package_xml_contents = package_xml_file.read() + root = ET.fromstring(package_xml_contents) + root = strip_namespace(root) + version_element = root.find(".//version") + if version_element is not None: + api_version = version_element.text + return api_version + else: + raise CumulusCIUsageError("API version not found in the package.xml file.") + + def zip_clean_invalid_references(zf: zipfile.ZipFile, context: TaskContext): + """Cleans the profiles and permissionset files in the zip file of any invalid references""" # Set API version - api_version = context.org_config.latest_api_version - + api_version = ret_api_version(zf) # Query and get entities sf = ret_sf(context, api_version) - userPermissions, fields, tabs, objects = entities_from_query(sf) + user_permissions, fields, tabs, objects = entities_from_api_calls(sf) # Update the target entites - target_entites = {} - target_entites.update(entities_from_package(zf, context, api_version)) - target_entites.setdefault("userPermissions", set()).update(userPermissions) - target_entites.setdefault("tabs", set()).update(tabs) - target_entites.setdefault("fields", set()).update(fields) - target_entites.setdefault("objects", set()).update(objects) + target_entities = {} + target_entities.update(entities_from_package(zf, context, api_version)) + target_entities.setdefault("tabs", set()).update(tabs) + target_entities.setdefault("fields", set()).update(fields) + target_entities.setdefault("objects", set()).update(objects) + target_entities["userPermissions"] = user_permissions # Clean the zip file - return clean_zip_file(zf, target_entites) + return clean_zip_file(zf, target_entities) diff --git a/cumulusci/utils/tests/test_clean_invalid_references.py b/cumulusci/utils/tests/test_clean_invalid_references.py index b9cceb1370..51586e94d1 100644 --- a/cumulusci/utils/tests/test_clean_invalid_references.py +++ b/cumulusci/utils/tests/test_clean_invalid_references.py @@ -6,25 +6,26 @@ from lxml import etree as ET from simple_salesforce.api import Salesforce +from cumulusci.core.exceptions import CumulusCIUsageError from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged from cumulusci.utils.clean_invalid_references import ( - CleanXML, PermissionElementXPath, + clean_xml, clean_zip_file, create_package_xml, + entities_from_api_calls, entities_from_package, - entities_from_query, fetch_permissionable_entity_names, get_fields_and_recordtypes, - get_tabs_from_app, get_target_entities_from_zip, process_fields, process_objects, process_tabs, process_user_permissions, + ret_api_version, ret_sf, return_package_xml_from_zip, - run_queries_in_parallel, + run_calls_in_parallel, strip_namespace, zip_clean_invalid_references, ) @@ -59,6 +60,12 @@ " \n" " CustomPermission1\n" " \n" + " \n" + " View\n" + " \n" + " \n" + " EditSetup\n" + " \n" "\n" ) @@ -127,24 +134,22 @@ def test_get_fields_and_recordtypes(sample_root): assert recordTypes == expected_recordTypes -@pytest.mark.parametrize("sample_xml", [SAMPLE_APP_XML]) -def test_get_tabs_from_app(sample_root): - tabs = get_tabs_from_app(sample_root) - expected_tabs = {"Tab1", "Tab2"} - assert tabs == expected_tabs - - @pytest.mark.parametrize("sample_xml", [SAMPLE_XML]) -def test_CleanXML(sample_root): +def test_clean_xml(sample_root): target_entities = { "fields": {"Field1", "Field2"}, "recordTypes": {"RecordType1", "RecordType2"}, "tabs": {"Tab1", "Tab2"}, "objects": {"Account"}, "customPermissions": {"CustomPermission1"}, + "userPermissions": { + "View": set(), + "Edit": set(), + "EditSetup": {"View", "Edit"}, + }, } - cleaned_root = CleanXML(sample_root, target_entities) + cleaned_root = clean_xml(sample_root, target_entities) for element in cleaned_root.findall(".//objectPermissions"): assert element.find("object").text in target_entities["objects"] for element in cleaned_root.findall(".//customPermissions"): @@ -154,6 +159,8 @@ def test_CleanXML(sample_root): assert "Contact" not in element.find("object").text for element in cleaned_root.findall(".//fieldPermissions"): assert "SomeField" not in element.find("field").text + for element in cleaned_root.findall(".//userPermissions"): + assert "EditSetup" not in element.find("name").text @pytest.fixture @@ -181,14 +188,11 @@ def sample_zip(elements): def test_get_target_entities_from_zip(sample_zip): with patch( "cumulusci.utils.clean_invalid_references.get_fields_and_recordtypes" - ) as mock_fields, patch( - "cumulusci.utils.clean_invalid_references.get_tabs_from_app" - ) as mock_tabs: + ) as mock_fields: mock_fields.return_value = ( {"sample.Field1", "sample.Field2"}, {"sample.RecordType1", "sample.RecordType2"}, ) - mock_tabs.return_value = {"Tab1", "Tab2"} target_entities = get_target_entities_from_zip(sample_zip) @@ -197,7 +201,7 @@ def test_get_target_entities_from_zip(sample_zip): "fields": {"sample.Field1", "sample.Field2"}, "recordTypes": {"sample.RecordType1", "sample.RecordType2"}, "applications": {"sample"}, - "tabs": {"Tab1", "Tab2"}, + "tabs": set(), "classes": set(), "customPermissions": set(), "flows": set(), @@ -245,7 +249,7 @@ def fetch_entities(root, perm, parent=False): ) -@patch("cumulusci.utils.clean_invalid_references.CleanXML") +@patch("cumulusci.utils.clean_invalid_references.clean_xml") @pytest.mark.parametrize( "elements", [ @@ -257,7 +261,7 @@ def fetch_entities(root, perm, parent=False): ], ) def test_clean_zip_file(mock_clean_xml, sample_zip): - # Mock CleanXML + # Mock clean_xml def clean_xml(root, target_entities): for element in root.findall(".//objectPermissions"): if element.find("object").text == "Contact": @@ -319,6 +323,16 @@ def test_strip_namespace(): "fields": [ {"name": "PermissionsEdit", "type": "boolean"}, {"name": "PermissionsView", "type": "boolean"}, + {"name": "PermissionsViewPackage", "type": "boolean"}, + ] +} +pd_result_dict = { + "records": [ + {"Permission": "ViewSetup", "RequiredPermission": "View"}, + {"Permission": "EditSetup", "RequiredPermission": "Edit"}, + {"Permission": "EditSetup", "RequiredPermission": "View"}, + {"Permission": "ManageSetup", "RequiredPermission": "Manage"}, + {"Permission": "ViewPackage", "RequiredPermission": "View"}, ] } field_result_dict = { @@ -346,8 +360,15 @@ def test_strip_namespace(): def test_process_user_permissions(): - result = process_user_permissions(up_result_dict) - expected = {"Edit", "View"} + result = process_user_permissions(up_result_dict, pd_result_dict) + expected = { + "Edit": set(), + "View": set(), + "ViewSetup": {"View"}, + "EditSetup": {"Edit", "View"}, + "ViewPackage": {"View"}, + "ManageSetup": {"Manage"}, + } assert result == expected @@ -378,13 +399,13 @@ def sample_run_query(query): raise ValueError("Some error occurred.") -def test_run_queries_in_parallel(): +def test_run_calls_in_parallel(): queries = { "query1": "query1", "query2": "query2", } - results = run_queries_in_parallel(queries, sample_run_query, num_threads=2) + results = run_calls_in_parallel(queries, sample_run_query, num_threads=2) assert "query1" in results assert "query2" in results @@ -392,14 +413,14 @@ def test_run_queries_in_parallel(): assert results["query2"] == ["Query 2 result"] -def test_run_queries_in_parallel_exception_handling(): +def test_run_calls_in_parallel_exception_handling(): queries = { "query_that_raises": "query_data_error", } with pytest.raises(Exception) as exc_info: - run_queries_in_parallel(queries, sample_run_query) - assert "Error executing query 'query_that_raises':" in str(exc_info.value) + run_calls_in_parallel(queries, sample_run_query) + assert "Error executing call 'query_that_raises':" in str(exc_info.value) class SampleResponse: @@ -412,7 +433,12 @@ def json(self): def call_salesforce(method, urlpath): result = { - "123" + "sobjects/PermissionSet/describe": SampleResponse(up_result_dict), + "123" + + "tooling/sobjects/PermissionSet/describe": SampleResponse(up_result_dict), + "123" + + "tooling/query/?q=SELECT+Permission,+RequiredPermission+FROM+PermissionDependency+WHERE+PermissionType+=+'User Permission'+AND+RequiredPermissionType+=+'User Permission'": SampleResponse( + pd_result_dict + ), "123" + "sobjects/FieldPermissions/describe": SampleResponse(field_result_dict), "123" + "sobjects/ObjectPermissions/describe": SampleResponse(objects_result_dict), @@ -424,12 +450,19 @@ def call_salesforce(method, urlpath): return result[urlpath] -def test_entities_from_query(task_context): +def test_entities_from_api_calls(task_context): sf = Mock() sf.base_url = "123" sf._call_salesforce = call_salesforce - result_userpermissions = {"Edit", "View"} + result_userpermissions = { + "Edit": set(), + "View": set(), + "ViewSetup": {"View"}, + "EditSetup": {"Edit", "View"}, + "ViewPackage": {"View"}, + "ManageSetup": {"Manage"}, + } result_fields = {"Option1", "Option2"} result_tabs = {"Tab1", "Tab2"} result_objects = {"Object1", "Object2"} @@ -447,13 +480,13 @@ def test_entities_from_query(task_context): "cumulusci.utils.clean_invalid_references.process_objects", return_value=result_objects, ) as mock_process_objects: - userpermissions, fields, tabs, objects = entities_from_query(sf) + userpermissions, fields, tabs, objects = entities_from_api_calls(sf) assert userpermissions == result_userpermissions assert fields == result_fields assert tabs == result_tabs assert objects == result_objects - mock_process_user_permissions.assert_called_with(up_result_dict) + mock_process_user_permissions.assert_called_with(up_result_dict, pd_result_dict) mock_process_fields.assert_called_with(field_result_dict) mock_process_tabs.assert_called_with(tabs_result_dict) mock_process_objects.assert_called_with(objects_result_dict) @@ -485,33 +518,62 @@ def test_entities_from_package(task_context, sample_zip): def test_zip_clean_invalid_references(task_context): - userPermissions = {"UserPermissions"} + userPermissions = {"UserPermissions": {"UP"}} fields = {"fields"} tabs = {"tabs"} objects = {"objects"} entities = {"applications": {"applications"}} expected_target_entites = {} - expected_target_entites.setdefault("userPermissions", set()).update(userPermissions) expected_target_entites.setdefault("tabs", set()).update(tabs) expected_target_entites.setdefault("fields", set()).update(fields) expected_target_entites.setdefault("objects", set()).update(objects) + expected_target_entites["userPermissions"] = userPermissions expected_target_entites.update(entities) zf = Mock() with patch("cumulusci.utils.clean_invalid_references.ret_sf"), patch( - "cumulusci.utils.clean_invalid_references.entities_from_query", + "cumulusci.utils.clean_invalid_references.entities_from_api_calls", return_value=(userPermissions, fields, tabs, objects), ), patch( "cumulusci.utils.clean_invalid_references.entities_from_package", return_value=entities, ), patch( "cumulusci.utils.clean_invalid_references.clean_zip_file" - ) as mock_clean_zip_file: + ) as mock_clean_zip_file, patch( + "cumulusci.utils.clean_invalid_references.ret_api_version", return_value="58.0" + ): zip_clean_invalid_references(zf, task_context) mock_clean_zip_file.assert_called_once_with(zf, expected_target_entites) +def test_ret_api_version_success(): + zf = zipfile.ZipFile(io.BytesIO(), "w") + package_xml_contents = ( + b"" + b"" + b" 52.0" + b"" + ) + zf.writestr("package.xml", package_xml_contents) + + api_version = ret_api_version(zf) + assert api_version == "52.0" + + +def test_ret_api_version_failure(): + zf = zipfile.ZipFile(io.BytesIO(), "w") + package_xml_contents = ( + b"" b"" b"" + ) + zf.writestr("package.xml", package_xml_contents) + + error_response = "API version not found in the package.xml file." + with pytest.raises(CumulusCIUsageError) as exc_info: + ret_api_version(zf) + assert error_response == str(exc_info.value) + + if __name__ == "__main": pytest.main()