From b64108642df94a0b1d02bcd1eb9bfcc317ae9a60 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 31 May 2017 15:11:51 -0400 Subject: [PATCH] Create deployment file directory so it's owned by the user (#559) * Create deployment file directory so it's owned by the user * Install ansible.kubernetes-modules role. --- container/core.py | 33 ++++++--- container/docker/engine.py | 6 ++ container/engine.py | 8 +++ container/k8s/base_engine.py | 38 +++++++++- docs/rst/reference/deploy.rst | 95 ++++++++++++++++++------- docs/rst/reference/deploy/index.rst | 8 --- docs/rst/reference/deploy/kube.rst | 62 ---------------- docs/rst/reference/deploy/openshift.rst | 60 ---------------- docs/rst/reference/index.rst | 1 - 9 files changed, 142 insertions(+), 169 deletions(-) delete mode 100644 docs/rst/reference/deploy/index.rst delete mode 100644 docs/rst/reference/deploy/kube.rst delete mode 100644 docs/rst/reference/deploy/openshift.rst diff --git a/container/core.py b/container/core.py index 44d28b69..7afbbd2a 100644 --- a/container/core.py +++ b/container/core.py @@ -455,6 +455,23 @@ def resolve_push_to(push_to, default_url, default_namespace): return registry_url, namespace +@conductor_only +def set_path_ownership(path, uid, gid): + """ + Starting with the path, recursively set ownership of files and subdirectories. + :param path: Root path + :param uid: User ID + :param gid: Group ID + :return: None + """ + os.chown(path, uid, gid) + for root, dirs, files in os.walk(path): + for d in dirs: + os.chown(os.path.join(root, d), uid, gid) + for f in files: + os.chown(os.path.join(root, f), uid, gid) + + @conductor_only def run_playbook(playbook, engine, service_map, ansible_options='', local_python=False, debug=False, deployment_output_path=None, tags=None, **kwargs): @@ -468,8 +485,6 @@ def run_playbook(playbook, engine, service_map, ansible_options='', local_python remove_tmpdir = True output_dir = tempfile.mkdtemp() - os.chown(output_dir, uid, gid) - playbook_path = os.path.join(output_dir, 'playbook.yml') logger.debug("writing playbook to {}".format(playbook_path)) logger.debug("playbook", playbook=playbook) @@ -493,14 +508,7 @@ def run_playbook(playbook, engine, service_map, ansible_options='', local_python if not os.path.exists(os.path.join(output_dir, 'templates')): os.mkdir(os.path.join(output_dir, 'templates')) - for root, dirs, files in os.walk(output_dir): - for d in dirs: - logger.debug('found dir %s', os.path.join(root, d)) - os.chown(os.path.join(root, d), uid, gid) - for f in files: - logger.debug('found file %s', os.path.join(root, f)) - os.chown(os.path.join(root, f), uid, gid) - + set_path_ownership(output_dir, uid, gid) rc = subprocess.call(['mount', '--bind', '/src', os.path.join(output_dir, 'files')]) @@ -785,6 +793,8 @@ def conductorcmd_destroy(engine_name, project_name, services, **kwargs): @conductor_only def conductorcmd_deploy(engine_name, project_name, services, **kwargs): + uid, gid = kwargs.get('host_user_uid', 1), kwargs.get('host_user_gid', 1) + engine = load_engine(['DEPLOY'], engine_name, project_name, services, **kwargs) logger.info(u'Engine integration loaded. Preparing deploy.', engine=engine.display_name) @@ -802,6 +812,8 @@ def conductorcmd_deploy(engine_name, project_name, services, **kwargs): deployment_output_path = kwargs.get('deployment_output_path') playbook = engine.generate_orchestration_playbook(**kwargs) + engine.pre_deployment_setup(project_name, services, **kwargs) + try: with open(os.path.join(deployment_output_path, '%s.yml' % project_name), 'w') as ofs: ofs.write(ruamel.yaml.round_trip_dump(playbook, indent=4, block_seq_indent=2, default_flow_style=False)) @@ -810,6 +822,7 @@ def conductorcmd_deploy(engine_name, project_name, services, **kwargs): logger.error(u'Failure writing deployment playbook', exc_info=True) raise + set_path_ownership(deployment_output_path, uid, gid) @conductor_only def conductorcmd_install(engine_name, project_name, services, **kwargs): diff --git a/container/docker/engine.py b/container/docker/engine.py index e165d9c1..8001560b 100644 --- a/container/docker/engine.py +++ b/container/docker/engine.py @@ -231,6 +231,8 @@ def run_conductor(self, command, config, base_path, params, engine_name=None, vo if params.get('deployment_output_path'): deployment_path = params['deployment_output_path'] + if not os.path.isdir(deployment_path): + os.mkdir(deployment_path, 0o755) volumes[deployment_path] = {'bind': deployment_path, 'mode': 'rw'} roles_path = None @@ -868,3 +870,7 @@ def _get_registry_auth(registry_url, config_path): if auth_key: username, password = base64.b64decode(auth_key).split(':', 1) return username, password + + @conductor_only + def pre_deployment_setup(self, project_name, services, **kwargs): + pass diff --git a/container/engine.py b/container/engine.py index 0e0577da..8bfb62f4 100644 --- a/container/engine.py +++ b/container/engine.py @@ -193,3 +193,11 @@ def get_registry_username(registry_url, config_path): return the username """ raise NotImplementedError() + + @conductor_only + def pre_deployment_setup(self, **kwargs): + """ + Perform any setup tasks required prior to writing the Ansible playbook. + return None + """ + raise NotImplementedError() \ No newline at end of file diff --git a/container/k8s/base_engine.py b/container/k8s/base_engine.py index b44c9f0c..9323e5cf 100644 --- a/container/k8s/base_engine.py +++ b/container/k8s/base_engine.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import logging +plainLogger = logging.getLogger(__name__) + import os +import subprocess from abc import ABCMeta, abstractproperty, abstractmethod from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -60,6 +64,38 @@ def k8s_client(self): def k8s_config_path(self): return os.path.normpath(os.path.expanduser('~/.kube/config')) + @conductor_only + def pre_deployment_setup(self, project_name, services, deployment_output_path=None, **kwargs): + # Prior to running the playbook, install the ansible.kubernetes-modules role + + if not os.path.isdir(os.path.join(deployment_output_path, 'roles')): + # Create roles subdirectory + os.mkdir(os.path.join(deployment_output_path, 'roles'), 0o777) + + role_path = os.path.join(deployment_output_path, 'roles', 'ansible.kubernetes-modules') + if deployment_output_path and not os.path.exists(role_path): + # Install the role, if not already installed + ansible_cmd = "ansible-galaxy -vvv install -p ./roles ansible.kubernetes-modules" + logger.debug('Running ansible-galaxy', command=ansible_cmd, cwd=deployment_output_path) + process = subprocess.Popen(ansible_cmd, + shell=True, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=deployment_output_path, + ) + log_iter = iter(process.stdout.readline, '') + while process.returncode is None: + try: + plainLogger.info(log_iter.next().rstrip()) + except StopIteration: + process.wait() + finally: + process.poll() + + if process.returncode: + raise exceptions.AnsibleContainerDeployException(u"Failed to install ansible.kubernetes-modules role") + @log_runs @host_only @abstractmethod @@ -131,7 +167,7 @@ def generate_orchestration_playbook(self, url=None, namespace=None, settings=Non play['roles'] = CommentedSeq() play['tasks'] = CommentedSeq() role = CommentedMap([ - ('role', 'kubernetes-modules') + ('role', 'ansible.kubernetes-modules') ]) play['roles'].append(role) play.yaml_set_comment_before_after_key( diff --git a/docs/rst/reference/deploy.rst b/docs/rst/reference/deploy.rst index 77766295..a38d82e6 100644 --- a/docs/rst/reference/deploy.rst +++ b/docs/rst/reference/deploy.rst @@ -3,45 +3,86 @@ deploy .. program:: ansible-container deploy -The ``ansible-container deploy`` command generates an Ansible role for deploying your app to the cloud. The role is -created based on the configuration found in container.yml +The ``deploy`` command pushes images to a remote registry, and generates an Ansible playbook that can be used to deploy the application to a cloud platform. -.. option:: --roles-path LOCAL_PATH +The cloud platform is determined by the engine used to generate the deployment. Supported engines include: docker, k8s, and openshift. The default is docker. +Use the ``--engine`` option to choose the engine. + +The ``deploy`` command maps your ``container.yml`` file to a cloud configuration, depending on the engine. See the :doc:`../../container_yml/reference` +for details on how directives are mapped, and for cloud specific options. + +The playbook name will match the name of the project, and have a ``.yml`` extension. The playbook is created in in the ``ansible-deployment`` directory, which will be +created automatically. Use the *deployment_output_path* option in the *settings* section of ``container.yml`` to write the playbook to a different directory. + +.. note:: + + For K8s and OpenShift, the generated playbook requires the ``ansible.kubernetes-modules`` role, which is automatically installed to ``ansible-deployment/roles``. + It contains the K8s and OpenShift modules, and by referencing the role in the generated playbook, subsequent tasks and roles can access the modules. + + For more information about the role, visit `ansible/ansible-kubernetes-modules `_. + + +Before ``deploy`` executes, the conductor container is started, and images are pushed to a registry. If you don't wish to push images, but intend to use images +local to the cluster, use the ``--local-images`` option. + +Use the ``--push-to`` option to specify the registry where images will be pushed. Pass either a registry name defined in the *registries* section +of ``container.yml``, or pass a URL. + +If using the URL form, include the namespace. For example: ``registry.example.com:5000/myproject``. If a namespace is not provided, +the default namespace for the engine will be used. In the case of docker, the default is the project name. In the case of k8s and openshift, the *k8s_namespace* defined +in the *settings* section of ``container.yml`` will be used. If that's not defined, the project name will be used. + +.. option:: --help -**New in version 0.9.0** +Display usage help. -If you have Ansible roles in a local path other than your `ansible/` directory that you wish to use -during your build/run/deploy, specify that path with this option. +.. option:: --engine -Immediately after *deploy* specify the cloud service provider. See the :ref:`deploy-engine-idx` for the supported *deploy* -engines. For example, to execute *deploy* with the Kubernetes engine: +Specify the deployment engine. The selected engine will be used to generate the Ansible playbook for deploying the application to the corresponding platform. +Valid engines include: docker, k8s, and openshift, with the default engine being docker. -.. code-block:: bash +.. option:: --local-images - $ ansible-container --engine k8s deploy +Use images directly from the local image cache managed by the Docker daemon. Prevents images from being automatically pushed. + +.. option:: --push-to + +Pass either a name defined in the *registries* key within container.yml, or the URL to the registry. + +If passing a URL, you can include the namespace. For example: ``registry.example.com:5000/myproject``. If the namespace is not included, the default namespace +for the engine will be used. For the docker engine, the default namespace is the project name. For k8s and openshift, the *k8s_namespace* +value will be used from the *settings* section of ``container.yml``. If the value is not set, then the project name will be used. + +When passing a registry name defined in the *registries* section of ``container.yml``, set the *namespace* as an attribute of the registry. + +If no ``--push-to`` option is passed, and the ``--local-images`` option is not passed, then the default registry will be used. The current default is +set to ``https://index.docker.io/v1/``. + +.. option:: --with-variables WITH_VARIABLES [WITH_VARIABLES ...] + +Define one or more environment variables in the Ansible Builder Container. Format each variable as a key=value string. + +.. option:: --with-volumes WITH_VOLUMES [WITH_VOLUMES ...] + +Mount one or more volumes to the Conductor container. Specify volumes as strings using the Docker volume format. + +.. option:: --roles-path LOCAL_PATH +If you have Ansible roles in a local path other than your `ansible/` directory that you wish to use, specify that path with this option. -When *deploy* executes it adds a *roles* directory and playbook to the ``ansible-deployment`` directory in the project path. -The playbook it creates will be in the format *.yml*. +.. option:: --username -Subsequent playbook runs will update the playbook and all of the role files as needed. -Each run will create a new tasks playbook. However, each run will also save the previous main.yml -file by appending a timestamp to the file name. If you make changes to the tasks playbook and then run -deploy, your changes will not be lost. They will exist in the most recent timestamped file. +If the registry requires authentication, pass the username. +.. option:: --password -Dependencies -```````````` +If the registry requires authentication, pass a password. If the ``--username`` is provided without the ``--password`` option, you will +be prompted for a password. -Execution of the role created by ``deploy`` depends on `Ansible `_ -2.3 or greater. Check the specific *deploy* engine for any additional dependencies. +.. option:: --email -.. _deploy-engine-idx: +If registry authentication requires an email address, use to pass the email address. -Engine Index -```````````` -.. toctree:: - :maxdepth: 1 +.. option:: --tag - deploy/kube - deploy/openshift +Tag the images prior to pushing. diff --git a/docs/rst/reference/deploy/index.rst b/docs/rst/reference/deploy/index.rst deleted file mode 100644 index 7a1889ee..00000000 --- a/docs/rst/reference/deploy/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -deploy engines -============== - -.. toctree:: - :maxdepth: 1 - - kube - openshift diff --git a/docs/rst/reference/deploy/kube.rst b/docs/rst/reference/deploy/kube.rst deleted file mode 100644 index 3e417624..00000000 --- a/docs/rst/reference/deploy/kube.rst +++ /dev/null @@ -1,62 +0,0 @@ -kube -==== - -.. program:: ansible-container --engine k8s deploy - -The ``ansible-container --engine k8s deploy`` command creates in the ``ansible`` directory an Ansible -playbook and role to deploy your application on Kubernetes. The name of the playbook is -*ansible-deployment/playbook.yml* and it will be put in the project path by default. - -The ``deploy`` commands maps your ``container.yml`` file to a cloud configuration. See the :doc:`../../container_yml/reference` -for details on how directives are mapped and for available Cloud options. - - -.. note:: - - The generated playbook requires that the Ansible kubernetes modules are installed. These are automatically available in the Conductor Container. - -.. note:: - - Before ``deploy`` starts, the conductor container is started, and images are pushed before - Ansible Playbook is run from inside the Conductor. Supply the same ``--roles-path``, - ``--with-volumes`` and ``--with-variables`` options passed to the ``build`` command to - ensure that ``main.yml`` can be parsed and interpreted. - -.. option:: --help - -Display usage help. - -.. option:: --pull-from - -Pass either a name defined in the *registries* key within container.yml or the actual URL the cluster will use to -pull images. If passing a URL, an example would be 'registry.example.com:5000/myproject'. - -.. option:: --with-variables WITH_VARIABLES [WITH_VARIABLES ...] - -**New in version 0.2.0** - -Define one or more environment variables in the Ansible Builder Container. Format each variable as a -key=value string. - -.. option:: --with-volumes WITH_VOLUMES [WITH_VOLUMES ...] - -**New in version 0.2.0** - -Mount one or more volumes to the Ansible Builder Container. Specify volumes as strings using the Docker -volume format. - -If ``--with-volumes`` was used during the `build` process to access roles or includes, then pass the same option to the `shipit` command so that ``main.yml`` can be parsed. - -.. option:: --roles-path LOCAL_PATH - -**New in version 0.2.0** - -If you have Ansible roles in a local path other than your `ansible/` directory that you wish to use, specify that path with this option. - -If ``--roles-path`` was used during the `build` process, then pass the same option to the `shipit` command so that ``main.yml`` can be parsed. - -.. option:: --local-images - -**New in version 0.3.0** - -Use images directly from the local image cache managed by the Docker daemon. Prevents the automatic use of the default registry URL. diff --git a/docs/rst/reference/deploy/openshift.rst b/docs/rst/reference/deploy/openshift.rst deleted file mode 100644 index 7aad0beb..00000000 --- a/docs/rst/reference/deploy/openshift.rst +++ /dev/null @@ -1,60 +0,0 @@ -openshift -========= - -.. program:: ansible-container --engine openshift deploy - -The ``ansible-container --engine openshift deploy`` command creates in the ``ansible`` directory an Ansible -playbook and role to deploy your application on Openshift. The name of the playbook is -*ansible-deployment/playbook.yml*, and the name of the role is *roles/-openshift*. - -The ``deploy`` commands maps your ``container.yml`` file to a cloud configuration. See the :doc:`../../container_yml/reference` -for details on how directives are mapped and for available Cloud options. - -.. note:: - The generated role requires that the OpenShift Ansible modules are installed. These are automatically included inside the Conductor container. - - -.. option:: --help - -Display usage help. - -.. option:: --save-config - -In addition to creating the playbook and role, creates a *shipit_config/openshift* directory and writes out each -JSON config file used to create the openshift services and deployments for your application. - -.. option:: --pull-from - -Pass either a name defined in the *registries* key within container.yml or the actual URL the cluster will use to -pull images. If passing a URL, an example would be 'registry.example.com:5000/myproject'. - -.. option:: --with-variables WITH_VARIABLES [WITH_VARIABLES ...] - -**New in version 0.2.0** - -Define one or more environment variables in the Ansible Builder Container. Format each variable as a -key=value string. - -.. option:: --with-volumes WITH_VOLUMES [WITH_VOLUMES ...] - -**New in version 0.2.0** - -Mount one or more volumes to the Ansible Builder Container. Specify volumes as strings using the Docker -volume format. - -If ``--with-volumes`` was used during the `build` process to access roles or includes, then pass the same option to the `shipit` command so that ``main.yml`` can be parsed. - -.. option:: --roles-path LOCAL_PATH - -**New in version 0.2.0** - -If you have Ansible roles in a local path other than your `ansible/` directory that you wish to use, specify that path with this option. - -If ``--roles-path`` was used during the `build` process, then pass the same option to the `shipit` command so that ``main.yml`` can be parsed. - -.. option:: --local-images - -**New in version 0.3.0** - -Use images directly from the local image cache managed by the Docker daemon. Prevents the automatic use of the default registry URL. - diff --git a/docs/rst/reference/index.rst b/docs/rst/reference/index.rst index aa271378..46a2c872 100644 --- a/docs/rst/reference/index.rst +++ b/docs/rst/reference/index.rst @@ -6,7 +6,6 @@ Command Reference build deploy - deploy/index destroy init install