Merge pull request #574 from meolu/2.0.0/feature/project

2.0.0/feature/project

migrations/versions/5ff964e844a7_07_project_include.py

@@ -0,0 +1,23 @@
+"""07_project_include
+
+Revision ID: 5ff964e844a7
+Revises: 0af33c7b8832
+Create Date: 2019-01-04 18:00:58.941866
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = '5ff964e844a7'
+down_revision = '0af33c7b8832'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column('projects', sa.Column('is_include', sa.Integer(), nullable=True, server_default='0'))
+
+
+def downgrade():
+ op.drop_column('projects', 'is_include')

walle/form/project.py

@@ -23,6 +23,7 @@ class ProjectForm(FlaskForm):
  space_id = StringField('space_id', [validators.Length(min=1, max=10)])
  status = StringField('status', [])
  excludes = StringField('excludes', [])
+ is_include = StringField('excludes', [])
  master = StringField('master', [])
  server_ids = StringField('server_ids', [validators.Length(min=1)])
  keep_version_num = StringField('keep_version_num', [])
@@ -70,6 +71,7 @@ def form2dict(self):
  'environment_id': self.environment_id.data if self.environment_id.data else '',
  'space_id': self.space_id.data if self.space_id.data else current_user.space_id(),
  'excludes': self.excludes.data if self.excludes.data else '',
+ 'is_include': self.is_include.data,
  'server_ids': self.server_ids.data if self.server_ids.data else '',
  'keep_version_num': self.keep_version_num.data if self.keep_version_num.data else 5,
 

walle/model/project.py

@@ -37,6 +37,7 @@ class ProjectModel(SurrogatePK, Model):
  master = db.Column(String(100))
  version = db.Column(String(40))
  excludes = db.Column(Text)
+ is_include = db.Column(Integer)
  target_root = db.Column(String(200))
  target_releases = db.Column(String(200))
  server_ids = db.Column(Text)
@@ -154,6 +155,7 @@ def to_json(self):
  'master': UserModel.fetch_by_uid(self.master.split(',')) if self.master else '',
  'version': self.version,
  'excludes': self.excludes,
+ 'is_include': self.is_include,
  'target_root': self.target_root,
  'target_releases': self.target_releases,
  'server_ids': self.server_ids,

walle/service/deployer.py

@@ -18,8 +18,8 @@
 from walle.model.task import TaskModel
 from walle.service.code import Code
 from walle.service.error import WalleError
-from walle.service.utils import color_clean, suffix_format
-from walle.service.utils import excludes_format
+from walle.service.utils import color_clean
+from walle.service.utils import excludes_format, includes_format
 from walle.service.notice import Notice
 from walle.service.waller import Waller
 from flask_login import current_user
@@ -45,6 +45,7 @@ class Deployer:
  TaskRecord = None
 
  console = False
+ custom_global_env = {}
 
  version = datetime.now().strftime('%Y%m%d%H%M%S')
 
@@ -56,7 +57,7 @@ class Deployer:
  local = None
 
  def __init__(self, task_id=None, project_id=None, console=False):
- self.local_codebase = current_app.config.get('CODE_BASE')
+ self.local_codebase = current_app.config.get('CODE_BASE').rstrip('/') + '/'
  self.localhost = Waller(host='127.0.0.1')
  self.TaskRecord = RecordModel()
 
@@ -69,6 +70,37 @@ def __init__(self, task_id=None, project_id=None, console=False):
  self.servers = self.taskMdl.get('servers_info')
  self.project_info = self.taskMdl.get('project_info')
 
+ # copy to a local version
+ self.release_version = '{project_id}_{task_id}_{timestamp}'.format(
+ project_id=self.project_info['id'],
+ task_id=self.task_id,
+ timestamp=time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())),
+ )
+ current_app.logger.info(self.taskMdl)
+
+ self.custom_global_env = {
+ 'WEBROOT': '"{}"'.format(self.project_info['target_root']),
+ 'CURRENT_RELEASE': '"{}"'.format(self.release_version),
+ 'BRANCH': '"{}"'.format(self.taskMdl.get('branch')),
+ 'TAG': '"{}"'.format(self.taskMdl.get('tag')),
+ 'COMMIT_ID': '"{}"'.format(self.taskMdl.get('commit_id')),
+ 'PROJECT_NAME': '"{}"'.format(self.project_info['name']),
+ 'PROJECT_ID': '"{}"'.format(self.project_info['id']),
+ 'TASK_NAME': '"{}"'.format(self.taskMdl.get('name')),
+ 'TASK_ID': '"{}"'.format(self.task_id),
+ 'DEPLOY_USER': '"{}"'.format(self.taskMdl.get('user_name')),
+ 'DEPLOY_TIME': '"{}"'.format(time.strftime('%Y%m%d-%H:%M:%S', time.localtime(time.time()))),
+ }
+ if self.project_info['task_vars']:
+ task_vars = [i.strip() for i in self.project_info['task_vars'].split('\n') if i.strip() and not i.strip().startswith('#')]
+ for var in task_vars:
+ var_list = var.split('=', 1)
+ if len(var_list) != 2:
+ continue
+ self.custom_global_env[var_list[0]] = var_list[1]
+
+ self.localhost.init_env(env=self.custom_global_env)
+
  if project_id:
  self.project_id = project_id
  self.project_info = ProjectModel(id=project_id).item()
@@ -118,11 +150,13 @@ def prev_deploy(self):
  self.init_repo()
 
  # 用户自定义命令
- command = self.project_info['prev_deploy']
- if command:
- current_app.logger.info(command)
- with self.localhost.cd(self.dir_codebase_project):
- result = self.localhost.local(command, wenv=self.config())
+ commands = self.project_info['prev_deploy']
+ if commands:
+ for command in commands.split('\n'):
+ if command.strip().startswith('#') or not command.strip():
+ continue
+ with self.localhost.cd(self.dir_codebase_project):
+ result = self.localhost.local(command, wenv=self.config())
 
  def deploy(self):
  '''
@@ -133,10 +167,10 @@ def deploy(self):
  '''
  self.stage = self.stage_deploy
  self.sequence = 2
-
- # copy to a local version
- self.release_version = '%s_%s_%s' % (
- self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
+#
+# # copy to a local version
+# self.release_version = '%s_%s_%s' % (
+# self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
 
  with self.localhost.cd(self.local_codebase):
  command = 'cp -rf %s %s' % (self.dir_codebase_project, self.release_version)
@@ -168,26 +202,25 @@ def post_deploy(self):
  self.sequence = 3
 
  # 用户自定义命令
- command = self.project_info['post_deploy']
- if command:
- with self.localhost.cd(self.local_codebase + self.release_version):
- result = self.localhost.local(command, wenv=self.config())
+ commands = self.project_info['post_deploy']
+ if commands:
+ for command in commands.split('\n'):
+ if command.strip().startswith('#') or not command.strip():
+ continue
+ with self.localhost.cd(self.local_codebase + self.release_version):
+ result = self.localhost.local(command, wenv=self.config())
 
  # 压缩打包
  # 排除文件发布
  self.release_version_tar = '%s.tgz' % (self.release_version)
  with self.localhost.cd(self.local_codebase):
- excludes = excludes_format(self.project_info['excludes'])
- command = 'tar zcf %s %s %s' % (self.release_version_tar, excludes, self.release_version)
+ if self.project_info['is_include']:
+ files = includes_format(self.release_version, self.project_info['excludes'])
+ else:
+ files = excludes_format(self.release_version, self.project_info['excludes'])
+ command = 'tar zcf %s/%s %s' % (self.local_codebase.rstrip('/'), self.release_version_tar, files)
  result = self.localhost.local(command, wenv=self.config())
 
- # # 指定文件发布
- # self.release_version_tar = '%s.tgz' % (self.release_version)
- # with self.localhost.cd(self.local_codebase):
- # excludes = suffix_format(self.dir_codebase_project, self.project_info['excludes'])
- # command = 'tar zcf %s %s %s' % (self.release_version_tar, excludes, self.release_version)
- # result = self.local.run(command, wenv=self.config())
-
  def prev_release(self, waller):
  '''
  4.部署代码到目标机器前做的任务
@@ -215,13 +248,15 @@ def prev_release(self, waller):
 
  def prev_release_custom(self, waller):
  # 用户自定义命令
- command = self.project_info['prev_release']
- if command:
- current_app.logger.info(command)
- # TODO
- target_release_version = "%s/%s" % (self.project_info['target_releases'], self.release_version)
- with waller.cd(target_release_version):
- result = waller.run(command, wenv=self.config())
+ commands = self.project_info['prev_release']
+ if commands:
+ for command in commands.split('\n'):
+ if command.strip().startswith('#') or not command.strip():
+ continue
+ # TODO
+ target_release_version = "%s/%s" % (self.project_info['target_releases'], self.release_version)
+ with waller.cd(target_release_version):
+ result = waller.run(command, wenv=self.config())
 
  def release(self, waller):
  '''
@@ -299,14 +334,16 @@ def post_release(self, waller):
  '''
  self.stage = self.stage_post_release
  self.sequence = 6
-
  # 用户自定义命令
- command = self.project_info['post_release']
- if command:
- current_app.logger.info(command)
- with waller.cd(self.project_info['target_root']):
- result = waller.run(command, wenv=self.config())
-
+ commands = self.project_info['post_release']
+ if commands:
+ for command in commands.split('\n'):
+ if command.strip().startswith('#') or not command.strip():
+ continue
+ # TODO
+ with waller.cd(self.project_info['target_root']):
+ pty = False if command.find('nohup') >= 0 else True
+ result = waller.run(command, wenv=self.config(), pty=pty)
  # 个性化，用户重启的不一定是NGINX，可能是tomcat, apache, php-fpm等
  # self.post_release_service(waller)
 
@@ -495,7 +532,10 @@ def walle_deploy(self):
  for server_info in self.servers:
  host = server_info['host']
  try:
- self.connections[host] = Waller(host=host, user=server_info['user'], port=server_info['port'])
+ waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
+ waller.init_env(env=self.custom_global_env)
+
+ self.connections[host] = waller
  self.prev_release(self.connections[host])
  self.release(self.connections[host])
  self.post_release(self.connections[host])
@@ -527,7 +567,10 @@ def walle_rollback(self):
  for server_info in self.servers:
  host = server_info['host']
  try:
- self.connections[host] = Waller(host=host, user=server_info['user'], port=server_info['port'])
+ waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
+ waller.init_env(env=self.custom_global_env)
+
+ self.connections[host] = waller 
  self.prev_release_custom(self.connections[host])
  self.release(self.connections[host])
  self.post_release(self.connections[host])

walle/service/utils.py

@@ -81,16 +81,39 @@ def say_yes():
  )
 
 
-def excludes_format(excludes_string):
- excludes = [i for i in excludes_string.split('\n') if i.strip()]
- if not excludes:
- return ''
- excludes = ' --exclude='.join(excludes)
- return ' --exclude=' + excludes
-
-
-# 指定发布文件，支持模糊匹配，如：*.war
-def suffix_format(path, suffix_file):
- for suffix_file in os.listdir('%s' % path):
- if fnmatch.fnmatch(suffix_file, '%s' % suffix_file):
- return suffix_file
+def excludes_format(path, excludes_string=None):
+ '''
+ 排除文件，支持正则匹配，支持多选字符串
+ @param path:
+ @param excludes_string:
+ @return:
+ '''
+ path = os.path.basename(path) + '/'
+ if not excludes_string:
+ return path
+
+ prefix = '--exclude='
+ excludes = [prefix + i for i in excludes_string.split('\n') if i.strip()]
+
+ return ' {excludes} {path} '.format(excludes=' '.join(excludes), path=path)
+
+
+def includes_format(path, includes_string=None):
+ '''
+ 指定发布文件，支持正则匹配，如：*.war。支持多行字符串。
+
+ @param path: release目录，非路径
+ @param includes_string:
+ @return:
+ '''
+ path = os.path.basename(path) + '/'
+ if not includes_string:
+ return path
+
+ prefix = path
+ includes = [prefix + i for i in includes_string.split('\n') if i.strip()]
+
+ if not includes:
+ return path
+
+ return ' '.join(includes)

walle/service/waller.py

@@ -23,6 +23,11 @@ class Waller(Connection):
  connections, success, errors = {}, {}, {}
  release_version_tar, release_version = None, None
 
+ custom_global_env = {}
+
+ def init_env(self, env):
+ self.custom_global_env = env
+
  def run(self, command, wenv=None, run_mode=run_mode_remote, pty=True, exception=True, **kwargs):
  '''
  pty=True/False是直接影响到输出.False较适合在获取文本,True更适合websocket
@@ -40,11 +45,12 @@ def run(self, command, wenv=None, run_mode=run_mode_remote, pty=True, exception=
  current_app.logger.info(message)
  try:
  if run_mode == self.run_mode_sudo:
- result = super(Waller, self).sudo(command, pty=pty, **kwargs)
+ result = super(Waller, self).sudo(command, pty=pty, env=self.custom_global_env, **kwargs)
  elif run_mode == self.run_mode_local:
- result = super(Waller, self).local(command, pty=pty, warn=True, watchers=[say_yes()], **kwargs)
+ current_app.logger.info(self.custom_global_env)
+ result = super(Waller, self).local(command, pty=pty, warn=True, watchers=[say_yes()], env=self.custom_global_env, **kwargs)
  else:
- result = super(Waller, self).run(command, pty=pty, warn=True, watchers=[say_yes()], **kwargs)
+ result = super(Waller, self).run(command, pty=pty, warn=True, watchers=[say_yes()], env=self.custom_global_env, **kwargs)
 
  if result.failed:
  exitcode, stdout, stderr = result.exited, '', result.stdout
@@ -174,17 +180,24 @@ def sync(self, wtype, remote=None, local=None, wenv=None):
  except Exception as e:
  # TODO 收尾下
  current_app.logger.info('put: %s, %s', e, dir(e))
+ if wtype == 'put':
+ op_user = pwd.getpwuid(os.getuid())[0]
+ op_host = '127.0.0.1'
+ else:
+ op_user = self.user
+ op_host = self.host
 
  # TODO command
  ws_dict = {
- 'user': self.user,
- 'host': self.host,
+ 'user': op_user,
+ 'host': op_host,
  'cmd': command,
  'status': 1,
  'stage': wenv['stage'],
  'sequence': wenv['sequence'],
  'success': '',
- 'error': e.message,
+ 'error': str(e),
  }
+ current_app.logger.info(ws_dict)
  if wenv['console']:
  emit('console', {'event': 'task:console', 'data': ws_dict}, room=wenv['task_id'])