Issue #256 的两项核心能力

spug_api/apps/app/views.py

@@ -7,6 +7,7 @@
 from apps.app.models import App, Deploy, DeployExtend1, DeployExtend2
 from apps.config.models import Config, ConfigHistory, Service
 from apps.app.utils import fetch_versions, remove_repo
+from apps.host.utils import parse_group_host_ids
 from apps.setting.utils import AppSetting
 import json
 import re
@@ -125,17 +126,23 @@ def post(self, request):
             Argument('id', type=int, required=False),
             Argument('app_id', type=int, help='请选择应用'),
             Argument('env_id', type=int, help='请选择环境'),
-            Argument('host_ids', type=list, filter=lambda x: len(x), help='请选择要部署的主机'),
+            Argument('host_ids', type=list, required=False, default=[]),
+            Argument('group_ids', type=list, required=False, default=[]),
             Argument('rst_notify', type=dict, help='请选择发布结果通知方式'),
             Argument('extend', filter=lambda x: x in dict(Deploy.EXTENDS), help='请选择发布类型'),
             Argument('is_parallel', type=bool, default=True),
             Argument('is_audit', type=bool, default=False)
         ).parse(request.body)
         if error is None:
+            host_ids = set(form.host_ids)
+            host_ids.update(parse_group_host_ids(form.group_ids))
+            if not host_ids:
+                return json_response(error='请选择要部署的主机')
             deploy = Deploy.objects.filter(app_id=form.app_id, env_id=form.env_id).first()
             if deploy and deploy.id != form.id:
                 return json_response(error='应用在该环境下已经存在发布配置')
-            form.host_ids = json.dumps(form.host_ids)
+            form.pop('group_ids')
+            form.host_ids = json.dumps(sorted(host_ids))
             form.rst_notify = json.dumps(form.rst_notify)
             if form.extend == '1':
                 extend_form, error = JsonParser(

spug_api/apps/deploy/utils.py

@@ -13,6 +13,7 @@
 from apps.deploy.helper import Helper, SpugError
 from concurrent import futures
 from functools import partial
+import requests
 import json
 import uuid
 import os
@@ -141,7 +142,10 @@ def _ext2_deploy(req, helper, env):
         helper.send_info('local', f'\033[32m完成√\033[0m\r\n')
         for action in server_actions:
             helper.send_step('local', step, f'{human_time()} {action["title"]}...\r\n')
-            helper.local(f'cd /tmp && {action["data"]}', env)
+            if action.get('type') == 'jenkins':
+                _deploy_jenkins(helper, action, env)
+            else:
+                helper.local(f'cd /tmp && {action["data"]}', env)
             step += 1
 
     for action in host_actions:
@@ -221,6 +225,39 @@ def _ext2_deploy(req, helper, env):
         helper.send_step('local', 100, f'\r\n{human_time()} ** 发布成功 **')
 
 
+def _deploy_jenkins(helper, action, env):
+    url = (action.get('url') or '').strip()
+    if not url:
+        helper.send_error('local', 'Jenkins URL is required')
+    auth = None
+    user = action.get('user')
+    token = action.get('token')
+    if user and token:
+        auth = (user, token)
+    try:
+        timeout = int(action.get('timeout') or 20)
+    except (TypeError, ValueError):
+        timeout = 20
+    params = {}
+    raw_params = action.get('params') or {}
+    if not isinstance(raw_params, dict):
+        raw_params = {}
+    for k, v in raw_params.items():
+        params[k] = render_str(str(v), env)
+    build_url = url.rstrip('/')
+    if params:
+        if not build_url.endswith('/buildWithParameters'):
+            build_url = f'{build_url}/buildWithParameters'
+    elif not build_url.endswith('/build'):
+        build_url = f'{build_url}/build'
+    helper.send_info('local', f'{human_time()} Trigger Jenkins: {build_url}\\r\\n')
+    response = requests.post(build_url, data=params or None, auth=auth, timeout=timeout)
+    if response.status_code not in (200, 201, 202):
+        helper.send_error('local', f'Jenkins trigger failed with status {response.status_code}: {response.text[:500]}')
+    queue_url = response.headers.get('Location') or '-'
+    helper.send_info('local', f'{human_time()} Jenkins accepted, queue url: {queue_url}\\r\\n')
+
+
 def _deploy_ext1_host(req, helper, h_id, env):
     helper.send_step(h_id, 1, f'\033[32m就绪√\033[0m\r\n{human_time()} 数据准备...        ')
     host = Host.objects.filter(pk=h_id).first()

spug_api/apps/deploy/views.py

@@ -11,6 +11,7 @@
 from apps.app.models import Deploy, DeployExtend2
 from apps.repository.models import Repository
 from apps.deploy.utils import dispatch, Helper
+from apps.host.utils import parse_group_host_ids
 from apps.host.models import Host
 from collections import defaultdict
 from threading import Thread
@@ -19,6 +20,13 @@
 import json
 import os
 
+def merge_host_ids(host_ids, group_ids, limited_host_ids=None):
+    ids = set(host_ids or [])
+    ids.update(parse_group_host_ids(group_ids))
+    if limited_host_ids is not None:
+        ids = ids.intersection(set(limited_host_ids))
+    return sorted(ids)
+
 
 class RequestView(View):
     @auth('deploy.request.view')
@@ -211,13 +219,17 @@ def post_request_ext1(request):
         Argument('deploy_id', type=int, help='参数错误'),
         Argument('name', help='请输入申请标题'),
         Argument('extra', type=list, help='请选择发布版本'),
-        Argument('host_ids', type=list, filter=lambda x: len(x), help='请选择要部署的主机'),
+        Argument('host_ids', type=list, required=False, default=[]),
+        Argument('group_ids', type=list, required=False, default=[]),
         Argument('type', default='1'),
         Argument('plan', required=False),
         Argument('desc', required=False),
     ).parse(request.body)
     if error is None:
         deploy = Deploy.objects.get(pk=form.deploy_id)
+        host_ids = merge_host_ids(form.host_ids, form.group_ids, json.loads(deploy.host_ids))
+        if not host_ids:
+            return json_response(error='请选择要部署的主机')
         form.spug_version = Repository.make_spug_version(deploy.id)
         if form.extra[0] == 'tag':
             if not form.extra[1]:
@@ -240,7 +252,8 @@ def post_request_ext1(request):
 
         form.extra = json.dumps(form.extra)
         form.status = '0' if deploy.is_audit else '1'
-        form.host_ids = json.dumps(sorted(form.host_ids))
+        form.pop('group_ids')
+        form.host_ids = json.dumps(host_ids)
         if form.id:
             req = DeployRequest.objects.get(pk=form.id)
             is_required_notify = deploy.is_audit and req.status == '-1'
@@ -258,18 +271,23 @@ def post_request_ext1_rollback(request):
     form, error = JsonParser(
         Argument('request_id', type=int, help='请选择要回滚的版本'),
         Argument('name', help='请输入申请标题'),
-        Argument('host_ids', type=list, filter=lambda x: len(x), help='请选择要部署的主机'),
+        Argument('host_ids', type=list, required=False, default=[]),
+        Argument('group_ids', type=list, required=False, default=[]),
         Argument('desc', required=False),
     ).parse(request.body)
     if error is None:
         req = DeployRequest.objects.get(pk=form.pop('request_id'))
+        host_ids = merge_host_ids(form.host_ids, form.group_ids, json.loads(req.deploy.host_ids))
+        if not host_ids:
+            return json_response(error='请选择要部署的主机')
         requests = DeployRequest.objects.filter(deploy=req.deploy, status__in=('3', '-3'))
         versions = list({x.spug_version: 1 for x in requests}.keys())
         if req.spug_version not in versions[:req.deploy.extend_obj.versions + 1]:
             return json_response(error='选择的版本超出了发布配置中设置的版本数量，无法快速回滚，可通过新建发布申请选择构建仓库里的该版本再次发布。')
 
         form.status = '0' if req.deploy.is_audit else '1'
-        form.host_ids = json.dumps(sorted(form.host_ids))
+        form.pop('group_ids')
+        form.host_ids = json.dumps(host_ids)
         new_req = DeployRequest.objects.create(
             deploy_id=req.deploy_id,
             repository_id=req.repository_id,
@@ -291,7 +309,8 @@ def post_request_ext2(request):
         Argument('id', type=int, required=False),
         Argument('deploy_id', type=int, help='缺少必要参数'),
         Argument('name', help='请输申请标题'),
-        Argument('host_ids', type=list, filter=lambda x: len(x), help='请选择要部署的主机'),
+        Argument('host_ids', type=list, required=False, default=[]),
+        Argument('group_ids', type=list, required=False, default=[]),
         Argument('extra', type=dict, required=False),
         Argument('version', default=''),
         Argument('type', default='1'),
@@ -302,6 +321,9 @@ def post_request_ext2(request):
         deploy = Deploy.objects.filter(pk=form.deploy_id).first()
         if not deploy:
             return json_response(error='未找到该发布配置')
+        host_ids = merge_host_ids(form.host_ids, form.group_ids, json.loads(deploy.host_ids))
+        if not host_ids:
+            return json_response(error='请选择要部署的主机')
         extra = form.pop('extra')
         if DeployExtend2.objects.filter(deploy=deploy, host_actions__contains='"src_mode": "1"').exists():
             if not extra:
@@ -312,7 +334,8 @@ def post_request_ext2(request):
             form.spug_version = Repository.make_spug_version(deploy.id)
         form.name = form.name.replace("'", '')
         form.status = '0' if deploy.is_audit else '1'
-        form.host_ids = json.dumps(form.host_ids)
+        form.pop('group_ids')
+        form.host_ids = json.dumps(host_ids)
         if form.id:
             req = DeployRequest.objects.get(pk=form.id)
             is_required_notify = deploy.is_audit and req.status == '-1'

spug_api/apps/exec/views.py

@@ -7,6 +7,7 @@
 from libs import json_response, JsonParser, Argument, human_datetime, auth
 from apps.exec.models import ExecTemplate, ExecHistory
 from apps.host.models import Host
+from apps.host.utils import parse_group_host_ids
 from apps.account.utils import has_host_perm
 import uuid
 import json
@@ -61,13 +62,18 @@ def get(self, request):
     @auth('exec.task.do')
     def post(self, request):
         form, error = JsonParser(
-            Argument('host_ids', type=list, filter=lambda x: len(x), help='请选择执行主机'),
+            Argument('host_ids', type=list, required=False, default=[]),
+            Argument('group_ids', type=list, required=False, default=[]),
             Argument('command', help='请输入执行命令内容'),
             Argument('interpreter', default='sh'),
             Argument('template_id', type=int, required=False),
             Argument('params', type=dict, handler=json.dumps, default={})
         ).parse(request.body)
         if error is None:
+            form.host_ids = sorted(set(form.host_ids).union(parse_group_host_ids(form.group_ids)))
+            if not form.host_ids:
+                return json_response(error='请选择执行主机')
+            form.pop('group_ids')
             if not has_host_perm(request.user, form.host_ids):
                 return json_response(error='无权访问主机，请联系管理员')
             token, rds = uuid.uuid4().hex, get_redis_connection()

spug_api/apps/host/utils.py

@@ -6,7 +6,7 @@
 from libs.ssh import SSH, AuthenticationException
 from libs.utils import AttrDict, human_datetime
 from libs.validators import ip_validator
-from apps.host.models import HostExtend
+from apps.host.models import HostExtend, Group
 from apps.setting.utils import AppSetting
 from collections import defaultdict
 from datetime import datetime, timezone
@@ -301,3 +301,14 @@ def _get_ssh(kwargs, pkey=None, private_key=None, public_key=None, password=None
                 ssh.add_public_key(public_key)
             return _get_ssh(kwargs, private_key)
         raise e
+
+
+def parse_group_host_ids(group_ids):
+    group_ids = {int(x) for x in (group_ids or [])}
+    if not group_ids:
+        return set()
+    all_ids, sub_ids = set(group_ids), set(group_ids)
+    while sub_ids:
+        sub_ids = {x.id for x in Group.objects.filter(parent_id__in=sub_ids)}
+        all_ids.update(sub_ids)
+    return set(x.host_id for x in Group.hosts.through.objects.filter(group_id__in=all_ids))

spug_web/src/pages/deploy/app/AddSelect.js

@@ -20,6 +20,7 @@ class AddSelect extends React.Component {
       is_audit: false,
       rst_notify: {mode: '0'},
       host_ids: [],
+      group_ids: [],
       filter_rule: {type: 'exclude', data: ''}
     }
   };
@@ -31,6 +32,7 @@ class AddSelect extends React.Component {
       is_audit: false,
       rst_notify: {mode: '0'},
       host_ids: [],
+      group_ids: [],
       host_actions: [],
       server_actions: []
     }

spug_web/src/pages/deploy/app/Ext1Setup1.js

@@ -10,6 +10,7 @@ import { Switch, Form, Input, Select, Button, Radio } from 'antd';
 import Repo from './Repo';
 import envStore from 'pages/config/environment/store';
 import HostSelector from 'pages/host/Selector';
+import hostStore from 'pages/host/store';
 import store from './store';
 
 export default observer(function Ext1Setup1() {
@@ -22,13 +23,29 @@ export default observer(function Ext1Setup1() {
   }
 
   useEffect(() => {
+    hostStore.initial()
+    if (!Array.isArray(store.deploy.group_ids)) {
+      store.deploy.group_ids = []
+    }
     if (store.currentRecord['deploys'] === undefined) {
       store.loadDeploys(store.app_id).then(updateEnvs)
     } else {
       updateEnvs()
     }
   }, [])
 
+  function handleChangeGroups(group_ids) {
+    info.group_ids = group_ids;
+    const host_ids = new Set(info.host_ids || []);
+    for (let id of group_ids) {
+      const counter = hostStore.counter[id];
+      if (counter) {
+        counter.forEach(h_id => host_ids.add(h_id))
+      }
+    }
+    info.host_ids = [...host_ids]
+  }
+
   const info = store.deploy;
   let modePlaceholder;
   switch (info['rst_notify']['mode']) {
@@ -64,6 +81,18 @@ export default observer(function Ext1Setup1() {
       <Form.Item required label="目标主机" tooltip="该发布配置作用于哪些目标主机。">
         <HostSelector value={info.host_ids} onChange={ids => info.host_ids = ids}/>
       </Form.Item>
+      <Form.Item label="机器组选择" tooltip="可按机器组批量加入目标主机，之后仍可手动调整主机列表。">
+        <Select
+          mode="multiple"
+          allowClear
+          value={info.group_ids || []}
+          onChange={handleChangeGroups}
+          placeholder="请选择机器组">
+          {Object.entries(hostStore.groups).map(([id, name]) => (
+            <Select.Option key={id} value={Number(id)}>{name}</Select.Option>
+          ))}
+        </Select>
+      </Form.Item>
       <Form.Item required label="Git仓库地址" extra={<span className="btn" onClick={() => setVisible(true)}>私有仓库？</span>}>
         <Input disabled={store.isReadOnly} value={info['git_repo']} onChange={e => info['git_repo'] = e.target.value}
                placeholder="请输入Git仓库地址"/>

spug_web/src/pages/deploy/app/Ext2Setup1.js

@@ -9,6 +9,7 @@ import { Link } from 'react-router-dom';
 import { Form, Switch, Select, Button, Input, Radio } from 'antd';
 import envStore from 'pages/config/environment/store';
 import HostSelector from 'pages/host/Selector';
+import hostStore from 'pages/host/store';
 import store from './store';
 
 export default observer(function Ext2Setup1() {
@@ -20,13 +21,29 @@ export default observer(function Ext2Setup1() {
   }
 
   useEffect(() => {
+    hostStore.initial()
+    if (!Array.isArray(store.deploy.group_ids)) {
+      store.deploy.group_ids = []
+    }
     if (store.currentRecord['deploys'] === undefined) {
       store.loadDeploys(store.app_id).then(updateEnvs)
     } else {
       updateEnvs()
     }
   }, [])
 
+  function handleChangeGroups(group_ids) {
+    info.group_ids = group_ids;
+    const host_ids = new Set(info.host_ids || []);
+    for (let id of group_ids) {
+      const counter = hostStore.counter[id];
+      if (counter) {
+        counter.forEach(h_id => host_ids.add(h_id))
+      }
+    }
+    info.host_ids = [...host_ids]
+  }
+
   const info = store.deploy;
   let modePlaceholder;
   switch (info['rst_notify']['mode']) {
@@ -62,6 +79,18 @@ export default observer(function Ext2Setup1() {
       <Form.Item required label="目标主机" tooltip="该发布配置作用于哪些目标主机。">
         <HostSelector value={info.host_ids} onChange={ids => info.host_ids = ids}/>
       </Form.Item>
+      <Form.Item label="机器组选择" tooltip="可按机器组批量加入目标主机，之后仍可手动调整主机列表。">
+        <Select
+          mode="multiple"
+          allowClear
+          value={info.group_ids || []}
+          onChange={handleChangeGroups}
+          placeholder="请选择机器组">
+          {Object.entries(hostStore.groups).map(([id, name]) => (
+            <Select.Option key={id} value={Number(id)}>{name}</Select.Option>
+          ))}
+        </Select>
+      </Form.Item>
       <Form.Item label="发布模式" tooltip="串行即发布时一台完成后再发布下一台，期间出现异常则终止发布。并行则每个主机相互独立发布同时进行。">
         <Radio.Group
           buttonStyle="solid"