Merge pull request #1489 from shapeblue/dynamicroles-master

CLOUDSTACK-8562: Dynamic Role-Based API Checker for CloudStack### CLOUDSTACK-8562: DB-Backed Dynamic Role Based API Access Checker

This feature allows root administrators to define new roles and associate API
permissions to them.

A limited form of role-based access control for the CloudStack management server
API is provided through a properties file, commands.properties, embedded in the
WAR distribution. Therefore, customizing API permissions requires unpacking the
distribution and modifying this file consistently on all servers. The old system
also does not permit the specification of additional roles.

FS:
https://cwiki.apache.org/confluence/display/CLOUDSTACK/Dynamic+Role+Based+API+Access+Checker+for+CloudStack

DB-Backed Dynamic Role Based API Access Checker for CloudStack brings following
changes, features and use-cases:
- Moves the API access definitions from commands.properties to the mgmt server DB
- Allows defining custom roles (such as a read-only ROOT admin) beyond the
  current set of four (4) roles
- All roles will resolve to one of the four known roles types (Admin, Resource
  Admin, Domain Admin and User) which maintains this association by requiring
  all new defined roles to specify a role type.
- Allows changes to roles and API permissions per role at runtime including additions or
  removal of roles and/or modifications of permissions, without the need
  of restarting management server(s)

Upgrade/installation notes:
- The feature will be enabled by default for new installations, existing
  deployments will continue to use the older static role based api access checker
  with an option to enable this feature
- During fresh installation or upgrade, the upgrade paths will add four default
  roles based on the four default role types
- For ease of migration, at the time of upgrade commands.properties will be used
  to add existing set of permissions to the default roles. cloud.account
  will have a new role_id column which will be populated based on default roles
  as well

Dynamic-roles migration tool: scripts/util/migrate-dynamicroles.py
- Allows admins to migrate to the dynamic role based checker at a future date
- Performs a harder one-way migrate and update
- Migrates rules from existing commands.properties file into db and deprecates it
- Enables an internal hidden switch to enable dynamic role based checker feature

* pr/1489:
  maven: Fix jstl version usage
  CLOUDSTACK-8562: Deprecate commands.properties
  CLOUDSTACK-8562: DB-Backed Dynamic Role Based API Access Checker
  CLOUDSTACK-9361: Centrally handle API validations

Signed-off-by: Will Stevens <williamstevens@gmail.com>

Author: swill

.travis.yml

@@ -36,10 +36,10 @@ env:
      - REGRESSION_INDEX=6
      - PATH=$HOME/.local/bin:$PATH
   matrix:
-    - TESTS="smoke/test_affinity_groups smoke/test_affinity_groups_projects smoke/test_deploy_vgpu_enabled_vm smoke/test_deploy_vm_iso smoke/test_deploy_vm_root_resize smoke/test_deploy_vm_with_userdata smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings smoke/test_guest_vlan_range"
+    - TESTS="smoke/test_affinity_groups smoke/test_affinity_groups_projects smoke/test_dynamicroles smoke/test_deploy_vgpu_enabled_vm smoke/test_deploy_vm_iso smoke/test_deploy_vm_root_resize smoke/test_deploy_vm_with_userdata smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings smoke/test_guest_vlan_range"
     - TESTS="smoke/test_hosts smoke/test_internal_lb smoke/test_iso smoke/test_list_ids_parameter smoke/test_loadbalance smoke/test_multipleips_per_nic smoke/test_network smoke/test_network_acl smoke/test_nic smoke/test_nic_adapter_type smoke/test_non_contigiousvlan"
     - TESTS="smoke/test_over_provisioning smoke/test_password_server smoke/test_portable_publicip smoke/test_primary_storage smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions smoke/test_reset_vm_on_reboot smoke/test_resource_detail"
-    - TESTS="smoke/test_router_dhcphosts smoke/test_routers smoke/test_routers_iptables_default_policy smoke/test_routers_network_ops smoke/test_scale_vm smoke/test_secondary_storage smoke/test_service_offerings smoke/test_snapshots smoke/test_ssvm smoke/test_templates"
+    - TESTS="smoke/test_router_dhcphosts smoke/test_routers smoke/test_routers_iptables_default_policy smoke/test_routers_network_ops smoke/test_staticroles smoke/test_scale_vm smoke/test_secondary_storage smoke/test_service_offerings smoke/test_snapshots smoke/test_ssvm smoke/test_templates"
     - TESTS="smoke/test_usage_events smoke/test_vm_life_cycle smoke/test_vm_snapshots smoke/test_volumes smoke/test_vpc_redundant smoke/test_vpc_router_nics smoke/test_vpc_vpn smoke/misc/test_deploy_vm smoke/misc/test_vm_ha smoke/misc/test_escalations_templates smoke/misc/test_vm_sync"
 
     - TESTS="component/test_mm_max_limits component/test_acl_isolatednetwork_delete"

api/src/com/cloud/event/EventTypes.java

@@ -66,6 +66,8 @@
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicSecondaryIp;
 import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.acl.Role;
+import org.apache.cloudstack.acl.RolePermission;
 import org.apache.cloudstack.config.Configuration;
 import org.apache.cloudstack.usage.Usage;
 
@@ -166,6 +168,14 @@ public class EventTypes {
     public static final String EVENT_GLOBAL_LOAD_BALANCER_DELETE = "GLOBAL.LB.DELETE";
     public static final String EVENT_GLOBAL_LOAD_BALANCER_UPDATE = "GLOBAL.LB.UPDATE";
 
+    // Role events
+    public static final String EVENT_ROLE_CREATE = "ROLE.CREATE";
+    public static final String EVENT_ROLE_UPDATE = "ROLE.UPDATE";
+    public static final String EVENT_ROLE_DELETE = "ROLE.DELETE";
+    public static final String EVENT_ROLE_PERMISSION_CREATE = "ROLE.PERMISSION.CREATE";
+    public static final String EVENT_ROLE_PERMISSION_UPDATE = "ROLE.PERMISSION.UPDATE";
+    public static final String EVENT_ROLE_PERMISSION_DELETE = "ROLE.PERMISSION.DELETE";
+
     // Account events
     public static final String EVENT_ACCOUNT_ENABLE = "ACCOUNT.ENABLE";
     public static final String EVENT_ACCOUNT_DISABLE = "ACCOUNT.DISABLE";
@@ -605,6 +615,14 @@ public class EventTypes {
         entityEventDetails.put(EVENT_LB_CERT_ASSIGN, LoadBalancer.class);
         entityEventDetails.put(EVENT_LB_CERT_REMOVE, LoadBalancer.class);
 
+        // Role events
+        entityEventDetails.put(EVENT_ROLE_CREATE, Role.class);
+        entityEventDetails.put(EVENT_ROLE_UPDATE, Role.class);
+        entityEventDetails.put(EVENT_ROLE_DELETE, Role.class);
+        entityEventDetails.put(EVENT_ROLE_PERMISSION_CREATE, RolePermission.class);
+        entityEventDetails.put(EVENT_ROLE_PERMISSION_UPDATE, RolePermission.class);
+        entityEventDetails.put(EVENT_ROLE_PERMISSION_DELETE, RolePermission.class);
+
         // Account events
         entityEventDetails.put(EVENT_ACCOUNT_ENABLE, Account.class);
         entityEventDetails.put(EVENT_ACCOUNT_DISABLE, Account.class);

api/src/com/cloud/user/Account.java

@@ -46,6 +46,8 @@ public enum State {
 
     public short getType();
 
+    public Long getRoleId();
+
     public State getState();
 
     public Date getRemoved();

api/src/com/cloud/user/AccountService.java

@@ -57,9 +57,9 @@ public interface AccountService {
      * @return the user if created successfully, null otherwise
      */
     UserAccount createUserAccount(String userName, String password, String firstName, String lastName, String email, String timezone, String accountName,
-        short accountType, Long domainId, String networkDomain, Map<String, String> details, String accountUUID, String userUUID);
+        short accountType, Long roleId, Long domainId, String networkDomain, Map<String, String> details, String accountUUID, String userUUID);
 
-    UserAccount createUserAccount(String userName, String password, String firstName, String lastName, String email, String timezone, String accountName, short accountType, Long domainId, String networkDomain,
+    UserAccount createUserAccount(String userName, String password, String firstName, String lastName, String email, String timezone, String accountName, short accountType, Long roleId, Long domainId, String networkDomain,
                                   Map<String, String> details, String accountUUID, String userUUID, User.Source source);
 
     /**

api/src/org/apache/cloudstack/acl/Role.java

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.acl;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface Role extends InternalIdentity, Identity {
+    String getName();
+    RoleType getRoleType();
+    String getDescription();
+}

api/src/org/apache/cloudstack/acl/RolePermission.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.acl;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface RolePermission extends InternalIdentity, Identity {
+    enum Permission {ALLOW, DENY}
+
+    long getRoleId();
+    Rule getRule();
+    Permission getPermission();
+    String getDescription();
+    long getSortOrder();
+}

api/src/org/apache/cloudstack/acl/RoleService.java

@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.acl;
+
+import org.apache.cloudstack.framework.config.ConfigKey;
+
+import java.util.List;
+
+public interface RoleService {
+
+    ConfigKey<Boolean> EnableDynamicApiChecker = new ConfigKey<>("Advanced", Boolean.class, "dynamic.apichecker.enabled", "false",
+            "If set to true, this enables the dynamic role-based api access checker and disables the default static role-based api access checker.",
+            true);
+
+    boolean isEnabled();
+    Role findRole(final Long id);
+    Role createRole(final String name, final RoleType roleType, final String description);
+    boolean updateRole(final Role role, final String name, final RoleType roleType, final String description);
+    boolean deleteRole(final Role role);
+
+    RolePermission findRolePermission(final Long id);
+    RolePermission findRolePermissionByUuid(final String uuid);
+
+    RolePermission createRolePermission(final Role role, final Rule rule, final RolePermission.Permission permission, final String description);
+    /**
+     * updateRolePermission updates the order/position of an role permission
+     * @param role The role whose permissions needs to be re-ordered
+     * @param newOrder The new list of ordered role permissions
+     */
+    boolean updateRolePermission(final Role role, final List<RolePermission> newOrder);
+    boolean deleteRolePermission(final RolePermission rolePermission);
+
+    List<Role> listRoles();
+    List<Role> findRolesByName(final String name);
+    List<Role> findRolesByType(final RoleType roleType);
+    List<RolePermission> findAllPermissionsBy(final Long roleId);
+}

api/src/org/apache/cloudstack/acl/RoleType.java

@@ -16,18 +16,90 @@
 // under the License.
 package org.apache.cloudstack.acl;
 
+import com.cloud.user.Account;
+import com.google.common.base.Enums;
+import com.google.common.base.Strings;
+
 // Enum for default roles in CloudStack
 public enum RoleType {
-    Admin(1), ResourceAdmin(2), DomainAdmin(4), User(8), Unknown(0);
+    Admin(1L, Account.ACCOUNT_TYPE_ADMIN, 1),
+    ResourceAdmin(2L, Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN, 2),
+    DomainAdmin(3L, Account.ACCOUNT_TYPE_DOMAIN_ADMIN, 4),
+    User(4L, Account.ACCOUNT_TYPE_NORMAL, 8),
+    Unknown(-1L, (short) -1, 0);
 
+    private long id;
+    private short accountType;
     private int mask;
 
-    private RoleType(int mask) {
+    RoleType(final long id, final short accountType, final int mask) {
+        this.id = id;
+        this.accountType = accountType;
         this.mask = mask;
     }
 
-    public int getValue() {
+    public long getId() {
+        return id;
+    }
+
+    public short getAccountType() {
+        return accountType;
+    }
+
+    public int getMask() {
         return mask;
     }
-}
 
+    public static RoleType fromString(final String name) {
+        if (!Strings.isNullOrEmpty(name)
+                && Enums.getIfPresent(RoleType.class, name).isPresent()) {
+            return RoleType.valueOf(name);
+        }
+        throw new IllegalStateException("Illegal RoleType name provided");
+    }
+
+    public static RoleType fromMask(int mask) {
+        for (RoleType roleType : RoleType.values()) {
+            if (roleType.getMask() == mask) {
+                return roleType;
+            }
+        }
+        return Unknown;
+    }
+
+    public static RoleType getByAccountType(final short accountType) {
+        RoleType roleType = RoleType.Unknown;
+        switch (accountType) {
+            case Account.ACCOUNT_TYPE_ADMIN:
+                roleType = RoleType.Admin;
+                break;
+            case Account.ACCOUNT_TYPE_DOMAIN_ADMIN:
+                roleType = RoleType.DomainAdmin;
+                break;
+            case Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN:
+                roleType = RoleType.ResourceAdmin;
+                break;
+            case Account.ACCOUNT_TYPE_NORMAL:
+                roleType = RoleType.User;
+                break;
+        }
+        return roleType;
+    }
+
+    public static Long getRoleByAccountType(final Long roleId, final Short accountType) {
+        if (roleId == null && accountType != null) {
+            RoleType defaultRoleType = RoleType.getByAccountType(accountType);
+            if (defaultRoleType != null && defaultRoleType != RoleType.Unknown) {
+                return defaultRoleType.getId();
+            }
+        }
+        return roleId;
+    }
+
+    public static Short getAccountTypeByRole(final Role role, final Short accountType) {
+        if (role != null && role.getId() > 0L) {
+            return role.getRoleType().getAccountType();
+        }
+        return accountType;
+    }
+}

api/src/org/apache/cloudstack/acl/Rule.java

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.acl;
+
+import com.cloud.exception.InvalidParameterValueException;
+import com.google.common.base.Strings;
+
+import java.util.regex.Pattern;
+
+public final class Rule {
+    private final String rule;
+    private final static Pattern ALLOWED_PATTERN = Pattern.compile("^[a-zA-Z0-9*]+$");
+
+    public Rule(final String rule) {
+        validate(rule);
+        this.rule = rule;
+    }
+
+    public boolean matches(final String commandName) {
+        return !Strings.isNullOrEmpty(commandName)
+                && commandName.toLowerCase().matches(rule.toLowerCase().replace("*", "\\w*"));
+    }
+
+    public String getRuleString() {
+        return rule;
+    }
+
+    @Override
+    public String toString() {
+        return rule;
+    }
+
+    private static boolean validate(final String rule) {
+        if (Strings.isNullOrEmpty(rule) || !ALLOWED_PATTERN.matcher(rule).matches()) {
+            throw new InvalidParameterValueException("Only API names and wildcards are allowed, invalid rule provided: " + rule);
+        }
+        return true;
+    }
+}

api/src/org/apache/cloudstack/api/ApiArgValidator.java

@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api;
+
+public enum ApiArgValidator {
+    NotNullOrEmpty, // does Strings.isNullOrEmpty check
+    PositiveNumber, // does != null and > 0 check
+}