diff --git a/README.md b/README.md index 95c2d08..8d77497 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ This module is designed so that different hosts can run the various components o * puppet-module-xdmod **6.x** supports Open XDMoD **9.0.x** * puppet-module-xdmod **7.x** supports Open XDMoD **9.5.x** * puppet-module-xdmod **8.x** and **9.x** supports Open XDMoD **10.0.x** -* puppet-module-xdmod **10.x** supports Open XDMoD **10.5.x** +* puppet-module-xdmod **10.x** and **11.x** supports Open XDMoD **10.5.x** +* puppet-module-xdmod **12.x** supports Open XDMoD **11.0.x** ### Optional Dependencies @@ -72,7 +73,7 @@ include xdmod Examples of some hiera values that may be useful to set globally ```yaml -xdmod::version: '10.5.0' +xdmod::version: '11.0.2' # Disable roles that are enabled by default xdmod::web: false xdmod::database: false diff --git a/manifests/config.pp b/manifests/config.pp index fa39e15..1e1f2ca 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -223,20 +223,6 @@ } } - # Hack until merged and released: - # https://github.com/ubccr/xdmod/pull/1827 - file_line { 'etl_overseer-db-log': - ensure => 'present', - path => '/usr/share/xdmod/tools/etl/etl_overseer.php', - line => " 'db' => false,", - after => "\s+'emailSubject'.+", - append_on_no_match => false, - before => [ - Exec['etl-bootstrap'], - Exec['etl-bootstrap-supremm'], - ], - } - # List from /usr/share/xdmod/classes/OpenXdmod/Setup/DatabaseSetup.php # Under 'ETLv2 database bootstrap start' $etl_bootstrap_sections = [ @@ -253,14 +239,14 @@ exec { 'etl-bootstrap': path => '/usr/bin:/bin:/usr/sbin:/sbin', - command => "/usr/share/xdmod/tools/etl/etl_overseer.php ${etl_bootstrap_args.join(' ')}", + command => "/usr/share/xdmod/tools/etl/etl_overseer.php --log-to-database no ${etl_bootstrap_args.join(' ')}", logoutput => true, refreshonly => true, notify => Exec['acl-config'], } exec { 'etl-bootstrap-supremm': path => '/usr/bin:/bin:/usr/sbin:/sbin', - command => '/usr/share/xdmod/tools/etl/etl_overseer.php -p supremm.bootstrap -p jobefficiency.bootstrap', + command => '/usr/share/xdmod/tools/etl/etl_overseer.php --log-to-database no -p supremm.bootstrap -p jobefficiency.bootstrap', logoutput => true, refreshonly => true, notify => Exec['acl-config'], @@ -298,6 +284,11 @@ Undef => 'HPC', default => $r['resource_type'], } + if $r['resource_allocation_type'] =~ Undef { + $resource_allocation_type = 'CPU' + } else { + $resource_allocation_type = $r['resource_allocation_type'] + } if $resource_type == 'HPC' { $pi_column = $r['pi_column'] ? { Undef => $xdmod::pi_column, @@ -311,6 +302,7 @@ 'name' => $r['name'], 'description' => $r['description'], 'resource_type' => $resource_type, + 'resource_allocation_type' => $resource_allocation_type, 'pi_column' => $pi_column, 'timezone' => $r['timezone'], 'shared_jobs' => $r['shared_jobs'], @@ -324,7 +316,37 @@ 'ppn' => 1, } } - $resource_specs = $xdmod::resource_specs + $ondemand_resource_specs + $resource_specs = $xdmod::resource_specs.map |$r| { + if $r['gpu_node_count'] =~ Undef { + $gpu_node_count = 0 + } else { + $gpu_node_count = $r['gpu_node_count'] + } + if $r['gpu_processor_count'] =~ Undef { + $gpu_processor_count = 0 + } else { + $gpu_processor_count = $r['gpu_processor_count'] + } + if $r['gpu_ppn'] =~ Undef { + $gpu_ppn = 0 + } else { + $gpu_ppn = $r['gpu_ppn'] + } + $data = { + 'resource' => $r['resource'], + 'start_date' => $r['start_date'], + 'end_date' => $r['end_date'], + 'processors' => $r['processors'], + 'nodes' => $r['nodes'], + 'ppn' => $r['ppn'], + 'cpu_node_count' => $r['cpu_node_count'], + 'cpu_processor_count' => $r['cpu_processor_count'], + 'cpu_ppn' => $r['cpu_ppn'], + 'gpu_node_count' => $gpu_node_count, + 'gpu_processor_count' => $gpu_processor_count, + 'gpu_ppn' => $gpu_ppn, + } + } + $ondemand_resource_specs file { '/etc/xdmod/resources.json': ensure => 'file', diff --git a/manifests/database.pp b/manifests/database.pp index e9915dd..5a2653f 100644 --- a/manifests/database.pp +++ b/manifests/database.pp @@ -8,8 +8,8 @@ user => $xdmod::database_user, password => $xdmod::database_password, host => $xdmod::web_host, - charset => 'latin1', - collate => 'latin1_swedish_ci', + charset => 'utf8', + collate => 'utf8_unicode_ci', grant => ['ALL'], } @@ -20,9 +20,7 @@ mysql::db { 'modw': } mysql::db { 'modw_aggregates': } mysql::db { 'modw_filters': } - if $xdmod::enable_cloud_realm { - mysql::db { 'modw_cloud': } - } + mysql::db { 'modw_cloud': } if $xdmod::enable_ondemand { mysql::db { 'modw_ondemand': } } diff --git a/manifests/init.pp b/manifests/init.pp index 33185a2..c215ff8 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -22,8 +22,6 @@ # Enable XDMoD AppKernel support # @param enable_supremm # Enable XDMod SUPReMM support -# @param enable_cloud_realm -# Enable the XDMoD cloud realm by adding the necessary database # @param enable_ondemand # Enable the XDMOD OnDemand plugin and necessary database # @param local_repo_name @@ -276,7 +274,6 @@ Boolean $enable_appkernel = false, Boolean $enable_supremm = false, - Boolean $enable_cloud_realm = false, Boolean $enable_ondemand = false, Optional[String] $local_repo_name = undef, Boolean $manage_epel = true, @@ -293,7 +290,7 @@ String $xdmod_supremm_package_name = $xdmod::params::xdmod_supremm_package_name, Variant[Stdlib::HTTPSUrl, Stdlib::HTTPUrl] $xdmod_supremm_package_url = $xdmod::params::xdmod_supremm_package_url, - String[1] $php_mongodb_version = '1.16.2', + String[1] $php_mongodb_version = '1.18.1', String $database_host = '127.0.0.1', Integer $database_port = 3306, String $database_user = 'xdmod', diff --git a/manifests/install.pp b/manifests/install.pp index 0c04466..f93877f 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -8,11 +8,19 @@ enable_only => true, provider => 'dnfmodule', } + package { 'php-module': + ensure => '7.4', + name => 'php', + enable_only => true, + provider => 'dnfmodule', + } if $xdmod::local_repo_name { Package['nodejs-module'] -> Package['xdmod'] + Package['php-module'] -> Package['xdmod'] } else { Package['nodejs-module'] -> Yum::Install[$xdmod::package_name] + Package['php-module'] -> Yum::Install[$xdmod::package_name] } $mongodb_dependencies = [ @@ -25,7 +33,6 @@ } } php::extension { 'mongodb': - # 1.17+ requires newer PHP than what's default for RHEL8 ensure => $xdmod::php_mongodb_version, provider => 'pecl', ini_prefix => '40-', diff --git a/manifests/params.pp b/manifests/params.pp index 5a9567e..ba96b1c 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -1,10 +1,10 @@ # @summary XDMoD module defaults # @api private class xdmod::params { - $version = '10.5.0' - $xdmod_appkernels_version = '10.5.0' - $xdmod_supremm_version = '10.5.0' - $xdmod_ondemand_version = '10.5.0' + $version = '11.0.2' + $xdmod_appkernels_version = '11.0.0' + $xdmod_supremm_version = '11.0.1' + $xdmod_ondemand_version = '11.0.1' $sender_email = "xdmod@xdmod.${facts['networking']['domain']}" $apache_vhost_name = "xdmod.${facts['networking']['domain']}" $portal_settings = {} @@ -30,13 +30,13 @@ $compute_only = false } $package_name = 'xdmod' - $package_url = "https://github.com/ubccr/xdmod/releases/download/vVERSION-1.0/xdmod-VERSION-1.0.el${facts['os']['release']['major']}.noarch.rpm" + $package_url = "https://github.com/ubccr/xdmod/releases/download/vVERSION-3/xdmod-VERSION-3.el${facts['os']['release']['major']}.noarch.rpm" $appkernels_package_name = 'xdmod-appkernels' $appkernels_package_url = "https://github.com/ubccr/xdmod-appkernels/releases/download/vVERSION-1.0/xdmod-appkernels-VERSION-1.0.el${facts['os']['release']['major']}.noarch.rpm" $xdmod_supremm_package_name = 'xdmod-supremm' - $xdmod_supremm_package_url = "https://github.com/ubccr/xdmod-supremm/releases/download/vVERSION-1.0/xdmod-supremm-VERSION-1.0.el${facts['os']['release']['major']}.noarch.rpm" + $xdmod_supremm_package_url = "https://github.com/ubccr/xdmod-supremm/releases/download/vVERSION-1/xdmod-supremm-VERSION-1.el${facts['os']['release']['major']}.noarch.rpm" $supremm_package_url = "https://github.com/ubccr/supremm/releases/download/SUPREMM_VERSION/supremm-SUPREMM_VERSION-1.el${facts['os']['release']['major']}.x86_64.rpm" - $ondemand_package_url = "https://github.com/ubccr/xdmod-ondemand/releases/download/vVERSION-1.0/xdmod-ondemand-VERSION-1.0.el${facts['os']['release']['major']}.noarch.rpm" + $ondemand_package_url = "https://github.com/ubccr/xdmod-ondemand/releases/download/vVERSION-1/xdmod-ondemand-VERSION-1.el${facts['os']['release']['major']}.noarch.rpm" } default: { diff --git a/metadata.json b/metadata.json index 87e33e0..526ee32 100644 --- a/metadata.json +++ b/metadata.json @@ -10,7 +10,7 @@ "dependencies": [ { "name": "puppetlabs/stdlib", - "version_requirement": ">= 9.0.0 <10.0.0" + "version_requirement": ">= 9.1.0 <10.0.0" }, { "name": "puppetlabs/inifile", @@ -22,15 +22,15 @@ }, { "name": "puppet/mongodb", - "version_requirement": ">= 1.0.0 <7.0.0" + "version_requirement": ">= 1.0.0 <8.0.0" }, { "name": "puppet/epel", - "version_requirement": ">= 3.0.0 <6.0.0" + "version_requirement": ">= 3.0.0 <7.0.0" }, { "name": "puppet/logrotate", - "version_requirement": ">= 1.0.0 <8.0.0" + "version_requirement": ">= 1.0.0 <10.0.0" }, { "name": "puppet/yum", @@ -38,11 +38,11 @@ }, { "name": "puppet/archive", - "version_requirement": ">= 4.0.0 <8.0.0" + "version_requirement": ">= 4.0.0 <9.0.0" }, { "name": "puppetlabs/apache", - "version_requirement": ">= 1.8.0 <13.0.0" + "version_requirement": ">= 1.8.0 <14.0.0" }, { "name": "puppet/augeasproviders_shellvar", @@ -50,7 +50,7 @@ }, { "name": "saz/sudo", - "version_requirement": ">= 3.0.0 <9.0.0" + "version_requirement": ">= 3.0.0 <10.0.0" }, { "name": "treydock/pcp", @@ -62,7 +62,7 @@ }, { "name": "puppet/php", - "version_requirement": ">= 8.0.0 <11.0.0" + "version_requirement": ">= 8.0.0 <13.0.0" } ], "operatingsystem_support": [ diff --git a/spec/classes/xdmod_ondemand_spec.rb b/spec/classes/xdmod_ondemand_spec.rb index 8e416d8..d13eba4 100644 --- a/spec/classes/xdmod_ondemand_spec.rb +++ b/spec/classes/xdmod_ondemand_spec.rb @@ -6,7 +6,7 @@ on_supported_os(supported_os: [ { 'operatingsystem' => 'RedHat', - 'operatingsystemrelease' => ['7', '8'], + 'operatingsystemrelease' => ['8'], }, ]).each do |os, facts| context "when #{os}" do diff --git a/spec/classes/xdmod_spec.rb b/spec/classes/xdmod_spec.rb index 6075bf0..d931121 100644 --- a/spec/classes/xdmod_spec.rb +++ b/spec/classes/xdmod_spec.rb @@ -6,7 +6,7 @@ on_supported_os(supported_os: [ { 'operatingsystem' => 'RedHat', - 'operatingsystemrelease' => ['7', '8'], + 'operatingsystemrelease' => ['8'], }, ]).each do |os, facts| context "when #{os}" do diff --git a/spec/shared_examples/config.rb b/spec/shared_examples/config.rb index ba9ad67..c5c7c10 100644 --- a/spec/shared_examples/config.rb +++ b/spec/shared_examples/config.rb @@ -90,11 +90,9 @@ mode: '0644',) end - it { is_expected.to contain_file_line('etl_overseer-db-log') } - it do expected_cmd = [ - '/usr/share/xdmod/tools/etl/etl_overseer.php', + '/usr/share/xdmod/tools/etl/etl_overseer.php --log-to-database no', '-p supremm.bootstrap', '-p jobefficiency.bootstrap', ] is_expected.to contain_exec('etl-bootstrap-supremm').with( @@ -108,7 +106,7 @@ it do expected_cmd = [ - '/usr/share/xdmod/tools/etl/etl_overseer.php', + '/usr/share/xdmod/tools/etl/etl_overseer.php --log-to-database no', '-p xdb-bootstrap', '-p jobs-xdw-bootstrap', '-p xdw-bootstrap-storage', '-p shredder-bootstrap', '-p staging-bootstrap', '-p hpcdb-bootstrap', '-p acls-xdmod-management', '-p logger-bootstrap', @@ -242,6 +240,7 @@ expected = [{ 'resource' => 'example', 'resource_type' => 'HPC', + 'resource_allocation_type' => 'CPU', 'name' => 'Example', 'pi_column' => 'account_name', }] @@ -266,12 +265,14 @@ { 'resource' => 'example1', 'resource_type' => 'HPC', + 'resource_allocation_type' => 'CPU', 'name' => 'Example1', 'pi_column' => 'account_name', }, { 'resource' => 'example2', 'resource_type' => 'HPC', + 'resource_allocation_type' => 'CPU', 'name' => 'Example2', 'pi_column' => 'account_name', }, @@ -294,9 +295,17 @@ value = JSON.parse(content) expected = [ 'resource' => 'example', + 'start_date' => nil, + 'end_date' => nil, 'processors' => 2, 'nodes' => 1, 'ppn' => 2, + 'cpu_node_count' => nil, + 'cpu_ppn' => nil, + 'cpu_processor_count' => nil, + 'gpu_node_count' => 0, + 'gpu_ppn' => 0, + 'gpu_processor_count' => 0, ] expect(value).to eq(expected) end @@ -603,6 +612,7 @@ expected = [{ 'resource' => 'ondemand', 'resource_type' => 'Gateway', + 'resource_allocation_type' => 'CPU', 'name' => 'OnDemand', }] expect(value).to eq(expected) @@ -611,18 +621,28 @@ it do content = catalogue.resource('file', '/etc/xdmod/resource_specs.json').send(:parameters)[:content] value = JSON.parse(content) - expected = [{ - 'resource' => 'example', - 'processors' => 2, - 'nodes' => 1, - 'ppn' => 2, - }, - { - 'resource' => 'ondemand', - 'processors' => 1, - 'nodes' => 1, - 'ppn' => 1, - },] + expected = [ + { + 'resource' => 'example', + 'start_date' => nil, + 'end_date' => nil, + 'processors' => 2, + 'nodes' => 1, + 'ppn' => 2, + 'cpu_node_count' => nil, + 'cpu_ppn' => nil, + 'cpu_processor_count' => nil, + 'gpu_node_count' => 0, + 'gpu_ppn' => 0, + 'gpu_processor_count' => 0, + }, + { + 'resource' => 'ondemand', + 'processors' => 1, + 'nodes' => 1, + 'ppn' => 1, + }, + ] expect(value).to eq(expected) end end diff --git a/spec/shared_examples/database.rb b/spec/shared_examples/database.rb index 99a26cd..9e090d3 100644 --- a/spec/shared_examples/database.rb +++ b/spec/shared_examples/database.rb @@ -8,39 +8,23 @@ 'moddb', 'modw', 'modw_aggregates', + 'modw_cloud', ].each do |db| it "creates Mysql::Db[#{db}]" do is_expected.to contain_mysql__db(db).with(ensure: 'present', user: 'xdmod', password: 'changeme', host: 'localhost', - charset: 'latin1', - collate: 'latin1_swedish_ci', + charset: 'utf8', + collate: 'utf8_unicode_ci', grant: ['ALL'],) end end - it { is_expected.not_to contain_mysql__db('modw_cloud') } it { is_expected.not_to contain_mysql__db('mod_appkernel') } it { is_expected.not_to contain_mysql__db('mod_akrr') } it { is_expected.not_to contain_mysql_grant('akrr@localhost/modw.resourcefact') } - context 'when enable_cloud_realm => true' do - let(:params) { { enable_cloud_realm: true } } - - it do - is_expected.to contain_mysql__db('modw_cloud').with( - ensure: 'present', - user: 'xdmod', - password: 'changeme', - host: 'localhost', - charset: 'latin1', - collate: 'latin1_swedish_ci', - grant: ['ALL'], - ) - end - end - context 'when enable_ondemand => true' do let(:params) { { enable_ondemand: true } } @@ -50,8 +34,8 @@ user: 'xdmod', password: 'changeme', host: 'localhost', - charset: 'latin1', - collate: 'latin1_swedish_ci', + charset: 'utf8', + collate: 'utf8_unicode_ci', grant: ['ALL'], ) end @@ -65,8 +49,8 @@ user: 'akrr', password: 'changeme', host: 'localhost', - charset: 'latin1', - collate: 'latin1_swedish_ci', + charset: 'utf8', + collate: 'utf8_unicode_ci', grant: ['ALL'],) end @@ -75,8 +59,8 @@ user: 'akrr', password: 'changeme', host: 'localhost', - charset: 'latin1', - collate: 'latin1_swedish_ci', + charset: 'utf8', + collate: 'utf8_unicode_ci', grant: ['ALL'],) end @@ -86,8 +70,8 @@ user: 'akrr', password: 'changeme', host: 'localhost', - charset: 'latin1', - collate: 'latin1_swedish_ci', + charset: 'utf8', + collate: 'utf8_unicode_ci', grant: ['SELECT'],) end diff --git a/spec/shared_examples/install.rb b/spec/shared_examples/install.rb index 6cb59a5..5080bef 100644 --- a/spec/shared_examples/install.rb +++ b/spec/shared_examples/install.rb @@ -4,7 +4,7 @@ it do is_expected.to contain_yum__install('xdmod').with( ensure: 'present', - source: "https://github.com/ubccr/xdmod/releases/download/v10.5.0-1.0/xdmod-10.5.0-1.0.el#{facts[:os]['release']['major']}.noarch.rpm", + source: "https://github.com/ubccr/xdmod/releases/download/v11.0.2-3/xdmod-11.0.2-3.el#{facts[:os]['release']['major']}.noarch.rpm", timeout: '0', require: ['Yumrepo[epel]'], notify: 'Exec[etl-bootstrap]', @@ -20,7 +20,7 @@ it do is_expected.to contain_yum__install('xdmod-appkernels').with( ensure: 'present', - source: "https://github.com/ubccr/xdmod-appkernels/releases/download/v10.5.0-1.0/xdmod-appkernels-10.5.0-1.0.el#{facts[:os]['release']['major']}.noarch.rpm", + source: "https://github.com/ubccr/xdmod-appkernels/releases/download/v11.0.0-1.0/xdmod-appkernels-11.0.0-1.0.el#{facts[:os]['release']['major']}.noarch.rpm", timeout: '0', require: ['Yumrepo[epel]'], ) @@ -33,7 +33,7 @@ it do is_expected.to contain_yum__install('xdmod-supremm').with( ensure: 'present', - source: "https://github.com/ubccr/xdmod-supremm/releases/download/v10.5.0-1.0/xdmod-supremm-10.5.0-1.0.el#{facts[:os]['release']['major']}.noarch.rpm", + source: "https://github.com/ubccr/xdmod-supremm/releases/download/v11.0.1-1/xdmod-supremm-11.0.1-1.el#{facts[:os]['release']['major']}.noarch.rpm", timeout: '0', require: ['Yumrepo[epel]'], notify: 'Exec[etl-bootstrap-supremm]', diff --git a/templates/ondemand/ingest.sh.erb b/templates/ondemand/ingest.sh.erb index 020751c..9e565b8 100755 --- a/templates/ondemand/ingest.sh.erb +++ b/templates/ondemand/ingest.sh.erb @@ -6,14 +6,13 @@ set -e BASE_CMD="xdmod-ondemand-ingestor" -OOD_HOSTNAME="$1" -RESOURCE="$2" -LOG_DIRECTORY="$3" -if [ "x${OOD_HOSTNAME}" != "x" -a "x${RESOURCE}" != "x" -a "x${LOG_DIRECTORY}" != "x" ]; then - $BASE_CMD -d $LOG_DIRECTORY -r $RESOURCE -u $OOD_HOSTNAME +RESOURCE="$1" +LOG_DIRECTORY="$2" +if [ "x${RESOURCE}" != "x" -a "x${LOG_DIRECTORY}" != "x" ]; then + $BASE_CMD -d $LOG_DIRECTORY -r $RESOURCE else <% scope['xdmod::ondemand_resources'].each do |resource| -%> <%- next if resource['shred_directory'].nil? -%> - $BASE_CMD -d <%= resource['shred_directory'] %> -r <%= resource['resource'] %> -u <%= resource['hostname'] %> + $BASE_CMD -d <%= resource['shred_directory'] %> -r <%= resource['resource'] %> <%- end -%> fi diff --git a/types/resource.pp b/types/resource.pp index bbbbbdd..f16ca62 100644 --- a/types/resource.pp +++ b/types/resource.pp @@ -3,9 +3,9 @@ name => String, Optional[description] => String, Optional[resource_type] => String, + Optional[resource_allocation_type] => String, Optional[pi_column] => String, Optional[timezone] => String, Optional[shared_jobs] => Boolean, Optional[shred_directory] => Stdlib::Absolutepath, - Optional[hostname] => Stdlib::HTTPSUrl, }] diff --git a/types/resource_spec.pp b/types/resource_spec.pp index cf0e314..49a3311 100644 --- a/types/resource_spec.pp +++ b/types/resource_spec.pp @@ -5,4 +5,10 @@ processors => Integer, nodes => Integer, ppn => Integer, + cpu_node_count => Optional[Integer], + cpu_processor_count => Optional[Integer], + cpu_ppn => Optional[Integer], + gpu_node_count => Optional[Integer], + gpu_processor_count => Optional[Integer], + gpu_ppn => Optional[Integer], }]