From bbdc9d5e4d1be5c47b26c680e9c763c6704d17d2 Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Fri, 16 Aug 2019 14:35:26 +0300 Subject: [PATCH 1/4] Add scripts to dump the OPL tables to a dump-file and to restore from such a file, as loading from a dump-file is much faster than running OPL-update. Modify the docker-entrypoint.sh to do such a dump after running OPL-update, to attempt to restore the OPL tables from a dump-file before trying to run OPL-update, and also to save/restore the JSON files generated by OPL-update. Also add a script to import that OPL global statistics without running the full update-OPL-statistics script which also works on local data. --- bin/dump-OPL-tables | 106 +++++++++++++++++++++++++++++++++ bin/load-OPL-global-statistics | 80 +++++++++++++++++++++++++ bin/restore-OPL-tables | 95 +++++++++++++++++++++++++++++ 3 files changed, 281 insertions(+) create mode 100755 bin/dump-OPL-tables create mode 100755 bin/load-OPL-global-statistics create mode 100755 bin/restore-OPL-tables diff --git a/bin/dump-OPL-tables b/bin/dump-OPL-tables new file mode 100755 index 0000000000..5222830f04 --- /dev/null +++ b/bin/dump-OPL-tables @@ -0,0 +1,106 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script dumps the OPL library tables to a dump file. +use strict; + +# Get the necessary packages, including adding webwork to our path. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use String::ShellQuote; +use DBI; + +# get course environment and configured OPL path + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +my $configured_OPL_path = $ce->{problemLibrary}{root}; + + +# Drop the "OpenProblemLibrary" from the end of the path + +$configured_OPL_path =~ s+OpenProblemLibrary++; + +# Check that it exists + +if ( -d "$configured_OPL_path" ) { + print "OPL path seems to be $configured_OPL_path\n"; +} else { + print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n"; + exit; +} + +# Set TABLE-DUMP path and make directory if necessary + +my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP"; +if ( ! -d "$prepared_OPL_tables_dir" ) { + `mkdir -p $prepared_OPL_tables_dir`; +} + +# Set dump file name + +my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql"; + +# Get DB connection settings + +my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); + +$host = 'localhost' unless $host; + +$port = 3306 unless $port; + +my $dbuser = $ce->{database_username}; +my $dbpass = $ce->{database_password}; + +$dbuser = shell_quote($dbuser); +$dbpass = shell_quote($dbpass); +$db = shell_quote($db); + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + +# Get mysqldump_command + +my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; + +# The tables to dump are: + +my $OPL_tables_to_dump = "OPL_DBsubject OPL_DBchapter OPL_DBsection OPL_author OPL_path OPL_pgfile OPL_keyword OPL_pgfile_keyword OPL_textbook OPL_chapter OPL_section OPL_problem OPL_morelt OPL_pgfile_problem"; + +# Tables NOT dumped: +# OPL_problem_user - is created by bin/update-OPL-statistics and need not be archived +# OPL_global_statistics - loaded from a special file provide by the OPL +# OPL_local_statistics - locally generated + +print "Dumping OPL tables\n"; + +`$mysqldump_command --host=$host --port=$port --user=$dbuser --password=$dbpass --default-character-set=$character_set $db $OPL_tables_to_dump > $prepared_OPL_tables_file`; + +print "OPL database dump created: $prepared_OPL_tables_file\n"; + +1; diff --git a/bin/load-OPL-global-statistics b/bin/load-OPL-global-statistics new file mode 100755 index 0000000000..d25ae13088 --- /dev/null +++ b/bin/load-OPL-global-statistics @@ -0,0 +1,80 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script loads the OPL global statistics, which is often done by bin/update-OPL-statistics but may need to be done outside of that setting. +use strict; + +# Get the necessary packages, including adding webwork to our path. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use String::ShellQuote; +use DBI; + +# get course environment and configured OPL path + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +my $dbh = DBI->connect( + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + AutoCommit => 0, + PrintError => 0, + RaiseError => 1, + }, +); + +# check to see if the global statistics file exists and if it does, upload it. + +my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql'; + +if (-e $global_sql_file) { + + my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); + + $host = 'localhost' unless $host; + + $port = 3306 unless $port; + + my $dbuser = $ce->{database_username}; + my $dbpass = $ce->{database_password}; + + + $dbh->do(<commit(); + + $dbuser = shell_quote($dbuser); + $dbpass = shell_quote($dbpass); + $db = shell_quote($db); + + my $mysql_command = $ce->{externalPrograms}->{mysql}; + + `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`; + +} + +1; diff --git a/bin/restore-OPL-tables b/bin/restore-OPL-tables new file mode 100755 index 0000000000..c4933ad4c9 --- /dev/null +++ b/bin/restore-OPL-tables @@ -0,0 +1,95 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script restores the OPL library tables from a dump file. +use strict; + +# Get the necessary packages, including adding webwork to our path. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use String::ShellQuote; +use DBI; + +# get course environment and configured OPL path + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +my $configured_OPL_path = $ce->{problemLibrary}{root}; + + +# Drop the "OpenProblemLibrary" from the end of the path + +$configured_OPL_path =~ s+OpenProblemLibrary++; + +# Check that it exists + +if ( -d "$configured_OPL_path" ) { + print "OPL path seems to be $configured_OPL_path\n"; +} else { + print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n"; + exit; +} + +# Set TABLE-DUMP path and make directory if necessary + +my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP"; +if ( ! -d "$prepared_OPL_tables_dir" ) { + `mkdir -p $prepared_OPL_tables_dir`; +} + +# Set dump file name + +my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql"; + +# Get DB connection settings + +my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); + +$host = 'localhost' unless $host; + +$port = 3306 unless $port; + +my $dbuser = $ce->{database_username}; +my $dbpass = $ce->{database_password}; + +$dbuser = shell_quote($dbuser); +$dbpass = shell_quote($dbpass); +$db = shell_quote($db); + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + +my $mysql_command = $ce->{externalPrograms}->{mysql}; + +# check to see if the prepared_OPL_tables_file exists and if it does load it in + +if (-e $prepared_OPL_tables_file) { + `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass --default-character-set=$character_set $db < $prepared_OPL_tables_file`; +} + +1; From f63ed9f96a81e42799a435a45b19bd94cd2a5f8a Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Fri, 16 Aug 2019 17:06:30 +0300 Subject: [PATCH 2/4] Have OPL-update call load-OPL-global-statistics to load the global statistics data from the OPL after running update-OPL-statistics. Remove the reloading of the global statistics data from update-OPL-statistics but print a warning that it no longer does that. Also fix the warning in lib/WeBWorK/Utils/LibraryStats.pm to inform users to use load-OPL-global-statistics instead of update-OPL-statistics when the global statistics data is missing. --- bin/OPL-update | 3 +++ bin/update-OPL-statistics | 33 +++---------------------------- lib/WeBWorK/Utils/LibraryStats.pm | 2 +- 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/bin/OPL-update b/bin/OPL-update index 91f727edc1..4a904aaaa7 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -986,6 +986,9 @@ if ($ce->{problemLibrary}{showLibraryLocalStats} || $ce->{problemLibrary}{showLibraryGlobalStats}) { print "\nUpdating Library Statistics.\n"; do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics'; + + print "\nLoading global statistics (if possible).\n"; + do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics'; } diff --git a/bin/update-OPL-statistics b/bin/update-OPL-statistics index f582fcce83..f0b9219844 100755 --- a/bin/update-OPL-statistics +++ b/bin/update-OPL-statistics @@ -192,35 +192,8 @@ EOS $dbh->commit(); -# check to see if the global statistics file exists and if it does, upload it. - -my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql'; - -if (-e $global_sql_file) { - - my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); - - $host = 'localhost' unless $host; - - $port = 3306 unless $port; - - my $dbuser = $ce->{database_username}; - my $dbpass = $ce->{database_password}; - - - $dbh->do(<commit(); - - $dbuser = shell_quote($dbuser); - $dbpass = shell_quote($dbpass); - $db = shell_quote($db); - - my $mysql_command = $ce->{externalPrograms}->{mysql}; - - `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`; - -} +# We no longer automatically load the global statistics data here. +print( "You may want to run load-OPL-global-statistics to update the global statistics data.\n", + "If this is being run by OPL-update, that will be done automatically.\n"); 1; diff --git a/lib/WeBWorK/Utils/LibraryStats.pm b/lib/WeBWorK/Utils/LibraryStats.pm index 7ef55e239a..aa08d06a64 100644 --- a/lib/WeBWorK/Utils/LibraryStats.pm +++ b/lib/WeBWorK/Utils/LibraryStats.pm @@ -92,7 +92,7 @@ sub getGlobalStats { unless ($selectstm->execute($source_file)) { if ($selectstm->errstr =~ /Table .* doesn't exist/) { - warn "Couldn't find the OPL global statistics table. Did you download the latest OPL and run update-OPL-statistics?" + warn "Couldn't find the OPL global statistics table. Did you download the latest OPL and run load-OPL-global-statistics?" } die $selectstm->errstr; } From 00d4f3017df3d65da87ab909419f7eac1956a5f6 Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Mon, 19 Aug 2019 16:35:19 +0300 Subject: [PATCH 3/4] Replace use of password on command line for mysql/mysqldump commands with the use of the MYSQL_PWD environment variable. See: https://github.com/openwebwork/webwork2/issues/987 --- bin/dump-OPL-tables | 4 +++- bin/load-OPL-global-statistics | 4 +++- bin/restore-OPL-tables | 4 +++- bin/upload-OPL-statistics | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bin/dump-OPL-tables b/bin/dump-OPL-tables index 5222830f04..d4ee88bc0f 100755 --- a/bin/dump-OPL-tables +++ b/bin/dump-OPL-tables @@ -77,6 +77,8 @@ $dbuser = shell_quote($dbuser); $dbpass = shell_quote($dbpass); $db = shell_quote($db); +$ENV{'MYSQL_PWD'}=$dbpass; + # decide whether the mysql installation can handle # utf8mb4 and that should be used for the OPL @@ -99,7 +101,7 @@ my $OPL_tables_to_dump = "OPL_DBsubject OPL_DBchapter OPL_DBsection OPL_author O print "Dumping OPL tables\n"; -`$mysqldump_command --host=$host --port=$port --user=$dbuser --password=$dbpass --default-character-set=$character_set $db $OPL_tables_to_dump > $prepared_OPL_tables_file`; +`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db $OPL_tables_to_dump > $prepared_OPL_tables_file`; print "OPL database dump created: $prepared_OPL_tables_file\n"; diff --git a/bin/load-OPL-global-statistics b/bin/load-OPL-global-statistics index d25ae13088..ad3bb62abc 100755 --- a/bin/load-OPL-global-statistics +++ b/bin/load-OPL-global-statistics @@ -70,10 +70,12 @@ EOS $dbuser = shell_quote($dbuser); $dbpass = shell_quote($dbpass); $db = shell_quote($db); + + $ENV{'MYSQL_PWD'}=$dbpass; my $mysql_command = $ce->{externalPrograms}->{mysql}; - `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`; + `$mysql_command --host=$host --port=$port --user=$dbuser $db < $global_sql_file`; } diff --git a/bin/restore-OPL-tables b/bin/restore-OPL-tables index c4933ad4c9..4c748e5878 100755 --- a/bin/restore-OPL-tables +++ b/bin/restore-OPL-tables @@ -77,6 +77,8 @@ $dbuser = shell_quote($dbuser); $dbpass = shell_quote($dbpass); $db = shell_quote($db); +$ENV{'MYSQL_PWD'}=$dbpass; + # decide whether the mysql installation can handle # utf8mb4 and that should be used for the OPL @@ -89,7 +91,7 @@ my $mysql_command = $ce->{externalPrograms}->{mysql}; # check to see if the prepared_OPL_tables_file exists and if it does load it in if (-e $prepared_OPL_tables_file) { - `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass --default-character-set=$character_set $db < $prepared_OPL_tables_file`; + `$mysql_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db < $prepared_OPL_tables_file`; } 1; diff --git a/bin/upload-OPL-statistics b/bin/upload-OPL-statistics index 49a354358c..37004296d3 100755 --- a/bin/upload-OPL-statistics +++ b/bin/upload-OPL-statistics @@ -49,9 +49,11 @@ $dbuser = shell_quote($dbuser); $dbpass = shell_quote($dbpass); $db = shell_quote($db); +$ENV{'MYSQL_PWD'}=$dbpass; + my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; -`$mysqldump_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db OPL_local_statistics > $output_file`; +`$mysqldump_command --host=$host --port=$port --user=$dbuser $db OPL_local_statistics > $output_file`; print "Database File Created\n"; From 70f2e2e21ec1b7dc6225c7a7ffd8d9266a8c4917 Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Wed, 21 Aug 2019 21:48:29 +0300 Subject: [PATCH 4/4] Rename serveral Perl scripts to end in .pl, and fix things which call them/refer to them --- bin/OPL-update | 2 +- bin/{dump-OPL-tables => dump-OPL-tables.pl} | 0 ...ad-OPL-global-statistics => load-OPL-global-statistics.pl} | 0 bin/{restore-OPL-tables => restore-OPL-tables.pl} | 0 bin/{update-OPL-statistics => update-OPL-statistics.pl} | 2 +- bin/{upload-OPL-statistics => upload-OPL-statistics.pl} | 0 lib/WeBWorK/Utils/LibraryStats.pm | 4 ++-- 7 files changed, 4 insertions(+), 4 deletions(-) rename bin/{dump-OPL-tables => dump-OPL-tables.pl} (100%) rename bin/{load-OPL-global-statistics => load-OPL-global-statistics.pl} (100%) rename bin/{restore-OPL-tables => restore-OPL-tables.pl} (100%) rename bin/{update-OPL-statistics => update-OPL-statistics.pl} (98%) rename bin/{upload-OPL-statistics => upload-OPL-statistics.pl} (100%) diff --git a/bin/OPL-update b/bin/OPL-update index 4a904aaaa7..79d6345342 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -988,7 +988,7 @@ if ($ce->{problemLibrary}{showLibraryLocalStats} || do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics'; print "\nLoading global statistics (if possible).\n"; - do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics'; + do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl'; } diff --git a/bin/dump-OPL-tables b/bin/dump-OPL-tables.pl similarity index 100% rename from bin/dump-OPL-tables rename to bin/dump-OPL-tables.pl diff --git a/bin/load-OPL-global-statistics b/bin/load-OPL-global-statistics.pl similarity index 100% rename from bin/load-OPL-global-statistics rename to bin/load-OPL-global-statistics.pl diff --git a/bin/restore-OPL-tables b/bin/restore-OPL-tables.pl similarity index 100% rename from bin/restore-OPL-tables rename to bin/restore-OPL-tables.pl diff --git a/bin/update-OPL-statistics b/bin/update-OPL-statistics.pl similarity index 98% rename from bin/update-OPL-statistics rename to bin/update-OPL-statistics.pl index f0b9219844..b50f4855cc 100755 --- a/bin/update-OPL-statistics +++ b/bin/update-OPL-statistics.pl @@ -193,7 +193,7 @@ BEGIN $dbh->commit(); # We no longer automatically load the global statistics data here. -print( "You may want to run load-OPL-global-statistics to update the global statistics data.\n", +print( "You may want to run load-OPL-global-statistics.pl to update the global statistics data.\n", "If this is being run by OPL-update, that will be done automatically.\n"); 1; diff --git a/bin/upload-OPL-statistics b/bin/upload-OPL-statistics.pl similarity index 100% rename from bin/upload-OPL-statistics rename to bin/upload-OPL-statistics.pl diff --git a/lib/WeBWorK/Utils/LibraryStats.pm b/lib/WeBWorK/Utils/LibraryStats.pm index aa08d06a64..dcf178aa17 100644 --- a/lib/WeBWorK/Utils/LibraryStats.pm +++ b/lib/WeBWorK/Utils/LibraryStats.pm @@ -66,7 +66,7 @@ sub getLocalStats { unless ($selectstm->execute($source_file)) { if ($selectstm->errstr =~ /Table .* doesn't exist/) { - warn "Couldn't find the OPL local statistics table. Did you download the latest OPL and run update-OPL-statistics?" + warn "Couldn't find the OPL local statistics table. Did you download the latest OPL and run update-OPL-statistics.pl?" } die $selectstm->errstr; } @@ -92,7 +92,7 @@ sub getGlobalStats { unless ($selectstm->execute($source_file)) { if ($selectstm->errstr =~ /Table .* doesn't exist/) { - warn "Couldn't find the OPL global statistics table. Did you download the latest OPL and run load-OPL-global-statistics?" + warn "Couldn't find the OPL global statistics table. Did you download the latest OPL and run load-OPL-global-statistics.pl?" } die $selectstm->errstr; }