new OPL scripts: dump/restore OPL tables, load global statistics without other local table work

bin/OPL-update

@@ -986,6 +986,9 @@ if ($ce->{problemLibrary}{showLibraryLocalStats} ||
     $ce->{problemLibrary}{showLibraryGlobalStats}) {
   print "\nUpdating Library Statistics.\n";
   do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics';
+
+  print "\nLoading global statistics (if possible).\n";
+  do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl';
 }
 
 

bin/dump-OPL-tables.pl

@@ -0,0 +1,108 @@
+#!/usr/bin/perl
+
+##############################################################################
+# WeBWorK Online Homework Delivery System
+# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of either: (a) the GNU General Public License as published by the
+# Free Software Foundation; either version 2, or (at your option) any later
+# version, or (b) the "Artistic License" which comes with this package.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the
+# Artistic License for more details.
+##############################################################################
+
+# This script dumps the OPL library tables to a dump file.
+use strict;
+
+# Get the necessary packages, including adding webwork to our path.  
+
+BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n')
+	   unless($ENV{WEBWORK_ROOT});}
+use lib "$ENV{WEBWORK_ROOT}/lib";
+
+use WeBWorK::CourseEnvironment;
+
+use String::ShellQuote;
+use DBI;
+
+# get course environment and configured OPL path
+
+my $ce = new WeBWorK::CourseEnvironment({
+	webwork_dir => $ENV{WEBWORK_ROOT},
+	});
+
+my $configured_OPL_path = $ce->{problemLibrary}{root};
+
+
+# Drop the "OpenProblemLibrary" from the end of the path
+
+$configured_OPL_path =~ s+OpenProblemLibrary++;
+
+# Check that it exists
+
+if ( -d "$configured_OPL_path" ) {
+	print "OPL path seems to be $configured_OPL_path\n";
+} else {
+	print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n";
+	exit;
+}
+
+# Set TABLE-DUMP path and make directory if necessary
+
+my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP";
+if ( ! -d "$prepared_OPL_tables_dir" ) {
+	`mkdir -p $prepared_OPL_tables_dir`;
+}
+
+# Set dump file name
+
+my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql";
+
+# Get DB connection settings
+
+my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
+
+$host = 'localhost' unless $host;
+
+$port = 3306 unless $port;
+
+my $dbuser = $ce->{database_username};
+my $dbpass = $ce->{database_password};
+
+$dbuser = shell_quote($dbuser);
+$dbpass = shell_quote($dbpass);
+$db = shell_quote($db);
+
+$ENV{'MYSQL_PWD'}=$dbpass;
+
+# decide whether the mysql installation can handle
+# utf8mb4 and that should be used for the OPL
+
+my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0;
+
+my $character_set =  ($ENABLE_UTF8MB4)? "utf8mb4":"utf8";
+
+# Get mysqldump_command
+
+my $mysqldump_command = $ce->{externalPrograms}->{mysqldump};
+
+# The tables to dump are:
+
+my $OPL_tables_to_dump = "OPL_DBsubject OPL_DBchapter OPL_DBsection OPL_author OPL_path OPL_pgfile OPL_keyword OPL_pgfile_keyword OPL_textbook OPL_chapter OPL_section OPL_problem OPL_morelt OPL_pgfile_problem";
+
+# Tables NOT dumped:
+# OPL_problem_user - is created by bin/update-OPL-statistics and need not be archived
+# OPL_global_statistics - loaded from a special file provide by the OPL
+# OPL_local_statistics - locally generated
+
+print "Dumping OPL tables\n";
+
+`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db $OPL_tables_to_dump  > $prepared_OPL_tables_file`;
+
+print "OPL database dump created: $prepared_OPL_tables_file\n";
+
+1;

bin/load-OPL-global-statistics.pl

@@ -0,0 +1,82 @@
+#!/usr/bin/perl
+
+##############################################################################
+# WeBWorK Online Homework Delivery System
+# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of either: (a) the GNU General Public License as published by the
+# Free Software Foundation; either version 2, or (at your option) any later
+# version, or (b) the "Artistic License" which comes with this package.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the
+# Artistic License for more details.
+##############################################################################
+
+# This script loads the OPL global statistics, which is often done by bin/update-OPL-statistics but may need to be done outside of that setting.
+use strict;
+
+# Get the necessary packages, including adding webwork to our path.  
+
+BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n')
+	   unless($ENV{WEBWORK_ROOT});}
+use lib "$ENV{WEBWORK_ROOT}/lib";
+
+use WeBWorK::CourseEnvironment;
+
+use String::ShellQuote;
+use DBI;
+
+# get course environment and configured OPL path
+
+my $ce = new WeBWorK::CourseEnvironment({
+	webwork_dir => $ENV{WEBWORK_ROOT},
+	});
+
+my $dbh = DBI->connect(
+        $ce->{problemLibrary_db}->{dbsource},
+        $ce->{problemLibrary_db}->{user},
+        $ce->{problemLibrary_db}->{passwd},
+        {
+		AutoCommit => 0,
+                PrintError => 0,
+                RaiseError => 1,
+        },
+);
+
+# check to see if the global statistics file exists and if it does, upload it.
+
+my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql';
+
+if (-e $global_sql_file) {
+
+  my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
+
+  $host = 'localhost' unless $host;
+
+  $port = 3306 unless $port;
+
+  my $dbuser = $ce->{database_username};
+  my $dbpass = $ce->{database_password};
+
+
+  $dbh->do(<<EOS);
+DROP TABLE IF EXISTS OPL_global_statistics;
+EOS
+  $dbh->commit();
+
+  $dbuser = shell_quote($dbuser);
+  $dbpass = shell_quote($dbpass);
+  $db = shell_quote($db);
+
+  $ENV{'MYSQL_PWD'}=$dbpass;
+
+  my $mysql_command = $ce->{externalPrograms}->{mysql};  
+
+  `$mysql_command --host=$host --port=$port --user=$dbuser $db < $global_sql_file`;
+
+}
+
+1;

bin/restore-OPL-tables.pl

@@ -0,0 +1,97 @@
+#!/usr/bin/perl
+
+##############################################################################
+# WeBWorK Online Homework Delivery System
+# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of either: (a) the GNU General Public License as published by the
+# Free Software Foundation; either version 2, or (at your option) any later
+# version, or (b) the "Artistic License" which comes with this package.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the
+# Artistic License for more details.
+##############################################################################
+
+# This script restores the OPL library tables from a dump file.
+use strict;
+
+# Get the necessary packages, including adding webwork to our path.  
+
+BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n')
+	   unless($ENV{WEBWORK_ROOT});}
+use lib "$ENV{WEBWORK_ROOT}/lib";
+
+use WeBWorK::CourseEnvironment;
+
+use String::ShellQuote;
+use DBI;
+
+# get course environment and configured OPL path
+
+my $ce = new WeBWorK::CourseEnvironment({
+	webwork_dir => $ENV{WEBWORK_ROOT},
+	});
+
+my $configured_OPL_path = $ce->{problemLibrary}{root};
+
+
+# Drop the "OpenProblemLibrary" from the end of the path
+
+$configured_OPL_path =~ s+OpenProblemLibrary++;
+
+# Check that it exists
+
+if ( -d "$configured_OPL_path" ) {
+	print "OPL path seems to be $configured_OPL_path\n";
+} else {
+	print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n";
+	exit;
+}
+
+# Set TABLE-DUMP path and make directory if necessary
+
+my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP";
+if ( ! -d "$prepared_OPL_tables_dir" ) {
+	`mkdir -p $prepared_OPL_tables_dir`;
+}
+
+# Set dump file name
+
+my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql";
+
+# Get DB connection settings
+
+my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
+
+$host = 'localhost' unless $host;
+
+$port = 3306 unless $port;
+
+my $dbuser = $ce->{database_username};
+my $dbpass = $ce->{database_password};
+
+$dbuser = shell_quote($dbuser);
+$dbpass = shell_quote($dbpass);
+$db = shell_quote($db);
+
+$ENV{'MYSQL_PWD'}=$dbpass;
+
+# decide whether the mysql installation can handle
+# utf8mb4 and that should be used for the OPL
+
+my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0;
+
+my $character_set =  ($ENABLE_UTF8MB4)? "utf8mb4":"utf8";
+
+my $mysql_command = $ce->{externalPrograms}->{mysql};
+
+# check to see if the prepared_OPL_tables_file exists and if it does load it in
+
+if (-e $prepared_OPL_tables_file) {
+  `$mysql_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db < $prepared_OPL_tables_file`;
+}
+
+1;

bin/update-OPL-statistics → bin/update-OPL-statistics.pl

@@ -192,35 +192,8 @@ BEGIN
 
 $dbh->commit();
 
-# check to see if the global statistics file exists and if it does, upload it.
-
-my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql';
-
-if (-e $global_sql_file) {
-
-  my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
-
-  $host = 'localhost' unless $host;
-
-  $port = 3306 unless $port;
-
-  my $dbuser = $ce->{database_username};
-  my $dbpass = $ce->{database_password};
-
-
-  $dbh->do(<<EOS);
-DROP TABLE IF EXISTS OPL_global_statistics;
-EOS
-  $dbh->commit();
-
-  $dbuser = shell_quote($dbuser);
-  $dbpass = shell_quote($dbpass);
-  $db = shell_quote($db);
-
-  my $mysql_command = $ce->{externalPrograms}->{mysql};  
-
-  `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`;
-
-}
+# We no longer automatically load the global statistics data here.
+print( "You may want to run load-OPL-global-statistics.pl to update the global statistics data.\n",
+	"If this is being run by OPL-update, that will be done automatically.\n");
 
 1;

bin/upload-OPL-statistics → bin/upload-OPL-statistics.pl

@@ -49,9 +49,11 @@
 $dbpass = shell_quote($dbpass);
 $db = shell_quote($db);
 
+$ENV{'MYSQL_PWD'}=$dbpass;
+
 my $mysqldump_command = $ce->{externalPrograms}->{mysqldump};  
 
-`$mysqldump_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db OPL_local_statistics > $output_file`;
+`$mysqldump_command --host=$host --port=$port --user=$dbuser $db OPL_local_statistics > $output_file`;
 
 print "Database File Created\n";
 

lib/WeBWorK/Utils/LibraryStats.pm

@@ -66,7 +66,7 @@ sub getLocalStats {
 
     unless ($selectstm->execute($source_file)) {
       if ($selectstm->errstr =~ /Table .* doesn't exist/) {
-	warn "Couldn't find the OPL local statistics table.  Did you download the latest OPL and run update-OPL-statistics?"
+	warn "Couldn't find the OPL local statistics table.  Did you download the latest OPL and run update-OPL-statistics.pl?"
       }
       die $selectstm->errstr;
     }
@@ -92,7 +92,7 @@ sub getGlobalStats {
 
     unless ($selectstm->execute($source_file)) {
       if ($selectstm->errstr =~ /Table .* doesn't exist/) {
-	warn "Couldn't find the OPL global statistics table.  Did you download the latest OPL and run update-OPL-statistics?"
+	warn "Couldn't find the OPL global statistics table.  Did you download the latest OPL and run load-OPL-global-statistics.pl?"
       }
       die $selectstm->errstr;
     }