From a086e50a68dbcbc598d4708b85c57d966776a604 Mon Sep 17 00:00:00 2001
From: Nathan Wallach <taniwallach@gmail.com>
Date: Mon, 23 Nov 2020 22:04:35 +0200
Subject: [PATCH 01/10] Modifications to allow use of the DBD::MariaDB driver
 instead of DBD:mysql. It handles UTF-8 by default, and does not need
 mysql_enable_utf8mb4 or mysql_enable_utf8. Also mods to prevent mysqldump of
 mysql 8+ for reporting errors about column-statistics being missing from
 MariaDB databases by disabling column_statistic when the installed mysqldump
 command would support it.

---
 bin/dump-OPL-tables.pl                        | 22 +++++++-
 bin/load-OPL-global-statistics.pl             | 12 +++++
 bin/restore-OPL-tables.pl                     | 10 ++++
 bin/upload-OPL-statistics.pl                  | 22 +++++++-
 lib/WeBWorK/DB/Driver/SQL.pm                  | 12 ++++-
 lib/WeBWorK/DB/Schema/NewSQL/Std.pm           | 50 ++++++++++++++-----
 .../Utils/CourseManagement/sql_single.pm      | 42 ++++++++++++----
 7 files changed, 143 insertions(+), 27 deletions(-)

diff --git a/bin/dump-OPL-tables.pl b/bin/dump-OPL-tables.pl
index d4ee88bc0f..80468bda8d 100755
--- a/bin/dump-OPL-tables.pl
+++ b/bin/dump-OPL-tables.pl
@@ -66,6 +66,16 @@
 
 my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
 
+# The MariaDB driver use a different DSN format
+# Ex: DBI:MariaDB:database=webwork;host=db;port=3306
+
+if ( $dbtype =~ /MariaDB/i ) {
+  ($db,$host,$port) = split(';',$db);
+  $db   =~ s/database=//;
+  $host =~ s/host=//;
+  $port =~ s/port=//;
+}
+
 $host = 'localhost' unless $host;
 
 $port = 3306 unless $port;
@@ -101,7 +111,17 @@
 
 print "Dumping OPL tables\n";
 
-`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db $OPL_tables_to_dump  > $prepared_OPL_tables_file`;
+# Conditionally add --column-statistics=0 as MariaDB databases do not support it
+# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109
+#      https://github.com/drush-ops/drush/issues/4410
+
+my $column_statistics_off = "";
+my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`;
+if ( $test_for_column_statistics ) {
+  $column_statistics_off = " --column-statistics=0 ";
+}
+
+`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $column_statistics_off $db $OPL_tables_to_dump  > $prepared_OPL_tables_file`;
 
 print "OPL database dump created: $prepared_OPL_tables_file\n";
 
diff --git a/bin/load-OPL-global-statistics.pl b/bin/load-OPL-global-statistics.pl
index ad3bb62abc..d0b136a5cd 100755
--- a/bin/load-OPL-global-statistics.pl
+++ b/bin/load-OPL-global-statistics.pl
@@ -53,6 +53,18 @@
 if (-e $global_sql_file) {
 
   my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
+
+  # The MariaDB driver use a different DSN format
+  # Ex: DBI:MariaDB:database=webwork;host=db;port=3306
+
+  if ( $dbtype =~ /MariaDB/i ) {
+    ($db,$host,$port) = split(';',$db);
+    $db   =~ s/database=//;
+    $host =~ s/host=//;
+    $port =~ s/port=//;
+  }
+
+
   
   $host = 'localhost' unless $host;
 
diff --git a/bin/restore-OPL-tables.pl b/bin/restore-OPL-tables.pl
index 4c748e5878..a7a91c9582 100755
--- a/bin/restore-OPL-tables.pl
+++ b/bin/restore-OPL-tables.pl
@@ -66,6 +66,16 @@
 
 my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
 
+# The MariaDB driver use a different DSN format
+# Ex: DBI:MariaDB:database=webwork;host=db;port=3306
+
+if ( $dbtype =~ /MariaDB/i ) {
+  ($db,$host,$port) = split(';',$db);
+  $db   =~ s/database=//;
+  $host =~ s/host=//;
+  $port =~ s/port=//;
+}
+
 $host = 'localhost' unless $host;
 
 $port = 3306 unless $port;
diff --git a/bin/upload-OPL-statistics.pl b/bin/upload-OPL-statistics.pl
index 37004296d3..541efbb51a 100755
--- a/bin/upload-OPL-statistics.pl
+++ b/bin/upload-OPL-statistics.pl
@@ -32,6 +32,16 @@
 
 my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
 
+# The MariaDB driver use a different DSN format
+# Ex: DBI:MariaDB:database=webwork;host=db;port=3306
+
+if ( $dbtype =~ /MariaDB/i ) {
+  ($db,$host,$port) = split(';',$db);
+  $db   =~ s/database=//;
+  $host =~ s/host=//;
+  $port =~ s/port=//;
+}
+
 $host = 'localhost' unless $host;
 
 $port = 3306 unless $port;
@@ -53,7 +63,17 @@
 
 my $mysqldump_command = $ce->{externalPrograms}->{mysqldump};  
 
-`$mysqldump_command --host=$host --port=$port --user=$dbuser $db OPL_local_statistics > $output_file`;
+# Conditionally add --column-statistics=0 as MariaDB databases do not support it
+# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109
+#      https://github.com/drush-ops/drush/issues/4410
+
+my $column_statistics_off = "";
+my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`;
+if ( $test_for_column_statistics ) {
+  $column_statistics_off = " --column-statistics=0 ";
+}
+
+`$mysqldump_command --host=$host --port=$port --user=$dbuser $column_statistics_off $db OPL_local_statistics > $output_file`;
 
 print "Database File Created\n";
 
diff --git a/lib/WeBWorK/DB/Driver/SQL.pm b/lib/WeBWorK/DB/Driver/SQL.pm
index 0301e6e8f0..f065e02c43 100644
--- a/lib/WeBWorK/DB/Driver/SQL.pm
+++ b/lib/WeBWorK/DB/Driver/SQL.pm
@@ -61,6 +61,15 @@ sub new($$$) {
 	
 	my $self = $proto->SUPER::new($source, $params);
 	
+	# The DBD::MariaDB driver should not get the
+	#    mysql_enable_utf8mb4 or mysql_enable_utf8 settings,
+	# but DBD::mysql should.
+	my %utf8_parameters = ();
+	if ( $source =~ /DBI:mysql/ ) {
+	  $utf8_parameters{mysql_enable_utf8mb4} = 1;
+	  $utf8_parameters{mysql_enable_utf8} = 1;
+	}
+
 	# add handle
 	$self->{handle} = DBI->connect_cached(
 		$source,
@@ -70,8 +79,7 @@ sub new($$$) {
 			PrintError => 0,
 			RaiseError => 1,
 
-			mysql_enable_utf8mb4 => 1,
-			mysql_enable_utf8 => 1,  # for older versions of DBD-mysql Perl modules
+			%utf8_parameters,
 		},
 	);
 	die $DBI::errstr unless defined $self->{handle};
diff --git a/lib/WeBWorK/DB/Schema/NewSQL/Std.pm b/lib/WeBWorK/DB/Schema/NewSQL/Std.pm
index c5140fe222..7ae827fce4 100644
--- a/lib/WeBWorK/DB/Schema/NewSQL/Std.pm
+++ b/lib/WeBWorK/DB/Schema/NewSQL/Std.pm
@@ -269,28 +269,52 @@ sub _get_db_info {
 	my $dsn = $self->{driver}{source};
 	my $username = $self->{params}{username};
 	my $password = $self->{params}{password};
-	
-	die "Can't call dump_table or restore_table on a table with a non-MySQL source"
-		unless $dsn =~ s/^dbi:mysql://i;
-	
-	# this is an internal function which we probably shouldn't be using here
-	# but it's quick and gets us what we want (FIXME what about sockets, etc?)
+
 	my %dsn;
-	DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']);
-	die "no database specified in DSN!" unless defined $dsn{database};
+	if (      $dsn =~ m/^dbi:mariadb:/i ) {
+		# Expect DBI:MariaDB:database=webwork;host=db;port=3306
+		my ($dbi,$dbtype,$temp1) = split(':',$dsn);
+		( $dsn{database}, $dsn{host}, $dsn{port} ) = split(';',$temp1);
+		$dsn{database} =~ s/database=//;
+		$dsn{host} =~ s/host=// if ( defined $dsn{host} );
+		$dsn{port} =~ s/port=// if ( defined $dsn{port} );
+	} elsif ( $dsn =~ m/^dbi:mysql:/i ) {
+		# This code works for DBD::mysql
+		# this is an internal function which we probably shouldn't be using here
+		# but it's quick and gets us what we want (FIXME what about sockets, etc?)
+		DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']);
+	} else {
+		die "Can't call dump_table or restore_table on a table with a non-MySQL/MariaDB source";
+	}
 	
+	die "no database specified in DSN!" unless defined $dsn{database};
+
+	my $mysqldump = $self->{params}{mysqldump_path};
+	# Conditionally add column-statistics=0 as MariaDB databases do not support it
+	# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109
+	#      https://github.com/drush-ops/drush/issues/4410
+
+	my $column_statistics_off = "";
+	my $test_for_column_statistics = `$mysqldump --help | grep 'column-statistics'`;
+	if ( $test_for_column_statistics ) {
+		$column_statistics_off = "[mysqldump]\ncolumn-statistics=0\n";
+		#warn "Setting in the temporary mysql config file for table dump/restore:\n$column_statistics_off\n\n";
+	}
+
 	# doing this securely is kind of a hassle...
 	my $my_cnf = new File::Temp;
 	$my_cnf->unlink_on_destroy(1);
 	chmod 0600, $my_cnf or die "failed to chmod 0600 $my_cnf: $!"; # File::Temp objects stringify with ->filename
 	print $my_cnf "[client]\n";
-	print $my_cnf "user=\"$username\"\n" if defined $username and length($username) > 0;
-	print $my_cnf "password=\"$password\"\n" if defined $password and length($password) > 0;
-	print $my_cnf "host=\"$dsn{host}\"\n" if defined $dsn{host} and length($dsn{host}) > 0;
-	print $my_cnf "port=\"$dsn{port}\"\n" if defined $dsn{port} and length($dsn{port}) > 0;
-	
+	print $my_cnf "user=$username\n" if defined $username and length($username) > 0;
+	print $my_cnf "password=$password\n" if defined $password and length($password) > 0;
+	print $my_cnf "host=$dsn{host}\n" if defined $dsn{host} and length($dsn{host}) > 0;
+	print $my_cnf "port=$dsn{port}\n" if defined $dsn{port} and length($dsn{port}) > 0;
+	print $my_cnf "$column_statistics_off" if $test_for_column_statistics;
+
 	return ($my_cnf, $dsn{database});
 }
+
 ####################################################
 # checking Fields
 ####################################################
diff --git a/lib/WeBWorK/Utils/CourseManagement/sql_single.pm b/lib/WeBWorK/Utils/CourseManagement/sql_single.pm
index 719432f21e..bde72d36b1 100644
--- a/lib/WeBWorK/Utils/CourseManagement/sql_single.pm
+++ b/lib/WeBWorK/Utils/CourseManagement/sql_single.pm
@@ -195,17 +195,38 @@ sub _get_db_info {
 	my $dsn = $ce->{database_dsn};
 	my $username = $ce->{database_username};
 	my $password = $ce->{database_password};
-	
-	die "Can't call dump_table or restore_table on a table with a non-MySQL source"
-		unless $dsn =~ s/^dbi:mysql://i;
-	
-	# this is an internal function which we probably shouldn't be using here
-	# but it's quick and gets us what we want (FIXME what about sockets, etc?)
+
 	my %dsn;
-	runtime_use "DBD::mysql";
-	DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']);
-	die "no database specified in DSN!" unless defined $dsn{database};
+	if (      $dsn =~ s/^dbi:mariadb://i ) {
+		my ($dbi,$dbtype,$temp1) = split(':',$dsn);
+		( $dsn{database}, $dsn{host}, $dsn{port} ) = split(';',$db);
+		$dsn{database} =~ s/database=//;
+		$dsn{host} =~ s/host=// if ( defined $dsn{host} );
+		$dsn{port} =~ s/port=// if ( defined $dsn{port} );
+	} elsif ( $dsn =~ s/^dbi:mysql://i ) {
+		# This code works for DBD::mysql
+		# this is an internal function which we probably shouldn't be using here
+		# but it's quick and gets us what we want (FIXME what about sockets, etc?)
+		runtime_use "DBD::mysql";
+		DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']);
+	} else {
+		die "Can't call dump_table or restore_table on a table with a non-MySQL/MariaDB source";
+	}
 	
+	die "no database specified in DSN!" unless defined $dsn{database};
+
+	my $mysqldump = $self->{params}{mysqldump_path};
+	# Conditionally add column-statistics=0 as MariaDB databases do not support it
+	# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109
+	#      https://github.com/drush-ops/drush/issues/4410
+
+	my $column_statistics_off = "";
+	my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`;
+	if ( $test_for_column_statistics ) {
+		$column_statistics_off = "[mysqldump]\ncolumn-statistics=0\n";
+warn "Setting in the temporary mysql config file for table dump/restore:\n$column_statistics_off\n\n";
+	}
+
 	# doing this securely is kind of a hassle...
 	my $my_cnf = new File::Temp;
 	$my_cnf->unlink_on_destroy(1);
@@ -215,7 +236,8 @@ sub _get_db_info {
 	print $my_cnf "password=$password\n" if defined $password and length($password) > 0;
 	print $my_cnf "host=$dsn{host}\n" if defined $dsn{host} and length($dsn{host}) > 0;
 	print $my_cnf "port=$dsn{port}\n" if defined $dsn{port} and length($dsn{port}) > 0;
-	
+	print $my_cnf "$column_statistics_off" if $test_for_column_statistics;
+
 	return ($my_cnf, $dsn{database});
 }
 

From 73912c9fb02d870dc0cb5c694ad122bffbb3cdf5 Mon Sep 17 00:00:00 2001
From: Nathan Wallach <taniwallach@gmail.com>
Date: Wed, 10 Mar 2021 18:10:46 +0200
Subject: [PATCH 02/10] Fix OPL-update for compatibility with either the
 DBD::MariaDB driver or the DBD::mysql driver.

---
 bin/OPL-update | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/bin/OPL-update b/bin/OPL-update
index d9da002e66..ad83da29f2 100755
--- a/bin/OPL-update
+++ b/bin/OPL-update
@@ -92,15 +92,28 @@ my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
 my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0;
 print  "using utf8mb4 \n\n" if $ENABLE_UTF8MB4;
 
+# The DBD::MariaDB driver should not get the
+#    mysql_enable_utf8mb4 or mysql_enable_utf8 settings,
+# but DBD::mysql should.
+my %utf8_parameters = ();
+if ( $ce->{database_dsn} =~ /DBI:mysql:/i ) {
+	# Only needed for older DBI:mysql driver
+	if ( $ENABLE_UTF8MB4 ) {
+		$utf8_parameters{mysql_enable_utf8mb4} = 1;
+	} else {
+		$utf8_parameters{mysql_enable_utf8} = 1;
+	}
+}
+
 my $dbh = DBI->connect(
-        $ce->{problemLibrary_db}->{dbsource},
-        $ce->{problemLibrary_db}->{user},
-        $ce->{problemLibrary_db}->{passwd},
-        {
-            PrintError => 0,
-            RaiseError => 1,
-            ($ENABLE_UTF8MB4)?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1),
-        },
+	$ce->{problemLibrary_db}->{dbsource},
+	$ce->{problemLibrary_db}->{user},
+	$ce->{problemLibrary_db}->{passwd},
+	{
+		PrintError => 0,
+		RaiseError => 1,
+		%utf8_parameters,
+	},
 );
 
 my $character_set='';

From 561f2d3a278e41157688c254a84b4ce7d6d978a2 Mon Sep 17 00:00:00 2001
From: Nathan Wallach <taniwallach@gmail.com>
Date: Wed, 10 Mar 2021 18:41:16 +0200
Subject: [PATCH 03/10] Provide instructions and samples for the database_dsn
 setting for the 2 drivers. Change a default switch setting for pdflatex.

---
 conf/defaults.config |  2 +-
 conf/site.conf.dist  | 18 ++++++++++++++----
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/conf/defaults.config b/conf/defaults.config
index 6db460802a..3412d6cd05 100644
--- a/conf/defaults.config
+++ b/conf/defaults.config
@@ -549,7 +549,7 @@ $default_status = "Enrolled";
 ################################################################################
 # set these variables in site.conf
 #
-# $database_dsn = "dbi:mysql:webwork";
+# $database_dsn = "dbi:mysql:webwork"; # Pay attention to the instructions in site.conf.dist about the format based on the DBD driver in use.
 # $database_username = "webworkWrite";
 # $database_password = ""; #set this in site.conf
 # $database_debug = 0;
diff --git a/conf/site.conf.dist b/conf/site.conf.dist
index 6e7497648b..62f9d8e8cd 100644
--- a/conf/site.conf.dist
+++ b/conf/site.conf.dist
@@ -105,10 +105,13 @@ $externalPrograms{git} = "/usr/bin/git";
 ####################################################
 $externalPrograms{latex}    ="/usr/bin/latex";
 
-$externalPrograms{pdflatex} ="/usr/bin/pdflatex --shell-escape";
+# Using --shell-escape can open security holes. It should typically be disabled,
+# and anyone using --shell-escape whould understand the potential risks.
+
+$externalPrograms{pdflatex} ="/usr/bin/pdflatex --no-shell-escape";
 # Consider using xelatex instead of pdflatex for multilingual use, and
 # use polyglossia and fontspec packages (which require xelatex or lualatex).
-#$externalPrograms{pdflatex} ="/usr/bin/xelatex --shell-escape";
+#$externalPrograms{pdflatex} ="/usr/bin/xelatex --no-shell-escape";
 
 $externalPrograms{dvipng}   ="/usr/bin/dvipng";
 
@@ -176,8 +179,15 @@ $externalPrograms{mysqldump} ="/usr/bin/mysqldump";
 # The database dsn is the path to the WeBWorK database which you have created.
 # Unless you have given the database a different name or the database resides on another
 # server you do not need to change this first value.
-# The format is dbi:mysql:[databasename] for databases on the local machine
-# For a remote database the format is dbi:mysql:[databasename]:[hostname]:[port]
+
+# When using the DBD:mysql driver the format is dbi:mysql:[databasename]:[hostname]:[port]
+#    example:  DBI:mysql:webwork:db:3306
+#    ** Only uses colons between fields.
+#    For a database on the local machine the format is dbi:mysql:[databasename]
+# When using the DBD::MariaDB driver the format is dbi:MariaDB:database=dbname;host=hostname;port=portnum
+#    example:  DBI:MariaDB:database=webwork;host=db;port=3306
+#    ** Pay attention to the semicolons after the second colon, as the dbname, hostname, portnum are all "one" parameter
+
 $database_dsn ="dbi:mysql:webwork";
 $database_storage_engine = 'myisam';
 

From 51976b069d7247e6b987f41c764cc891a21de0b4 Mon Sep 17 00:00:00 2001
From: Nathan Wallach <taniwallach@gmail.com>
Date: Fri, 12 Mar 2021 00:28:23 +0200
Subject: [PATCH 04/10] Changes to DB connection settings configuration. Build
 DSN from single variables, and avoid needed to parse DSN in the bin scripts.
 That remains unavoidable in lib/WeBWorK/DB/Schema/NewSQL/Std.pm and
 lib/WeBWorK/Utils/CourseManagement/sql_single.pm. Based on suggestions from
 @dlglin in
 https://github.com/openwebwork/webwork2/pull/1160#discussion_r591722721

---
 bin/OPL-update                    |  3 ++-
 bin/dump-OPL-tables.pl            | 19 +++-------------
 bin/load-OPL-global-statistics.pl | 22 +++---------------
 bin/restore-OPL-tables.pl         | 19 +++-------------
 bin/upload-OPL-statistics.pl      | 22 +++++-------------
 conf/database.conf.dist           | 14 +++++++++---
 conf/defaults.config              | 23 +------------------
 conf/site.conf.dist               | 37 ++++++++++++++++++++-----------
 8 files changed, 53 insertions(+), 106 deletions(-)

diff --git a/bin/OPL-update b/bin/OPL-update
index ad83da29f2..e1d1569df0 100755
--- a/bin/OPL-update
+++ b/bin/OPL-update
@@ -96,7 +96,8 @@ print  "using utf8mb4 \n\n" if $ENABLE_UTF8MB4;
 #    mysql_enable_utf8mb4 or mysql_enable_utf8 settings,
 # but DBD::mysql should.
 my %utf8_parameters = ();
-if ( $ce->{database_dsn} =~ /DBI:mysql:/i ) {
+
+if ( $ce->{database_driver} =~ /^mysql$/i ) {
 	# Only needed for older DBI:mysql driver
 	if ( $ENABLE_UTF8MB4 ) {
 		$utf8_parameters{mysql_enable_utf8mb4} = 1;
diff --git a/bin/dump-OPL-tables.pl b/bin/dump-OPL-tables.pl
index 80468bda8d..13ede25802 100755
--- a/bin/dump-OPL-tables.pl
+++ b/bin/dump-OPL-tables.pl
@@ -64,22 +64,9 @@
 
 # Get DB connection settings
 
-my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
-
-# The MariaDB driver use a different DSN format
-# Ex: DBI:MariaDB:database=webwork;host=db;port=3306
-
-if ( $dbtype =~ /MariaDB/i ) {
-  ($db,$host,$port) = split(';',$db);
-  $db   =~ s/database=//;
-  $host =~ s/host=//;
-  $port =~ s/port=//;
-}
-
-$host = 'localhost' unless $host;
-
-$port = 3306 unless $port;
-
+my $db     = $ce->{database_name};
+my $host   = $ce->{database_host};
+my $port   = $ce->{database_port};
 my $dbuser = $ce->{database_username};
 my $dbpass = $ce->{database_password};
 
diff --git a/bin/load-OPL-global-statistics.pl b/bin/load-OPL-global-statistics.pl
index d0b136a5cd..995aae8402 100755
--- a/bin/load-OPL-global-statistics.pl
+++ b/bin/load-OPL-global-statistics.pl
@@ -52,27 +52,11 @@
 
 if (-e $global_sql_file) {
 
-  my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
-
-  # The MariaDB driver use a different DSN format
-  # Ex: DBI:MariaDB:database=webwork;host=db;port=3306
-
-  if ( $dbtype =~ /MariaDB/i ) {
-    ($db,$host,$port) = split(';',$db);
-    $db   =~ s/database=//;
-    $host =~ s/host=//;
-    $port =~ s/port=//;
-  }
-
-
-  
-  $host = 'localhost' unless $host;
-
-  $port = 3306 unless $port;
-  
+  my $db     = $ce->{database_name};
+  my $host   = $ce->{database_host};
+  my $port   = $ce->{database_port};
   my $dbuser = $ce->{database_username};
   my $dbpass = $ce->{database_password};
-
   
   $dbh->do(<<EOS);
 DROP TABLE IF EXISTS OPL_global_statistics;
diff --git a/bin/restore-OPL-tables.pl b/bin/restore-OPL-tables.pl
index a7a91c9582..2e107de50d 100755
--- a/bin/restore-OPL-tables.pl
+++ b/bin/restore-OPL-tables.pl
@@ -64,22 +64,9 @@
 
 # Get DB connection settings
 
-my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
-
-# The MariaDB driver use a different DSN format
-# Ex: DBI:MariaDB:database=webwork;host=db;port=3306
-
-if ( $dbtype =~ /MariaDB/i ) {
-  ($db,$host,$port) = split(';',$db);
-  $db   =~ s/database=//;
-  $host =~ s/host=//;
-  $port =~ s/port=//;
-}
-
-$host = 'localhost' unless $host;
-
-$port = 3306 unless $port;
-
+my $db     = $ce->{database_name};
+my $host   = $ce->{database_host};
+my $port   = $ce->{database_port};
 my $dbuser = $ce->{database_username};
 my $dbpass = $ce->{database_password};
 
diff --git a/bin/upload-OPL-statistics.pl b/bin/upload-OPL-statistics.pl
index 541efbb51a..bfe851fba8 100755
--- a/bin/upload-OPL-statistics.pl
+++ b/bin/upload-OPL-statistics.pl
@@ -28,24 +28,14 @@
 use String::ShellQuote;
 
 my $ce = new WeBWorK::CourseEnvironment({
-					 webwork_dir => $ENV{WEBWORK_ROOT},});
+	webwork_dir => $ENV{WEBWORK_ROOT},
+	});
 
-my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
-
-# The MariaDB driver use a different DSN format
-# Ex: DBI:MariaDB:database=webwork;host=db;port=3306
-
-if ( $dbtype =~ /MariaDB/i ) {
-  ($db,$host,$port) = split(';',$db);
-  $db   =~ s/database=//;
-  $host =~ s/host=//;
-  $port =~ s/port=//;
-}
-
-$host = 'localhost' unless $host;
-
-$port = 3306 unless $port;
+# Get DB connection settings
 
+my $db     = $ce->{database_name};
+my $host   = $ce->{database_host};
+my $port   = $ce->{database_port};
 my $dbuser = $ce->{database_username};
 my $dbpass = $ce->{database_password};
 
diff --git a/conf/database.conf.dist b/conf/database.conf.dist
index 10fc07e3be..7e1429df51 100644
--- a/conf/database.conf.dist
+++ b/conf/database.conf.dist
@@ -1,8 +1,7 @@
 #!perl
 ################################################################################
 # WeBWorK Online Homework Delivery System
-# Copyright &copy; 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/
-# $CVSHeader: webwork2/conf/database.conf.dist,v 1.38 2007/08/13 22:59:51 sh002i Exp $
+# Copyright &copy; 2000-2021 The WeBWorK Project, http://github.com/openwebwork
 # 
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of either: (a) the GNU General Public License as published by the
@@ -80,12 +79,21 @@ my %sqlParams = (
 	username => $database_username,
 	password => $database_password,
 	debug    => $database_debug,
-    ($ENABLE_UTF8MB4)?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1),
 	# kinda hacky, but needed for table dumping
 	mysql_path     => $externalPrograms{mysql},
 	mysqldump_path => $externalPrograms{mysqldump},
 );
 
+if ( $ce->{database_driver} =~ /^mysql$/i ) {
+	# The extra UTF8 connection setting is ONLY needed for older DBD:mysql driver
+	# and forbidden by the newer DBD::MariaDB driver
+	if ( $ENABLE_UTF8MB4 ) {
+		$sqlParams{mysql_enable_utf8mb4} = 1; # Full 4-bit UTF-8
+	} else {
+		$sqlParams{mysql_enable_utf8} = 1; # Only the partial 3-bit mySQL UTF-8
+	}
+}
+
 $dbLayouts{sql_single} = {
         locations => {
                 record => "WeBWorK::DB::Record::Locations",
diff --git a/conf/defaults.config b/conf/defaults.config
index 3412d6cd05..bc9fe3fd03 100644
--- a/conf/defaults.config
+++ b/conf/defaults.config
@@ -1,8 +1,7 @@
 #!perl
 ################################################################################
 # WeBWorK Online Homework Delivery System
-# Copyright &copy; 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/
-# $CVSHeader: webwork2/conf/defaults.config,v 1.225 2010/05/18 18:03:31 apizer Exp $
+# Copyright &copy; 2000-2021 The WeBWorK Project, http://github.com/openwebwork
 # 
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of either: (a) the GNU General Public License as published by the
@@ -538,29 +537,9 @@ $default_status = "Enrolled";
 # Database options
 ################################################################################
 
-# these variables are used by database.conf. we define them here so that editing
-# database.conf isn't necessary.
-
-# required permissions
-# GRANT SELECT ON webwork.* TO webworkRead@localhost IDENTIFIED BY 'passwordRO';
-# GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER, DROP, INDEX, LOCK TABLES ON webwork.* TO webworkWrite@localhost IDENTIFIED BY 'passwordRW';
-
-
-################################################################################
-# set these variables in site.conf
-#
-# $database_dsn = "dbi:mysql:webwork"; # Pay attention to the instructions in site.conf.dist about the format based on the DBD driver in use.
-# $database_username = "webworkWrite";
-# $database_password = ""; #set this in site.conf
-# $database_debug = 0;
-################################################################################
-
-
-
 # Database schemas are defined in the file conf/database.conf and stored in the
 # hash %dbLayouts. The standard schema is called "sql_single";
 
-
 include( "./conf/database.conf.dist");  # always include database.conf.dist
 
                                     # in the rare case where you want local overrides
diff --git a/conf/site.conf.dist b/conf/site.conf.dist
index 62f9d8e8cd..e710a41cd2 100644
--- a/conf/site.conf.dist
+++ b/conf/site.conf.dist
@@ -1,8 +1,7 @@
 #!perl
 ################################################################################
 # WeBWorK Online Homework Delivery System
-# Copyright &copy; 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/
-# $CVSHeader: webwork2/conf/site.conf.dist,v 1.225 2010/05/18 18:03:31 apizer Exp $
+# Copyright &copy; 2000-2021 The WeBWorK Project, http://github.com/openwebwork
 # 
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of either: (a) the GNU General Public License as published by the
@@ -163,6 +162,7 @@ $externalPrograms{mysqldump} ="/usr/bin/mysqldump";
 # Database options
 ################################################################################
 
+# $database_debug = 0;
 
 # Standard permissions command used to initialize the webwork database
 # GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER, DROP, INDEX, LOCK TABLES ON webwork.* TO webworkWrite@localhost IDENTIFIED BY 'passwordRW';
@@ -176,19 +176,30 @@ $externalPrograms{mysqldump} ="/usr/bin/mysqldump";
 # Edit the $database_password line and replace 'passwordRW' by the actual password used in the GRANT command above
 ################################################################################
 
-# The database dsn is the path to the WeBWorK database which you have created.
-# Unless you have given the database a different name or the database resides on another
-# server you do not need to change this first value.
+# The database DSN is the path to the WeBWorK database which you have created.
 
-# When using the DBD:mysql driver the format is dbi:mysql:[databasename]:[hostname]:[port]
-#    example:  DBI:mysql:webwork:db:3306
-#    ** Only uses colons between fields.
-#    For a database on the local machine the format is dbi:mysql:[databasename]
-# When using the DBD::MariaDB driver the format is dbi:MariaDB:database=dbname;host=hostname;port=portnum
-#    example:  DBI:MariaDB:database=webwork;host=db;port=3306
-#    ** Pay attention to the semicolons after the second colon, as the dbname, hostname, portnum are all "one" parameter
+# Modern database DSN format:
+#	DBI:driver:database=$database;host=$hostname;port=$port
 
-$database_dsn ="dbi:mysql:webwork";
+# One thing on which it depends is the driver name, which you may want to modify.
+# It also depends on the database name, which may be non-standard in some settings,
+# as may be the hostname and port of the database server.
+
+# driver should be one of:
+#	"mysql"   for the DBD:mysql driver
+#	"MariaDB" for the DBD:mysql driver
+
+# Select the desired DB driver:
+$database_driver="mysql";
+#$database_driver="MariaDB";
+
+$database_host="localhost";
+$database_port="3306";
+$database_name="webwork";
+
+$database_dsn="DBI:$database_driver:database=$database_name;host=$database_host;port=$database_port";
+
+# The default storange engine to use is set here:
 $database_storage_engine = 'myisam';
 
 #########################

From e4b0ad7d4f7964d66179f44fafe1705e53a13f72 Mon Sep 17 00:00:00 2001
From: Nathan Wallach <taniwallach@gmail.com>
Date: Wed, 17 Mar 2021 23:14:21 +0200
Subject: [PATCH 05/10] Change  default to MariaDB

---
 conf/site.conf.dist | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/conf/site.conf.dist b/conf/site.conf.dist
index e710a41cd2..72b7170ca6 100644
--- a/conf/site.conf.dist
+++ b/conf/site.conf.dist
@@ -190,8 +190,8 @@ $externalPrograms{mysqldump} ="/usr/bin/mysqldump";
 #	"MariaDB" for the DBD:mysql driver
 
 # Select the desired DB driver:
-$database_driver="mysql";
-#$database_driver="MariaDB";
+#$database_driver="mysql";
+$database_driver="MariaDB";
 
 $database_host="localhost";
 $database_port="3306";

From 99da07e5c15e04bc2132c3b191d9bf45fda236e4 Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Thu, 18 Mar 2021 10:16:56 -0400
Subject: [PATCH 06/10] fixed updateOPLextras to work with MariaDB

---
 bin/OPLUtils.pm        |  0
 bin/updateOPLextras.pl | 36 ++++++++++++++++++++++++++++--------
 2 files changed, 28 insertions(+), 8 deletions(-)
 mode change 100755 => 100644 bin/OPLUtils.pm

diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm
old mode 100755
new mode 100644
diff --git a/bin/updateOPLextras.pl b/bin/updateOPLextras.pl
index 73430eb880..4669c2098f 100755
--- a/bin/updateOPLextras.pl
+++ b/bin/updateOPLextras.pl
@@ -76,15 +76,35 @@ BEGIN
 
 my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
 
+# decide whether the mysql installation can handle 
+# utf8mb4 and that should be used for the OPL
+
+my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0;
+print  "using utf8mb4 \n\n" if $ENABLE_UTF8MB4;
+
+# The DBD::MariaDB driver should not get the
+#    mysql_enable_utf8mb4 or mysql_enable_utf8 settings,
+# but DBD::mysql should.
+my %utf8_parameters = ();
+
+if ( $ce->{database_driver} =~ /^mysql$/i ) {
+	# Only needed for older DBI:mysql driver
+	if ( $ENABLE_UTF8MB4 ) {
+		$utf8_parameters{mysql_enable_utf8mb4} = 1;
+	} else {
+		$utf8_parameters{mysql_enable_utf8} = 1;
+	}
+}
+
 my $dbh = DBI->connect(
-				$ce->{problemLibrary_db}->{dbsource},
-				$ce->{problemLibrary_db}->{user},
-				$ce->{problemLibrary_db}->{passwd},
-				{
-						PrintError => 0,
-						RaiseError => 1,
-						($ce->{ENABLE_UTF8MB4})?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1),
-				},
+	$ce->{problemLibrary_db}->{dbsource},
+	$ce->{problemLibrary_db}->{user},
+	$ce->{problemLibrary_db}->{passwd},
+	{
+		PrintError => 0,
+		RaiseError => 1,
+		%utf8_parameters,
+	},
 );
 
 build_library_textbook_tree($ce,$dbh,$verbose) if ($all || $textbooks);

From a695569887d8914f2f588ccc0a0d44d95756b30b Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Thu, 18 Mar 2021 10:21:14 -0400
Subject: [PATCH 07/10] added verbosity mode to the help pages.

---
 bin/updateOPLextras.pl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/updateOPLextras.pl b/bin/updateOPLextras.pl
index 4669c2098f..dd6c84b883 100755
--- a/bin/updateOPLextras.pl
+++ b/bin/updateOPLextras.pl
@@ -14,7 +14,7 @@ =head1 SYNOPSIS
    -d --directories      (rebuild directory tree)
    -a --all              (rebuild all trees)
    -h --help             (display this text)
- 
+   -v --verbose          (turn on verbosity mode) 
 =head1 OPTIONS
  
 =over 8
@@ -30,7 +30,11 @@ =head1 OPTIONS
 =item B<-d> I<--directories>
 
 Rebuild the directory tree and write to a JSON file.
-  
+
+=item B<-v> I<--directories>
+
+Turn on verbosity mode.
+
 =back
  
 =head1 DESCRIPTION

From fcbf60089a990575209bef2dc2d46c4809b33c6d Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Thu, 18 Mar 2021 10:21:40 -0400
Subject: [PATCH 08/10] fixed another typo

---
 bin/updateOPLextras.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/updateOPLextras.pl b/bin/updateOPLextras.pl
index dd6c84b883..0970e77f81 100755
--- a/bin/updateOPLextras.pl
+++ b/bin/updateOPLextras.pl
@@ -31,7 +31,7 @@ =head1 OPTIONS
 
 Rebuild the directory tree and write to a JSON file.
 
-=item B<-v> I<--directories>
+=item B<-v> I<--verbosity>
 
 Turn on verbosity mode.
 

From 68b8b64c53d77f406bfeca5af3dbfebacea0b8c3 Mon Sep 17 00:00:00 2001
From: Nathan Wallach <taniwallach@gmail.com>
Date: Sun, 21 Mar 2021 23:17:23 +0200
Subject: [PATCH 09/10]  settings needs to be different when database is on
 localhost vs. on another host

---
 conf/site.conf.dist | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/conf/site.conf.dist b/conf/site.conf.dist
index e9804b9a12..5185b19c5d 100644
--- a/conf/site.conf.dist
+++ b/conf/site.conf.dist
@@ -177,7 +177,12 @@ $externalPrograms{mysqldump} ="/usr/bin/mysqldump";
 # The database DSN is the path to the WeBWorK database which you have created.
 
 # Modern database DSN format:
-#	DBI:driver:database=$database;host=$hostname;port=$port
+#    DBI:driver:database=$database;host=$hostname;port=$port  (when DB not on localhost)
+# or DBI:driver:database=$database;host=127.0.0.1;port=$port  (when DB on localhost, using TCP)
+#         See: https://metacpan.org/pod/DBD::MariaDB#port
+#              "To connect to a MariaDB or MySQL server on localhost using TCP/IP,
+#		you must specify the host as 127.0.0.1 with the optional port, e.g. 3306."
+# or DBI:driver:database=$database                            (when DB on localhost, using socket)
 
 # One thing on which it depends is the driver name, which you may want to modify.
 # It also depends on the database name, which may be non-standard in some settings,
@@ -195,7 +200,19 @@ $database_host="localhost";
 $database_port="3306";
 $database_name="webwork";
 
-$database_dsn="DBI:$database_driver:database=$database_name;host=$database_host;port=$database_port";
+# For a DB on localhost - default to using Unix socket.
+# Change to 0 to use a TCP connection to 127.0.0.1.
+$database_use_socket_if_localhost=1;
+
+if ( $database_host eq "localhost" ) {
+	if ( $database_use_socket_if_localhost ) {
+		$database_dsn="DBI:$database_driver:database=$database_name";
+	} else {
+		$database_dsn="DBI:$database_driver:database=$database_name;host=127.0.0.1;port=$database_port";
+	}
+} else {
+	$database_dsn="DBI:$database_driver:database=$database_name;host=$database_host;port=$database_port";
+}
 
 # The default storange engine to use is set here:
 $database_storage_engine = 'myisam';

From 521c451c135f289305ffe0c442c268ceeb10dcad Mon Sep 17 00:00:00 2001
From: Nathan Wallach <taniwallach@gmail.com>
Date: Mon, 22 Mar 2021 22:52:40 +0200
Subject: [PATCH 10/10] Comment out and indent a warn which was added during
 testing and left active

---
 lib/WeBWorK/Utils/CourseManagement/sql_single.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/WeBWorK/Utils/CourseManagement/sql_single.pm b/lib/WeBWorK/Utils/CourseManagement/sql_single.pm
index bde72d36b1..82480dcaac 100644
--- a/lib/WeBWorK/Utils/CourseManagement/sql_single.pm
+++ b/lib/WeBWorK/Utils/CourseManagement/sql_single.pm
@@ -224,7 +224,7 @@ sub _get_db_info {
 	my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`;
 	if ( $test_for_column_statistics ) {
 		$column_statistics_off = "[mysqldump]\ncolumn-statistics=0\n";
-warn "Setting in the temporary mysql config file for table dump/restore:\n$column_statistics_off\n\n";
+		#warn "Setting in the temporary mysql config file for table dump/restore:\n$column_statistics_off\n\n";
 	}
 
 	# doing this securely is kind of a hassle...