diff --git a/bin/OPL-update b/bin/OPL-update index 1ab22f35bb..6508c6d509 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -92,15 +92,29 @@ my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0; print "using utf8mb4 \n\n" if $ENABLE_UTF8MB4; +# The DBD::MariaDB driver should not get the +# mysql_enable_utf8mb4 or mysql_enable_utf8 settings, +# but DBD::mysql should. +my %utf8_parameters = (); + +if ( $ce->{database_driver} =~ /^mysql$/i ) { + # Only needed for older DBI:mysql driver + if ( $ENABLE_UTF8MB4 ) { + $utf8_parameters{mysql_enable_utf8mb4} = 1; + } else { + $utf8_parameters{mysql_enable_utf8} = 1; + } +} + my $dbh = DBI->connect( - $ce->{problemLibrary_db}->{dbsource}, - $ce->{problemLibrary_db}->{user}, - $ce->{problemLibrary_db}->{passwd}, - { - PrintError => 0, - RaiseError => 1, - ($ENABLE_UTF8MB4)?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1), - }, + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + PrintError => 0, + RaiseError => 1, + %utf8_parameters, + }, ); my $character_set=''; diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm old mode 100755 new mode 100644 diff --git a/bin/dump-OPL-tables.pl b/bin/dump-OPL-tables.pl index 6767486944..997be3d8c4 100755 --- a/bin/dump-OPL-tables.pl +++ b/bin/dump-OPL-tables.pl @@ -64,12 +64,9 @@ # Get DB connection settings -my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); - -$host = 'localhost' unless $host; - -$port = 3306 unless $port; - +my $db = $ce->{database_name}; +my $host = $ce->{database_host}; +my $port = $ce->{database_port}; my $dbuser = $ce->{database_username}; my $dbpass = $ce->{database_password}; @@ -100,7 +97,17 @@ print "Dumping OPL tables\n"; -`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db $OPL_tables_to_dump > $prepared_OPL_tables_file`; +# Conditionally add --column-statistics=0 as MariaDB databases do not support it +# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109 +# https://github.com/drush-ops/drush/issues/4410 + +my $column_statistics_off = ""; +my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`; +if ( $test_for_column_statistics ) { + $column_statistics_off = " --column-statistics=0 "; +} + +`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $column_statistics_off $db $OPL_tables_to_dump > $prepared_OPL_tables_file`; print "OPL database dump created: $prepared_OPL_tables_file\n"; diff --git a/bin/load-OPL-global-statistics.pl b/bin/load-OPL-global-statistics.pl index d0b2405938..8c8d909d08 100755 --- a/bin/load-OPL-global-statistics.pl +++ b/bin/load-OPL-global-statistics.pl @@ -52,15 +52,11 @@ if (-e $global_sql_file) { - my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); - - $host = 'localhost' unless $host; - - $port = 3306 unless $port; - + my $db = $ce->{database_name}; + my $host = $ce->{database_host}; + my $port = $ce->{database_port}; my $dbuser = $ce->{database_username}; my $dbpass = $ce->{database_password}; - $dbh->do(<{database_dsn}); - -$host = 'localhost' unless $host; - -$port = 3306 unless $port; - +my $db = $ce->{database_name}; +my $host = $ce->{database_host}; +my $port = $ce->{database_port}; my $dbuser = $ce->{database_username}; my $dbpass = $ce->{database_password}; diff --git a/bin/updateOPLextras.pl b/bin/updateOPLextras.pl index 73430eb880..0970e77f81 100755 --- a/bin/updateOPLextras.pl +++ b/bin/updateOPLextras.pl @@ -14,7 +14,7 @@ =head1 SYNOPSIS -d --directories (rebuild directory tree) -a --all (rebuild all trees) -h --help (display this text) - + -v --verbose (turn on verbosity mode) =head1 OPTIONS =over 8 @@ -30,7 +30,11 @@ =head1 OPTIONS =item B<-d> I<--directories> Rebuild the directory tree and write to a JSON file. - + +=item B<-v> I<--verbosity> + +Turn on verbosity mode. + =back =head1 DESCRIPTION @@ -76,15 +80,35 @@ BEGIN my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0; +print "using utf8mb4 \n\n" if $ENABLE_UTF8MB4; + +# The DBD::MariaDB driver should not get the +# mysql_enable_utf8mb4 or mysql_enable_utf8 settings, +# but DBD::mysql should. +my %utf8_parameters = (); + +if ( $ce->{database_driver} =~ /^mysql$/i ) { + # Only needed for older DBI:mysql driver + if ( $ENABLE_UTF8MB4 ) { + $utf8_parameters{mysql_enable_utf8mb4} = 1; + } else { + $utf8_parameters{mysql_enable_utf8} = 1; + } +} + my $dbh = DBI->connect( - $ce->{problemLibrary_db}->{dbsource}, - $ce->{problemLibrary_db}->{user}, - $ce->{problemLibrary_db}->{passwd}, - { - PrintError => 0, - RaiseError => 1, - ($ce->{ENABLE_UTF8MB4})?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1), - }, + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + PrintError => 0, + RaiseError => 1, + %utf8_parameters, + }, ); build_library_textbook_tree($ce,$dbh,$verbose) if ($all || $textbooks); diff --git a/bin/upload-OPL-statistics.pl b/bin/upload-OPL-statistics.pl index d1ca9e37ef..3cee5efedc 100755 --- a/bin/upload-OPL-statistics.pl +++ b/bin/upload-OPL-statistics.pl @@ -28,14 +28,14 @@ use String::ShellQuote; my $ce = new WeBWorK::CourseEnvironment({ - webwork_dir => $ENV{WEBWORK_ROOT},}); + webwork_dir => $ENV{WEBWORK_ROOT}, + }); -my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); - -$host = 'localhost' unless $host; - -$port = 3306 unless $port; +# Get DB connection settings +my $db = $ce->{database_name}; +my $host = $ce->{database_host}; +my $port = $ce->{database_port}; my $dbuser = $ce->{database_username}; my $dbpass = $ce->{database_password}; @@ -52,7 +52,17 @@ my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; -`$mysqldump_command --host=$host --port=$port --user=$dbuser $db OPL_local_statistics > $output_file`; +# Conditionally add --column-statistics=0 as MariaDB databases do not support it +# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109 +# https://github.com/drush-ops/drush/issues/4410 + +my $column_statistics_off = ""; +my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`; +if ( $test_for_column_statistics ) { + $column_statistics_off = " --column-statistics=0 "; +} + +`$mysqldump_command --host=$host --port=$port --user=$dbuser $column_statistics_off $db OPL_local_statistics > $output_file`; print "Database File Created\n"; diff --git a/conf/database.conf.dist b/conf/database.conf.dist index 10fc07e3be..7e1429df51 100644 --- a/conf/database.conf.dist +++ b/conf/database.conf.dist @@ -1,8 +1,7 @@ #!perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/conf/database.conf.dist,v 1.38 2007/08/13 22:59:51 sh002i Exp $ +# Copyright © 2000-2021 The WeBWorK Project, http://github.com/openwebwork # # This program is free software; you can redistribute it and/or modify it under # the terms of either: (a) the GNU General Public License as published by the @@ -80,12 +79,21 @@ my %sqlParams = ( username => $database_username, password => $database_password, debug => $database_debug, - ($ENABLE_UTF8MB4)?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1), # kinda hacky, but needed for table dumping mysql_path => $externalPrograms{mysql}, mysqldump_path => $externalPrograms{mysqldump}, ); +if ( $ce->{database_driver} =~ /^mysql$/i ) { + # The extra UTF8 connection setting is ONLY needed for older DBD:mysql driver + # and forbidden by the newer DBD::MariaDB driver + if ( $ENABLE_UTF8MB4 ) { + $sqlParams{mysql_enable_utf8mb4} = 1; # Full 4-bit UTF-8 + } else { + $sqlParams{mysql_enable_utf8} = 1; # Only the partial 3-bit mySQL UTF-8 + } +} + $dbLayouts{sql_single} = { locations => { record => "WeBWorK::DB::Record::Locations", diff --git a/conf/defaults.config b/conf/defaults.config index 0eacc5d75f..0f67b40740 100644 --- a/conf/defaults.config +++ b/conf/defaults.config @@ -1,9 +1,8 @@ #!perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/conf/defaults.config,v 1.225 2010/05/18 18:03:31 apizer Exp $ -# +# Copyright © 2000-2021 The WeBWorK Project, http://github.com/openwebwork +# # This program is free software; you can redistribute it and/or modify it under # the terms of either: (a) the GNU General Public License as published by the # Free Software Foundation; either version 2, or (at your option) any later @@ -555,29 +554,9 @@ $default_status = "Enrolled"; # Database options ################################################################################ -# these variables are used by database.conf. we define them here so that editing -# database.conf isn't necessary. - -# required permissions -# GRANT SELECT ON webwork.* TO webworkRead@localhost IDENTIFIED BY 'passwordRO'; -# GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER, DROP, INDEX, LOCK TABLES ON webwork.* TO webworkWrite@localhost IDENTIFIED BY 'passwordRW'; - - -################################################################################ -# set these variables in site.conf -# -# $database_dsn = "dbi:mysql:webwork"; -# $database_username = "webworkWrite"; -# $database_password = ""; #set this in site.conf -# $database_debug = 0; -################################################################################ - - - # Database schemas are defined in the file conf/database.conf and stored in the # hash %dbLayouts. The standard schema is called "sql_single"; - include( "./conf/database.conf.dist"); # always include database.conf.dist # in the rare case where you want local overrides diff --git a/conf/site.conf.dist b/conf/site.conf.dist index f8ae911c1c..5185b19c5d 100644 --- a/conf/site.conf.dist +++ b/conf/site.conf.dist @@ -1,8 +1,7 @@ #!perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/conf/site.conf.dist,v 1.225 2010/05/18 18:03:31 apizer Exp $ +# Copyright © 2000-2021 The WeBWorK Project, http://github.com/openwebwork # # This program is free software; you can redistribute it and/or modify it under # the terms of either: (a) the GNU General Public License as published by the @@ -161,6 +160,7 @@ $externalPrograms{mysqldump} ="/usr/bin/mysqldump"; # Database options ################################################################################ +# $database_debug = 0; # Standard permissions command used to initialize the webwork database # GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER, DROP, INDEX, LOCK TABLES ON webwork.* TO webworkWrite@localhost IDENTIFIED BY 'passwordRW'; @@ -174,12 +174,47 @@ $externalPrograms{mysqldump} ="/usr/bin/mysqldump"; # Edit the $database_password line and replace 'passwordRW' by the actual password used in the GRANT command above ################################################################################ -# The database dsn is the path to the WeBWorK database which you have created. -# Unless you have given the database a different name or the database resides on another -# server you do not need to change this first value. -# The format is dbi:mysql:[databasename] for databases on the local machine -# For a remote database the format is dbi:mysql:[databasename]:[hostname]:[port] -$database_dsn ="dbi:mysql:webwork"; +# The database DSN is the path to the WeBWorK database which you have created. + +# Modern database DSN format: +# DBI:driver:database=$database;host=$hostname;port=$port (when DB not on localhost) +# or DBI:driver:database=$database;host=127.0.0.1;port=$port (when DB on localhost, using TCP) +# See: https://metacpan.org/pod/DBD::MariaDB#port +# "To connect to a MariaDB or MySQL server on localhost using TCP/IP, +# you must specify the host as 127.0.0.1 with the optional port, e.g. 3306." +# or DBI:driver:database=$database (when DB on localhost, using socket) + +# One thing on which it depends is the driver name, which you may want to modify. +# It also depends on the database name, which may be non-standard in some settings, +# as may be the hostname and port of the database server. + +# driver should be one of: +# "mysql" for the DBD:mysql driver +# "MariaDB" for the DBD:mysql driver + +# Select the desired DB driver: +#$database_driver="mysql"; +$database_driver="MariaDB"; + +$database_host="localhost"; +$database_port="3306"; +$database_name="webwork"; + +# For a DB on localhost - default to using Unix socket. +# Change to 0 to use a TCP connection to 127.0.0.1. +$database_use_socket_if_localhost=1; + +if ( $database_host eq "localhost" ) { + if ( $database_use_socket_if_localhost ) { + $database_dsn="DBI:$database_driver:database=$database_name"; + } else { + $database_dsn="DBI:$database_driver:database=$database_name;host=127.0.0.1;port=$database_port"; + } +} else { + $database_dsn="DBI:$database_driver:database=$database_name;host=$database_host;port=$database_port"; +} + +# The default storange engine to use is set here: $database_storage_engine = 'myisam'; ######################### diff --git a/lib/WeBWorK/DB/Driver/SQL.pm b/lib/WeBWorK/DB/Driver/SQL.pm index 0301e6e8f0..f065e02c43 100644 --- a/lib/WeBWorK/DB/Driver/SQL.pm +++ b/lib/WeBWorK/DB/Driver/SQL.pm @@ -61,6 +61,15 @@ sub new($$$) { my $self = $proto->SUPER::new($source, $params); + # The DBD::MariaDB driver should not get the + # mysql_enable_utf8mb4 or mysql_enable_utf8 settings, + # but DBD::mysql should. + my %utf8_parameters = (); + if ( $source =~ /DBI:mysql/ ) { + $utf8_parameters{mysql_enable_utf8mb4} = 1; + $utf8_parameters{mysql_enable_utf8} = 1; + } + # add handle $self->{handle} = DBI->connect_cached( $source, @@ -70,8 +79,7 @@ sub new($$$) { PrintError => 0, RaiseError => 1, - mysql_enable_utf8mb4 => 1, - mysql_enable_utf8 => 1, # for older versions of DBD-mysql Perl modules + %utf8_parameters, }, ); die $DBI::errstr unless defined $self->{handle}; diff --git a/lib/WeBWorK/DB/Schema/NewSQL/Std.pm b/lib/WeBWorK/DB/Schema/NewSQL/Std.pm index 1cbe1ea878..5db40d7ae0 100644 --- a/lib/WeBWorK/DB/Schema/NewSQL/Std.pm +++ b/lib/WeBWorK/DB/Schema/NewSQL/Std.pm @@ -269,28 +269,52 @@ sub _get_db_info { my $dsn = $self->{driver}{source}; my $username = $self->{params}{username}; my $password = $self->{params}{password}; - - die "Can't call dump_table or restore_table on a table with a non-MySQL source" - unless $dsn =~ s/^dbi:mysql://i; - - # this is an internal function which we probably shouldn't be using here - # but it's quick and gets us what we want (FIXME what about sockets, etc?) + my %dsn; - DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']); - die "no database specified in DSN!" unless defined $dsn{database}; + if ( $dsn =~ m/^dbi:mariadb:/i ) { + # Expect DBI:MariaDB:database=webwork;host=db;port=3306 + my ($dbi,$dbtype,$temp1) = split(':',$dsn); + ( $dsn{database}, $dsn{host}, $dsn{port} ) = split(';',$temp1); + $dsn{database} =~ s/database=//; + $dsn{host} =~ s/host=// if ( defined $dsn{host} ); + $dsn{port} =~ s/port=// if ( defined $dsn{port} ); + } elsif ( $dsn =~ m/^dbi:mysql:/i ) { + # This code works for DBD::mysql + # this is an internal function which we probably shouldn't be using here + # but it's quick and gets us what we want (FIXME what about sockets, etc?) + DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']); + } else { + die "Can't call dump_table or restore_table on a table with a non-MySQL/MariaDB source"; + } + die "no database specified in DSN!" unless defined $dsn{database}; + + my $mysqldump = $self->{params}{mysqldump_path}; + # Conditionally add column-statistics=0 as MariaDB databases do not support it + # see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109 + # https://github.com/drush-ops/drush/issues/4410 + + my $column_statistics_off = ""; + my $test_for_column_statistics = `$mysqldump --help | grep 'column-statistics'`; + if ( $test_for_column_statistics ) { + $column_statistics_off = "[mysqldump]\ncolumn-statistics=0\n"; + #warn "Setting in the temporary mysql config file for table dump/restore:\n$column_statistics_off\n\n"; + } + # doing this securely is kind of a hassle... my $my_cnf = new File::Temp; $my_cnf->unlink_on_destroy(1); chmod 0600, $my_cnf or die "failed to chmod 0600 $my_cnf: $!"; # File::Temp objects stringify with ->filename print $my_cnf "[client]\n"; - print $my_cnf "user=\"$username\"\n" if defined $username and length($username) > 0; - print $my_cnf "password=\"$password\"\n" if defined $password and length($password) > 0; - print $my_cnf "host=\"$dsn{host}\"\n" if defined $dsn{host} and length($dsn{host}) > 0; - print $my_cnf "port=\"$dsn{port}\"\n" if defined $dsn{port} and length($dsn{port}) > 0; - + print $my_cnf "user=$username\n" if defined $username and length($username) > 0; + print $my_cnf "password=$password\n" if defined $password and length($password) > 0; + print $my_cnf "host=$dsn{host}\n" if defined $dsn{host} and length($dsn{host}) > 0; + print $my_cnf "port=$dsn{port}\n" if defined $dsn{port} and length($dsn{port}) > 0; + print $my_cnf "$column_statistics_off" if $test_for_column_statistics; + return ($my_cnf, $dsn{database}); } + #################################################### # checking Fields #################################################### diff --git a/lib/WeBWorK/Utils/CourseManagement/sql_single.pm b/lib/WeBWorK/Utils/CourseManagement/sql_single.pm index 719432f21e..82480dcaac 100644 --- a/lib/WeBWorK/Utils/CourseManagement/sql_single.pm +++ b/lib/WeBWorK/Utils/CourseManagement/sql_single.pm @@ -195,17 +195,38 @@ sub _get_db_info { my $dsn = $ce->{database_dsn}; my $username = $ce->{database_username}; my $password = $ce->{database_password}; - - die "Can't call dump_table or restore_table on a table with a non-MySQL source" - unless $dsn =~ s/^dbi:mysql://i; - - # this is an internal function which we probably shouldn't be using here - # but it's quick and gets us what we want (FIXME what about sockets, etc?) + my %dsn; - runtime_use "DBD::mysql"; - DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']); - die "no database specified in DSN!" unless defined $dsn{database}; + if ( $dsn =~ s/^dbi:mariadb://i ) { + my ($dbi,$dbtype,$temp1) = split(':',$dsn); + ( $dsn{database}, $dsn{host}, $dsn{port} ) = split(';',$db); + $dsn{database} =~ s/database=//; + $dsn{host} =~ s/host=// if ( defined $dsn{host} ); + $dsn{port} =~ s/port=// if ( defined $dsn{port} ); + } elsif ( $dsn =~ s/^dbi:mysql://i ) { + # This code works for DBD::mysql + # this is an internal function which we probably shouldn't be using here + # but it's quick and gets us what we want (FIXME what about sockets, etc?) + runtime_use "DBD::mysql"; + DBD::mysql->_OdbcParse($dsn, \%dsn, ['database', 'host', 'port']); + } else { + die "Can't call dump_table or restore_table on a table with a non-MySQL/MariaDB source"; + } + die "no database specified in DSN!" unless defined $dsn{database}; + + my $mysqldump = $self->{params}{mysqldump_path}; + # Conditionally add column-statistics=0 as MariaDB databases do not support it + # see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109 + # https://github.com/drush-ops/drush/issues/4410 + + my $column_statistics_off = ""; + my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`; + if ( $test_for_column_statistics ) { + $column_statistics_off = "[mysqldump]\ncolumn-statistics=0\n"; + #warn "Setting in the temporary mysql config file for table dump/restore:\n$column_statistics_off\n\n"; + } + # doing this securely is kind of a hassle... my $my_cnf = new File::Temp; $my_cnf->unlink_on_destroy(1); @@ -215,7 +236,8 @@ sub _get_db_info { print $my_cnf "password=$password\n" if defined $password and length($password) > 0; print $my_cnf "host=$dsn{host}\n" if defined $dsn{host} and length($dsn{host}) > 0; print $my_cnf "port=$dsn{port}\n" if defined $dsn{port} and length($dsn{port}) > 0; - + print $my_cnf "$column_statistics_off" if $test_for_column_statistics; + return ($my_cnf, $dsn{database}); }