Develop candidate 02 24 2019

Dockerfile

@@ -78,15 +78,20 @@ RUN apt-get update \
        mysql-client \
     && rm -fr /var/lib/apt/lists/*
 
+# Warning - when I tried to include XML::Simple near the start of the first "cpanm install" line, there was an error:
+#       Building and testing XMLRPC-Lite-0.717 ... ! Installing XMLRPC::Lite failed. See /root/.cpanm/work/1551887935.125/build.log for details. Retry with --force to force install it.
+# so it was put into a second "cpanm install" line.
+
 RUN curl -Lk https://cpanmin.us | perl - App::cpanminus \
     && cpanm install XML::Parser::EasyTree Iterator Iterator::Util Pod::WSDL Array::Utils HTML::Template XMLRPC::Lite Mail::Sender Email::Sender::Simple Data::Dump Statistics::R::IO \
+    && cpanm install XML::Simple \
     && rm -fr ./cpanm /root/.cpanm /tmp/*
 
 RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/webwork2
 
 # Block to include webwork2 in the container, when needed, instead of  getting it from a bind mount.
 #    Uncomment when needed, and set the correct branch name on the following line.
-#ENV WEBWORK_BRANCH=master   # need a valid branch name from https://github.com/openwebwork/webwork2
+#ENV WEBWORK_BRANCH=develop   # need a valid branch name from https://github.com/openwebwork/webwork2
 #RUN curl -fSL https://github.com/openwebwork/webwork2/archive/${WEBWORK_BRANCH}.tar.gz -o /tmp/${WEBWORK_BRANCH}.tar.gz \
 #    && cd /tmp \
 #    && tar xzf /tmp/${WEBWORK_BRANCH}.tar.gz \

PG_VERSION

@@ -1,4 +1,4 @@
-$PG_VERSION  ='PG-2.14';
+$PG_VERSION  ='develop';
 $PG_COPYRIGHT_YEARS = '1996-2019';
 
 1;

README

@@ -3,8 +3,8 @@
                                    Version 2.*
                         Branch:  github.com/openwebwork 
 
-	          http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.13
-                    Copyright 2000-2017, The WeBWorK Project
+	          http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.14
+                    Copyright 2000-2019, The WeBWorK Project
                              http://webwork.maa.org
                               All rights reserved.                              
 

bin/OPL-update

@@ -21,7 +21,6 @@ use File::Basename;
 use Cwd;
 use DBI;
 
-
  #(maximum varchar length is 255 for mysql version < 5.0.3.  
  #You can increase path length to  4096 for mysql > 5.0.3)
 
@@ -93,9 +92,12 @@ my $dbh = DBI->connect(
         {
                 PrintError => 0,
                 RaiseError => 1,
+		mysql_enable_utf8mb4 => 1,
         },
 );
 
+$dbh->prepare("SET NAMES 'utf8mb4'")->execute();
+
 my $libraryRoot = $ce->{problemLibrary}->{root};
 $libraryRoot =~ s|/+$||;
 my $libraryVersion = $ce->{problemLibrary}->{version};
@@ -133,21 +135,21 @@ if($libraryVersion eq '2.5') {
 @create_tables = (
 [$tables{dbsubject}, '
 	DBsubject_id int(15) NOT NULL auto_increment,
-	name varchar(255) NOT NULL,
+	name varchar(245) NOT NULL,
 	KEY DBsubject (name),
 	PRIMARY KEY (DBsubject_id)
 '],
 [$tables{dbchapter}, '
 	DBchapter_id int(15) NOT NULL auto_increment,
-	name varchar(255) NOT NULL,
+	name varchar(245) NOT NULL,
 	DBsubject_id int(15) DEFAULT 0 NOT NULL,
 	KEY DBchapter (name),
 	KEY (DBsubject_id),
 	PRIMARY KEY (DBchapter_id)
 '],
 [$tables{dbsection}, '
 	DBsection_id int(15) NOT NULL auto_increment,
-	name varchar(255) NOT NULL,
+	name varchar(245) NOT NULL,
 	DBchapter_id int(15) DEFAULT 0 NOT NULL,
 	KEY DBsection (name),
 	KEY (DBchapter_id),
@@ -164,7 +166,7 @@ if($libraryVersion eq '2.5') {
 '],
 [$tables{path}, '
 	path_id int(15) NOT NULL auto_increment,
-	path varchar(255) NOT NULL,
+	path varchar(245) NOT NULL,
 	machine varchar(255),
 	user varchar(255),
 	KEY (path),
@@ -210,7 +212,7 @@ if($libraryVersion eq '2.5') {
 	chapter_id int (15) NOT NULL auto_increment,
 	textbook_id int (15),
 	number int(3),
-	name varchar(255) NOT NULL,
+	name varchar(245) NOT NULL,
 	page int(4),
 	KEY (textbook_id, name),
 	KEY (number),
@@ -220,7 +222,7 @@ if($libraryVersion eq '2.5') {
 	section_id int(15) NOT NULL auto_increment,
 	chapter_id int (15),
 	number int(3),
-	name varchar(255) NOT NULL,
+	name varchar(245) NOT NULL,
 	page int(4),
 	KEY (chapter_id, name),
 	KEY (number),
@@ -237,7 +239,7 @@ if($libraryVersion eq '2.5') {
 '],
 [$tables{morelt}, '
 	morelt_id int(15) NOT NULL auto_increment,
-	name varchar(255) NOT NULL,
+	name varchar(245) NOT NULL,
 	DBsection_id int(15),
 	leader int(15), # pgfile_id of the MLT leader
 	KEY (name),
@@ -346,7 +348,7 @@ my ($name,$pgfile,$pgpath);
 
 #### First read in textbook information
 
-if(open(IN, "$libraryRoot/Textbooks")) {
+if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Textbooks")) {
 	print "Reading in textbook data from Textbooks in the library $libraryRoot.\n";
 	my %textinfo = ( TitleText => '', EditionText =>'', AuthorText=>'');
 	my $bookid = undef;
@@ -445,10 +447,10 @@ my $tagtaxo = [];
 my ($chaplist, $seclist) = ([],[]);
 
 my $canopenfile = 0;
-if(open(IN, "$libraryRoot/Taxonomy2")) {
+if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy2")) {
 	print "Reading in OPL taxonomy from Taxonomy2 in the library $libraryRoot.\n";
 	$canopenfile = 1;
-} elsif(open(IN, "$libraryRoot/Taxonomy")) {
+} elsif(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy")) {
 	print "Reading in OPL taxonomy from Taxonomy in the library $libraryRoot.\n";
 	$canopenfile = 1;
 } else {
@@ -516,6 +518,7 @@ if($canopenfile) {
 my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
 my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json";
 open(OUTF, ">$file") or die "Cannot open $file";
+binmode(OUTF,':encoding(UTF-8)'); 
 print OUTF to_json($tagtaxo,{pretty=>1}) or die "Cannot write to $file";
 close(OUTF);
 print "Saved taxonomy to $file.\n";

bin/fix_copyright.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+sed -i .bak '/Copyright/c\
+# Copyright © 2000-2019. The WeBWorK Project. https://github.com/openwebwork/webwork2\
+' $1
+
+
+
+#obtained by trial and error after much toil -- mostly error.
+# this version works on a mac
+# the space after -i might need to be removed for linux.
+# produces $1.bak file
+
+# use with the find command: 
+# find . -name course.conf -exec /opt/webwork/webwork2/bin/fix_copyright.sh {} ';'

bin/update-OPL-statistics

@@ -79,9 +79,9 @@ CREATE TABLE IF NOT EXISTS `OPL_problem_user` (
   `attempted` int(11) DEFAULT NULL,
   `num_correct` int(11) DEFAULT NULL,
   `num_incorrect` int(11) DEFAULT NULL,
-  UNIQUE KEY `unique_key_idx` (`course_id`(100),`user_id`(100),`set_id`(100),`due_date`,`problem_id`),
-  KEY `source_file_idx` (`source_file`(255)))
-  CHARACTER SET ascii
+  UNIQUE KEY `unique_key_idx` (`course_id`(80),`user_id`(80),`set_id`(80),`due_date`,`problem_id`),
+  KEY `source_file_idx` (`source_file`(245)))
+  CHARACTER SET utf8mb4
 EOS
 
 $dbh->do(<<EOS);
@@ -90,12 +90,12 @@ EOS
 
 $dbh->do(<<EOS);
 CREATE TABLE `OPL_local_statistics` (
-  `source_file` varchar(255) NOT NULL,
+  `source_file` varchar(245) NOT NULL,
   `students_attempted` int(11) DEFAULT NULL,
   `average_attempts` float DEFAULT NULL,
   `average_status` float DEFAULT NULL,
-  PRIMARY KEY (`source_file`(255))
-)
+  PRIMARY KEY (`source_file`(245))
+)  ENGINE=MyISAM
 EOS
 
 $dbh->commit();

clients/sendXMLRPC.pl

@@ -269,6 +269,7 @@ BEGIN
 }
 $ENV{MOD_PERL_API_VERSION} = 2;
 use lib "$main::dirname";
+print "home directory ".$main::dirname;
 #use lib "."; # is this needed?
 
 # some files such as FormatRenderedProblem.pm may need to be in the same directory
@@ -290,7 +291,7 @@ BEGIN
 	}
 
 }
-
+use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib";
 
 
 use Carp;

conf/defaults.config

@@ -223,6 +223,14 @@ $pg{options}{periodicRandomizationPeriod} = 5;
 
 $language = "en";   # tr = turkish,  en=english
 
+# $perProblemLangAndDirSettingMode controls how and whether LANG and/or DIR
+#    attributes are added to the DIV element enveloping a problem
+#    which helps handle proper display of problems with a text direction
+#    different from that used course-wide. Ex: enables English problems to
+#    be displayed properly in a Hebrew course site, which helps select problems
+#    to be translated to Hebrew.
+$perProblemLangAndDirSettingMode = "force::ltr";
+
 ################################################################################
 # System-wide locations (directories and URLs)
 ################################################################################
@@ -1211,6 +1219,8 @@ $pg{specialPGEnvironmentVars}{problemPostamble} = { TeX => '', HTML=>'' };
 # should appear as [qw(Mymodule.pm, Dependency1.pm, Dependency2.pm)]
 
 ${pg}{modules} = [
+    [qw(Encode)],
+	[qw(Encode::Encoding)],
         [qw(HTML::Parser)],
         [qw(HTML::Entities)],
 	[qw(DynaLoader)],
@@ -1439,6 +1449,12 @@ $ConfigValues = [
 		  values => [qw(en tr es fr zh_hk heb)],
 		  type => 'popuplist'
 		},
+		{ var => 'perProblemLangAndDirSettingMode',
+		  doc => 'Mode in which the LANG and DIR settings for a single problem are determined.',
+		  doc2 => 'Mode in which the LANG and DIR settings for a single problem are determined.<p>The system will set the LANGuage attribute to either a value determined from the problem, a course-wide default, or the system default of en-US, depending on the mode selected. The tag will only be added to the DIV enclosing the problem if it is different than the value which should be set in the main HTML tag set for the entire course based on the course language.<p>There are two options for the DIRection attribute: \"ltr\" for left-to-write sripts, and \"rtl\" for right-to-left scripts like Arabic and Hebrew.<p>The DIRection attribute is needed to trigger proper display of the question text when the problem text-direction is different than that used by the current language of the course. For example, English problems from the library browser would display improperly in RTL mode for a Hebrew course, unless the problen Direction is set to LTR.<p>The feature to set a problem language and direction was only added in 2018 to the PG language, so most problems will not declare their language, and the system needs to fall back to determining the language and direction in a different manner. The OPL itself is all English, so the system wide fallback is to en-US in LTR mode.<p>Since the defaults fall back to the LTR direction, most sites should be fine with the \"auto::\" mode, but may want to select the one which matches their course language. The mode \"force::ltr\" would also be an option for a course which runs into trouble with the \"auto\" modes.<p>Modes:<br><ul><li> \"none\" prevents any additional LANG and/or DIR tag being added. The browser will use the main setting which was applied to the entire HTML page. This is likely to cause trouble when a problem of the other direction is displayed.</li><li> \"auto::\" allows the system to make the settings based on the language and direction reported by the problem (a new feature, so not set in almost all existing problems) and falling back to the expected default of en-US in LTR mode. </li><li> \"auto:LangCode:Dir\" allows the system to make the settings based on the language and direction reported by the problem (a new feature, so not set in almost all existing problems) but falling back to the language with the given LangCode and the direction Dir when problem settings are not available from PG.</li><li> \"auto::Dir\" for problems without PG settings, this will use the default en=english language, but force the direction to Dir. Problems with PG settings will get those settings.</li><li> \"auto:LangCode:\" for problems without PG settings, this will use the default LTR direction, but will set the language to LangCode.Problems with PG settings will get those settings.</li><li> \"force:LangCode:Dir\" will <b>ignore</b> any setting made by the PG code of the problem, and will force the system to set the language with the given LangCode and the direction to Dir for <b>all</b> problems.</li><li> \"force::Dir\" will <b>ignore</b> any setting made by the PG code of the problem, and will force the system to set the direction to Dir for <b>all</b> problems, but will avoid setting any language attribute for individual problem.</li></ul>',
+		  values => [qw(none auto:: force::ltr force::rtl force:en:ltr auto:en:ltr force:tr:ltr auto:tr:ltr force:es:ltr auto:es:ltr force:fr:ltr auto:fr:ltr force:zh_hk:ltr auto:zh_hk:ltr force:he:rtl auto:he:rtl   )],
+		  type => 'popuplist'
+		},
 		{ var => 'sessionKeyTimeout',
 		  doc => 'Inactivity time before a user is required to login again',
 		  doc2 => 'Length of time, in seconds, a user has to be inactive before he is required to login again.<p> This value should be entered as a number, so as 3600 instead of 60*60 for one hour',

conf/snippets/hardcopyThemes/oneColumn/hardcopyPreamble.tex

@@ -11,7 +11,13 @@
 \usepackage{epsfig}
 \usepackage{pslatex}
 \usepackage{fullpage}
+
 \usepackage[utf8]{inputenc}
+
+\usepackage{eurosym}                   % the euro symbol
+\DeclareUnicodeCharacter{20AC}{\euro}  % make it possible to use the UTF-8 character for the euro symbol in problems
+
+
 \pagestyle{plain}
 \def\endline{\bigskip\hrule width \hsize height 0.8pt }
 \newcommand{\lt}{<}

conf/snippets/hardcopyThemes/twoColumn/hardcopyPreamble.tex

@@ -10,7 +10,12 @@
 \usepackage{epsf}
 \usepackage{epsfig}
 \usepackage{pslatex}
+
 \usepackage[utf8]{inputenc}
+
+\usepackage{eurosym}                   % the euro symbol
+\DeclareUnicodeCharacter{20AC}{\euro}  % make it possible to use the UTF-8 character for the euro symbol in problems
+
 \pagestyle{plain}
 \textheight 9in
 \oddsidemargin = -0.42in

docker-compose.yml

@@ -4,6 +4,9 @@ services:
     image: mariadb:10.1
     volumes:
       - mysql:/var/lib/mysql
+      # Set up UTF8MB4 in config file for the container.
+      # Needs to be done BEFORE the database is created.
+      - "./docker-config/db/mariadb.cnf:/etc/mysql/conf.d/mariadb.cnf"
     restart: always
     environment:
       MYSQL_ROOT_PASSWORD: randomepassword

docker-compose.yml.specialUTF8MB4-db-storage

@@ -0,0 +1,54 @@
+version: '2'
+services:
+  # We have renamed the database service from "db" to "dbMB4",
+  # changed the name of the data volume to "mysqlMB4" and set
+  # environment variables to use the "dbMB4" server. All this was needed
+  # ONLY to allow leaving a version of the "db" container which does not
+  # use utf8mb4 available until the UTF8MB4 changes become mainstream.
+  dbMB4:
+    image: mariadb:10.1
+    volumes:
+      - mysqlMB4:/var/lib/mysql
+      # Set up UTF8MB4 in config file for the container.
+      # Needs to be done BEFORE the database is created.
+      - "./docker-config/db/mariadb.cnf:/etc/mysql/conf.d/mariadb.cnf"
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: randomepassword
+      MYSQL_DATABASE: webwork
+      MYSQL_USER: webworkWrite
+      MYSQL_PASSWORD: passwordRW
+  app:
+    build: .
+    image: webwork
+    depends_on:
+      - dbMB4
+      - r
+    volumes:
+      - ".:/opt/webwork/webwork2"
+
+      # OLD approach put the courses tree under webwork2/.data/courses
+      #- "./.data/courses:/opt/webwork/courses"
+      # NEW appoach puts the courses tree in a separate tree outside of webwork2/
+      - "../ww-docker-data/courses:/opt/webwork/courses"
+
+      # Uncomment the line below to use local OPL for development
+      #- "../opl:/opt/webwork/libraries/webwork-open-problem-library"
+      # Uncomment the line below to use local PG for development
+      #- "../pg:/opt/webwork/pg"
+    ports:
+      - "8080:80"
+    environment:
+      - DEV=0
+      - WEBWORK_DB_HOST=dbMB4
+      # - WEBWORK_DB_PORT=3306
+      # - WEBWORK_DB_NAME=webwork
+      - WEBWORK_DB_DSN=DBI:mysql:webwork:dbMB4:3306
+  r:
+    image: ubcctlt/rserve
+    ports:
+      - "6311:6311"
+
+volumes:
+  mysqlMB4:
+

docker-config/db/mariadb.cnf

@@ -0,0 +1,34 @@
+# MariaDB-specific config file.
+# Read by /etc/mysql/my.cnf
+
+[client]
+# Default is Latin1, if you need UTF-8 set this (also in server section)
+#default-character-set = utf8 
+
+# Based on:  https://salsa.debian.org/mariadb-team/mariadb-10.1/commit/e6ade2be57856736e8bc8039d71b35f9ffcde48e
+default-character-set = utf8mb4
+
+[mysql]
+# Based on:  https://salsa.debian.org/mariadb-team/mariadb-10.1/commit/e6ade2be57856736e8bc8039d71b35f9ffcde48e
+default-character-set = utf8mb4
+
+[mysqld]
+#
+# * Character sets
+# 
+# Default is Latin1, if you need UTF-8 set all this (also in client section)
+#
+#character-set-server  = utf8 
+#collation-server      = utf8_general_ci 
+#character_set_server   = utf8 
+#collation_server       = utf8_general_ci 
+
+# MySQL/MariaDB default is Latin1, but we want the full utf8 4-bit character set. 
+# See also client.cnf
+# Based on:  https://salsa.debian.org/mariadb-team/mariadb-10.1/commit/e6ade2be57856736e8bc8039d71b35f9ffcde48e
+character-set-server  = utf8mb4
+collation-server      = utf8mb4_general_ci
+character_set_server   = utf8mb4
+collation_server       = utf8mb4_general_ci
+init-connect='SET NAMES utf8mb4'
+

htdocs/themes/math4-ar/gateway.template

@@ -104,7 +104,7 @@
 	<!--#endif-->
 
 	<!--#if can="body"-->
-	<div class="Body">
+	<div class="Body" <!--#output_problem_lang_and_dir-->>
 		<!--#body-->
 	</div>
 	<!--#endif-->