diff --git a/Dockerfile b/Dockerfile
index 8b3df7e988..d7470274b5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -78,15 +78,22 @@ RUN apt-get update \
        mysql-client \
     && rm -fr /var/lib/apt/lists/*
 
+# Warning - when I tried to include XML::Simple near the start of the first "cpanm install" line, there was an error:
+#       Building and testing XMLRPC-Lite-0.717 ... ! Installing XMLRPC::Lite failed. See /root/.cpanm/work/1551887935.125/build.log for details. Retry with --force to force install it.
+# so it was put into a second "cpanm install" line.
+
 RUN curl -Lk https://cpanmin.us | perl - App::cpanminus \
-    && cpanm install XML::Parser::EasyTree Iterator Iterator::Util Pod::WSDL Array::Utils HTML::Template XMLRPC::Lite Mail::Sender Email::Sender::Simple Data::Dump Statistics::R::IO \
-    && rm -fr ./cpanm /root/.cpanm /tmp/*
+     && cpanm install XML::Parser::EasyTree Iterator Iterator::Util Pod::WSDL Array::Utils HTML::Template Mail::Sender Email::Sender::Simple Data::Dump Statistics::R::IO 
+     
+RUN cpanm install  XMLRPC::Lite XML::Simple \
+     && rm -fr ./cpanm /root/.cpanm /tmp/*
+
 
 RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/webwork2
 
 # Block to include webwork2 in the container, when needed, instead of  getting it from a bind mount.
 #    Uncomment when needed, and set the correct branch name on the following line.
-#ENV WEBWORK_BRANCH=master   # need a valid branch name from https://github.com/openwebwork/webwork2
+#ENV WEBWORK_BRANCH=develop   # need a valid branch name from https://github.com/openwebwork/webwork2
 #RUN curl -fSL https://github.com/openwebwork/webwork2/archive/${WEBWORK_BRANCH}.tar.gz -o /tmp/${WEBWORK_BRANCH}.tar.gz \
 #    && cd /tmp \
 #    && tar xzf /tmp/${WEBWORK_BRANCH}.tar.gz \
diff --git a/PG_VERSION b/PG_VERSION
index c2a29f7f1a..260d6729c8 100644
--- a/PG_VERSION
+++ b/PG_VERSION
@@ -1,4 +1,5 @@
-$PG_VERSION  ='PG-2.14';
+$PG_VERSION  ='develop';
 $PG_COPYRIGHT_YEARS = '1996-2019';
-
+# this file is not being used (I believe)
+# it should be deleted. 
 1;
diff --git a/README b/README
index ca428fb85d..0777afbc52 100644
--- a/README
+++ b/README
@@ -3,8 +3,8 @@
                                    Version 2.*
                         Branch:  github.com/openwebwork 
 
-	          http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.13
-                    Copyright 2000-2017, The WeBWorK Project
+	          http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.14
+                    Copyright 2000-2019, The WeBWorK Project
                              http://webwork.maa.org
                               All rights reserved.                              
 
diff --git a/bin/OPL-update-latin1 b/bin/OPL-update-latin1
new file mode 100755
index 0000000000..72382f38cd
--- /dev/null
+++ b/bin/OPL-update-latin1
@@ -0,0 +1,976 @@
+#!/usr/bin/env perl
+
+# This is the script formerly known as loadDB2, and then known as NPL-update.  
+
+# It is used to update
+# the database when it comes to the WeBWorK Open Problem Library (OPL).
+# This should be run after doing a git clone or pull for the OPL
+# files.
+
+# In order for this script to work:
+#   1) The OPL downloaded to your machine (the .pg files)
+#   2) The environment variable WEBWORK_ROOT needs to be 
+#      correctly defined (as with other scripts here).
+#   3) Configuration for the OPL in site.conf needs to be
+#      done (basically just setting the path to the OPL files).
+
+#use strict;
+use File::Find;
+use File::Find::Rule;
+use File::Basename;
+use Cwd;
+use DBI;
+
+
+ #(maximum varchar length is 255 for mysql version < 5.0.3.  
+ #You can increase path length to  4096 for mysql > 5.0.3)
+
+BEGIN {
+        die "WEBWORK_ROOT not found in environment.\n"
+                unless exists $ENV{WEBWORK_ROOT};
+	# Unused variable, but define it to avoid an error message.
+	$WeBWorK::Constants::WEBWORK_DIRECTORY = '';
+}
+
+# Taxonomy global variables
+# Make a hash of hashes of hashes to record what is legal
+# Also create list for json file
+my $taxo={};
+#my $taxsubs = [];
+
+
+### Data for creating the database tables
+
+my %OPLtables = (
+ dbsubject => 'OPL_DBsubject',
+ dbchapter => 'OPL_DBchapter',
+ dbsection => 'OPL_DBsection',
+ author => 'OPL_author',
+ path => 'OPL_path',
+ pgfile => 'OPL_pgfile',
+ keyword => 'OPL_keyword',
+ pgfile_keyword => 'OPL_pgfile_keyword',
+ textbook => 'OPL_textbook',
+ chapter => 'OPL_chapter',
+ section => 'OPL_section',
+ problem => 'OPL_problem',
+ morelt => 'OPL_morelt',
+ pgfile_problem => 'OPL_pgfile_problem',
+);
+
+
+my %NPLtables = (
+ dbsubject => 'NPL-DBsubject',
+ dbchapter => 'NPL-DBchapter',
+ dbsection => 'NPL-DBsection',
+ author => 'NPL-author',
+ path => 'NPL-path',
+ pgfile => 'NPL-pgfile',
+ keyword => 'NPL-keyword',
+ pgfile_keyword => 'NPL-pgfile-keyword',
+ textbook => 'NPL-textbook',
+ chapter => 'NPL-chapter',
+ section => 'NPL-section',
+ problem => 'NPL-problem',
+ morelt => 'NPL-morelt',
+ pgfile_problem => 'NPL-pgfile-problem',
+);
+
+
+# Get database connection
+
+use lib "$ENV{WEBWORK_ROOT}/lib";
+use lib "$ENV{WEBWORK_ROOT}/bin";
+use WeBWorK::CourseEnvironment;
+use WeBWorK::Utils::Tags;
+use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/;
+
+my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
+my $dbh = DBI->connect(
+        $ce->{problemLibrary_db}->{dbsource},
+        $ce->{problemLibrary_db}->{user},
+        $ce->{problemLibrary_db}->{passwd},
+        {
+                PrintError => 0,
+                RaiseError => 1,
+        },
+);
+
+my $libraryRoot = $ce->{problemLibrary}->{root};
+$libraryRoot =~ s|/+$||;
+my $libraryVersion = $ce->{problemLibrary}->{version};
+my $db_storage_engine = $ce->{problemLibrary_db}->{storage_engine};
+
+my $verbose = 0;
+my $cnt2 = 0;
+# Can force library version
+$libraryVersion = $ARGV[0] if(@ARGV);
+
+# auto flush printing
+my $old_fh = select(STDOUT);
+$| = 1;
+select($old_fh);
+
+sub dbug {
+	my $msg = shift;
+	my $insignificance = shift || 2;
+	print $msg if($verbose>=$insignificance);
+}
+
+##Figure out which set of tables to use
+
+my %tables;
+if($libraryVersion eq '2.5') {
+	%tables = %OPLtables;
+	my $lib = 'OPL';
+	warn "Library version is $libraryVersion; using OPLtables!\n";
+} else {
+	%tables = %NPLtables;
+	my $lib = 'NPL';
+	print "Library version is $libraryVersion; NPLtables! \n";
+}
+
+@create_tables = (
+[$tables{dbsubject}, '
+	DBsubject_id int(15) NOT NULL auto_increment,
+	name varchar(255) NOT NULL,
+	KEY DBsubject (name),
+	PRIMARY KEY (DBsubject_id)
+'],
+[$tables{dbchapter}, '
+	DBchapter_id int(15) NOT NULL auto_increment,
+	name varchar(255) NOT NULL,
+	DBsubject_id int(15) DEFAULT 0 NOT NULL,
+	KEY DBchapter (name),
+	KEY (DBsubject_id),
+	PRIMARY KEY (DBchapter_id)
+'],
+[$tables{dbsection}, '
+	DBsection_id int(15) NOT NULL auto_increment,
+	name varchar(255) NOT NULL,
+	DBchapter_id int(15) DEFAULT 0 NOT NULL,
+	KEY DBsection (name),
+	KEY (DBchapter_id),
+	PRIMARY KEY (DBsection_id)
+'],
+[$tables{author}, '
+	author_id int (15) NOT NULL auto_increment,
+	institution tinyblob,
+	lastname varchar (255) NOT NULL,
+	firstname varchar (255) NOT NULL,
+	email varchar (255),
+	KEY author (lastname(100), firstname(100)),
+	PRIMARY KEY (author_id)
+'],
+[$tables{path}, '
+	path_id int(15) NOT NULL auto_increment,
+	path varchar(255) NOT NULL,
+	machine varchar(255),
+	user varchar(255),
+	KEY (path),
+	PRIMARY KEY (path_id)
+'],
+[$tables{pgfile}, '
+	pgfile_id int(15) NOT NULL auto_increment,
+	DBsection_id int(15) NOT NULL,
+	author_id int(15),
+	institution tinyblob,
+	path_id int(15) NOT NULL,
+	filename varchar(255) NOT NULL,
+	morelt_id int(127) DEFAULT 0 NOT NULL,
+	level int(15),
+	language varchar(255),
+	static TINYINT,
+	MO TINYINT,
+	PRIMARY KEY (pgfile_id)
+'],
+[$tables{keyword}, '
+	keyword_id int(15) NOT NULL auto_increment,
+	keyword varchar(256) NOT NULL,
+	KEY (keyword),
+	PRIMARY KEY (keyword_id)
+'],
+[$tables{pgfile_keyword}, '
+	pgfile_id int(15) DEFAULT 0 NOT NULL,
+	keyword_id int(15) DEFAULT 0 NOT NULL,
+	KEY pgfile_keyword (keyword_id, pgfile_id),
+	KEY pgfile (pgfile_id)
+'],
+[$tables{textbook}, '
+	textbook_id int (15) NOT NULL auto_increment,
+	title varchar (255) NOT NULL,
+	edition int (15) DEFAULT 0 NOT NULL,
+	author varchar (255) NOT NULL,
+	publisher varchar (255),
+	isbn char (15),
+	pubdate varchar (255),
+	PRIMARY KEY (textbook_id)
+'],
+[$tables{chapter}, '
+	chapter_id int (15) NOT NULL auto_increment,
+	textbook_id int (15),
+	number int(3),
+	name varchar(255) NOT NULL,
+	page int(4),
+	KEY (textbook_id, name),
+	KEY (number),
+	PRIMARY KEY (chapter_id)
+'],
+[$tables{section}, '
+	section_id int(15) NOT NULL auto_increment,
+	chapter_id int (15),
+	number int(3),
+	name varchar(255) NOT NULL,
+	page int(4),
+	KEY (chapter_id, name),
+	KEY (number),
+	PRIMARY KEY section (section_id)
+'],
+[$tables{problem}, '
+	problem_id int(15) NOT NULL auto_increment,
+	section_id int(15),
+	number int(4) NOT NULL,
+	page int(4),
+	#KEY (page, number),
+	KEY (section_id),
+	PRIMARY KEY (problem_id)
+'],
+[$tables{morelt}, '
+	morelt_id int(15) NOT NULL auto_increment,
+	name varchar(255) NOT NULL,
+	DBsection_id int(15),
+	leader int(15), # pgfile_id of the MLT leader
+	KEY (name),
+	PRIMARY KEY (morelt_id)
+'],
+[$tables{pgfile_problem}, '
+	pgfile_id int(15) DEFAULT 0 NOT NULL,
+	problem_id int(15) DEFAULT 0 NOT NULL,
+	PRIMARY KEY (pgfile_id, problem_id)
+']);
+
+### End of database data
+
+## Resetting the database tables.
+#  First take care of tables which are no longer used
+
+$dbh->do("DROP TABLE IF EXISTS `NPL-institution`");
+$dbh->do("DROP TABLE IF EXISTS `NPL-pgfile-institution`");
+
+for my $tableinfo (@create_tables) {
+	my $tabname = $tableinfo->[0];
+	my $tabinit = $tableinfo->[1];
+	my $query = "DROP TABLE IF EXISTS `$tabname`";
+	$dbh->do($query);
+	$query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine";
+	$dbh->do($query);
+	if($lib eq 'OPL') {
+		$old_tabname = $tabname;
+		$old_tabname =~ s/OPL/NPL/;
+		$old_tabname =~ s/_/-/g;
+		$query = "DROP TABLE IF EXISTS `$old_tabname`";
+		$dbh -> do($query);
+	}
+}
+
+
+print "Mysql database reinitialized.\n";
+
+# From pgfile
+## DBchapter('Limits and Derivatives')
+## DBsection('Calculating Limits using the Limit Laws')
+## Date('6/3/2002')
+## Author('Tangan Gao')
+## Institution('csulb')
+## TitleText1('Calculus Early Transcendentals')
+## EditionText1('4')
+## AuthorText1('Stewart')
+## Section1('2.3')
+## Problem1('7')
+
+# Get an id, and add entry to the database if needed
+sub safe_get_id {
+	my $tablename = shift;
+	my $idname = shift;
+	my $whereclause = shift;
+	my $wherevalues = shift;
+	my $addifnew = shift;
+	my @insertvalues = @_;
+#print "\nCalled with $tablename, $idname, $whereclause, [".join(',', @$wherevalues)."], (".join(',', @insertvalues).")\n";
+	for my $j (0..$#insertvalues) {
+		$insertvalues[$j] =~ s/"/\\\"/g;
+	}
+
+	my $query = "SELECT $idname FROM `$tablename` ".$whereclause;
+	my $sth = $dbh->prepare($query);
+	$sth->execute(@$wherevalues);
+	my $idvalue, @row;
+	unless(@row = $sth->fetchrow_array()) {
+		return 0 unless $addifnew;
+		for my $j (0..$#insertvalues) {
+			#print "Looking at ".$insertvalues[$j]."\n";
+			if ($insertvalues[$j] ne "") {
+				$insertvalues[$j] = '"'.$insertvalues[$j].'"';
+			} else {
+				$insertvalues[$j] = NULL;
+			}
+		}
+		$dbh->do("INSERT INTO `$tablename` VALUES(". join(',',@insertvalues) .")");
+		dbug "INSERT INTO $tablename VALUES( ".join(',',@insertvalues).")\n";
+		$sth = $dbh->prepare($query);
+		$sth->execute(@$wherevalues);
+		@row = $sth->fetchrow_array();
+	}
+	$idvalue = $row[0];
+	return($idvalue);
+}
+
+sub isvalid {
+	my $tags = shift;
+	if(not defined $taxo->{$tags->{DBsubject}}) {
+		print "\nInvalid subject ".$tags->{DBsubject}."\n";
+		return 0;
+	}
+	if(not ($tags->{DBchapter} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}) {
+		print "\nInvalid chapter ".$tags->{DBchapter}."\n";
+		return 0;
+	}
+	if(not ($tags->{DBsection} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}) {
+		print "\nInvalid section ".$tags->{DBsection}."\n";
+		return 0;
+	}
+	return 1;
+}
+
+my ($name,$pgfile,$pgpath);
+
+#### First read in textbook information
+
+if(open(IN, "$libraryRoot/Textbooks")) {
+	print "Reading in textbook data from Textbooks in the library $libraryRoot.\n";
+	my %textinfo = ( TitleText => '', EditionText =>'', AuthorText=>'');
+	my $bookid = undef;
+	while (my $line = <IN>) {
+		$line =~ s|#*$||;
+		if($line =~ /^\s*(.*?)\s*>>>\s*(.*?)\s*$/) { # Should have chapter or section information
+			my $chapsec = $1;
+			my $title = $2;
+			if($chapsec=~ /(\d+)\.(\d+)/) { # We have a section
+				if(defined($bookid)) {
+					my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\"";
+					my $chapid = $dbh->selectrow_array($query);
+					if(defined($chapid)) {
+						my $sectid = safe_get_id($tables{section}, 'section_id',
+							qq(WHERE chapter_id = ? and name = ?), [$chapid, $title], 1, "", $chapid, $2, $title, "");
+					} else {
+						print "Cannot enter section $chapsec because textbook information is missing the chapter entry\n";
+					}
+				} else {
+					print "Cannot enter section $chapsec because textbook information is incomplete\n";
+				}
+			} else { # We have a chapter entry
+				if(defined($bookid)) {
+					my $chapid = safe_get_id($tables{chapter}, 'chapter_id',
+						qq(WHERE textbook_id = ? AND number = ?), [$bookid, $chapsec], 1, "", $bookid, $chapsec, $title, "");
+
+						# Add dummy section entry for problems tagged to the chapter
+						# without a section
+						$query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapid\" AND number = -1";
+						my $sectid = $dbh->selectrow_array($query);
+						if (!defined($sectid)) {
+							$dbh->do("INSERT INTO `$tables{section}`
+						VALUES(
+							NULL,
+							\"$chapid\",
+							\"-1\",
+							\"\",
+							NULL
+						)"
+								);
+						dbug "INSERT INTO section VALUES(\"\", \"$chapid\", \"-1\", \"\", \"\" )\n";
+						}
+				} else {
+					print "Cannot enter chapter $chapsec because textbook information is incomplete\n";
+				}
+			}
+		} elsif($line =~ /^\s*(TitleText|EditionText|AuthorText)\(\s*'(.*?)'\s*\)/) {
+			# Textbook information, maybe new
+			my $type = $1;
+			if(defined($textinfo{$type})) { # signals new text
+				%textinfo = ( TitleText => undef, 
+							  EditionText =>undef, 
+							  AuthorText=> undef);
+				$textinfo{$type} = $2;
+				$bookid = undef;
+			} else {
+				$textinfo{$type} = $2;
+				if(defined($textinfo{TitleText}) and 
+				   defined($textinfo{AuthorText}) and 
+				   defined($textinfo{EditionText})) {
+					my $query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$textinfo{TitleText}\" AND edition = \"$textinfo{EditionText}\" AND author=\"$textinfo{AuthorText}\"";
+					$bookid = $dbh->selectrow_array($query);
+					if (!defined($bookid)) {
+						$dbh->do("INSERT INTO `$tables{textbook}`
+					VALUES(
+						NULL,
+						\"$textinfo{TitleText}\",
+						\"$textinfo{EditionText}\",
+						\"$textinfo{AuthorText}\",
+						NULL,
+						NULL,
+						NULL
+					)"
+							);
+						dbug "INSERT INTO textbook VALUES( \"\", \"$textinfo{TitleText}\", \"$textinfo{EditionText}\", \"$textinfo{AuthorText}\", \"\", \"\", \"\" )\n";
+						$bookid = $dbh->selectrow_array($query);
+					}
+				}
+			}
+		}
+	}
+	close(IN);
+} else {
+	print "Textbooks file was not found in library $libraryRoot. If the path to the problem library doesn't seem
+	correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}).  If that is correct then
+	updating from git should download the Textbooks file.\n";
+}
+#### End of textbooks
+
+#### Next read in the taxonomy
+my $clsep = '<<<';
+my $clinner = '__';
+my @cllist = ();
+# Record full taxonomy for tagging menus (does not include cross-lists)
+my $tagtaxo = [];
+my ($chaplist, $seclist) = ([],[]);
+
+my $canopenfile = 0;
+if(open(IN, "$libraryRoot/Taxonomy2")) {
+	print "Reading in OPL taxonomy from Taxonomy2 in the library $libraryRoot.\n";
+	$canopenfile = 1;
+} elsif(open(IN, "$libraryRoot/Taxonomy")) {
+	print "Reading in OPL taxonomy from Taxonomy in the library $libraryRoot.\n";
+	$canopenfile = 1;
+} else {
+	print "Taxonomy file was not found in library $libraryRoot. If the path to the problem library doesn't seem
+	correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}).  If that is correct then
+	updating from git should download the Taxonomy file.\n";
+} 
+
+# Taxonomy is a subset of Taxonomy2, so we can use the same code either way
+if($canopenfile) {
+	my ($cursub,$curchap); # these are strings
+	my ($subj, $chap, $sect); # these are indeces
+	while(my $line = <IN>) {
+		$line =~ /^(\t*)/;
+		my $count = length($1);
+		my $oktag = 1;
+		chomp($line);
+		if($line =~ m/$clsep/) {
+			$oktag = 0;
+			my @cross = split $clsep, $line;
+			@cross = map(trim($_), @cross);
+			if(scalar(@cross) > 1) {
+				push @cllist, [join($clinner, ($cursub,$curchap,$cross[0])) ,$cross[1]];
+			}
+			$line = $cross[0];
+		}
+		$line = trim($line);
+
+		# We put the line in the database in all cases
+		# but crosslists are not put in the heierarchy of legal tags
+		# instead they go in a list of crosslists to deal with after
+		# the full taxonomy is read in
+		if($count == 0) { #DBsubject
+			$cursub = $line;
+			if($oktag) {
+				$taxo->{$line} = {};
+				($chaplist, $seclist) = ([],[]);
+				push @{$tagtaxo}, {name=>$line, subfields=>$chaplist};
+			}
+			$subj = safe_get_id($tables{dbsubject}, 'DBsubject_id',
+				qq(WHERE name = ?), [$line], 1, "", $line);
+		} elsif($count == 1) { #DBchapter
+			if($oktag) {
+				$taxo->{$cursub}->{$line} = {};
+				$seclist=[];
+				push @{$chaplist}, {name=>$line, subfields=>$seclist};
+			}
+			$curchap = $line;
+			$chap = safe_get_id($tables{dbchapter}, 'DBchapter_id',
+				qq(WHERE name = ? and DBsubject_id = ?), [$line, $subj], 1, "", $line, $subj);
+		} else { #DBsection
+			if($oktag) {
+				$taxo->{$cursub}->{$curchap}->{$line} = [];
+				push @{$seclist}, {name=>$line};
+			}
+			$sect = safe_get_id($tables{dbsection}, 'DBsection_id',
+				qq(WHERE name = ? and DBchapter_id = ?), [$line, $chap], 1, "", $line, $chap);
+		}
+	}
+	close(IN);
+}
+#### End of taxonomy/taxonomy2
+
+#### Save the official taxonomy in json format
+my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
+my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json";
+open(OUTF, ">$file") or die "Cannot open $file";
+print OUTF to_json($tagtaxo,{pretty=>1}) or die "Cannot write to $file";
+close(OUTF);
+print "Saved taxonomy to $file.\n";
+
+#### Now deal with cross-listed sections
+for my $clinfo (@cllist) {
+	my @scs = split /$clinner/, $clinfo->[1];
+	if(defined $taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}) {
+			push @{$taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}}, $clinfo->[0];
+	} else {
+		print "Faulty cross-list: pointing to $scs[0] / $scs[1] / $scs[2]\n";
+	}
+}
+
+print "Converting data from tagged pgfiles into mysql.\n";
+print "Number of files processed:\n";
+
+#### Now search for tagged problems
+#recursive search for all pg files
+
+File::Find::find({ wanted => \&pgfiles, follow_fast=> 1}, $libraryRoot);
+
+sub trim {
+	my $str = shift;
+	$str =~ s/^\s+//;
+	$str =~ s/\s+$//;
+	return $str;
+}
+
+sub kwtidy {
+	my $s = shift;
+	$s =~ s/\W//g;
+	$s =~ s/_//g;
+	$s = lc($s);
+	return($s);
+}
+
+sub keywordcleaner {
+	my $string = shift;
+	my @spl1 = split /,/, $string;
+	my @spl2 = map(kwtidy($_), @spl1);
+	return(@spl2);
+}
+
+# Save on passing these values around
+my %textinfo;
+
+# Initialize, if needed more text-info information;
+sub maybenewtext {
+	my $textno = shift;
+	return if defined($textinfo{$textno});
+	# So, not defined yet
+	$textinfo{$textno} = { title => '', author =>'', edition =>'',
+						   section => '', chapter =>'', problems => [] };
+}
+
+# process each file returned by the find command.
+sub pgfiles {
+	my $name = $File::Find::name;
+	my ($text, $edition, $textauthor, $textsection, $textproblem);
+	%textinfo=();
+	my @textproblems = (-1);
+#print "\n$name";
+
+	if ($name =~ /swf$/) { # Found a flash applet
+		my $applet_file = basename($name);
+		symlink($name,$ce->{webworkDirs}->{htdocs}."/applets/".$applet_file);
+	}
+
+	if ($name =~ /\.pg$/) {
+		$pgfile = basename($name);
+		$pgpath = dirname($name);
+		$cnt2++;
+		printf("%6d", $cnt2) if(($cnt2 % 100) == 0);
+		print "\n" if(($cnt2 % 1000) == 0);
+
+		$pgpath =~ s|^$libraryRoot/||;
+		my $tags = WeBWorK::Utils::Tags->new($name);
+
+		if ($tags->istagged()) {
+			# Fill in missing data with Misc. instead of blank
+			print "\nNO SUBJECT $name\n" unless ($tags->{DBsubject} =~ /\S/);
+
+			$tags->{DBchapter} = 'Misc.' unless $tags->{DBchapter} =~ /\S/;
+			$tags->{DBsection} = 'Misc.' unless $tags->{DBsection} =~ /\S/;
+
+			# DBsubject table
+
+			if(isvalid($tags)) {
+				my $DBsubject_id = safe_get_id($tables{dbsubject}, 'DBsubject_id',
+					qq(WHERE BINARY name = ?), [$tags->{DBsubject}], 1, "", $tags->{DBsubject});
+				if(not $DBsubject_id) {
+					print "\nInvalid subject '$tags->{DBsubject}' in $name\n";
+					next;
+				}
+
+				# DBchapter table
+
+				$DBchapter_id = safe_get_id($tables{dbchapter}, 'DBchapter_id',
+					qq(WHERE BINARY name = ? and DBsubject_id = ?), [$tags->{DBchapter}, $DBsubject_id], 1, "", $tags->{DBchapter}, $DBsubject_id);
+				if(not $DBchapter_id) {
+					print "\nInvalid chapter '$tags->{DBchapter}' in $name\n";
+					next;
+				}
+
+				# DBsection table
+
+				$aDBsection_id = safe_get_id($tables{dbsection}, 'DBsection_id',
+					qq(WHERE BINARY name = ? and DBchapter_id = ?), [$tags->{DBsection}, $DBchapter_id], 1, "", $tags->{DBsection}, $DBchapter_id);
+				if(not $aDBsection_id) {
+					print "\nInvalid section '$tags->{DBsection}' in $name\n";
+					next;
+				}
+			} else { # tags are not valid, error printed by validation part
+				print "File $name\n";
+				next;
+			}
+
+			my @DBsection_ids=($aDBsection_id);
+			# Now add crosslisted section
+			my @CL_array = @{$taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}};
+			for my $clsect (@CL_array) {
+				my @np = split /$clinner/, $clsect;
+				@np = map(trim($_), @np);
+				my $new_dbsubj_id = safe_get_id($tables{dbsubject}, 'DBsubject_id',
+					qq(WHERE name = ?), [$np[0]], 1, "", $np[0]);
+				my $new_dbchap_id = safe_get_id($tables{dbchapter}, 'DBchapter_id',
+					qq(WHERE name = ? and DBsubject_id = ?), [$np[1], $new_dbsubj_id], 1, "", $np[1], $new_dbsubj_id);
+				my $new_dbsect_id = safe_get_id($tables{dbsection}, 'DBsection_id',
+					qq(WHERE name = ? and DBchapter_id = ?), [$np[2], $new_dbchap_id], 1, "", $np[2], $new_dbchap_id);
+				push @DBsection_ids, $new_dbsect_id;
+			}
+
+			# author table
+
+			$tags->{Author} =~ /(.*?)\s(\w+)\s*$/;
+			my $firstname = $1;
+			my $lastname = $2;
+			#remove leading and trailing spaces from firstname, which includes any middle name too.
+			$firstname =~ s/^\s*//;
+			$firstname =~ s/\s*$//;
+			$lastname =~ s/^\s*//;
+			$lastname =~ s/\s*$//;
+			my $author_id = 0;
+			if($lastname) {
+				$author_id = safe_get_id($tables{author}, 'author_id',
+					qq(WHERE lastname = ? AND firstname = ?), [$lastname, $firstname], 1, "", $tags->{Institution}, $lastname, $firstname,"");
+			}
+
+			# path table
+
+			my $path_id = safe_get_id($tables{path}, 'path_id',
+				qq(WHERE path = ?), [$pgpath], 1, "", $pgpath, "", "");
+
+			# pgfile table -- set 4 defaults first
+
+			## TODO this is where we have to deal with crosslists, and pgfileid
+			##      will be an array of id's
+			##      Make an array of DBsection-id's above
+
+			my $level = $tags->{Level} || 0;
+			# Default language is English
+			my $lang = $tags->{Language} || "en";
+			my $mathobj = $tags->{MO} || 0;
+			my $static = $tags->{Static} || 0;
+
+			my @pgfile_ids = ();
+
+			for my $DBsection_id (@DBsection_ids) {
+				my $pgfile_id = safe_get_id($tables{pgfile}, 'pgfile_id',
+					qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? ), [$pgfile, $path_id, $DBsection_id], 1, "", $DBsection_id, $author_id, $tags->{Institution}, $path_id, $pgfile, 0, $level, $lang, $static, $mathobj);
+				push @pgfile_ids, $pgfile_id;
+			}
+#if (scalar(@pgfile_ids)>1) {
+ # print "\npg ".join(', ', @pgfile_ids)."\n";
+#}
+
+			# morelt table
+
+			my $morelt_id;
+			if($tags->{MLT}) {
+				for my $DBsection_id (@DBsection_ids) {
+					$morelt_id = safe_get_id($tables{morelt}, 'morelt_id',
+						qq(WHERE name = ?), [$tags->{MLT}], 1, "", $tags->{MLT}, $DBsection_id, "");
+
+					for my $pgfile_id (@pgfile_ids) {
+						$dbh->do("UPDATE `$tables{pgfile}` SET
+							morelt_id = \"$morelt_id\" WHERE pgfile_id = \"$pgfile_id\" ");
+
+						dbug "UPDATE pgfile morelt_id for $pgfile_id to $morelt_id\n";
+						if($tags->{MLTleader}) {
+							$dbh->do("UPDATE `$tables{morelt}` SET
+									leader = \"$pgfile_id\" WHERE morelt_id = \"$morelt_id\" ");
+							dbug "UPDATE morelt leader for $morelt_id to $pgfile_id\n";
+						}
+					}
+				}
+			}
+
+			# keyword table, and problem_keyword many-many table
+
+			foreach my $keyword (@{$tags->{keywords}}) {
+				$keyword =~ s/[\'\"]//g;
+				$keyword = kwtidy($keyword);
+                                # skip it if it is empty
+                                next unless $keyword;
+				my $keyword_id = safe_get_id($tables{keyword}, 'keyword_id',
+					qq(WHERE keyword = ?), [$keyword], 1, "", $keyword);
+
+				for my $pgfile_id (@pgfile_ids) {
+					$query = "SELECT pgfile_id FROM `$tables{pgfile_keyword}` WHERE keyword_id = \"$keyword_id\" and pgfile_id=\"$pgfile_id\"";
+					my $ok = $dbh->selectrow_array($query);
+					if (!defined($ok)) {
+						$dbh->do("INSERT INTO `$tables{pgfile_keyword}`
+							VALUES(
+								\"$pgfile_id\",
+								\"$keyword_id\"
+							)"
+								);
+						dbug "INSERT INTO pgfile_keyword VALUES( \"$pgfile_id\", \"$keyword_id\" )\n";
+					}
+				}
+			}					#end foreach keyword
+
+			# Textbook section
+			# problem table contains textbook problems
+
+			for my $texthashref (@{$tags->{textinfo}}) {
+
+				# textbook table
+
+				$text = $texthashref->{TitleText};
+				$edition = $texthashref->{EditionText} || 0;
+				$edition =~ s/[^\d\.]//g;
+				$textauthor = $texthashref->{AuthorText};
+				next unless($text and $textauthor);
+				my $chapnum = $texthashref->{chapter} || -1;
+				my $secnum = $texthashref->{section} || -1;
+				$query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$text\" AND edition = \"$edition\" AND author=\"$textauthor\"";
+				my $textbook_id = $dbh->selectrow_array($query);
+				if (!defined($textbook_id)) {
+ 				        # make sure edition is an integer
+				        $edition = 0 unless $edition;
+					$dbh->do("INSERT INTO `$tables{textbook}`
+					VALUES(
+						NULL,
+						\"$text\",
+						\"$edition\",
+						\"$textauthor\",
+						NULL,
+						NULL,
+						NULL
+					)"
+							);
+					dbug "INSERT INTO textbook VALUES( \"\", \"$text\", \"$edition\", \"$textauthor\", \"\", \"\", \"\" )\n";
+					dbug "\nLate add into $tables{textbook} \"$text\", \"$edition\", \"$textauthor\"\n", 1;
+					$textbook_id = $dbh->selectrow_array($query);
+				}
+
+				# chapter weak table of textbook
+				#
+				$query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$textbook_id\" AND number = \"$chapnum\"";
+				my $chapter_id = $dbh->selectrow_array($query);
+				if (!defined($chapter_id)) {
+					$dbh->do("INSERT INTO `$tables{chapter}`
+					VALUES(
+						NULL,
+						\"$textbook_id\",
+						\"".$chapnum."\",
+						\"$tags->{DBchapter}\",
+						NULL
+					)"
+							);
+					dbug "\nLate add into $tables{chapter} \"$text\", \"$edition\", \"$textauthor\", $chapnum $tags->{chapter} from $name\n", 1;
+					dbug "INSERT INTO chapter VALUES(\"\", \"$textbook_id\", \"".$chapnum."\", \"$tags->{DBchapter}\", \"\" )\n";
+					$chapter_id = $dbh->selectrow_array($query);
+				}
+
+				# section weak table of textbook
+				#
+				$tags->{DBsection} = '' if ($secnum < 0);
+				$query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapter_id\" AND number = \"$secnum\"";
+				my $section_id = $dbh->selectrow_array($query);
+				if (!defined($section_id)) {
+					$dbh->do("INSERT INTO `$tables{section}`
+					VALUES(
+						NULL,
+						\"$chapter_id\",
+						\"$secnum\",
+						\"$tags->{DBsection}\",
+						NULL
+					)"
+							);
+					dbug "INSERT INTO section VALUES(\"\", \"$textbook_id\", \"$secnum\", \"$tags->{DBsection}\", \"\" )\n";
+					dbug "\nLate add into $tables{section} \"$text\", \"$edition\", \"$textauthor\", $secnum $tags->{DBsection} from $name\n", 1;
+					$section_id = $dbh->selectrow_array($query);
+				}
+
+				@textproblems = @{$texthashref->{problems}};
+				for my $tp (@textproblems) {
+					$query = "SELECT problem_id FROM `$tables{problem}` WHERE section_id = \"$section_id\" AND number = \"$tp\"";
+					my $problem_id = $dbh->selectrow_array($query);
+					if (!defined($problem_id)) {
+						$dbh->do("INSERT INTO `$tables{problem}`
+					VALUES(
+						NULL,
+						\"$section_id\",
+						\"$tp\",
+						NULL
+					)"
+								);
+						dbug "INSERT INTO problem VALUES( \"\", \"$section_id\", \"$tp\", \"\" )\n";
+						$problem_id = $dbh->selectrow_array($query);
+					}
+
+					# pgfile_problem table associates pgfiles with textbook problems
+					for my $pgfile_id (@pgfile_ids) {
+						$query = "SELECT problem_id FROM `$tables{pgfile_problem}` WHERE problem_id = \"$problem_id\" AND pgfile_id = \"$pgfile_id\"";
+						my $pg_problem_id = $dbh->selectrow_array($query);  
+						if (!defined($pg_problem_id)) {
+							$dbh->do("INSERT INTO `$tables{pgfile_problem}`
+						VALUES(
+							\"$pgfile_id\",
+							\"$problem_id\"
+						)"
+									);
+							dbug "INSERT INTO pgfile_problem VALUES( \"$pgfile_id\", \"$problem_id\" )\n";
+						}
+					}
+				}
+
+				#reset tag vars, they may not match the next text/file
+				$textauthor=""; $textsection="";
+			}
+		} else { # This file was not tagged
+			# Message if not a pointer
+			# print STDERR "File $name is not tagged\n" if not $tags->isplaceholder();
+			;
+		}
+	}
+}
+
+print "\n\n";
+
+# Now prune away DBsection, etc, which do not appear in any files
+#%my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\"";
+#%my $chapid = $dbh->selectrow_array($query);
+
+#select dbs.DBsection_id from OPL_DBsection dbs;
+#select COUNT(*) from OPL_pgfile where DBsection_id=857;
+
+my $dbsects = $dbh->selectall_arrayref("SELECT DBsection_id from `$tables{dbsection}`");
+for my $sect (@{$dbsects}) {
+	$sect = $sect->[0];
+	my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{pgfile}` WHERE DBsection_id=$sect");
+	if(scalar(@{$srar})==0) {
+		$dbh->do("DELETE FROM `$tables{dbsection}` WHERE DBsection_id=$sect");
+	}
+}
+
+my $dbchaps = $dbh->selectall_arrayref("SELECT DBchapter_id from `$tables{dbchapter}`");
+for my $chap (@{$dbchaps}) {
+	$chap = $chap->[0];
+	my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chap");
+	if(scalar(@{$srar})==0) {
+		$dbh->do("DELETE FROM `$tables{dbchapter}` WHERE DBchapter_id=$chap");
+	}
+}
+
+# Now run the script build-library-tree
+# This is used to create the file library-tree.json which can be used to 
+# load in subject-chapter-section information for the OPL
+
+use strict;
+use warnings;
+use JSON;
+
+
+my $tree;  # the library subject tree will be stored as arrays of objects. 
+
+my $sth = $dbh->prepare("select * from OPL_DBsubject");
+$sth->execute;
+
+my @subjects = ();
+my @subject_names = ();
+while ( my @row = $sth->fetchrow_array ) {
+	push(@subjects,$row[0]);
+	push(@subject_names,$row[1]);
+  }
+
+
+my @subject_tree;  # array to store the individual library tree for each subject 
+
+foreach my $i (0..$#subjects){
+
+	my $subject_row = $subjects[$i];
+	my $subject_name = $subject_names[$i];
+
+	my $sth = $dbh->prepare("select * from OPL_DBchapter where DBsubject_id = $subject_row;");
+	$sth->execute;
+
+	my @chapters = ();
+	my @chapter_names = ();
+	while ( my @row = $sth->fetchrow_array ) {
+		push(@chapters,$row[0]);
+		push(@chapter_names,$row[1]);
+	  }
+
+
+	my @chapter_tree; # array to store the individual library tree for each chapter
+
+	foreach my $j (0..$#chapters) {
+		my $chapter_row = $chapters[$j];
+		my $chapter_name = $chapter_names[$j];
+		my $sth = $dbh->prepare("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chapter_row");
+		$sth->execute;
+
+		my @subfields = ();
+		while ( my @row = $sth->fetchrow_array ) {
+			my $section_name;
+			$section_name->{name} = $row[1];
+			my $clone = { %{ $section_name } };  # need to clone it before pushing into the @subfields array.
+		    push(@subfields,$clone);
+	    }
+
+		my $chapter_tree;
+		$chapter_tree->{name} = $chapter_name;
+		$chapter_tree->{subfields} = \@subfields;
+
+		my $clone = { %{ $chapter_tree } };  # need to clone it before pushing into the @chapter_tree array.
+		push(@chapter_tree,$clone);
+
+
+
+	}
+
+	my $subject_tree; 
+	$subject_tree->{name} = $subject_name;
+	$subject_tree->{subfields} = \@chapter_tree; 
+
+	my $clone = { % {$subject_tree}};
+	push (@subject_tree, $clone);
+}
+
+build_library_directory_tree($ce);
+build_library_subject_tree($ce,$dbh);
+build_library_textbook_tree($ce,$dbh);
+
+$dbh->disconnect;
+
+if ($ce->{problemLibrary}{showLibraryLocalStats} ||
+    $ce->{problemLibrary}{showLibraryGlobalStats}) {
+  print "\nUpdating Library Statistics.\n";
+  do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics';
+}
+
+
+print "\nDone.\n";
diff --git a/bin/OPL-update-utfmb4 b/bin/OPL-update-utfmb4
new file mode 100755
index 0000000000..811c809ba4
--- /dev/null
+++ b/bin/OPL-update-utfmb4
@@ -0,0 +1,979 @@
+#!/usr/bin/env perl
+
+# This is the script formerly known as loadDB2, and then known as NPL-update.  
+
+# It is used to update
+# the database when it comes to the WeBWorK Open Problem Library (OPL).
+# This should be run after doing a git clone or pull for the OPL
+# files.
+
+# In order for this script to work:
+#   1) The OPL downloaded to your machine (the .pg files)
+#   2) The environment variable WEBWORK_ROOT needs to be 
+#      correctly defined (as with other scripts here).
+#   3) Configuration for the OPL in site.conf needs to be
+#      done (basically just setting the path to the OPL files).
+
+#use strict;
+use File::Find;
+use File::Find::Rule;
+use File::Basename;
+use Cwd;
+use DBI;
+
+ #(maximum varchar length is 255 for mysql version < 5.0.3.  
+ #You can increase path length to  4096 for mysql > 5.0.3)
+
+BEGIN {
+        die "WEBWORK_ROOT not found in environment.\n"
+                unless exists $ENV{WEBWORK_ROOT};
+	# Unused variable, but define it to avoid an error message.
+	$WeBWorK::Constants::WEBWORK_DIRECTORY = '';
+}
+
+# Taxonomy global variables
+# Make a hash of hashes of hashes to record what is legal
+# Also create list for json file
+my $taxo={};
+#my $taxsubs = [];
+
+
+### Data for creating the database tables
+
+my %OPLtables = (
+ dbsubject => 'OPL_DBsubject',
+ dbchapter => 'OPL_DBchapter',
+ dbsection => 'OPL_DBsection',
+ author => 'OPL_author',
+ path => 'OPL_path',
+ pgfile => 'OPL_pgfile',
+ keyword => 'OPL_keyword',
+ pgfile_keyword => 'OPL_pgfile_keyword',
+ textbook => 'OPL_textbook',
+ chapter => 'OPL_chapter',
+ section => 'OPL_section',
+ problem => 'OPL_problem',
+ morelt => 'OPL_morelt',
+ pgfile_problem => 'OPL_pgfile_problem',
+);
+
+
+my %NPLtables = (
+ dbsubject => 'NPL-DBsubject',
+ dbchapter => 'NPL-DBchapter',
+ dbsection => 'NPL-DBsection',
+ author => 'NPL-author',
+ path => 'NPL-path',
+ pgfile => 'NPL-pgfile',
+ keyword => 'NPL-keyword',
+ pgfile_keyword => 'NPL-pgfile-keyword',
+ textbook => 'NPL-textbook',
+ chapter => 'NPL-chapter',
+ section => 'NPL-section',
+ problem => 'NPL-problem',
+ morelt => 'NPL-morelt',
+ pgfile_problem => 'NPL-pgfile-problem',
+);
+
+
+# Get database connection
+
+use lib "$ENV{WEBWORK_ROOT}/lib";
+use lib "$ENV{WEBWORK_ROOT}/bin";
+use WeBWorK::CourseEnvironment;
+use WeBWorK::Utils::Tags;
+use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/;
+
+my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
+my $dbh = DBI->connect(
+        $ce->{problemLibrary_db}->{dbsource},
+        $ce->{problemLibrary_db}->{user},
+        $ce->{problemLibrary_db}->{passwd},
+        {
+                PrintError => 0,
+                RaiseError => 1,
+		mysql_enable_utf8mb4 => 1,
+        },
+);
+
+$dbh->prepare("SET NAMES 'utf8mb4'")->execute();
+
+my $libraryRoot = $ce->{problemLibrary}->{root};
+$libraryRoot =~ s|/+$||;
+my $libraryVersion = $ce->{problemLibrary}->{version};
+my $db_storage_engine = $ce->{problemLibrary_db}->{storage_engine};
+
+my $verbose = 0;
+my $cnt2 = 0;
+# Can force library version
+$libraryVersion = $ARGV[0] if(@ARGV);
+
+# auto flush printing
+my $old_fh = select(STDOUT);
+$| = 1;
+select($old_fh);
+
+sub dbug {
+	my $msg = shift;
+	my $insignificance = shift || 2;
+	print $msg if($verbose>=$insignificance);
+}
+
+##Figure out which set of tables to use
+
+my %tables;
+if($libraryVersion eq '2.5') {
+	%tables = %OPLtables;
+	my $lib = 'OPL';
+	warn "Library version is $libraryVersion; using OPLtables!\n";
+} else {
+	%tables = %NPLtables;
+	my $lib = 'NPL';
+	print "Library version is $libraryVersion; NPLtables! \n";
+}
+
+@create_tables = (
+[$tables{dbsubject}, '
+	DBsubject_id int(15) NOT NULL auto_increment,
+	name varchar(245) NOT NULL,
+	KEY DBsubject (name),
+	PRIMARY KEY (DBsubject_id)
+'],
+[$tables{dbchapter}, '
+	DBchapter_id int(15) NOT NULL auto_increment,
+	name varchar(245) NOT NULL,
+	DBsubject_id int(15) DEFAULT 0 NOT NULL,
+	KEY DBchapter (name),
+	KEY (DBsubject_id),
+	PRIMARY KEY (DBchapter_id)
+'],
+[$tables{dbsection}, '
+	DBsection_id int(15) NOT NULL auto_increment,
+	name varchar(245) NOT NULL,
+	DBchapter_id int(15) DEFAULT 0 NOT NULL,
+	KEY DBsection (name),
+	KEY (DBchapter_id),
+	PRIMARY KEY (DBsection_id)
+'],
+[$tables{author}, '
+	author_id int (15) NOT NULL auto_increment,
+	institution tinyblob,
+	lastname varchar (255) NOT NULL,
+	firstname varchar (255) NOT NULL,
+	email varchar (255),
+	KEY author (lastname(100), firstname(100)),
+	PRIMARY KEY (author_id)
+'],
+[$tables{path}, '
+	path_id int(15) NOT NULL auto_increment,
+	path varchar(245) NOT NULL,
+	machine varchar(255),
+	user varchar(255),
+	KEY (path),
+	PRIMARY KEY (path_id)
+'],
+[$tables{pgfile}, '
+	pgfile_id int(15) NOT NULL auto_increment,
+	DBsection_id int(15) NOT NULL,
+	author_id int(15),
+	institution tinyblob,
+	path_id int(15) NOT NULL,
+	filename varchar(255) NOT NULL,
+	morelt_id int(127) DEFAULT 0 NOT NULL,
+	level int(15),
+	language varchar(255),
+	static TINYINT,
+	MO TINYINT,
+	PRIMARY KEY (pgfile_id)
+'],
+[$tables{keyword}, '
+	keyword_id int(15) NOT NULL auto_increment,
+	keyword varchar(256) NOT NULL,
+	KEY (keyword),
+	PRIMARY KEY (keyword_id)
+'],
+[$tables{pgfile_keyword}, '
+	pgfile_id int(15) DEFAULT 0 NOT NULL,
+	keyword_id int(15) DEFAULT 0 NOT NULL,
+	KEY pgfile_keyword (keyword_id, pgfile_id),
+	KEY pgfile (pgfile_id)
+'],
+[$tables{textbook}, '
+	textbook_id int (15) NOT NULL auto_increment,
+	title varchar (255) NOT NULL,
+	edition int (15) DEFAULT 0 NOT NULL,
+	author varchar (255) NOT NULL,
+	publisher varchar (255),
+	isbn char (15),
+	pubdate varchar (255),
+	PRIMARY KEY (textbook_id)
+'],
+[$tables{chapter}, '
+	chapter_id int (15) NOT NULL auto_increment,
+	textbook_id int (15),
+	number int(3),
+	name varchar(245) NOT NULL,
+	page int(4),
+	KEY (textbook_id, name),
+	KEY (number),
+	PRIMARY KEY (chapter_id)
+'],
+[$tables{section}, '
+	section_id int(15) NOT NULL auto_increment,
+	chapter_id int (15),
+	number int(3),
+	name varchar(245) NOT NULL,
+	page int(4),
+	KEY (chapter_id, name),
+	KEY (number),
+	PRIMARY KEY section (section_id)
+'],
+[$tables{problem}, '
+	problem_id int(15) NOT NULL auto_increment,
+	section_id int(15),
+	number int(4) NOT NULL,
+	page int(4),
+	#KEY (page, number),
+	KEY (section_id),
+	PRIMARY KEY (problem_id)
+'],
+[$tables{morelt}, '
+	morelt_id int(15) NOT NULL auto_increment,
+	name varchar(245) NOT NULL,
+	DBsection_id int(15),
+	leader int(15), # pgfile_id of the MLT leader
+	KEY (name),
+	PRIMARY KEY (morelt_id)
+'],
+[$tables{pgfile_problem}, '
+	pgfile_id int(15) DEFAULT 0 NOT NULL,
+	problem_id int(15) DEFAULT 0 NOT NULL,
+	PRIMARY KEY (pgfile_id, problem_id)
+']);
+
+### End of database data
+
+## Resetting the database tables.
+#  First take care of tables which are no longer used
+
+$dbh->do("DROP TABLE IF EXISTS `NPL-institution`");
+$dbh->do("DROP TABLE IF EXISTS `NPL-pgfile-institution`");
+
+for my $tableinfo (@create_tables) {
+	my $tabname = $tableinfo->[0];
+	my $tabinit = $tableinfo->[1];
+	my $query = "DROP TABLE IF EXISTS `$tabname`";
+	$dbh->do($query);
+	$query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine";
+	$dbh->do($query);
+	if($lib eq 'OPL') {
+		$old_tabname = $tabname;
+		$old_tabname =~ s/OPL/NPL/;
+		$old_tabname =~ s/_/-/g;
+		$query = "DROP TABLE IF EXISTS `$old_tabname`";
+		$dbh -> do($query);
+	}
+}
+
+
+print "Mysql database reinitialized.\n";
+
+# From pgfile
+## DBchapter('Limits and Derivatives')
+## DBsection('Calculating Limits using the Limit Laws')
+## Date('6/3/2002')
+## Author('Tangan Gao')
+## Institution('csulb')
+## TitleText1('Calculus Early Transcendentals')
+## EditionText1('4')
+## AuthorText1('Stewart')
+## Section1('2.3')
+## Problem1('7')
+
+# Get an id, and add entry to the database if needed
+sub safe_get_id {
+	my $tablename = shift;
+	my $idname = shift;
+	my $whereclause = shift;
+	my $wherevalues = shift;
+	my $addifnew = shift;
+	my @insertvalues = @_;
+#print "\nCalled with $tablename, $idname, $whereclause, [".join(',', @$wherevalues)."], (".join(',', @insertvalues).")\n";
+	for my $j (0..$#insertvalues) {
+		$insertvalues[$j] =~ s/"/\\\"/g;
+	}
+
+	my $query = "SELECT $idname FROM `$tablename` ".$whereclause;
+	my $sth = $dbh->prepare($query);
+	$sth->execute(@$wherevalues);
+	my $idvalue, @row;
+	unless(@row = $sth->fetchrow_array()) {
+		return 0 unless $addifnew;
+		for my $j (0..$#insertvalues) {
+			#print "Looking at ".$insertvalues[$j]."\n";
+			if ($insertvalues[$j] ne "") {
+				$insertvalues[$j] = '"'.$insertvalues[$j].'"';
+			} else {
+				$insertvalues[$j] = NULL;
+			}
+		}
+		$dbh->do("INSERT INTO `$tablename` VALUES(". join(',',@insertvalues) .")");
+		dbug "INSERT INTO $tablename VALUES( ".join(',',@insertvalues).")\n";
+		$sth = $dbh->prepare($query);
+		$sth->execute(@$wherevalues);
+		@row = $sth->fetchrow_array();
+	}
+	$idvalue = $row[0];
+	return($idvalue);
+}
+
+sub isvalid {
+	my $tags = shift;
+	if(not defined $taxo->{$tags->{DBsubject}}) {
+		print "\nInvalid subject ".$tags->{DBsubject}."\n";
+		return 0;
+	}
+	if(not ($tags->{DBchapter} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}) {
+		print "\nInvalid chapter ".$tags->{DBchapter}."\n";
+		return 0;
+	}
+	if(not ($tags->{DBsection} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}) {
+		print "\nInvalid section ".$tags->{DBsection}."\n";
+		return 0;
+	}
+	return 1;
+}
+
+my ($name,$pgfile,$pgpath);
+
+#### First read in textbook information
+
+if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Textbooks")) {
+	print "Reading in textbook data from Textbooks in the library $libraryRoot.\n";
+	my %textinfo = ( TitleText => '', EditionText =>'', AuthorText=>'');
+	my $bookid = undef;
+	while (my $line = <IN>) {
+		$line =~ s|#*$||;
+		if($line =~ /^\s*(.*?)\s*>>>\s*(.*?)\s*$/) { # Should have chapter or section information
+			my $chapsec = $1;
+			my $title = $2;
+			if($chapsec=~ /(\d+)\.(\d+)/) { # We have a section
+				if(defined($bookid)) {
+					my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\"";
+					my $chapid = $dbh->selectrow_array($query);
+					if(defined($chapid)) {
+						my $sectid = safe_get_id($tables{section}, 'section_id',
+							qq(WHERE chapter_id = ? and name = ?), [$chapid, $title], 1, "", $chapid, $2, $title, "");
+					} else {
+						print "Cannot enter section $chapsec because textbook information is missing the chapter entry\n";
+					}
+				} else {
+					print "Cannot enter section $chapsec because textbook information is incomplete\n";
+				}
+			} else { # We have a chapter entry
+				if(defined($bookid)) {
+					my $chapid = safe_get_id($tables{chapter}, 'chapter_id',
+						qq(WHERE textbook_id = ? AND number = ?), [$bookid, $chapsec], 1, "", $bookid, $chapsec, $title, "");
+
+						# Add dummy section entry for problems tagged to the chapter
+						# without a section
+						$query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapid\" AND number = -1";
+						my $sectid = $dbh->selectrow_array($query);
+						if (!defined($sectid)) {
+							$dbh->do("INSERT INTO `$tables{section}`
+						VALUES(
+							NULL,
+							\"$chapid\",
+							\"-1\",
+							\"\",
+							NULL
+						)"
+								);
+						dbug "INSERT INTO section VALUES(\"\", \"$chapid\", \"-1\", \"\", \"\" )\n";
+						}
+				} else {
+					print "Cannot enter chapter $chapsec because textbook information is incomplete\n";
+				}
+			}
+		} elsif($line =~ /^\s*(TitleText|EditionText|AuthorText)\(\s*'(.*?)'\s*\)/) {
+			# Textbook information, maybe new
+			my $type = $1;
+			if(defined($textinfo{$type})) { # signals new text
+				%textinfo = ( TitleText => undef, 
+							  EditionText =>undef, 
+							  AuthorText=> undef);
+				$textinfo{$type} = $2;
+				$bookid = undef;
+			} else {
+				$textinfo{$type} = $2;
+				if(defined($textinfo{TitleText}) and 
+				   defined($textinfo{AuthorText}) and 
+				   defined($textinfo{EditionText})) {
+					my $query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$textinfo{TitleText}\" AND edition = \"$textinfo{EditionText}\" AND author=\"$textinfo{AuthorText}\"";
+					$bookid = $dbh->selectrow_array($query);
+					if (!defined($bookid)) {
+						$dbh->do("INSERT INTO `$tables{textbook}`
+					VALUES(
+						NULL,
+						\"$textinfo{TitleText}\",
+						\"$textinfo{EditionText}\",
+						\"$textinfo{AuthorText}\",
+						NULL,
+						NULL,
+						NULL
+					)"
+							);
+						dbug "INSERT INTO textbook VALUES( \"\", \"$textinfo{TitleText}\", \"$textinfo{EditionText}\", \"$textinfo{AuthorText}\", \"\", \"\", \"\" )\n";
+						$bookid = $dbh->selectrow_array($query);
+					}
+				}
+			}
+		}
+	}
+	close(IN);
+} else {
+	print "Textbooks file was not found in library $libraryRoot. If the path to the problem library doesn't seem
+	correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}).  If that is correct then
+	updating from git should download the Textbooks file.\n";
+}
+#### End of textbooks
+
+#### Next read in the taxonomy
+my $clsep = '<<<';
+my $clinner = '__';
+my @cllist = ();
+# Record full taxonomy for tagging menus (does not include cross-lists)
+my $tagtaxo = [];
+my ($chaplist, $seclist) = ([],[]);
+
+my $canopenfile = 0;
+if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy2")) {
+	print "Reading in OPL taxonomy from Taxonomy2 in the library $libraryRoot.\n";
+	$canopenfile = 1;
+} elsif(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy")) {
+	print "Reading in OPL taxonomy from Taxonomy in the library $libraryRoot.\n";
+	$canopenfile = 1;
+} else {
+	print "Taxonomy file was not found in library $libraryRoot. If the path to the problem library doesn't seem
+	correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}).  If that is correct then
+	updating from git should download the Taxonomy file.\n";
+} 
+
+# Taxonomy is a subset of Taxonomy2, so we can use the same code either way
+if($canopenfile) {
+	my ($cursub,$curchap); # these are strings
+	my ($subj, $chap, $sect); # these are indeces
+	while(my $line = <IN>) {
+		$line =~ /^(\t*)/;
+		my $count = length($1);
+		my $oktag = 1;
+		chomp($line);
+		if($line =~ m/$clsep/) {
+			$oktag = 0;
+			my @cross = split $clsep, $line;
+			@cross = map(trim($_), @cross);
+			if(scalar(@cross) > 1) {
+				push @cllist, [join($clinner, ($cursub,$curchap,$cross[0])) ,$cross[1]];
+			}
+			$line = $cross[0];
+		}
+		$line = trim($line);
+
+		# We put the line in the database in all cases
+		# but crosslists are not put in the heierarchy of legal tags
+		# instead they go in a list of crosslists to deal with after
+		# the full taxonomy is read in
+		if($count == 0) { #DBsubject
+			$cursub = $line;
+			if($oktag) {
+				$taxo->{$line} = {};
+				($chaplist, $seclist) = ([],[]);
+				push @{$tagtaxo}, {name=>$line, subfields=>$chaplist};
+			}
+			$subj = safe_get_id($tables{dbsubject}, 'DBsubject_id',
+				qq(WHERE name = ?), [$line], 1, "", $line);
+		} elsif($count == 1) { #DBchapter
+			if($oktag) {
+				$taxo->{$cursub}->{$line} = {};
+				$seclist=[];
+				push @{$chaplist}, {name=>$line, subfields=>$seclist};
+			}
+			$curchap = $line;
+			$chap = safe_get_id($tables{dbchapter}, 'DBchapter_id',
+				qq(WHERE name = ? and DBsubject_id = ?), [$line, $subj], 1, "", $line, $subj);
+		} else { #DBsection
+			if($oktag) {
+				$taxo->{$cursub}->{$curchap}->{$line} = [];
+				push @{$seclist}, {name=>$line};
+			}
+			$sect = safe_get_id($tables{dbsection}, 'DBsection_id',
+				qq(WHERE name = ? and DBchapter_id = ?), [$line, $chap], 1, "", $line, $chap);
+		}
+	}
+	close(IN);
+}
+#### End of taxonomy/taxonomy2
+
+#### Save the official taxonomy in json format
+my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
+my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json";
+open(OUTF, ">$file") or die "Cannot open $file";
+binmode(OUTF,':encoding(UTF-8)'); 
+print OUTF to_json($tagtaxo,{pretty=>1}) or die "Cannot write to $file";
+close(OUTF);
+print "Saved taxonomy to $file.\n";
+
+#### Now deal with cross-listed sections
+for my $clinfo (@cllist) {
+	my @scs = split /$clinner/, $clinfo->[1];
+	if(defined $taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}) {
+			push @{$taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}}, $clinfo->[0];
+	} else {
+		print "Faulty cross-list: pointing to $scs[0] / $scs[1] / $scs[2]\n";
+	}
+}
+
+print "Converting data from tagged pgfiles into mysql.\n";
+print "Number of files processed:\n";
+
+#### Now search for tagged problems
+#recursive search for all pg files
+
+File::Find::find({ wanted => \&pgfiles, follow_fast=> 1}, $libraryRoot);
+
+sub trim {
+	my $str = shift;
+	$str =~ s/^\s+//;
+	$str =~ s/\s+$//;
+	return $str;
+}
+
+sub kwtidy {
+	my $s = shift;
+	$s =~ s/\W//g;
+	$s =~ s/_//g;
+	$s = lc($s);
+	return($s);
+}
+
+sub keywordcleaner {
+	my $string = shift;
+	my @spl1 = split /,/, $string;
+	my @spl2 = map(kwtidy($_), @spl1);
+	return(@spl2);
+}
+
+# Save on passing these values around
+my %textinfo;
+
+# Initialize, if needed more text-info information;
+sub maybenewtext {
+	my $textno = shift;
+	return if defined($textinfo{$textno});
+	# So, not defined yet
+	$textinfo{$textno} = { title => '', author =>'', edition =>'',
+						   section => '', chapter =>'', problems => [] };
+}
+
+# process each file returned by the find command.
+sub pgfiles {
+	my $name = $File::Find::name;
+	my ($text, $edition, $textauthor, $textsection, $textproblem);
+	%textinfo=();
+	my @textproblems = (-1);
+#print "\n$name";
+
+	if ($name =~ /swf$/) { # Found a flash applet
+		my $applet_file = basename($name);
+		symlink($name,$ce->{webworkDirs}->{htdocs}."/applets/".$applet_file);
+	}
+
+	if ($name =~ /\.pg$/) {
+		$pgfile = basename($name);
+		$pgpath = dirname($name);
+		$cnt2++;
+		printf("%6d", $cnt2) if(($cnt2 % 100) == 0);
+		print "\n" if(($cnt2 % 1000) == 0);
+
+		$pgpath =~ s|^$libraryRoot/||;
+		my $tags = WeBWorK::Utils::Tags->new($name);
+
+		if ($tags->istagged()) {
+			# Fill in missing data with Misc. instead of blank
+			print "\nNO SUBJECT $name\n" unless ($tags->{DBsubject} =~ /\S/);
+
+			$tags->{DBchapter} = 'Misc.' unless $tags->{DBchapter} =~ /\S/;
+			$tags->{DBsection} = 'Misc.' unless $tags->{DBsection} =~ /\S/;
+
+			# DBsubject table
+
+			if(isvalid($tags)) {
+				my $DBsubject_id = safe_get_id($tables{dbsubject}, 'DBsubject_id',
+					qq(WHERE BINARY name = ?), [$tags->{DBsubject}], 1, "", $tags->{DBsubject});
+				if(not $DBsubject_id) {
+					print "\nInvalid subject '$tags->{DBsubject}' in $name\n";
+					next;
+				}
+
+				# DBchapter table
+
+				$DBchapter_id = safe_get_id($tables{dbchapter}, 'DBchapter_id',
+					qq(WHERE BINARY name = ? and DBsubject_id = ?), [$tags->{DBchapter}, $DBsubject_id], 1, "", $tags->{DBchapter}, $DBsubject_id);
+				if(not $DBchapter_id) {
+					print "\nInvalid chapter '$tags->{DBchapter}' in $name\n";
+					next;
+				}
+
+				# DBsection table
+
+				$aDBsection_id = safe_get_id($tables{dbsection}, 'DBsection_id',
+					qq(WHERE BINARY name = ? and DBchapter_id = ?), [$tags->{DBsection}, $DBchapter_id], 1, "", $tags->{DBsection}, $DBchapter_id);
+				if(not $aDBsection_id) {
+					print "\nInvalid section '$tags->{DBsection}' in $name\n";
+					next;
+				}
+			} else { # tags are not valid, error printed by validation part
+				print "File $name\n";
+				next;
+			}
+
+			my @DBsection_ids=($aDBsection_id);
+			# Now add crosslisted section
+			my @CL_array = @{$taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}};
+			for my $clsect (@CL_array) {
+				my @np = split /$clinner/, $clsect;
+				@np = map(trim($_), @np);
+				my $new_dbsubj_id = safe_get_id($tables{dbsubject}, 'DBsubject_id',
+					qq(WHERE name = ?), [$np[0]], 1, "", $np[0]);
+				my $new_dbchap_id = safe_get_id($tables{dbchapter}, 'DBchapter_id',
+					qq(WHERE name = ? and DBsubject_id = ?), [$np[1], $new_dbsubj_id], 1, "", $np[1], $new_dbsubj_id);
+				my $new_dbsect_id = safe_get_id($tables{dbsection}, 'DBsection_id',
+					qq(WHERE name = ? and DBchapter_id = ?), [$np[2], $new_dbchap_id], 1, "", $np[2], $new_dbchap_id);
+				push @DBsection_ids, $new_dbsect_id;
+			}
+
+			# author table
+
+			$tags->{Author} =~ /(.*?)\s(\w+)\s*$/;
+			my $firstname = $1;
+			my $lastname = $2;
+			#remove leading and trailing spaces from firstname, which includes any middle name too.
+			$firstname =~ s/^\s*//;
+			$firstname =~ s/\s*$//;
+			$lastname =~ s/^\s*//;
+			$lastname =~ s/\s*$//;
+			my $author_id = 0;
+			if($lastname) {
+				$author_id = safe_get_id($tables{author}, 'author_id',
+					qq(WHERE lastname = ? AND firstname = ?), [$lastname, $firstname], 1, "", $tags->{Institution}, $lastname, $firstname,"");
+			}
+
+			# path table
+
+			my $path_id = safe_get_id($tables{path}, 'path_id',
+				qq(WHERE path = ?), [$pgpath], 1, "", $pgpath, "", "");
+
+			# pgfile table -- set 4 defaults first
+
+			## TODO this is where we have to deal with crosslists, and pgfileid
+			##      will be an array of id's
+			##      Make an array of DBsection-id's above
+
+			my $level = $tags->{Level} || 0;
+			# Default language is English
+			my $lang = $tags->{Language} || "en";
+			my $mathobj = $tags->{MO} || 0;
+			my $static = $tags->{Static} || 0;
+
+			my @pgfile_ids = ();
+
+			for my $DBsection_id (@DBsection_ids) {
+				my $pgfile_id = safe_get_id($tables{pgfile}, 'pgfile_id',
+					qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? ), [$pgfile, $path_id, $DBsection_id], 1, "", $DBsection_id, $author_id, $tags->{Institution}, $path_id, $pgfile, 0, $level, $lang, $static, $mathobj);
+				push @pgfile_ids, $pgfile_id;
+			}
+#if (scalar(@pgfile_ids)>1) {
+ # print "\npg ".join(', ', @pgfile_ids)."\n";
+#}
+
+			# morelt table
+
+			my $morelt_id;
+			if($tags->{MLT}) {
+				for my $DBsection_id (@DBsection_ids) {
+					$morelt_id = safe_get_id($tables{morelt}, 'morelt_id',
+						qq(WHERE name = ?), [$tags->{MLT}], 1, "", $tags->{MLT}, $DBsection_id, "");
+
+					for my $pgfile_id (@pgfile_ids) {
+						$dbh->do("UPDATE `$tables{pgfile}` SET
+							morelt_id = \"$morelt_id\" WHERE pgfile_id = \"$pgfile_id\" ");
+
+						dbug "UPDATE pgfile morelt_id for $pgfile_id to $morelt_id\n";
+						if($tags->{MLTleader}) {
+							$dbh->do("UPDATE `$tables{morelt}` SET
+									leader = \"$pgfile_id\" WHERE morelt_id = \"$morelt_id\" ");
+							dbug "UPDATE morelt leader for $morelt_id to $pgfile_id\n";
+						}
+					}
+				}
+			}
+
+			# keyword table, and problem_keyword many-many table
+
+			foreach my $keyword (@{$tags->{keywords}}) {
+				$keyword =~ s/[\'\"]//g;
+				$keyword = kwtidy($keyword);
+                                # skip it if it is empty
+                                next unless $keyword;
+				my $keyword_id = safe_get_id($tables{keyword}, 'keyword_id',
+					qq(WHERE keyword = ?), [$keyword], 1, "", $keyword);
+
+				for my $pgfile_id (@pgfile_ids) {
+					$query = "SELECT pgfile_id FROM `$tables{pgfile_keyword}` WHERE keyword_id = \"$keyword_id\" and pgfile_id=\"$pgfile_id\"";
+					my $ok = $dbh->selectrow_array($query);
+					if (!defined($ok)) {
+						$dbh->do("INSERT INTO `$tables{pgfile_keyword}`
+							VALUES(
+								\"$pgfile_id\",
+								\"$keyword_id\"
+							)"
+								);
+						dbug "INSERT INTO pgfile_keyword VALUES( \"$pgfile_id\", \"$keyword_id\" )\n";
+					}
+				}
+			}					#end foreach keyword
+
+			# Textbook section
+			# problem table contains textbook problems
+
+			for my $texthashref (@{$tags->{textinfo}}) {
+
+				# textbook table
+
+				$text = $texthashref->{TitleText};
+				$edition = $texthashref->{EditionText} || 0;
+				$edition =~ s/[^\d\.]//g;
+				$textauthor = $texthashref->{AuthorText};
+				next unless($text and $textauthor);
+				my $chapnum = $texthashref->{chapter} || -1;
+				my $secnum = $texthashref->{section} || -1;
+				$query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$text\" AND edition = \"$edition\" AND author=\"$textauthor\"";
+				my $textbook_id = $dbh->selectrow_array($query);
+				if (!defined($textbook_id)) {
+ 				        # make sure edition is an integer
+				        $edition = 0 unless $edition;
+					$dbh->do("INSERT INTO `$tables{textbook}`
+					VALUES(
+						NULL,
+						\"$text\",
+						\"$edition\",
+						\"$textauthor\",
+						NULL,
+						NULL,
+						NULL
+					)"
+							);
+					dbug "INSERT INTO textbook VALUES( \"\", \"$text\", \"$edition\", \"$textauthor\", \"\", \"\", \"\" )\n";
+					dbug "\nLate add into $tables{textbook} \"$text\", \"$edition\", \"$textauthor\"\n", 1;
+					$textbook_id = $dbh->selectrow_array($query);
+				}
+
+				# chapter weak table of textbook
+				#
+				$query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$textbook_id\" AND number = \"$chapnum\"";
+				my $chapter_id = $dbh->selectrow_array($query);
+				if (!defined($chapter_id)) {
+					$dbh->do("INSERT INTO `$tables{chapter}`
+					VALUES(
+						NULL,
+						\"$textbook_id\",
+						\"".$chapnum."\",
+						\"$tags->{DBchapter}\",
+						NULL
+					)"
+							);
+					dbug "\nLate add into $tables{chapter} \"$text\", \"$edition\", \"$textauthor\", $chapnum $tags->{chapter} from $name\n", 1;
+					dbug "INSERT INTO chapter VALUES(\"\", \"$textbook_id\", \"".$chapnum."\", \"$tags->{DBchapter}\", \"\" )\n";
+					$chapter_id = $dbh->selectrow_array($query);
+				}
+
+				# section weak table of textbook
+				#
+				$tags->{DBsection} = '' if ($secnum < 0);
+				$query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapter_id\" AND number = \"$secnum\"";
+				my $section_id = $dbh->selectrow_array($query);
+				if (!defined($section_id)) {
+					$dbh->do("INSERT INTO `$tables{section}`
+					VALUES(
+						NULL,
+						\"$chapter_id\",
+						\"$secnum\",
+						\"$tags->{DBsection}\",
+						NULL
+					)"
+							);
+					dbug "INSERT INTO section VALUES(\"\", \"$textbook_id\", \"$secnum\", \"$tags->{DBsection}\", \"\" )\n";
+					dbug "\nLate add into $tables{section} \"$text\", \"$edition\", \"$textauthor\", $secnum $tags->{DBsection} from $name\n", 1;
+					$section_id = $dbh->selectrow_array($query);
+				}
+
+				@textproblems = @{$texthashref->{problems}};
+				for my $tp (@textproblems) {
+					$query = "SELECT problem_id FROM `$tables{problem}` WHERE section_id = \"$section_id\" AND number = \"$tp\"";
+					my $problem_id = $dbh->selectrow_array($query);
+					if (!defined($problem_id)) {
+						$dbh->do("INSERT INTO `$tables{problem}`
+					VALUES(
+						NULL,
+						\"$section_id\",
+						\"$tp\",
+						NULL
+					)"
+								);
+						dbug "INSERT INTO problem VALUES( \"\", \"$section_id\", \"$tp\", \"\" )\n";
+						$problem_id = $dbh->selectrow_array($query);
+					}
+
+					# pgfile_problem table associates pgfiles with textbook problems
+					for my $pgfile_id (@pgfile_ids) {
+						$query = "SELECT problem_id FROM `$tables{pgfile_problem}` WHERE problem_id = \"$problem_id\" AND pgfile_id = \"$pgfile_id\"";
+						my $pg_problem_id = $dbh->selectrow_array($query);  
+						if (!defined($pg_problem_id)) {
+							$dbh->do("INSERT INTO `$tables{pgfile_problem}`
+						VALUES(
+							\"$pgfile_id\",
+							\"$problem_id\"
+						)"
+									);
+							dbug "INSERT INTO pgfile_problem VALUES( \"$pgfile_id\", \"$problem_id\" )\n";
+						}
+					}
+				}
+
+				#reset tag vars, they may not match the next text/file
+				$textauthor=""; $textsection="";
+			}
+		} else { # This file was not tagged
+			# Message if not a pointer
+			# print STDERR "File $name is not tagged\n" if not $tags->isplaceholder();
+			;
+		}
+	}
+}
+
+print "\n\n";
+
+# Now prune away DBsection, etc, which do not appear in any files
+#%my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\"";
+#%my $chapid = $dbh->selectrow_array($query);
+
+#select dbs.DBsection_id from OPL_DBsection dbs;
+#select COUNT(*) from OPL_pgfile where DBsection_id=857;
+
+my $dbsects = $dbh->selectall_arrayref("SELECT DBsection_id from `$tables{dbsection}`");
+for my $sect (@{$dbsects}) {
+	$sect = $sect->[0];
+	my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{pgfile}` WHERE DBsection_id=$sect");
+	if(scalar(@{$srar})==0) {
+		$dbh->do("DELETE FROM `$tables{dbsection}` WHERE DBsection_id=$sect");
+	}
+}
+
+my $dbchaps = $dbh->selectall_arrayref("SELECT DBchapter_id from `$tables{dbchapter}`");
+for my $chap (@{$dbchaps}) {
+	$chap = $chap->[0];
+	my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chap");
+	if(scalar(@{$srar})==0) {
+		$dbh->do("DELETE FROM `$tables{dbchapter}` WHERE DBchapter_id=$chap");
+	}
+}
+
+# Now run the script build-library-tree
+# This is used to create the file library-tree.json which can be used to 
+# load in subject-chapter-section information for the OPL
+
+use strict;
+use warnings;
+use JSON;
+
+
+my $tree;  # the library subject tree will be stored as arrays of objects. 
+
+my $sth = $dbh->prepare("select * from OPL_DBsubject");
+$sth->execute;
+
+my @subjects = ();
+my @subject_names = ();
+while ( my @row = $sth->fetchrow_array ) {
+	push(@subjects,$row[0]);
+	push(@subject_names,$row[1]);
+  }
+
+
+my @subject_tree;  # array to store the individual library tree for each subject 
+
+foreach my $i (0..$#subjects){
+
+	my $subject_row = $subjects[$i];
+	my $subject_name = $subject_names[$i];
+
+	my $sth = $dbh->prepare("select * from OPL_DBchapter where DBsubject_id = $subject_row;");
+	$sth->execute;
+
+	my @chapters = ();
+	my @chapter_names = ();
+	while ( my @row = $sth->fetchrow_array ) {
+		push(@chapters,$row[0]);
+		push(@chapter_names,$row[1]);
+	  }
+
+
+	my @chapter_tree; # array to store the individual library tree for each chapter
+
+	foreach my $j (0..$#chapters) {
+		my $chapter_row = $chapters[$j];
+		my $chapter_name = $chapter_names[$j];
+		my $sth = $dbh->prepare("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chapter_row");
+		$sth->execute;
+
+		my @subfields = ();
+		while ( my @row = $sth->fetchrow_array ) {
+			my $section_name;
+			$section_name->{name} = $row[1];
+			my $clone = { %{ $section_name } };  # need to clone it before pushing into the @subfields array.
+		    push(@subfields,$clone);
+	    }
+
+		my $chapter_tree;
+		$chapter_tree->{name} = $chapter_name;
+		$chapter_tree->{subfields} = \@subfields;
+
+		my $clone = { %{ $chapter_tree } };  # need to clone it before pushing into the @chapter_tree array.
+		push(@chapter_tree,$clone);
+
+
+
+	}
+
+	my $subject_tree; 
+	$subject_tree->{name} = $subject_name;
+	$subject_tree->{subfields} = \@chapter_tree; 
+
+	my $clone = { % {$subject_tree}};
+	push (@subject_tree, $clone);
+}
+
+build_library_directory_tree($ce);
+build_library_subject_tree($ce,$dbh);
+build_library_textbook_tree($ce,$dbh);
+
+$dbh->disconnect;
+
+if ($ce->{problemLibrary}{showLibraryLocalStats} ||
+    $ce->{problemLibrary}{showLibraryGlobalStats}) {
+  print "\nUpdating Library Statistics.\n";
+  do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics';
+}
+
+
+print "\nDone.\n";
diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm
old mode 100644
new mode 100755
index 323436ab48..0355322044
--- a/bin/OPLUtils.pm
+++ b/bin/OPLUtils.pm
@@ -89,7 +89,7 @@ sub build_library_directory_tree {
 
 	# use the three arguments version of open
 	# and check for errors
-	open $OUTFILE, '>', $file  or die "Cannot open $file";
+	open $OUTFILE, '>encoding(UTF-8)', $file  or die "Cannot open $file";
 
 	# you can check for errors (e.g., if after opening the disk gets full)
 	print { $OUTFILE } to_json(\@dirArray) or die "Cannot write to $file";
@@ -279,7 +279,7 @@ sub build_library_subject_tree {
 
 	# use the three arguments version of open
 	# and check for errors
-	open $OUTFILE, '>', $file  or die "Cannot open $file";
+	open $OUTFILE, '>encoding(UTF-8)', $file  or die "Cannot open $file";
 
 	# you can check for errors (e.g., if after opening the disk gets full)
 	print { $OUTFILE } to_json(\@subject_tree,{pretty=>1}) or die "Cannot write to $file";
diff --git a/bin/fix_copyright.sh b/bin/fix_copyright.sh
new file mode 100755
index 0000000000..a44a16034e
--- /dev/null
+++ b/bin/fix_copyright.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+sed -i .bak '/Copyright/c\
+# Copyright &copy; 2000-2019. The WeBWorK Project. https://github.com/openwebwork/webwork2\
+' $1
+
+
+
+#obtained by trial and error after much toil -- mostly error.
+# this version works on a mac
+# the space after -i might need to be removed for linux.
+# produces $1.bak file
+
+# use with the find command: 
+# find . -name course.conf -exec /opt/webwork/webwork2/bin/fix_copyright.sh {} ';'
\ No newline at end of file
diff --git a/bin/pg-append-textbook-tags b/bin/pg-append-textbook-tags
index 4275f831b0..952d8ff567 100755
--- a/bin/pg-append-textbook-tags
+++ b/bin/pg-append-textbook-tags
@@ -43,7 +43,7 @@ sub main {
 sub add_tags_to_file {
 	my ($new_tags, $file) = @_;
 	
-	my $pgfile = new IO::File($file, '+<:utf8') or do {
+	my $pgfile = new IO::File($file, '+<encoding("UTF-8")') or do {
 		warn "Failed to open file $file for editing: $!\n";
 		warn "New tags will not be written to this file.\n";
 		return;
diff --git a/bin/update-OPL-statistics-latin1 b/bin/update-OPL-statistics-latin1
new file mode 100755
index 0000000000..6e59e2a2bc
--- /dev/null
+++ b/bin/update-OPL-statistics-latin1
@@ -0,0 +1,214 @@
+#!/usr/bin/perl
+
+##############################################################################
+# WeBWorK Online Homework Delivery System
+# Copyright &copy; 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/
+# $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of either: (a) the GNU General Public License as published by the
+# Free Software Foundation; either version 2, or (at your option) any later
+# version, or (b) the "Artistic License" which comes with this package.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the
+# Artistic License for more details.
+##############################################################################
+
+use strict; 
+
+# Get the necessary packages, including adding
+# webwork and pg library to our path.  
+BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n')
+	   unless($ENV{WEBWORK_ROOT});}
+use lib "$ENV{WEBWORK_ROOT}/lib";
+use WeBWorK::CourseEnvironment;
+use String::ShellQuote;
+
+# hack to set version so that the script runs without warnings in 
+# earlier versions of WeBWorK, e.g. WW 2.7
+
+BEGIN { $main::VERSION = "2.4"; } 
+
+BEGIN{ 
+    my $ce = new WeBWorK::CourseEnvironment({
+	webwork_dir => $ENV{WEBWORK_ROOT},
+					 });
+    
+    my $pg_dir = $ce->{pg_dir};
+    eval "use lib '$pg_dir/lib'"; 
+    die $@ if $@;
+}
+
+use DBI;
+use WeBWorK::Utils::CourseIntegrityCheck;
+use WeBWorK::Utils::CourseManagement qw/listCourses/;
+
+my $time = time();
+
+# get course environment and open up database
+my $ce = new WeBWorK::CourseEnvironment({
+    webwork_dir => $ENV{WEBWORK_ROOT},
+					});
+
+my $dbh = DBI->connect(
+        $ce->{problemLibrary_db}->{dbsource},
+        $ce->{problemLibrary_db}->{user},
+        $ce->{problemLibrary_db}->{passwd},
+        {
+		AutoCommit => 0,
+                PrintError => 0,
+                RaiseError => 1,
+        },
+);
+
+# get course list
+my @courses = listCourses($ce);
+
+# create tables.  We always redo the statistics table.  
+$dbh->do(<<EOS);
+CREATE TABLE IF NOT EXISTS `OPL_problem_user` (
+  `course_id` tinyblob NOT NULL,
+  `user_id` tinyblob NOT NULL,
+  `set_id` tinyblob NOT NULL,
+  `due_date` int(11) NOT NULL, 
+  `problem_id` int(11) NOT NULL,
+  `source_file` text,
+  `status` float DEFAULT NULL,
+  `attempted` int(11) DEFAULT NULL,
+  `num_correct` int(11) DEFAULT NULL,
+  `num_incorrect` int(11) DEFAULT NULL,
+  UNIQUE KEY `unique_key_idx` (`course_id`(100),`user_id`(100),`set_id`(100),`due_date`,`problem_id`),
+  KEY `source_file_idx` (`source_file`(255)))
+  CHARACTER SET ascii
+EOS
+
+$dbh->do(<<EOS);
+DROP TABLE IF EXISTS `OPL_local_statistics`
+EOS
+
+$dbh->do(<<EOS);
+CREATE TABLE `OPL_local_statistics` (
+  `source_file` varchar(255) NOT NULL,
+  `students_attempted` int(11) DEFAULT NULL,
+  `average_attempts` float DEFAULT NULL,
+  `average_status` float DEFAULT NULL,
+  PRIMARY KEY (`source_file`(255))
+)
+EOS
+
+$dbh->commit();
+
+# for each course get the data from the user problem table into the
+# opl user problem table.
+
+print "Importing statistics for ".scalar(@courses)." courses.\n";
+
+my $counter = 0;
+
+foreach my $courseID (@courses) {
+    $counter++;
+    print sprintf("%*d",4,$counter);
+    if ($counter % 10 == 0) {
+	print "\n";
+    }
+
+    next if $courseID eq 'admin' || $courseID eq 'modelCourse';
+
+    # we extract the identifying information of the problem,
+    # the status, attempted flag, number of attempts.
+    # and the source_file
+    # we strip of the local in front of the source file
+    # (assuming that these are mostly the same as their library counterparts
+    $dbh->do(<<EOS);
+INSERT IGNORE INTO OPL_problem_user 
+  (course_id,
+  user_id,
+  set_id,
+  due_date,
+  problem_id,
+  source_file,
+  status, 
+  attempted,
+  num_correct,
+  num_incorrect)
+SELECT
+  '$courseID' AS `course_id`,
+  `${courseID}_problem_user`.user_id,
+  `${courseID}_problem`.set_id,
+  `${courseID}_set`.due_date,
+  `${courseID}_problem`.problem_id,
+  REPLACE(`${courseID}_problem`.source_file,'local/Library/','Library/'),
+  `${courseID}_problem_user`.status,
+  `${courseID}_problem_user`.attempted,
+  `${courseID}_problem_user`.num_correct,
+  `${courseID}_problem_user`.num_incorrect
+FROM `${courseID}_problem_user`
+  JOIN `${courseID}_problem`
+  ON `${courseID}_problem_user`.set_id = `${courseID}_problem`.set_id
+  AND `${courseID}_problem_user`.problem_id = `${courseID}_problem`.problem_id
+  JOIN `${courseID}_set`
+  ON `${courseID}_problem_user`.set_id = `${courseID}_set`.set_id
+WHERE (`${courseID}_problem`.source_file LIKE 'Library/%'
+  OR `${courseID}_problem`.source_file LIKE 'local/Library/%')
+  AND `${courseID}_set`.due_date < $time;
+EOS
+
+}
+
+print "\n\n";
+
+$dbh->commit();
+
+# compile desired statistics from opl problem user table.  
+$dbh->do(<<EOS);
+INSERT INTO OPL_local_statistics
+  (source_file,
+  students_attempted,
+  average_attempts,
+  average_status)
+SELECT
+  source_file,
+  COUNT(*),
+  AVG(num_correct+num_incorrect),
+  AVG(status)
+FROM OPL_problem_user
+WHERE attempted=1
+GROUP BY source_file
+EOS
+
+$dbh->commit();
+
+# check to see if the global statistics file exists and if it does, upload it.
+
+my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql';
+
+if (-e $global_sql_file) {
+
+  my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
+  
+  $host = 'localhost' unless $host;
+
+  $port = 3306 unless $port;
+  
+  my $dbuser = $ce->{database_username};
+  my $dbpass = $ce->{database_password};
+
+  
+  $dbh->do(<<EOS);
+DROP TABLE IF EXISTS OPL_global_statistics;
+EOS
+  $dbh->commit();
+
+  $dbuser = shell_quote($dbuser);
+  $dbpass = shell_quote($dbpass);
+  $db = shell_quote($db);
+  
+  my $mysql_command = $ce->{externalPrograms}->{mysql};  
+
+  `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`;
+
+}
+
+1;
diff --git a/bin/update-OPL-statistics-utf8mb4 b/bin/update-OPL-statistics-utf8mb4
new file mode 100755
index 0000000000..316c6f3b53
--- /dev/null
+++ b/bin/update-OPL-statistics-utf8mb4
@@ -0,0 +1,214 @@
+#!/usr/bin/perl
+
+##############################################################################
+# WeBWorK Online Homework Delivery System
+# Copyright &copy; 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/
+# $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of either: (a) the GNU General Public License as published by the
+# Free Software Foundation; either version 2, or (at your option) any later
+# version, or (b) the "Artistic License" which comes with this package.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the
+# Artistic License for more details.
+##############################################################################
+
+use strict; 
+
+# Get the necessary packages, including adding
+# webwork and pg library to our path.  
+BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n')
+	   unless($ENV{WEBWORK_ROOT});}
+use lib "$ENV{WEBWORK_ROOT}/lib";
+use WeBWorK::CourseEnvironment;
+use String::ShellQuote;
+
+# hack to set version so that the script runs without warnings in 
+# earlier versions of WeBWorK, e.g. WW 2.7
+
+BEGIN { $main::VERSION = "2.4"; } 
+
+BEGIN{ 
+    my $ce = new WeBWorK::CourseEnvironment({
+	webwork_dir => $ENV{WEBWORK_ROOT},
+					 });
+    
+    my $pg_dir = $ce->{pg_dir};
+    eval "use lib '$pg_dir/lib'"; 
+    die $@ if $@;
+}
+
+use DBI;
+use WeBWorK::Utils::CourseIntegrityCheck;
+use WeBWorK::Utils::CourseManagement qw/listCourses/;
+
+my $time = time();
+
+# get course environment and open up database
+my $ce = new WeBWorK::CourseEnvironment({
+    webwork_dir => $ENV{WEBWORK_ROOT},
+					});
+
+my $dbh = DBI->connect(
+        $ce->{problemLibrary_db}->{dbsource},
+        $ce->{problemLibrary_db}->{user},
+        $ce->{problemLibrary_db}->{passwd},
+        {
+		AutoCommit => 0,
+                PrintError => 0,
+                RaiseError => 1,
+        },
+);
+
+# get course list
+my @courses = listCourses($ce);
+
+# create tables.  We always redo the statistics table.  
+$dbh->do(<<EOS);
+CREATE TABLE IF NOT EXISTS `OPL_problem_user` (
+  `course_id` tinyblob NOT NULL,
+  `user_id` tinyblob NOT NULL,
+  `set_id` tinyblob NOT NULL,
+  `due_date` int(11) NOT NULL, 
+  `problem_id` int(11) NOT NULL,
+  `source_file` text,
+  `status` float DEFAULT NULL,
+  `attempted` int(11) DEFAULT NULL,
+  `num_correct` int(11) DEFAULT NULL,
+  `num_incorrect` int(11) DEFAULT NULL,
+  UNIQUE KEY `unique_key_idx` (`course_id`(80),`user_id`(80),`set_id`(80),`due_date`,`problem_id`),
+  KEY `source_file_idx` (`source_file`(245)))
+  CHARACTER SET utf8mb4
+EOS
+
+$dbh->do(<<EOS);
+DROP TABLE IF EXISTS `OPL_local_statistics`
+EOS
+
+$dbh->do(<<EOS);
+CREATE TABLE `OPL_local_statistics` (
+  `source_file` varchar(245) NOT NULL,
+  `students_attempted` int(11) DEFAULT NULL,
+  `average_attempts` float DEFAULT NULL,
+  `average_status` float DEFAULT NULL,
+  PRIMARY KEY (`source_file`(245))
+)  ENGINE=MyISAM
+EOS
+
+$dbh->commit();
+
+# for each course get the data from the user problem table into the
+# opl user problem table.
+
+print "Importing statistics for ".scalar(@courses)." courses.\n";
+
+my $counter = 0;
+
+foreach my $courseID (@courses) {
+    $counter++;
+    print sprintf("%*d",4,$counter);
+    if ($counter % 10 == 0) {
+	print "\n";
+    }
+
+    next if $courseID eq 'admin' || $courseID eq 'modelCourse';
+
+    # we extract the identifying information of the problem,
+    # the status, attempted flag, number of attempts.
+    # and the source_file
+    # we strip of the local in front of the source file
+    # (assuming that these are mostly the same as their library counterparts
+    $dbh->do(<<EOS);
+INSERT IGNORE INTO OPL_problem_user 
+  (course_id,
+  user_id,
+  set_id,
+  due_date,
+  problem_id,
+  source_file,
+  status, 
+  attempted,
+  num_correct,
+  num_incorrect)
+SELECT
+  '$courseID' AS `course_id`,
+  `${courseID}_problem_user`.user_id,
+  `${courseID}_problem`.set_id,
+  `${courseID}_set`.due_date,
+  `${courseID}_problem`.problem_id,
+  REPLACE(`${courseID}_problem`.source_file,'local/Library/','Library/'),
+  `${courseID}_problem_user`.status,
+  `${courseID}_problem_user`.attempted,
+  `${courseID}_problem_user`.num_correct,
+  `${courseID}_problem_user`.num_incorrect
+FROM `${courseID}_problem_user`
+  JOIN `${courseID}_problem`
+  ON `${courseID}_problem_user`.set_id = `${courseID}_problem`.set_id
+  AND `${courseID}_problem_user`.problem_id = `${courseID}_problem`.problem_id
+  JOIN `${courseID}_set`
+  ON `${courseID}_problem_user`.set_id = `${courseID}_set`.set_id
+WHERE (`${courseID}_problem`.source_file LIKE 'Library/%'
+  OR `${courseID}_problem`.source_file LIKE 'local/Library/%')
+  AND `${courseID}_set`.due_date < $time;
+EOS
+
+}
+
+print "\n\n";
+
+$dbh->commit();
+
+# compile desired statistics from opl problem user table.  
+$dbh->do(<<EOS);
+INSERT INTO OPL_local_statistics
+  (source_file,
+  students_attempted,
+  average_attempts,
+  average_status)
+SELECT
+  source_file,
+  COUNT(*),
+  AVG(num_correct+num_incorrect),
+  AVG(status)
+FROM OPL_problem_user
+WHERE attempted=1
+GROUP BY source_file
+EOS
+
+$dbh->commit();
+
+# check to see if the global statistics file exists and if it does, upload it.
+
+my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql';
+
+if (-e $global_sql_file) {
+
+  my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn});
+  
+  $host = 'localhost' unless $host;
+
+  $port = 3306 unless $port;
+  
+  my $dbuser = $ce->{database_username};
+  my $dbpass = $ce->{database_password};
+
+  
+  $dbh->do(<<EOS);
+DROP TABLE IF EXISTS OPL_global_statistics;
+EOS
+  $dbh->commit();
+
+  $dbuser = shell_quote($dbuser);
+  $dbpass = shell_quote($dbpass);
+  $db = shell_quote($db);
+  
+  my $mysql_command = $ce->{externalPrograms}->{mysql};  
+
+  `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`;
+
+}
+
+1;
diff --git a/clients/sendXMLRPC.pl b/clients/sendXMLRPC.pl
index 68b6a8573a..6e8f355cac 100755
--- a/clients/sendXMLRPC.pl
+++ b/clients/sendXMLRPC.pl
@@ -269,6 +269,7 @@ BEGIN
 }
 $ENV{MOD_PERL_API_VERSION} = 2;
 use lib "$main::dirname";
+print "home directory ".$main::dirname;
 #use lib "."; # is this needed?
 
 # some files such as FormatRenderedProblem.pm may need to be in the same directory
@@ -290,7 +291,7 @@ BEGIN
 	}
 
 }
-
+use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib";
 
 
 use Carp;
diff --git a/conf/defaults.config b/conf/defaults.config
index 5c7fd72453..1872eca661 100644
--- a/conf/defaults.config
+++ b/conf/defaults.config
@@ -30,8 +30,10 @@
 
 include("conf/site.conf");
 include("VERSION");  # get WW version 
-include("PG_VERSION"); # get PG VERSION -- # these addresses are fragile.  
-                                           # Should we change the way include() works?
+#include("PG_VERSION"); 
+# The version of PG is now obtained from pg/VERSION
+# using code inside CourseEnvironment.pm  
+# include can only read files under the webwork2 directory 
 
 ################################################################################
 # site.conf  should contain basic information about directories and URLs on
@@ -223,6 +225,14 @@ $pg{options}{periodicRandomizationPeriod} = 5;
 
 $language = "en";   # tr = turkish,  en=english
 
+# $perProblemLangAndDirSettingMode controls how and whether LANG and/or DIR
+#    attributes are added to the DIV element enveloping a problem
+#    which helps handle proper display of problems with a text direction
+#    different from that used course-wide. Ex: enables English problems to
+#    be displayed properly in a Hebrew course site, which helps select problems
+#    to be translated to Hebrew.
+$perProblemLangAndDirSettingMode = "force::ltr";
+
 ################################################################################
 # System-wide locations (directories and URLs)
 ################################################################################
@@ -1211,10 +1221,13 @@ $pg{specialPGEnvironmentVars}{problemPostamble} = { TeX => '', HTML=>'' };
 # should appear as [qw(Mymodule.pm, Dependency1.pm, Dependency2.pm)]
 
 ${pg}{modules} = [
+    [qw(Encode)],
+	[qw(Encode::Encoding)],
         [qw(HTML::Parser)],
         [qw(HTML::Entities)],
 	[qw(DynaLoader)],
-	[qw(Exporter)],
+	[qw(Encode)],
+	[qw(Exporter )],
 	[qw(GD)],
 	[qw(AlgParser AlgParserWithImplicitExpand Expr ExprWithImplicitExpand utf8)],
 	[qw(AnswerHash AnswerEvaluator)],
@@ -1439,6 +1452,12 @@ $ConfigValues = [
 		  values => [qw(en tr es fr zh_hk heb)],
 		  type => 'popuplist'
 		},
+		{ var => 'perProblemLangAndDirSettingMode',
+		  doc => 'Mode in which the LANG and DIR settings for a single problem are determined.',
+		  doc2 => 'Mode in which the LANG and DIR settings for a single problem are determined.<p>The system will set the LANGuage attribute to either a value determined from the problem, a course-wide default, or the system default of en-US, depending on the mode selected. The tag will only be added to the DIV enclosing the problem if it is different than the value which should be set in the main HTML tag set for the entire course based on the course language.<p>There are two options for the DIRection attribute: \"ltr\" for left-to-write sripts, and \"rtl\" for right-to-left scripts like Arabic and Hebrew.<p>The DIRection attribute is needed to trigger proper display of the question text when the problem text-direction is different than that used by the current language of the course. For example, English problems from the library browser would display improperly in RTL mode for a Hebrew course, unless the problen Direction is set to LTR.<p>The feature to set a problem language and direction was only added in 2018 to the PG language, so most problems will not declare their language, and the system needs to fall back to determining the language and direction in a different manner. The OPL itself is all English, so the system wide fallback is to en-US in LTR mode.<p>Since the defaults fall back to the LTR direction, most sites should be fine with the \"auto::\" mode, but may want to select the one which matches their course language. The mode \"force::ltr\" would also be an option for a course which runs into trouble with the \"auto\" modes.<p>Modes:<br><ul><li> \"none\" prevents any additional LANG and/or DIR tag being added. The browser will use the main setting which was applied to the entire HTML page. This is likely to cause trouble when a problem of the other direction is displayed.</li><li> \"auto::\" allows the system to make the settings based on the language and direction reported by the problem (a new feature, so not set in almost all existing problems) and falling back to the expected default of en-US in LTR mode. </li><li> \"auto:LangCode:Dir\" allows the system to make the settings based on the language and direction reported by the problem (a new feature, so not set in almost all existing problems) but falling back to the language with the given LangCode and the direction Dir when problem settings are not available from PG.</li><li> \"auto::Dir\" for problems without PG settings, this will use the default en=english language, but force the direction to Dir. Problems with PG settings will get those settings.</li><li> \"auto:LangCode:\" for problems without PG settings, this will use the default LTR direction, but will set the language to LangCode.Problems with PG settings will get those settings.</li><li> \"force:LangCode:Dir\" will <b>ignore</b> any setting made by the PG code of the problem, and will force the system to set the language with the given LangCode and the direction to Dir for <b>all</b> problems.</li><li> \"force::Dir\" will <b>ignore</b> any setting made by the PG code of the problem, and will force the system to set the direction to Dir for <b>all</b> problems, but will avoid setting any language attribute for individual problem.</li></ul>',
+		  values => [qw(none auto:: force::ltr force::rtl force:en:ltr auto:en:ltr force:tr:ltr auto:tr:ltr force:es:ltr auto:es:ltr force:fr:ltr auto:fr:ltr force:zh_hk:ltr auto:zh_hk:ltr force:he:rtl auto:he:rtl   )],
+		  type => 'popuplist'
+		},
 		{ var => 'sessionKeyTimeout',
 		  doc => 'Inactivity time before a user is required to login again',
 		  doc2 => 'Length of time, in seconds, a user has to be inactive before he is required to login again.<p> This value should be entered as a number, so as 3600 instead of 60*60 for one hour',
diff --git a/conf/snippets/hardcopyThemes/oneColumn/hardcopyPreamble.tex b/conf/snippets/hardcopyThemes/oneColumn/hardcopyPreamble.tex
index 1cf495954b..95e98fe653 100644
--- a/conf/snippets/hardcopyThemes/oneColumn/hardcopyPreamble.tex
+++ b/conf/snippets/hardcopyThemes/oneColumn/hardcopyPreamble.tex
@@ -11,7 +11,13 @@
 \usepackage{epsfig}
 \usepackage{pslatex}
 \usepackage{fullpage}
+
 \usepackage[utf8]{inputenc}
+
+\usepackage{eurosym}                   % the euro symbol
+\DeclareUnicodeCharacter{20AC}{\euro}  % make it possible to use the UTF-8 character for the euro symbol in problems
+
+
 \pagestyle{plain}
 \def\endline{\bigskip\hrule width \hsize height 0.8pt }
 \newcommand{\lt}{<}
diff --git a/conf/snippets/hardcopyThemes/twoColumn/hardcopyPreamble.tex b/conf/snippets/hardcopyThemes/twoColumn/hardcopyPreamble.tex
index dd6522997a..8c57181151 100644
--- a/conf/snippets/hardcopyThemes/twoColumn/hardcopyPreamble.tex
+++ b/conf/snippets/hardcopyThemes/twoColumn/hardcopyPreamble.tex
@@ -10,7 +10,12 @@
 \usepackage{epsf}
 \usepackage{epsfig}
 \usepackage{pslatex}
+
 \usepackage[utf8]{inputenc}
+
+\usepackage{eurosym}                   % the euro symbol
+\DeclareUnicodeCharacter{20AC}{\euro}  % make it possible to use the UTF-8 character for the euro symbol in problems
+
 \pagestyle{plain}
 \textheight 9in
 \oddsidemargin = -0.42in
diff --git a/docker-compose.yml b/docker-compose.yml
index fd3cdfd4b0..d4da0e3777 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,6 +4,9 @@ services:
     image: mariadb:10.1
     volumes:
       - mysql:/var/lib/mysql
+      # Set up UTF8MB4 in config file for the container.
+      # Needs to be done BEFORE the database is created.
+      - "./docker-config/db/mariadb.cnf:/etc/mysql/conf.d/mariadb.cnf"
     restart: always
     environment:
       MYSQL_ROOT_PASSWORD: randomepassword
diff --git a/docker-compose.yml.specialUTF8MB4-db-storage b/docker-compose.yml.specialUTF8MB4-db-storage
new file mode 100644
index 0000000000..53dc498be0
--- /dev/null
+++ b/docker-compose.yml.specialUTF8MB4-db-storage
@@ -0,0 +1,54 @@
+version: '2'
+services:
+  # We have renamed the database service from "db" to "dbMB4",
+  # changed the name of the data volume to "mysqlMB4" and set
+  # environment variables to use the "dbMB4" server. All this was needed
+  # ONLY to allow leaving a version of the "db" container which does not
+  # use utf8mb4 available until the UTF8MB4 changes become mainstream.
+  dbMB4:
+    image: mariadb:10.1
+    volumes:
+      - mysqlMB4:/var/lib/mysql
+      # Set up UTF8MB4 in config file for the container.
+      # Needs to be done BEFORE the database is created.
+      - "./docker-config/db/mariadb.cnf:/etc/mysql/conf.d/mariadb.cnf"
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: randomepassword
+      MYSQL_DATABASE: webwork
+      MYSQL_USER: webworkWrite
+      MYSQL_PASSWORD: passwordRW
+  app:
+    build: .
+    image: webwork
+    depends_on:
+      - dbMB4
+      - r
+    volumes:
+      - ".:/opt/webwork/webwork2"
+
+      # OLD approach put the courses tree under webwork2/.data/courses
+      #- "./.data/courses:/opt/webwork/courses"
+      # NEW appoach puts the courses tree in a separate tree outside of webwork2/
+      - "../ww-docker-data/courses:/opt/webwork/courses"
+
+      # Uncomment the line below to use local OPL for development
+      #- "../opl:/opt/webwork/libraries/webwork-open-problem-library"
+      # Uncomment the line below to use local PG for development
+      #- "../pg:/opt/webwork/pg"
+    ports:
+      - "8080:80"
+    environment:
+      - DEV=0
+      - WEBWORK_DB_HOST=dbMB4
+      # - WEBWORK_DB_PORT=3306
+      # - WEBWORK_DB_NAME=webwork
+      - WEBWORK_DB_DSN=DBI:mysql:webwork:dbMB4:3306
+  r:
+    image: ubcctlt/rserve
+    ports:
+      - "6311:6311"
+
+volumes:
+  mysqlMB4:
+
diff --git a/docker-config/db/mariadb.cnf b/docker-config/db/mariadb.cnf
new file mode 100644
index 0000000000..22c4abce0b
--- /dev/null
+++ b/docker-config/db/mariadb.cnf
@@ -0,0 +1,34 @@
+# MariaDB-specific config file.
+# Read by /etc/mysql/my.cnf
+
+[client]
+# Default is Latin1, if you need UTF-8 set this (also in server section)
+#default-character-set = utf8 
+
+# Based on:  https://salsa.debian.org/mariadb-team/mariadb-10.1/commit/e6ade2be57856736e8bc8039d71b35f9ffcde48e
+default-character-set = utf8mb4
+
+[mysql]
+# Based on:  https://salsa.debian.org/mariadb-team/mariadb-10.1/commit/e6ade2be57856736e8bc8039d71b35f9ffcde48e
+default-character-set = utf8mb4
+
+[mysqld]
+#
+# * Character sets
+# 
+# Default is Latin1, if you need UTF-8 set all this (also in client section)
+#
+#character-set-server  = utf8 
+#collation-server      = utf8_general_ci 
+#character_set_server   = utf8 
+#collation_server       = utf8_general_ci 
+
+# MySQL/MariaDB default is Latin1, but we want the full utf8 4-bit character set. 
+# See also client.cnf
+# Based on:  https://salsa.debian.org/mariadb-team/mariadb-10.1/commit/e6ade2be57856736e8bc8039d71b35f9ffcde48e
+character-set-server  = utf8mb4
+collation-server      = utf8mb4_general_ci
+character_set_server   = utf8mb4
+collation_server       = utf8mb4_general_ci
+init-connect='SET NAMES utf8mb4'
+
diff --git a/htdocs/themes/math4-ar/gateway.template b/htdocs/themes/math4-ar/gateway.template
index 9f38b9c59e..a9d37822ab 100644
--- a/htdocs/themes/math4-ar/gateway.template
+++ b/htdocs/themes/math4-ar/gateway.template
@@ -104,7 +104,7 @@
 	<!--#endif-->
 
 	<!--#if can="body"-->
-	<div class="Body">
+	<div class="Body" <!--#output_problem_lang_and_dir-->>
 		<!--#body-->
 	</div>
 	<!--#endif-->
diff --git a/htdocs/themes/math4-ar/lbtwo.template b/htdocs/themes/math4-ar/lbtwo.template
index 2c9a2bfea1..a700faf43a 100644
--- a/htdocs/themes/math4-ar/lbtwo.template
+++ b/htdocs/themes/math4-ar/lbtwo.template
@@ -96,13 +96,13 @@
 	
 	    <!--#if can="body"-->
 				<!--#if warnings="1"-->
-				  <div class="Body" style="background-color:#ffcccc">
+				  <div class="Body" style="background-color:#ffcccc" <!--#output_problem_lang_and_dir-->>
 				  <p style="font-size:larger">
 				  Warning -- there may be something wrong with this question. Please inform your instructor
 				  including the warning messages below.
 				  </p>
 				<!--#else-->
-				  <div class="Body" style="background-color:#ffffff">
+				  <div class="Body" style="background-color:#ffffff" <!--#output_problem_lang_and_dir-->>
 				<!--#endif-->
 				
 				<!--#body-->
diff --git a/htdocs/themes/math4-ar/math4.css b/htdocs/themes/math4-ar/math4.css
index 3a286be034..3ff65e9f26 100644
--- a/htdocs/themes/math4-ar/math4.css
+++ b/htdocs/themes/math4-ar/math4.css
@@ -12,7 +12,14 @@
  * FOR A PARTICULAR PURPOSE.  See either the GNU General Public License or the
  * Artistic License for more details.
  */
+
+/* CodeMirror overrides */
  
+.CodeMirror-code {
+  outline: none;
+  direction: ltr !important;
+}
+
 /* Bootstrap overrides */
 a:focus {
     outline-style:solid;
diff --git a/htdocs/themes/math4-ar/simple.template b/htdocs/themes/math4-ar/simple.template
index 22aca91b37..2e21ef4253 100644
--- a/htdocs/themes/math4-ar/simple.template
+++ b/htdocs/themes/math4-ar/simple.template
@@ -152,7 +152,7 @@ var tabberOptions = {manualStartup:true};
 				<!--#if can="output_problem_body" can="output_message" can="output_editorLink"-->
 					<div class="problem">
 						<!--#if can="output_problem_body"-->
-						<div class="problem-content">
+						<div id="problem_body" class="problem-content" <!--#output_problem_lang_and_dir--> >
 							<!--#output_problem_body-->
 						</div>
 						<!--#endif-->
diff --git a/htdocs/themes/math4-ar/system.template b/htdocs/themes/math4-ar/system.template
index 42a62b965a..47ec242876 100644
--- a/htdocs/themes/math4-ar/system.template
+++ b/htdocs/themes/math4-ar/system.template
@@ -251,7 +251,7 @@ var tabberOptions = {manualStartup:true};
 				<!--#if can="output_problem_body" can="output_message" can="output_editorLink"-->
 					<div class="problem">
 						<!--#if can="output_problem_body"-->
-						<div id="problem_body" class="problem-content">
+						<div id="problem_body" class="problem-content" <!--#output_problem_lang_and_dir--> >
 							<!--#output_problem_body-->
 						</div>
 						<!--#endif-->
diff --git a/htdocs/themes/math4/achievements.css b/htdocs/themes/math4/achievements.css
index 1f0a8f61f8..862dfb219d 100644
--- a/htdocs/themes/math4/achievements.css
+++ b/htdocs/themes/math4/achievements.css
@@ -8,7 +8,7 @@
 }
 
 .cheevobigbox {
-    width:700px;
+    display:block;
 }
 
 .levelbox {
@@ -47,6 +47,7 @@
 .cheevoouterbox {
     height:50px;
     margin-bottom:15px;
+    clear:both;
 }
 
 .cheevoouterbox img {
@@ -54,7 +55,6 @@
     height:50px;
     width:50px;
     margin-top:5px;
-    
 }
 
 .unlocked {
@@ -76,7 +76,8 @@
 }
 
 .cheevotextbox {
-    margin-left:15px;
+    margin-left:10px;
+    margin-top:5px;
     float:left;
 }
 
diff --git a/htdocs/themes/math4/gateway.template b/htdocs/themes/math4/gateway.template
index 9e91c9db39..e8a9ca19cd 100644
--- a/htdocs/themes/math4/gateway.template
+++ b/htdocs/themes/math4/gateway.template
@@ -1,5 +1,8 @@
 <!DOCTYPE html>
-<html lang="en-US">
+<html <!--#output_course_lang_and_dir-->>
+<!--  Note that the lang and dir attributes are now set during the
+      processing of the template and not hard-coded above.
+-->
 <head>
 <meta charset='utf-8'>
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -104,7 +107,7 @@
 	<!--#endif-->
 
 	<!--#if can="body"-->
-	<div class="Body">
+	<div class="Body" <!--#output_problem_lang_and_dir-->>
 		<!--#body-->
 	</div>
 	<!--#endif-->
diff --git a/htdocs/themes/math4/lbtwo.template b/htdocs/themes/math4/lbtwo.template
index 03eccfb595..36f94c3cb0 100644
--- a/htdocs/themes/math4/lbtwo.template
+++ b/htdocs/themes/math4/lbtwo.template
@@ -21,7 +21,7 @@
 -->
 
 
-<html xmlns="http://www.w3.org/1999/xhtml" lang="en-US">
+<html xmlns="http://www.w3.org/1999/xhtml" <!--#output_course_lang_and_dir-->>
 <head>
 <meta http-equiv="content-type" content="text/html; charset=utf-8" />
 <link rel="stylesheet" type="text/css" href="<!--#url type="webwork" name="htdocs"-->/js/vendor/bootstrap/css/bootstrap.css"/>
@@ -96,13 +96,13 @@
 	
 	    <!--#if can="body"-->
 				<!--#if warnings="1"-->
-				  <div class="Body" style="background-color:#ffcccc">
+				  <div class="Body" style="background-color:#ffcccc" <!--#output_problem_lang_and_dir-->>
 				  <p style="font-size:larger">
 				  Warning -- there may be something wrong with this question. Please inform your instructor
 				  including the warning messages below.
 				  </p>
 				<!--#else-->
-				  <div class="Body" style="background-color:#ffffff">
+				  <div class="Body" style="background-color:#ffffff" <!--#output_problem_lang_and_dir-->>
 				<!--#endif-->
 				
 				<!--#body-->
diff --git a/htdocs/themes/math4/math4.css b/htdocs/themes/math4/math4.css
index 3a286be034..3ff65e9f26 100644
--- a/htdocs/themes/math4/math4.css
+++ b/htdocs/themes/math4/math4.css
@@ -12,7 +12,14 @@
  * FOR A PARTICULAR PURPOSE.  See either the GNU General Public License or the
  * Artistic License for more details.
  */
+
+/* CodeMirror overrides */
  
+.CodeMirror-code {
+  outline: none;
+  direction: ltr !important;
+}
+
 /* Bootstrap overrides */
 a:focus {
     outline-style:solid;
diff --git a/htdocs/themes/math4/simple.template b/htdocs/themes/math4/simple.template
index 6542bb1a53..9934c0ceab 100644
--- a/htdocs/themes/math4/simple.template
+++ b/htdocs/themes/math4/simple.template
@@ -1,5 +1,8 @@
 <!DOCTYPE html>
-<html lang="en-US">
+<html <!--#output_course_lang_and_dir-->>
+<!--  Note that the lang and dir attributes are now set during the
+      processing of the template and not hard-coded above.
+-->
 <head>
 <meta charset='utf-8'>
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -152,7 +155,7 @@ var tabberOptions = {manualStartup:true};
 				<!--#if can="output_problem_body" can="output_message" can="output_editorLink"-->
 					<div class="problem">
 						<!--#if can="output_problem_body"-->
-						<div class="problem-content">
+						<div id="problem_body" class="problem-content" <!--#output_problem_lang_and_dir--> >
 							<!--#output_problem_body-->
 						</div>
 						<!--#endif-->
diff --git a/htdocs/themes/math4/system.template b/htdocs/themes/math4/system.template
index c90db501c7..ed7d411688 100644
--- a/htdocs/themes/math4/system.template
+++ b/htdocs/themes/math4/system.template
@@ -1,5 +1,8 @@
 <!DOCTYPE html>
-<html lang="en-US">
+<html <!--#output_course_lang_and_dir-->>
+<!--  Note that the lang and dir attributes are now set during the
+      processing of the template and not hard-coded above.
+-->
 <head>
 <meta charset='utf-8'>
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -210,7 +213,6 @@ var tabberOptions = {manualStartup:true};
 		<!--#output_tag_info-->
 	</div></div>
 	<!--#endif-->
-
 	 <!--#if can="output_problem_body"--> 
 	    <!-- ==== in this case print body parts ELSE print entire body -->		<div class="row-fluid">
 		  <div id="problem_body" class="Body span12">
@@ -252,7 +254,7 @@ var tabberOptions = {manualStartup:true};
 				<!--#if can="output_problem_body" can="output_message" can="output_editorLink"-->
 					<div class="problem">
 						<!--#if can="output_problem_body"-->
-						<div id="problem_body" class="problem-content">
+						<div id="problem_body" class="problem-content" <!--#output_problem_lang_and_dir--> >
 							<!--#output_problem_body-->
 						</div>
 						<!--#endif-->
diff --git a/lib/Apache/WeBWorK.pm b/lib/Apache/WeBWorK.pm
index b41ff7330d..a81a2d802f 100644
--- a/lib/Apache/WeBWorK.pm
+++ b/lib/Apache/WeBWorK.pm
@@ -35,6 +35,8 @@ use HTML::Entities;
 use HTML::Scrubber;
 use Date::Format;
 use WeBWorK;
+use Encode;
+use utf8;
 
 use mod_perl;
 use constant MP2 => ( exists $ENV{MOD_PERL_API_VERSION} and $ENV{MOD_PERL_API_VERSION} >= 2 );
@@ -65,10 +67,10 @@ sub handler($) {
 	my $log = $r->log;
 	my $uri = $r->uri;
 
+
 	# We set the bimode for print to utf8 because some language options
 	# use utf8 characters
-	binmode(STDOUT, ":utf8");
-
+	binmode(STDOUT, "encoding(UTF-8)");
 	# the warning handler accumulates warnings in $r->notes("warnings") for
 	# later cumulative reporting
 	my $warning_handler;
@@ -76,11 +78,12 @@ sub handler($) {
 		$warning_handler = sub {
 			my ($warning) = @_;
 			chomp $warning;
-			
 			my $warnings = $r->notes->get("warnings");
+			$warnings = Encode::decode_utf8($warnings);
 			$warnings .= "$warning\n";
 			#my $backtrace = join("\n",backtrace());
 			#$warnings .= "$backtrace\n\n";
+			$warnings = Encode::encode_utf8($warnings);
 			$r->notes->set(warnings => $warnings);
 			
 			$log->warn("[$uri] $warning");
diff --git a/lib/WWSafe.pm b/lib/WWSafe.pm
index 24b4c97f0e..516d0e038f 100644
--- a/lib/WWSafe.pm
+++ b/lib/WWSafe.pm
@@ -1,8 +1,9 @@
 package WWSafe;
 
-use 5.003_11;
+#use 5.003_11;
+use 5.12.0;
 use strict;
-
+use utf8;
 $Safe::VERSION = "2.16";
 
 # *** Don't declare any lexicals above this point ***
diff --git a/lib/WeBWorK.pm b/lib/WeBWorK.pm
index 469301b1b4..7f0a17a3b6 100644
--- a/lib/WeBWorK.pm
+++ b/lib/WeBWorK.pm
@@ -1,6 +1,6 @@
 ################################################################################
 # WeBWorK Online Homework Delivery System
-# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/
+# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/
 # $CVSHeader: webwork2/lib/WeBWorK.pm,v 1.104 2010/05/15 18:44:26 gage Exp $
 # 
 # This program is free software; you can redistribute it and/or modify it under
diff --git a/lib/WeBWorK/AchievementEvaluator.pm b/lib/WeBWorK/AchievementEvaluator.pm
index 72a51720f7..a065b9293a 100644
--- a/lib/WeBWorK/AchievementEvaluator.pm
+++ b/lib/WeBWorK/AchievementEvaluator.pm
@@ -26,11 +26,10 @@ use base qw(WeBWorK);
 use strict;
 use warnings;
 use WeBWorK::CGI;
-use WeBWorK::Utils qw(before after readFile sortAchievements);
+use WeBWorK::Utils qw(before after readFile sortAchievements nfreeze_base64 thaw_base64);
 use WeBWorK::Utils::Tags;
 
 use WWSafe;
-use Storable qw(nfreeze thaw);
 
 sub checkForAchievements {
 
@@ -118,9 +117,10 @@ sub checkForAchievements {
     #Methods alowed in the safe container
     $compartment->permit(qw(time localtime));
 
-    #Thaw globalData hash
-    if ($globalUserAchievement->frozen_hash) {       
-		$globalData = thaw($globalUserAchievement->frozen_hash);
+    #Thaw_Base64 globalData hash
+    if ($globalUserAchievement->frozen_hash) {
+
+		$globalData = thaw_base64($globalUserAchievement->frozen_hash);
     }
 
     #Update a couple of "standard" variables in globalData hash.
@@ -217,9 +217,9 @@ sub checkForAchievements {
 	my $setType = $set->assignment_type;
 	next unless $achievement->assignment_type =~ /$setType/;
 
-	#thaw localData hash
+	#thaw_base64 localData hash
 	if ($userAchievement->frozen_hash) {
-	    $localData = thaw($userAchievement->frozen_hash);
+	    $localData = thaw_base64($userAchievement->frozen_hash);
 	}
 
 	#recover counter information (for progress bar achievements)
@@ -310,15 +310,15 @@ sub checkForAchievements {
 	    $achievementPoints += $points;
 	}    
 	
-	#update counter, nfreeze localData and store
+	#update counter, nfreeze_base64 localData and store
 	$userAchievement->counter($counter);
-	$userAchievement->frozen_hash(nfreeze($localData));	
+	$userAchievement->frozen_hash(nfreeze_base64($localData));	
 	$db->putUserAchievement($userAchievement);
 	
     }  #end for loop
     
-    #nfreeze globalData and store
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    #nfreeze_base64 globalData and store
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     if ($cheevoMessage) {
diff --git a/lib/WeBWorK/AchievementItems.pm b/lib/WeBWorK/AchievementItems.pm
index 22f568b48f..5019bf92c8 100644
--- a/lib/WeBWorK/AchievementItems.pm
+++ b/lib/WeBWorK/AchievementItems.pm
@@ -17,11 +17,11 @@
 package WeBWorK::AchievementItems;
 use base qw(WeBWorK);
 
+use WeBWorK::Utils qw(nfreeze_base64 thaw_base64);
+
 use strict;
 use warnings;
 
-use Storable qw(nfreeze thaw);
-
 # have to add any new items to this list, furthermore
 # the elements of this list have to match the class name/id of the
 # item classes defined below. 
@@ -70,7 +70,7 @@ sub UserItems {
     
     return unless ($globalUserAchievement->frozen_hash);
 
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
     my @items;
 
     # ugly eval to get a new item object for each type of item.  
@@ -86,8 +86,8 @@ sub UserItems {
 
 package WeBWorK::AchievementItems::ResurrectHW;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -139,7 +139,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -167,7 +167,7 @@ sub use_item {
     }
 
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -177,8 +177,8 @@ sub use_item {
 
 package WeBWorK::AchievementItems::ExtendDueDate;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -229,7 +229,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -250,7 +250,7 @@ sub use_item {
     $db->putUserSet($userSet);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -260,8 +260,8 @@ sub use_item {
 
 package WeBWorK::AchievementItems::SuperExtendDueDate;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -312,7 +312,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -333,7 +333,7 @@ sub use_item {
     $db->putUserSet($userSet);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -343,8 +343,8 @@ sub use_item {
 
 package WeBWorK::AchievementItems::ReducedCred;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -400,7 +400,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "This item won't work unless your instructor enables the reduced scoring feature.  Let them know that you recieved this message." unless $ce->{pg}{ansEvalDefaults}{reducedScoringPeriod};
 	
@@ -428,7 +428,7 @@ sub use_item {
     $db->putUserSet($userSet);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -438,8 +438,8 @@ sub use_item {
 
 package WeBWorK::AchievementItems::DoubleSet;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -491,7 +491,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -516,7 +516,7 @@ sub use_item {
     }
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -525,8 +525,8 @@ sub use_item {
 #Item to reset number of incorrect attempts.
 package WeBWorK::AchievementItems::ResetIncorrectAttempts;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -604,7 +604,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -627,7 +627,7 @@ sub use_item {
     $db->putUserProblem($problem);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -636,8 +636,8 @@ sub use_item {
 #Item to make a problem worth double.  
 package WeBWorK::AchievementItems::DoubleProb;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -716,7 +716,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -740,7 +740,7 @@ sub use_item {
     $db->putUserProblem($problem);
 
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -749,8 +749,8 @@ sub use_item {
 #Item to give half credit on a single problem.
 package WeBWorK::AchievementItems::HalfCreditProb;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -829,7 +829,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -856,7 +856,7 @@ sub use_item {
     $db->putUserProblem($problem);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -865,8 +865,8 @@ sub use_item {
 #Item to give half credit on all problems in a homework set.
 package WeBWorK::AchievementItems::HalfCreditSet;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -917,7 +917,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -946,7 +946,7 @@ sub use_item {
     }
     
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -955,8 +955,8 @@ sub use_item {
 #Item to give full credit on a single problem
 package WeBWorK::AchievementItems::FullCreditProb;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -1034,7 +1034,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -1057,7 +1057,7 @@ sub use_item {
     $db->putUserProblem($problem);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -1066,8 +1066,8 @@ sub use_item {
 #Item to give half credit on all problems in a homework set.
 package WeBWorK::AchievementItems::FullCreditSet;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -1118,7 +1118,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -1142,7 +1142,7 @@ sub use_item {
     }
     
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -1151,8 +1151,8 @@ sub use_item {
 #Item to turn one problem into another problem
 package WeBWorK::AchievementItems::DuplicateProb;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -1237,7 +1237,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -1267,7 +1267,7 @@ sub use_item {
     $db->putUserProblem($problem2);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     return;
@@ -1276,8 +1276,8 @@ sub use_item {
 #Item to print a suprise message
 package WeBWorK::AchievementItems::Surprise;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -1328,8 +1328,8 @@ sub use_item {
 #Item to allow students to take an addition test
 package WeBWorK::AchievementItems::AddNewTestGW;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -1397,7 +1397,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -1418,7 +1418,7 @@ sub use_item {
     $db->putUserSet($userSet);
     
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
 
     
@@ -1429,8 +1429,8 @@ sub use_item {
 #Item to extend the due date on a gateway 
 package WeBWorK::AchievementItems::ExtendDueDateGW;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -1498,7 +1498,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -1531,7 +1531,7 @@ sub use_item {
     }
     
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
     
     return;
@@ -1540,8 +1540,8 @@ sub use_item {
 #Item to extend the due date on a gateway 
 package WeBWorK::AchievementItems::ResurrectGW;
 our @ISA = qw(WeBWorK::AchievementItems);
-use Storable qw(nfreeze thaw);
-use WeBWorK::Utils qw(sortByName before after between x);
+
+use WeBWorK::Utils qw(sortByName before after between x nfreeze_base64 thaw_base64);
 
 sub new {
     my $class = shift;
@@ -1599,7 +1599,7 @@ sub use_item {
     my $globalUserAchievement = $db->getGlobalUserAchievement($userName);
     return "No achievement data?!?!?!" 
 	unless ($globalUserAchievement->frozen_hash);
-    my $globalData = thaw($globalUserAchievement->frozen_hash);
+    my $globalData = thaw_base64($globalUserAchievement->frozen_hash);
 
     return "You are $self->{id} trying to use an item you don't have" unless
 	($globalData->{$self->{id}});
@@ -1619,7 +1619,7 @@ sub use_item {
     $db->putUserSet($set);
 	
     $globalData->{$self->{id}} = 0;
-    $globalUserAchievement->frozen_hash(nfreeze($globalData));
+    $globalUserAchievement->frozen_hash(nfreeze_base64($globalData));
     $db->putGlobalUserAchievement($globalUserAchievement);
     
     return;
diff --git a/lib/WeBWorK/ContentGenerator.pm b/lib/WeBWorK/ContentGenerator.pm
index 088dae67ba..4e41268281 100644
--- a/lib/WeBWorK/ContentGenerator.pm
+++ b/lib/WeBWorK/ContentGenerator.pm
@@ -61,7 +61,8 @@ use HTML::Entities;
 use HTML::Scrubber;
 use WeBWorK::Utils qw(jitar_id_to_seq);
 use WeBWorK::Authen::LTIAdvanced::SubmitGrade;
-  
+use Encode; 
+
 our $TRACE_WARNINGS = 0;   # set to 1 to trace channel used by warning message
 
 
@@ -514,6 +515,53 @@ HTTP header is sent but before any content is sent.
 
 #sub initialize {  }
 
+=item output_course_lang_and_dir()
+
+Defined in this package.
+
+Sets the LANG attribute and when needed the DIR attribute based
+on the language set in the course configuration.
+
+The intended use is to set these tags in the main HTML tag of the generated
+web page when the template files calls this function.
+
+It selects the language based on the setting in the course configuration
+file (when it is set) and otherwise defaults back to
+	lang="en-US"
+which was the old hard-coded setting.
+
+When the language chosen is a known right to left language, it will also set
+the DIR attribute to "rtl". Currently, only Hebrew ("heb" or "he") and
+Arabic ("ar") trigger the RTL direction setting.
+
+=cut
+
+sub output_course_lang_and_dir{
+        my $self = shift;
+        my $master_lang_setting = "lang=\"en-US\""; # default setting
+        my $master_dir_setting  = "";               # default is NOT set
+
+        my $ce_lang = $self->r->ce->{language};
+
+        if ( $ce_lang eq "en" ) {
+          $master_lang_setting = "lang=\"en-US\""; # as in default
+        } elsif ( $ce_lang =~ /^he/i ) { # supports also the current "heb" option
+          # Hebrew - requires RTL direction
+          $master_lang_setting = "lang=\"he\""; # Hebrew
+          $master_dir_setting  = "dir=\"rtl\""; # RTL
+        } elsif ( $ce_lang =~ /^ar/i ) {
+          # Hebrew - requires RTL direction
+          $master_lang_setting = "lang=\"ar\""; # Arabic
+          $master_dir_setting  = "dir=\"rtl\""; # RTL
+        } else {
+          # use the language setting of the course, with NO direction setting
+          $master_lang_setting = "lang=\"${ce_lang}\"";
+        }
+
+        print "$master_lang_setting $master_dir_setting";
+        return "";
+}
+
 =item content()
 
 Defined in this package.
@@ -1213,6 +1261,7 @@ sub warnings {
 	print CGI::p("Entering ContentGenerator::warnings") if $TRACE_WARNINGS;
 	print "\n<!-- BEGIN " . __PACKAGE__ . "::warnings -->\n";
 	my $warnings = MP2 ? $r->notes->get("warnings") : $r->notes("warnings");
+	$warnings = Encode::decode_utf8($warnings);
 	print $self->warningOutput($warnings) if $warnings;
 	print "<!-- END " . __PACKAGE__ . "::warnings -->\n";
 	
@@ -1804,7 +1853,7 @@ sub url_args {
 	foreach my $param (@fields) {
 		my @values = $r->param($param);
 		foreach my $value (@values) {
-			push @pairs, uri_escape($param) . "=" . uri_escape($value);
+			push @pairs, uri_escape_utf8($param) . "=" . uri_escape($value);
 		}
 	}
 	
@@ -2160,7 +2209,6 @@ sub warningOutput($$) {
 	
 	foreach my $warning (@warnings) {
             # Since these warnings have html they look better scrubbed
-
 	    #$warning = HTML::Entities::encode_entities($warning);  
 	    $warning = $scrubber->scrub($warning);
 	    $warning = CGI::li(CGI::code($warning));
diff --git a/lib/WeBWorK/ContentGenerator/Achievements.pm b/lib/WeBWorK/ContentGenerator/Achievements.pm
index fd6f208ee7..9a084fe2c0 100644
--- a/lib/WeBWorK/ContentGenerator/Achievements.pm
+++ b/lib/WeBWorK/ContentGenerator/Achievements.pm
@@ -240,7 +240,7 @@ sub body {
 		    print CGI::start_div({id=>"modal_".$item->id(),class=>"modal hide fade"});
 		    print CGI::start_div({class=>'modal-header'});
 		    print CGI::a({href=>"#",class=>"close","data-dismiss"=>"modal", "aria-hidden"=>"true"},CGI::span({class=>"icon icon-remove"}),CGI::div({class=>"sr-only"},$r->maketext("close")));
-		    print CGI::h3($item->name()); 
+		    print CGI::h3($r->maketext($item->name())); 
 		    print CGI::end_div();
 		    print CGI::start_form({method=>"post", action=>$self->systemLink($urlpath,authen=>0), name=>"itemform_$itemnumber", class=>"achievementitemform"});
 		    print CGI::start_div({class=>"modal-body"});
@@ -303,7 +303,7 @@ sub body {
 			$imgSrc = $ce->{webworkURLs}->{htdocs}."/images/defaulticon.png";
 			}
 	
-			print CGI::img({src=>$imgSrc, alt=>$userAchievement->earned ? 'Achievement Earned' : 'Achievement Unearned'});
+			print CGI::div(CGI::img({src=>$imgSrc, alt=>$userAchievement->earned ? 'Achievement Earned' : 'Achievement Unearned'}));
 			print CGI::start_div({class=>'cheevotextbox'});
 			print CGI::h3($achievement->name);
 			print CGI::div(CGI::i($r->maketext("[_1] Points:", $achievement->{points})).' '.$achievement->{description});
diff --git a/lib/WeBWorK/ContentGenerator/Grades.pm b/lib/WeBWorK/ContentGenerator/Grades.pm
index dca20ea279..809735365c 100644
--- a/lib/WeBWorK/ContentGenerator/Grades.pm
+++ b/lib/WeBWorK/ContentGenerator/Grades.pm
@@ -109,7 +109,7 @@ sub scoring_info {
 	my $header = '';
 	local(*FILE);
 	if (-e "$filePath" and -r "$filePath") {
-		open FILE, "$filePath" || return("Can't open $filePath"); 
+		open FILE, "<:utf8", "$filePath" || return("Can't open $filePath"); 
 		while ($header !~ s/Message:\s*$//m and not eof(FILE)) { 
 			$header .= <FILE>; 
 		}
diff --git a/lib/WeBWorK/ContentGenerator/Hardcopy.pm b/lib/WeBWorK/ContentGenerator/Hardcopy.pm
index 6352c789e8..04293a0c00 100644
--- a/lib/WeBWorK/ContentGenerator/Hardcopy.pm
+++ b/lib/WeBWorK/ContentGenerator/Hardcopy.pm
@@ -692,7 +692,7 @@ sub generate_hardcopy {
 	# create TeX file  (callback  write_multiuser_tex,  or ??)
 	#######################################
 	
-	my $open_result = open my $FH, ">", $tex_file_path;
+	my $open_result = open my $FH, ">:utf8", $tex_file_path;
 	unless ($open_result) {
 		$self->add_errors("Failed to open file '".CGI::code(CGI::escapeHTML($tex_file_path))
 			."' for writing: ".CGI::code(CGI::escapeHTML($!)));
@@ -859,7 +859,7 @@ sub generate_hardcopy_pdf {
 		# read hardcopy.log and report first error
 		my $hardcopy_log = "$temp_dir_path/hardcopy.log";
 		if (-e $hardcopy_log) {
-			if (open my $LOG, "<", $hardcopy_log) {
+			if (open my $LOG, "<:utf8", $hardcopy_log) {
 				my $line;
 				while ($line = <$LOG>) {
 					last if $line =~ /^!\s+/;
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/AchievementEditor.pm b/lib/WeBWorK/ContentGenerator/Instructor/AchievementEditor.pm
index 679fbf925f..9147a655d6 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/AchievementEditor.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/AchievementEditor.pm
@@ -659,7 +659,7 @@ sub save_as_handler {
 	my $viewURL = $self->systemLink($problemPage, 
 					params=>{
 					    sourceFilePath     => $relativeOutputFilePath,
-					    status_message     => uri_escape($self->{status_message})}
+					    status_message     => uri_escape_utf8($self->{status_message})}
 					
 	    );
 	
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/Config.pm b/lib/WeBWorK/ContentGenerator/Instructor/Config.pm
index c85d176708..072ae82775 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/Config.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/Config.pm
@@ -245,9 +245,10 @@ sub save_string {
 	my $varname = $self->{var};
 	my $newval = $self->convert_newval_source($newvalsource);
 	my $displayoldval = $self->comparison_value($oldval);
+	my $r = $self->{Module}->r;
 	return '' if($displayoldval eq $newval);
 	my $str = '$'. $varname . " = '$newval';\n";
-	$str = '$'. $varname . " = undef;\n" if $newval eq 'nobody';
+	$str = '$'. $varname . " = undef;\n" if $newval eq $r->maketext('nobody');
 	return($str);
 }
 
@@ -486,7 +487,7 @@ sub writeFile {
 	my $writeFileErrors;
 	eval {                                                          
 		local *OUTPUTFILE;
-		if( open OUTPUTFILE, ">", $outputFilePath) {
+		if( open OUTPUTFILE, ">utf8:", $outputFilePath) {
 			print OUTPUTFILE $contents;
 			close OUTPUTFILE;
 		} else {
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm b/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm
index 1a28983930..41469f88b4 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm
@@ -17,6 +17,7 @@
 package WeBWorK::ContentGenerator::Instructor::FileManager;
 use base qw(WeBWorK::ContentGenerator::Instructor);
 
+use utf8;
 use WeBWorK::Utils qw(readDirectory readFile sortByName listFilesRecursive);
 use WeBWorK::Upload;
 use File::Path;
@@ -538,7 +539,7 @@ sub Save {
 	if (defined($data)) {
 		$data =~ s/\r\n?/\n/g;  # convert DOS and Mac line ends to unix
 		local (*OUTFILE);
-		if (open(OUTFILE,">$file")) {
+		if (open(OUTFILE,":encoding(UTF-8)",$file)) {
 			eval {print OUTFILE $data; close(OUTFILE)};
 			if ($@) {$self->addbadmessage($r->maketext("Failed to save: [_1]",$@))}
 			   else {$self->addgoodmessage($r->maketext("File saved"))}
@@ -819,7 +820,7 @@ sub NewFile {
 		my $name = $self->r->param('name');
 		if (my $file = $self->verifyName($name,"file")) {
 			local (*NEWFILE);
-			if (open(NEWFILE,">$file")) {
+			if (open(NEWFILE,">:encoding(UTF-8)",$file)) {
 				close(NEWFILE);
 				$self->RefreshEdit("",$name);
 				return;
@@ -925,7 +926,16 @@ sub Upload {
 	if ($type eq 'Text') {
 		$upload->dispose;
 		$data =~ s/\r\n?/\n/g;
-		if (open(UPLOAD,">$file")) {print UPLOAD $data; close(UPLOAD)}
+		if (open(UPLOAD,">:encoding(UTF-8)",$file)) {
+			my $backup_data=$data; 
+			my $success= utf8::decode($data); # try to decode as utf8
+			unless ($success){
+				warn "Trying to convert file $file from latin1? to UTF-8";
+				utf8::upgrade($backup_data); # try to convert data from latin1 to utf8.
+				$data=$backup_data;
+			}
+		  print UPLOAD $data; # print massaged data to file. 
+		  close(UPLOAD)}
 		  else {$self->addbadmessage($r->maketext("Can't create file '[_1]': [_2]", $name, $!))}
 	} else {
 		$upload->disposeTo($file);
@@ -1269,12 +1279,13 @@ sub showHTML {
 ##################################################
 #
 # Check if a string is plain text
-# (i.e., doesn't contain four non-regular
-# characters in a row.)
 #
 sub isText {
 	my $string = shift;
-	return $string !~ m/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]{2}/;
+
+	#	return $string !~ m/[^\s\x20-\x7E]{4}/;
+	return utf8::is_utf8($string);
+	# return $string !~ m/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]{2}/;
 }
 
 ##################################################
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/GetLibrarySetProblems.pm b/lib/WeBWorK/ContentGenerator/Instructor/GetLibrarySetProblems.pm
index 4f45337a64..6a306c9c3b 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/GetLibrarySetProblems.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/GetLibrarySetProblems.pm
@@ -34,6 +34,7 @@ use WeBWorK::Debug;
 use WeBWorK::Form;
 use WeBWorK::Utils qw(readDirectory max sortByName);
 use WeBWorK::Utils::Tasks qw(renderProblems);
+use WeBWorK::Utils::DetermineProblemLangAndDirection;
 use File::Find;
 
 require WeBWorK::Utils::ListingDB;
@@ -280,9 +281,22 @@ sub make_data_row {
 	my $isGatewaySet = ( defined($setRecord) && 
 			     $setRecord->assignment_type =~ /gateway/ );
 
+	my %problem_div_settings = ( -class=>"RenderSolo" );
+        # Add what is needed for lang and dir settings
+	my @to_set_lang_dir = get_problem_lang_and_dir( $self, $pg );
+	my $to_set_tag;
+	my $to_set_val;
+	while ( scalar(@to_set_lang_dir) > 0 ) {
+	  $to_set_tag = shift( @to_set_lang_dir );
+	  $to_set_val = shift( @to_set_lang_dir );
+	  if ( defined( $to_set_val ) ) {
+	    $problem_div_settings{ "$to_set_tag" } = "$to_set_val";
+	  }
+	}
+
 	my $problem_output = $pg->{flags}->{error_flag} ?
 		CGI::div({class=>"ResultsWithError"}, CGI::em("This problem produced an error"))
-		: CGI::div({class=>"RenderSolo"}, $pg->{body_text});
+		: CGI::div( \%problem_div_settings, $pg->{body_text});
 	$problem_output .= $pg->{flags}->{comment} if($pg->{flags}->{comment});
 
 
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/GetTargetSetProblems.pm b/lib/WeBWorK/ContentGenerator/Instructor/GetTargetSetProblems.pm
index c1e60ed886..51f7484381 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/GetTargetSetProblems.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/GetTargetSetProblems.pm
@@ -34,6 +34,7 @@ use WeBWorK::Debug;
 use WeBWorK::Form;
 use WeBWorK::Utils qw(readDirectory max sortByName);
 use WeBWorK::Utils::Tasks qw(renderProblems);
+use WeBWorK::Utils::DetermineProblemLangAndDirection;
 use File::Find;
 
 require WeBWorK::Utils::ListingDB;
@@ -98,9 +99,22 @@ sub make_myset_data_row {
 	my $isGatewaySet = ( defined($setRecord) && 
 			     $setRecord->assignment_type =~ /gateway/ );
 
+	my %problem_div_settings = ( -class=>"RenderSolo", -dir=>"ltr" );
+        # Add what is needed for lang and dir settings
+	my @to_set_lang_dir = get_problem_lang_and_dir( $self, $pg );
+	my $to_set_tag;
+	my $to_set_val;
+	while ( scalar(@to_set_lang_dir) > 0 ) {
+	  $to_set_tag = shift( @to_set_lang_dir );
+	  $to_set_val = shift( @to_set_lang_dir );
+	  if ( defined( $to_set_val ) ) {
+	    $problem_div_settings{ "$to_set_tag" } = "$to_set_val";
+	  }
+	}
+
 	my $problem_output = $pg->{flags}->{error_flag} ?
 		CGI::div({class=>"ResultsWithError"}, CGI::em("This problem produced an error"))
-		: CGI::div({class=>"RenderSolo"}, $pg->{body_text});
+		: CGI::div( \%problem_div_settings, $pg->{body_text});
 	$problem_output .= $pg->{flags}->{comment} if($pg->{flags}->{comment});
 
 
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor.pm b/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor.pm
index abbd7f1010..7327db362f 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor.pm
@@ -1079,7 +1079,7 @@ sub saveFileChanges {
 
 		eval {
 			local *OUTPUTFILE;
-			open OUTPUTFILE,  ">$outputFilePath"
+			open OUTPUTFILE,  ">:utf8", $outputFilePath
 					or die "Failed to open $outputFilePath";
 			print OUTPUTFILE $problemContents;
 			close OUTPUTFILE;		
@@ -1291,7 +1291,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);
@@ -1308,7 +1308,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1325,7 +1325,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1340,7 +1340,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'options_info') {  # redirec to Options.pm
@@ -1352,7 +1352,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} else {
@@ -1445,7 +1445,7 @@ sub add_problem_handler {
 					editMode           => "savedFile",
 					edit_level         => $edit_level,
 					sourceFilePath     => $relativeSourceFilePath,
-					status_message     => uri_escape($self->{status_message})
+					status_message     => uri_escape_utf8($self->{status_message})
 	
 				}
 		);
@@ -1472,7 +1472,7 @@ sub add_problem_handler {
 					displayMode        => $displayMode,
 					editMode           => "savedFile",
 					edit_level         => $edit_level,
-					status_message     => uri_escape($self->{status_message})
+					status_message     => uri_escape_utf8($self->{status_message})
 				}
 		);
 	} else {
@@ -1556,7 +1556,7 @@ sub save_handler {
 				editMode           => "savedFile",
 				edit_level         => 0,
 				sourceFilePath     => $relativeEditFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);
@@ -1571,7 +1571,7 @@ sub save_handler {
 				problemSeed        => $problemSeed,
 				editMode           => "savedFile",
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1586,7 +1586,7 @@ sub save_handler {
 				problemSeed        => $problemSeed,
 				editMode           => "savedFile",
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1598,7 +1598,7 @@ sub save_handler {
 			params => {
 				editMode           => ("savedFile"),
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'options_info') {  # redirect to Options.pm
@@ -1608,7 +1608,7 @@ sub save_handler {
 			params => {
 				editMode           => ("savedFile"),
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'source_path_for_problem_file') {  # redirect to ProblemSets.pm
@@ -1623,7 +1623,7 @@ sub save_handler {
 				edit_level         => 0,
 				sourceFilePath     => $outputFilePath, #The path relative to the templates directory is required.
 				file_type          => 'source_path_for_problem_file',
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 
 			 }
 	);
@@ -1918,7 +1918,7 @@ sub save_as_handler {
 									 problemSeed        => $problemSeed,
 									 edit_level         => $edit_level,
 									 file_type          => $new_file_type,
-									 status_message     => uri_escape($self->{status_message})
+									 status_message     => uri_escape_utf8($self->{status_message})
 
 								 }
 	);
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor2.pm b/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor2.pm
index b5ea7a32fe..585eb9a139 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor2.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor2.pm
@@ -1076,7 +1076,7 @@ sub saveFileChanges {
 
 		eval {
 			local *OUTPUTFILE;
-			open OUTPUTFILE,  ">$outputFilePath"
+			open OUTPUTFILE,  ">:utf8", $outputFilePath
 					or die "Failed to open $outputFilePath";
 			print OUTPUTFILE $problemContents;
 			close OUTPUTFILE;		
@@ -1292,7 +1292,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);
@@ -1309,7 +1309,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1326,7 +1326,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1341,7 +1341,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'options_info') {  # redirec to Options.pm
@@ -1353,7 +1353,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} else {
@@ -1460,7 +1460,7 @@ sub add_problem_handler {
 					editMode           => "savedFile",
 					edit_level         => $edit_level,
 					sourceFilePath     => $relativeSourceFilePath,
-					status_message     => uri_escape($self->{status_message}),
+					status_message     => uri_escape_utf8($self->{status_message}),
 					file_type          => 'problem',
 	
 				}
@@ -1489,7 +1489,7 @@ sub add_problem_handler {
 					displayMode        => $displayMode,
 					editMode           => "savedFile",
 					edit_level         => $edit_level,
-					status_message     => uri_escape($self->{status_message}),
+					status_message     => uri_escape_utf8($self->{status_message}),
 						}
 		);
 	} elsif ($targetFileType eq 'hardcopy_header')  {
@@ -1515,7 +1515,7 @@ sub add_problem_handler {
 					displayMode        => $displayMode,
 					editMode           => "savedFile",
 					edit_level         => $edit_level,
-					status_message     => uri_escape($self->{status_message}),
+					status_message     => uri_escape_utf8($self->{status_message}),
 						}
 		);
 	} else {
@@ -1599,7 +1599,7 @@ sub save_handler {
 				editMode           => "savedFile",
 				edit_level         => 0,
 				sourceFilePath     => $relativeEditFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);
@@ -1614,7 +1614,7 @@ sub save_handler {
 				problemSeed        => $problemSeed,
 				editMode           => "savedFile",
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1629,7 +1629,7 @@ sub save_handler {
 				problemSeed        => $problemSeed,
 				editMode           => "savedFile",
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1641,7 +1641,7 @@ sub save_handler {
 			params => {
 				editMode           => ("savedFile"),
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'options_info') {  # redirect to Options.pm
@@ -1651,7 +1651,7 @@ sub save_handler {
 			params => {
 				editMode           => ("savedFile"),
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'source_path_for_problem_file') {  # redirect to ProblemSets.pm
@@ -1666,7 +1666,7 @@ sub save_handler {
 				edit_level         => 0,
 				sourceFilePath     => $outputFilePath, #The path relative to the templates directory is required.
 				file_type          => 'source_path_for_problem_file',
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 
 			 }
 	);
@@ -1810,7 +1810,7 @@ sub save_as_handler {
 		# setting $do_not_save stops saving and any redirects
 		$do_not_save = 1;
 		$self->addbadmessage(CGI::p($r->maketext("File '[_1]' exists. File not saved. No changes have been made.  You can change the file path for this problem manually from the 'Hmwk Sets Editor' page", $self->shortPath($outputFilePath))));
-		$self->addgoodmessage(CGI::p($r->maketext($r->maketext("The text box now contains the source of the original problem. You can recover lost edits by using the Back button on your browser."))));
+		$self->addgoodmessage(CGI::p($r->maketext("The text box now contains the source of the original problem. You can recover lost edits by using the Back button on your browser.")));
 	} else {
 		$self->{editFilePath} = $outputFilePath;
 		$self->{tempFilePath} = ''; # nothing needs to be unlinked.
@@ -1938,7 +1938,7 @@ sub save_as_handler {
 									 problemSeed        => $problemSeed,
 									 edit_level         => $edit_level,
 									 file_type          => $new_file_type,
-									 status_message     => uri_escape($self->{status_message})
+									 status_message     => uri_escape_utf8($self->{status_message})
 
 								 }
 	);
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor3.pm b/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor3.pm
index a8e5be966d..43fc31caf4 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor3.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/PGProblemEditor3.pm
@@ -1154,8 +1154,8 @@ sub saveFileChanges {
 		die "outputFilePath is unsafe!" unless path_is_subdir($outputFilePath, $ce->{courseDirs}->{templates}, 1); # 1==path can be relative to dir
 
 		eval {
-			local *OUTPUTFILE;
-			open OUTPUTFILE,  ">$outputFilePath"
+		    local *OUTPUTFILE;
+		    open (OUTPUTFILE,  ">:encoding(UTF-8)",$outputFilePath)
 					or die "Failed to open $outputFilePath";
 			print OUTPUTFILE $problemContents;
 			close OUTPUTFILE;		
@@ -1368,7 +1368,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);
@@ -1385,7 +1385,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1402,7 +1402,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1417,7 +1417,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'options_info') {  # redirec to Options.pm
@@ -1429,7 +1429,7 @@ sub view_handler {
 				editMode           => "temporaryFile",
 				edit_level         => $edit_level,
 				sourceFilePath     => $relativeTempFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} else {
@@ -1535,7 +1535,7 @@ sub add_problem_handler {
 					edit_level         => $edit_level,
 					sourceFilePath     => $relativeSourceFilePath,
 					file_type          => 'problem',
-					status_message     => uri_escape($self->{status_message})
+					status_message     => uri_escape_utf8($self->{status_message})
 	
 				}
 		);
@@ -1562,7 +1562,7 @@ sub add_problem_handler {
 					displayMode        => $displayMode,
 					editMode           => "savedFile",
 					edit_level         => $edit_level,
-					status_message     => uri_escape($self->{status_message})
+					status_message     => uri_escape_utf8($self->{status_message})
 				}
 		);
 	} elsif ($targetFileType eq 'hardcopy_header')  {
@@ -1588,7 +1588,7 @@ sub add_problem_handler {
 					displayMode        => $displayMode,
 					editMode           => "savedFile",
 					edit_level         => $edit_level,
-					status_message     => uri_escape($self->{status_message}),
+					status_message     => uri_escape_utf8($self->{status_message}),
 						}
 		);
 	} else {
@@ -1672,7 +1672,7 @@ sub save_handler {
 				editMode           => "savedFile",
 				edit_level         => 0,
 				sourceFilePath     => $relativeEditFilePath,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);
@@ -1687,7 +1687,7 @@ sub save_handler {
 				problemSeed        => $problemSeed,
 				editMode           => "savedFile",
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1702,7 +1702,7 @@ sub save_handler {
 				problemSeed        => $problemSeed,
 				editMode           => "savedFile",
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 	
 			}
 		);	
@@ -1714,7 +1714,7 @@ sub save_handler {
 			params => {
 				editMode           => ("savedFile"),
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'options_info') {  # redirect to Options.pm
@@ -1724,7 +1724,7 @@ sub save_handler {
 			params => {
 				editMode           => ("savedFile"),
 				edit_level         => 0,
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 			}
 		);
 	} elsif ($file_type eq 'source_path_for_problem_file') {  # redirect to ProblemSets.pm
@@ -1739,7 +1739,7 @@ sub save_handler {
 				edit_level         => 0,
 				sourceFilePath     => $outputFilePath, #The path relative to the templates directory is required.
 				file_type          => 'source_path_for_problem_file',
-				status_message     => uri_escape($self->{status_message})
+				status_message     => uri_escape_utf8($self->{status_message})
 
 			 }
 	);
@@ -1883,7 +1883,7 @@ sub save_as_handler {
 		# setting $do_not_save stops saving and any redirects
 		$do_not_save = 1;
 		$self->addbadmessage(CGI::p($r->maketext("File '[_1]' exists. File not saved. No changes have been made.  You can change the file path for this problem manually from the 'Hmwk Sets Editor' page", $self->shortPath($outputFilePath))));
-		$self->addgoodmessage(CGI::p($r->maketext($r->maketext("The text box now contains the source of the original problem. You can recover lost edits by using the Back button on your browser."))));
+		$self->addgoodmessage(CGI::p($r->maketext("The text box now contains the source of the original problem. You can recover lost edits by using the Back button on your browser.")));
 	} else {
 		$self->{editFilePath} = $outputFilePath;
 		$self->{tempFilePath} = ''; # nothing needs to be unlinked.
@@ -2014,7 +2014,7 @@ sub save_as_handler {
 									 problemSeed        => $problemSeed,
 									 edit_level         => $edit_level,
 									 file_type          => $new_file_type,
-									 status_message     => uri_escape($self->{status_message})
+									 status_message     => uri_escape_utf8($self->{status_message})
 
 								 }
 	);
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/ProblemSetDetail.pm b/lib/WeBWorK/ContentGenerator/Instructor/ProblemSetDetail.pm
index 1d3107fee4..4fc4f0991c 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/ProblemSetDetail.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/ProblemSetDetail.pm
@@ -2291,6 +2291,10 @@ sub body {
 					($error ? 
 						CGI::div({class=>"ResultsWithError", style=>"font-weight: bold"}, $error) 
 						: CGI::div({class=> "RenderSolo"}, $problem_html[0]->{body_text})
+					 # did not add code to the div above to handle problem language and text direction
+					 # other RenderSolo objects make use of get_problem_lang_and_dir( $self, $pg )
+					 # and code to build a hash which sets the lang and dir attributes in addition to
+					 # the class RenderSolo. 
 					) .
 					($repeatFile ? CGI::div({class=>"ResultsWithError", style=>"font-weight: bold"}, $repeatFile) : ''),
 			]));
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/Scoring.pm b/lib/WeBWorK/ContentGenerator/Instructor/Scoring.pm
index 76b43035f5..ef9aa80bf2 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/Scoring.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/Scoring.pm
@@ -747,7 +747,7 @@ sub writeCSV {
 		rename $filename, $bakFileName or warn "Unable to rename $filename to $bakFileName";
 	}
 
-	open my $fh, ">", $filename or warn "Unable to open $filename for writing";
+	open my $fh, ">:utf8", $filename or warn "Unable to open $filename for writing";
 	foreach my $row (@csv) {
 		my @rowPadded = ();
 		foreach (my $column = 0; $column < @$row; $column++) {
@@ -775,7 +775,7 @@ sub readStandardCSV {
 
 sub writeStandardCSV {
 	my ($self, $filename, @csv) = @_;
-	open my $fh, ">", $filename;
+	open my $fh, ">:utf8", $filename;
 	foreach my $row (@csv) {
 		print $fh (join ",", map {$self->quote($_)} @$row);
 		print $fh "\n";
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/ScoringDownload.pm b/lib/WeBWorK/ContentGenerator/Instructor/ScoringDownload.pm
index a0d3f6de1e..4123b1b3a8 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/ScoringDownload.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/ScoringDownload.pm
@@ -89,7 +89,7 @@ sub content {
 		print "You do not have permission to access scoring data";
 	} else {
 		my $file = $r->param('getFile');
-		open my $fh, "<", "$scoringDir/$file";
+		open my $fh, "<:utf8", "$scoringDir/$file";
 		print while (<$fh>);
 		close $fh;
 	}
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm b/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm
index 6fafff810e..39625ef4c5 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm
@@ -776,7 +776,7 @@ sub saveProblem {
     my $self      = shift;
 	my ($body, $probFileName)= @_;
 	local(*PROBLEM);
-	open (PROBLEM, ">$probFileName") ||
+	open (PROBLEM, ">:utf8",$probFileName) ||
 		$self->addbadmessage(CGI::p("Could not open $probFileName for writing.
 						Check that the  permissions for this problem are 660 (-rw-rw----)"));
 	print PROBLEM $body if -w $probFileName;
@@ -794,9 +794,9 @@ sub read_input_file {
 	my ($subject, $from, $replyTo);
 	local(*FILE);
 	if (-e "$filePath" and -r "$filePath") {
-		open FILE, "$filePath" || do { $self->addbadmessage(CGI::p($r->maketext("Can't open [_1]",$filePath))); return};
-		while ($header !~ s/Message:\s*$//m and not eof(FILE)) {
-			$header .= <FILE>;
+		open FILE, "<:utf8", $filePath || do { $self->addbadmessage(CGI::p($r->maketext("Can't open [_1]",$filePath))); return};
+		while ($header !~ s/Message:\s*$//m and not eof(FILE)) { 
+			$header .= <FILE>; 
 		}
 		$text = join( '', <FILE>);
 		$text =~ s/^\s*//; # remove initial white space if any.
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm
index 42d7acbf63..29867cef96 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm
@@ -36,8 +36,10 @@ use WeBWorK::Utils qw(readDirectory max sortByName wwRound x);
 use WeBWorK::Utils::Tasks qw(renderProblems);
 use WeBWorK::Utils::Tags;
 use WeBWorK::Utils::LibraryStats;
+use WeBWorK::Utils::DetermineProblemLangAndDirection;
 use File::Find;
 use MIME::Base64 qw(encode_base64);
+use Encode;
 
 require WeBWorK::Utils::ListingDB;
 
@@ -403,7 +405,7 @@ sub browse_local_panel {
 	my $r = $self->r;	
 	my $library_selected = shift;
 	my $lib = shift || ''; $lib =~ s/^browse_//;
-	my $name = ($lib eq '')? $r->maketext('Local') : $problib{$lib};
+	my $name = ($lib eq '')? $r->maketext('Local') : Encode::decode_utf8($problib{$lib});
     
 	my $list_of_prob_dirs= get_problem_directories($r,$lib);
 	if(scalar(@$list_of_prob_dirs) == 0) {
@@ -834,7 +836,7 @@ sub make_top_row {
 	##	Make buttons for additional problem libraries
 	my $libs = '';
 	foreach my $lib (sort(keys(%problib))) {
-		$libs .= ' '. CGI::submit(-name=>"browse_$lib", -value=>$problib{$lib},
+		$libs .= ' '. CGI::submit(-name=>"browse_$lib", -value=>Encode::decode_utf8($problib{$lib}),
 																 ($browse_which eq "browse_$lib")? (-disabled=>1): ())
 			if (-d "$ce->{courseDirs}{templates}/$lib");
 	}
@@ -955,6 +957,7 @@ sub make_top_row {
 	CGI::end_table()));
 }
 
+
 sub make_data_row {
 	my $self = shift;
 	my $r = $self->r;
@@ -986,9 +989,22 @@ sub make_data_row {
 	my $isGatewaySet = ( defined($setRecord) && 
 			     $setRecord->assignment_type =~ /gateway/ );
 
+	my %problem_div_settings = ( class=>"RenderSolo", id=>"render$cnt" );
+        # Add what is needed for lang and dir settings
+	my @to_set_lang_dir = get_problem_lang_and_dir( $self, $pg );
+	my $to_set_tag;
+	my $to_set_val;
+	while ( scalar(@to_set_lang_dir) > 0 ) {
+	  $to_set_tag = shift( @to_set_lang_dir );
+	  $to_set_val = shift( @to_set_lang_dir );
+	  if ( defined( $to_set_val ) ) {
+	    $problem_div_settings{ "$to_set_tag" } = "$to_set_val";
+	  }
+	}
+
 	my $problem_output = $pg->{flags}->{error_flag} ?
 		CGI::div({class=>"ResultsWithError"}, CGI::em("This problem produced an error"))
-		: CGI::div({class=>"RenderSolo", id=>"render$cnt"}, $pg->{body_text});
+		: CGI::div( \%problem_div_settings, $pg->{body_text});
 	$problem_output .= $pg->{flags}->{comment} if($pg->{flags}->{comment});
 
 	my $problem_seed = $self->{'problem_seed'} || 1234;
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker2.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker2.pm
index dbe7169fdc..f970722571 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker2.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker2.pm
@@ -34,6 +34,7 @@ use WeBWorK::Debug;
 use WeBWorK::Form;
 use WeBWorK::Utils qw(readDirectory max sortByName);
 use WeBWorK::Utils::Tasks qw(renderProblems);
+use WeBWorK::Utils::DetermineProblemLangAndDirection;
 use File::Find;
 
 require WeBWorK::Utils::ListingDB;
@@ -1015,9 +1016,22 @@ sub make_data_row {
 	my $isGatewaySet = ( defined($setRecord) && 
 			     $setRecord->assignment_type =~ /gateway/ );
 
+	my %problem_div_settings = ( -class=>"RenderSolo" );
+        # Add what is needed for lang and dir settings
+	my @to_set_lang_dir = get_problem_lang_and_dir( $self, $pg );
+	my $to_set_tag;
+	my $to_set_val;
+	while ( scalar(@to_set_lang_dir) > 0 ) {
+	  $to_set_tag = shift( @to_set_lang_dir );
+	  $to_set_val = shift( @to_set_lang_dir );
+	  if ( defined( $to_set_val ) ) {
+	    $problem_div_settings{ "$to_set_tag" } = "$to_set_val";
+	  }
+	}
+
 	my $problem_output = $pg->{flags}->{error_flag} ?
 		CGI::div({class=>"ResultsWithError"}, CGI::em("This problem produced an error"))
-		: CGI::div({class=>"RenderSolo"}, $pg->{body_text});
+		: CGI::div( \%problem_div_settings, $pg->{body_text});
 	$problem_output .= $pg->{flags}->{comment} if($pg->{flags}->{comment});
 
 
@@ -1117,9 +1131,22 @@ sub make_myset_data_row {
 	my $isGatewaySet = ( defined($setRecord) && 
 			     $setRecord->assignment_type =~ /gateway/ );
 
+	my %problem_div_settings = ( -class=>"RenderSolo" );
+        # Add what is needed for lang and dir settings
+	my @to_set_lang_dir = get_problem_lang_and_dir( $self, $pg );
+	my $to_set_tag;
+	my $to_set_val;
+	while ( scalar(@to_set_lang_dir) > 0 ) {
+	  $to_set_tag = shift( @to_set_lang_dir );
+	  $to_set_val = shift( @to_set_lang_dir );
+	  if ( defined( $to_set_val ) ) {
+	    $problem_div_settings{ "$to_set_tag" } = "$to_set_val";
+	  }
+	}
+
 	my $problem_output = $pg->{flags}->{error_flag} ?
 		CGI::div({class=>"ResultsWithError"}, CGI::em("This problem produced an error"))
-		: CGI::div({class=>"RenderSolo"}, $pg->{body_text});
+		: CGI::div( \%problem_div_settings, $pg->{body_text});
 	$problem_output .= $pg->{flags}->{comment} if($pg->{flags}->{comment});
 
 
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMakernojs.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMakernojs.pm
index 269c5952bc..c5f9586184 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/SetMakernojs.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMakernojs.pm
@@ -34,6 +34,7 @@ use WeBWorK::Debug;
 use WeBWorK::Form;
 use WeBWorK::Utils qw(readDirectory max sortByName);
 use WeBWorK::Utils::Tasks qw(renderProblems);
+use WeBWorK::Utils::DetermineProblemLangAndDirection;
 use File::Find;
 
 require WeBWorK::Utils::ListingDB;
@@ -880,9 +881,22 @@ sub make_data_row {
 	my $isGatewaySet = ( defined($setRecord) && 
 			     $setRecord->assignment_type =~ /gateway/ );
 
+	my %problem_div_settings = ( -class=>"RenderSolo", -dir=>"ltr" );
+        # Add what is needed for lang and dir settings
+	my @to_set_lang_dir = get_problem_lang_and_dir( $self, $pg );
+	my $to_set_tag;
+	my $to_set_val;
+	while ( scalar(@to_set_lang_dir) > 0 ) {
+	  $to_set_tag = shift( @to_set_lang_dir );
+	  $to_set_val = shift( @to_set_lang_dir );
+	  if ( defined( $to_set_val ) ) {
+	    $problem_div_settings{ "$to_set_tag" } = "$to_set_val";
+	  }
+	}
+
 	my $problem_output = $pg->{flags}->{error_flag} ?
 		CGI::div({class=>"ResultsWithError"}, CGI::em("This problem produced an error"))
-		: CGI::div({class=>"RenderSolo"}, $pg->{body_text});
+		: CGI::div( (%problem_div_settings), $pg->{body_text});
 	$problem_output .= $pg->{flags}->{comment} if($pg->{flags}->{comment});
 
 
diff --git a/lib/WeBWorK/ContentGenerator/Instructor/ShowAnswers.pm b/lib/WeBWorK/ContentGenerator/Instructor/ShowAnswers.pm
index 83de755a72..d4f8e7c00e 100644
--- a/lib/WeBWorK/ContentGenerator/Instructor/ShowAnswers.pm
+++ b/lib/WeBWorK/ContentGenerator/Instructor/ShowAnswers.pm
@@ -210,7 +210,7 @@ sub initialize {
 
 	    $filename .= '.csv';
 
-	    open my $fh, ">", $fullFilename or warn "Unable to open $fullFilename for writing";
+	    open my $fh, ">:utf8", $fullFilename or warn "Unable to open $fullFilename for writing";
 
 	    my $csv = Text::CSV->new({"eol"=>"\n"});
 	    my @columns;
diff --git a/lib/WeBWorK/ContentGenerator/Login.pm b/lib/WeBWorK/ContentGenerator/Login.pm
index f4f6404665..d2aa0de845 100644
--- a/lib/WeBWorK/ContentGenerator/Login.pm
+++ b/lib/WeBWorK/ContentGenerator/Login.pm
@@ -29,9 +29,12 @@ use warnings;
 #use CGI qw(-nosticky );
 use WeBWorK::CGI;
 use WeBWorK::Utils qw(readFile dequote jitar_id_to_seq);
+use Encode;
 
 use mod_perl;
 use constant MP2 => ( exists $ENV{MOD_PERL_API_VERSION} and $ENV{MOD_PERL_API_VERSION} >= 2 );
+use Encode;
+
 
 # This content generator is NOT logged in.
 # BUT one must return a 1 so that error messages can be displayed.
@@ -185,6 +188,8 @@ sub body {
 	# us to yell at the user for doing that, since Authen isn't a content-
 	# generating module.
 	my $authen_error = MP2 ? $r->notes->get("authen_error") : $r->notes("authen_error");
+	$authen_error = Encode::decode_utf8($authen_error);
+
 	if ($authen_error) {
 		print CGI::div({class=>"ResultsWithError", tabindex=>'0'},
 			CGI::p($authen_error)
diff --git a/lib/WeBWorK/ContentGenerator/Problem.pm b/lib/WeBWorK/ContentGenerator/Problem.pm
index 6d3dbc4aa7..06c2e7ea4f 100644
--- a/lib/WeBWorK/ContentGenerator/Problem.pm
+++ b/lib/WeBWorK/ContentGenerator/Problem.pm
@@ -42,9 +42,13 @@ require WeBWorK::Utils::ListingDB;
 use URI::Escape;
 use WeBWorK::Localize;
 use WeBWorK::Utils::Tasks qw(fake_set fake_problem);
+use WeBWorK::Utils::DetermineProblemLangAndDirection;
 use WeBWorK::AchievementEvaluator;
 use WeBWorK::Utils::AttemptsTable;
 
+use utf8;
+#use open ':encoding(utf8)';
+binmode(STDOUT, ":utf8");
 ################################################################################
 # CGI param interface to this module (up-to-date as of v1.153)
 ################################################################################
@@ -1212,6 +1216,37 @@ sub output_form_start{
 	return "";
 }
 
+# output_problem_lang_and_dir subroutine
+
+# adds a lang and maybe also a dir setting to the DIV tag attributes, if
+# needed by the PROBLEM language
+
+sub output_problem_lang_and_dir {
+    my $self = shift;
+    my $pg = $self->{pg};
+
+    my @to_set_lang_dir = get_problem_lang_and_dir( $self, $pg );
+    my $to_set_tag;
+    my $to_set_val;
+
+    # String with the HTML attributes to add
+    my $to_set = " ";
+
+    # Put the requested tags and values into the string format
+    while ( scalar(@to_set_lang_dir) > 0 ) {
+	$to_set_tag = shift( @to_set_lang_dir );
+	$to_set_val = shift( @to_set_lang_dir );
+	if ( defined( $to_set_val ) ) {
+	    $to_set .= " ${to_set_tag}=\"${to_set_val}\"";
+	}
+    }
+
+    print "$to_set";
+    return "";
+}
+
+
+
 # output_problem_body subroutine
 
 # prints out the body of the current problem
@@ -1222,6 +1257,7 @@ sub output_problem_body{
 	my %will = %{ $self->{will} };
 
 	print "\n";
+
 	print CGI::div({id=>'output_problem_body'},$pg->{body_text});
 
 	return "";
@@ -2183,7 +2219,7 @@ sub output_JS{
 
 	# This is for tagging menus (if allowed)
 	if ($r->authz->hasPermissions($r->param('user'), "modify_tags")) {
-		if (open(TAXONOMY,  $ce->{webworkDirs}{root}.'/htdocs/DATA/tagging-taxonomy.json') ) {
+		if (open(TAXONOMY, "<:encoding(utf8)", $ce->{webworkDirs}{root}.'/htdocs/DATA/tagging-taxonomy.json') ) {
 			my $taxo = '[]';
 			$taxo = join("", <TAXONOMY>);
 			close TAXONOMY;
diff --git a/lib/WeBWorK/ContentGenerator/instructorXMLHandler.pm b/lib/WeBWorK/ContentGenerator/instructorXMLHandler.pm
index 7c7c06d648..956a640c26 100644
--- a/lib/WeBWorK/ContentGenerator/instructorXMLHandler.pm
+++ b/lib/WeBWorK/ContentGenerator/instructorXMLHandler.pm
@@ -25,7 +25,6 @@ use warnings;
 
 package WeBWorK::ContentGenerator::instructorXMLHandler;
 use base qw(WeBWorK::ContentGenerator);
-use MIME::Base64 qw( encode_base64 decode_base64);
 use WeBWorK::Debug;
 use WeBWorK::Utils qw(readFile);
 use PGUtil qw(not_null);
@@ -34,8 +33,6 @@ our $UNIT_TESTS_ON      = 0;  # should be called DEBUG??  FIXME
 
 #use Crypt::SSLeay;
 #use XMLRPC::Lite;
-#use MIME::Base64 qw( encode_base64 decode_base64);
-
 
 use strict;
 use warnings;
@@ -438,7 +435,8 @@ sub content {
 		# it behaves differently when re-randomization in the library takes place
 		# then during the initial rendering. 
 		# print only the text field (not the ra_out field)
-		# and print the text directly without formatting.
+	        # and print the text directly without formatting.
+	    
 		if ($xmlrpc_client->return_object->{problem_out}->{text}) {
 			print $xmlrpc_client->return_object->{problem_out}->{text};
 		} else {
diff --git a/lib/WeBWorK/CourseEnvironment.pm b/lib/WeBWorK/CourseEnvironment.pm
index 6c07f32aaf..d5c9878e8f 100644
--- a/lib/WeBWorK/CourseEnvironment.pm
+++ b/lib/WeBWorK/CourseEnvironment.pm
@@ -166,6 +166,9 @@ sub new {
 	$safe->reval($globalFileContents);
 	# warn "end the evaluation\n";
 	
+
+	
+	
 	# if that evaluation failed, we can't really go on...
 	# we need a global environment!
 	$@ and croak "Could not evaluate global environment file $globalEnvironmentFile: $@";
@@ -179,12 +182,16 @@ sub new {
 		${*{${$safe->root."::"}{courseFiles}}}{simpleConfig};
 	use strict 'refs';
 	
-	# read and evaluate the course environment file
-	# if readFile failed, we don't bother trying to reval
-	my $courseFileContents = eval { readFile($courseEnvironmentFile) }; # catch exceptions
-	$@ or $safe->reval($courseFileContents);
-	my $courseWebConfigContents = eval { readFile($courseWebConfigFile) }; # catch exceptions
-	$@ or $safe->reval($courseWebConfigContents);
+	# make sure the course environment file actually exists (it might not if we don't have a real course)
+	# before we try to read it
+	if(-r $courseEnvironmentFile){
+		# read and evaluate the course environment file
+		# if readFile failed, we don't bother trying to reval
+		my $courseFileContents = eval { readFile($courseEnvironmentFile) }; # catch exceptions
+		$@ or $safe->reval($courseFileContents);
+		my $courseWebConfigContents = eval { readFile($courseWebConfigFile) }; # catch exceptions
+		$@ or $safe->reval($courseWebConfigContents);
+	}
 	
 	# get the safe compartment's namespace as a hash
 	no strict 'refs';
@@ -210,6 +217,25 @@ sub new {
 			$self->{$name} = \%hash;
 		}
 	}
+	# now that we know the name of the pg_dir we can get the pg VERSION file
+	my $PG_version_file = $self->{'pg_dir'}."/VERSION";
+	
+	# #	We'll get the pg version here and read it into the safe symbol table
+	if (-r $PG_version_file){
+		#print STDERR ( "\n\nread PG_version file $PG_version_file\n\n");
+		my $PG_version_file_contents = readFile($PG_version_file)//'';
+		$safe->reval($PG_version_file_contents);
+		#print STDERR ("\n contents: $PG_version_file_contents");
+		
+		no strict 'refs';
+		my %symbolHash2 = %{$safe->root."::"};
+		#print STDERR "symbolHash".join(' ', keys %symbolHash2);
+		use strict 'refs';
+		$self->{PG_VERSION}=${*{$symbolHash2{PG_VERSION}}};
+	} else {
+		croak "Cannot read PG version file $PG_version_file";
+	}
+ 
 	
 	bless $self, $class;
 	
diff --git a/lib/WeBWorK/DB/Driver/SQL.pm b/lib/WeBWorK/DB/Driver/SQL.pm
index cac7b24b6f..85d1e83183 100644
--- a/lib/WeBWorK/DB/Driver/SQL.pm
+++ b/lib/WeBWorK/DB/Driver/SQL.pm
@@ -69,6 +69,7 @@ sub new($$$) {
 		{
 			PrintError => 0,
 			RaiseError => 1,
+			mysql_enable_utf8mb4 => 1,
 		},
 	);
 	die $DBI::errstr unless defined $self->{handle};
diff --git a/lib/WeBWorK/DB/Record/LocationAddresses.pm b/lib/WeBWorK/DB/Record/LocationAddresses.pm
index 4bac5ebac9..eb807fadf6 100644
--- a/lib/WeBWorK/DB/Record/LocationAddresses.pm
+++ b/lib/WeBWorK/DB/Record/LocationAddresses.pm
@@ -27,8 +27,8 @@ use warnings;
 
 BEGIN { 
 	__PACKAGE__->_fields(
-		location_id => { type=>"TINYBLOB NOT NULL", key=> 1 },
-		ip_mask     => { type=>"VARCHAR(255)", key=> 1 },
+		location_id => { type=>"TINYBLOB NOT NULL", key=> 1 }, # requires up to 256 bytes
+		ip_mask     => { type=>"VARCHAR(180)", key=> 1 }, # was VARCHAR(255), reduced to VARCHAR(180) for utf8mb4
 	);
 }
 
diff --git a/lib/WeBWorK/DB/Record/PastAnswer.pm b/lib/WeBWorK/DB/Record/PastAnswer.pm
index 03fa727f29..986bc3da2b 100644
--- a/lib/WeBWorK/DB/Record/PastAnswer.pm
+++ b/lib/WeBWorK/DB/Record/PastAnswer.pm
@@ -30,9 +30,9 @@ BEGIN {
 	__PACKAGE__->_fields(
 
 	    answer_id         => {   type=>"INT AUTO_INCREMENT", key=>1},
-	    course_id         => { type=>"VARCHAR(100) NOT NULL", key=>1},
-	    user_id           => { type=>"VARCHAR(100) NOT NULL", key=>1},	
-	    set_id            => { type=>"VARCHAR(100) NOT NULL", key=>1},
+	    course_id         => { type=>"VARCHAR(80) NOT NULL", key=>1},
+	    user_id           => { type=>"VARCHAR(80) NOT NULL", key=>1},
+	    set_id            => { type=>"VARCHAR(80) NOT NULL", key=>1},
 	    problem_id        => { type=>"INT NOT NULL", key=>1},
 	    source_file       => { type=>"TEXT"},
 	    timestamp         => { type=>"INT" }, 
diff --git a/lib/WeBWorK/DB/Record/Setting.pm b/lib/WeBWorK/DB/Record/Setting.pm
index ab3700e760..736e98ea16 100644
--- a/lib/WeBWorK/DB/Record/Setting.pm
+++ b/lib/WeBWorK/DB/Record/Setting.pm
@@ -30,7 +30,7 @@ use WeBWorK::Utils::DBUpgrade;
 
 BEGIN {
 	__PACKAGE__->_fields(
-		name  => { type=>"VARCHAR(255) NOT NULL", key=>1 },
+		name  => { type=>"VARCHAR(240) NOT NULL", key=>1 },
 		value => { type=>"TEXT" },
 	);
 	__PACKAGE__->_initial_records(
diff --git a/lib/WeBWorK/Localize.pm b/lib/WeBWorK/Localize.pm
index 5c51f2cfae..ad921676e9 100644
--- a/lib/WeBWorK/Localize.pm
+++ b/lib/WeBWorK/Localize.pm
@@ -137,6 +137,13 @@ my $ConfigStrings = [
 		  values => [qw(en tr es fr zh_hk heb)],
 		  type => 'popuplist'
 		},
+		{ var => 'perProblemLangAndDirSettingMode',
+		  doc => x('Mode in which the LANG and DIR settings for a single problem are determined.'),
+		  # doc2 is very long so is being commented out here for now
+		  # doc2 => x('Mode in which the LANG and DIR settings for a single problem are determined.<p>The system will set the LANGuage attribute to either a value determined from the problem, a course-wide default, or the system default of en-US, depending on the mode selected. The tag will only be added to the DIV enclosing the problem if it is different than the value which should be set in the main HTML tag set for the entire course based on the course language.<p>There are two options for the DIRection attribute: \"ltr\" for left-to-write sripts, and \"rtl\" for right-to-left scripts like Arabic and Hebrew.<p>The DIRection attribute is needed to trigger proper display of the question text when the problem text-direction is different than that used by the current language of the course. For example, English problems from the library browser would display improperly in RTL mode for a Hebrew course, unless the problen Direction is set to LTR.<p>The feature to set a problem language and direction was only added in 2018 to the PG language, so most problems will not declare their language, and the system needs to fall back to determining the language and direction in a different manner. The OPL itself is all English, so the system wide fallback is to en-US in LTR mode.<p>Since the defaults fall back to the LTR direction, most sites should be fine with the \"auto::\" mode, but may want to select the one which matches their course language. The mode \"force::ltr\" would also be an option for a course which runs into trouble with the \"auto\" modes.<p>Modes:<br><ul><li> \"none\" prevents any additional LANG and/or DIR tag being added. The browser will use the main setting which was applied to the entire HTML page. This is likely to cause trouble when a problem of the other direction is displayed.</li><li> \"auto::\" allows the system to make the settings based on the language and direction reported by the problem (a new feature, so not set in almost all existing problems) and falling back to the expected default of en-US in LTR mode. </li><li> \"auto:LangCode:Dir\" allows the system to make the settings based on the language and direction reported by the problem (a new feature, so not set in almost all existing problems) but falling back to the language with the given LangCode and the direction Dir when problem settings are not available from PG.</li><li> \"auto::Dir\" for problems without PG settings, this will use the default en=english language, but force the direction to Dir. Problems with PG settings will get those settings.</li><li> \"auto:LangCode:\" for problems without PG settings, this will use the default LTR direction, but will set the language to LangCode.Problems with PG settings will get those settings.</li><li> \"force:LangCode:Dir\" will <b>ignore</b> any setting made by the PG code of the problem, and will force the system to set the language with the given LangCode and the direction to Dir for <b>all</b> problems.</li><li> \"force::Dir\" will <b>ignore</b> any setting made by the PG code of the problem, and will force the system to set the direction to Dir for <b>all</b> problems, but will avoid setting any language attribute for individual problem.</li></ul>'),
+		  values => [qw(none auto:: force::ltr force::rtl force:en:ltr auto:en:ltr force:tr:ltr auto:tr:ltr force:es:ltr auto:es:ltr force:fr:ltr auto:fr:ltr force:zh_hk:ltr auto:zh_hk:ltr force:he:rtl auto:he:rtl   )],
+		  type => 'popuplist'
+		},
 		{ var => 'sessionKeyTimeout',
 		  doc => x('Inactivity time before a user is required to login again'),
 		  doc2 => x('Length of time, in seconds, a user has to be inactive before he is required to login again.<p> This value should be entered as a number, so as 3600 instead of 60*60 for one hour'),
diff --git a/lib/WeBWorK/Localize/en.po b/lib/WeBWorK/Localize/en.po
index 066e2e2773..281021e529 100644
--- a/lib/WeBWorK/Localize/en.po
+++ b/lib/WeBWorK/Localize/en.po
@@ -8733,7 +8733,8 @@ msgstr ""
 "any users answers using the form below and the answers will be colored "
 "according to correctness."
 
-#: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:467
+#
+#: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:464
 msgid "_CLASSLIST_EDITOR_DESCRIPTION"
 msgstr ""
 "This is the classlist editor page, where you can view and edit the records "
@@ -8753,6 +8754,7 @@ msgstr ""
 "%1 uses an external authentication system.  You've authenticated through "
 "that system, but aren't allowed to log in to this course."
 
+#
 #. (CGI::b($r->maketext("Guest Login")
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Login.pm:285
 msgid "_GUEST_LOGIN_MESSAGE"
@@ -8760,6 +8762,7 @@ msgstr ""
 "This course supports guest logins. Click %1 to log into this course as a "
 "guest."
 
+#
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/ProblemSetList2.pm:528
 msgid "_HMWKSETS_EDITOR_DESCRIPTION"
 msgstr ""
@@ -8798,6 +8801,7 @@ msgstr ""
 "in the edit assigned users field contains links which take you to a page "
 "where you can edit what students the set is assigned to."
 
+#
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator.pm:2094
 msgid "_REQUEST_ERROR"
 msgstr ""
@@ -8807,8 +8811,9 @@ msgstr ""
 "corrected. If you are a professor, please consult the error output below for "
 "more information."
 
-#: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:1871
-#: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:1873
+#
+#: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:1868
+#: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:1870
 msgid "_USER_TABLE_SUMMARY"
 msgstr ""
 "A table showing all the current users along with several fields of user "
diff --git a/lib/WeBWorK/Localize/en_us.po b/lib/WeBWorK/Localize/en_us.po
index 066e2e2773..4deffc900a 100644
--- a/lib/WeBWorK/Localize/en_us.po
+++ b/lib/WeBWorK/Localize/en_us.po
@@ -8712,6 +8712,7 @@ msgid "[Edit]"
 msgstr "[Edit]"
 
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/AchievementList.pm:265
+#, fuzzy
 msgid "_ACHIEVEMENTS_EDITOR_DESCRIPTION"
 msgstr ""
 "This is the homework sets editor page where you can view and edit the "
@@ -8737,8 +8738,8 @@ msgstr ""
 msgid "_CLASSLIST_EDITOR_DESCRIPTION"
 msgstr ""
 "This is the classlist editor page, where you can view and edit the records "
-"of all the students currently enrolled in this course.  The top of the page "
-"contains forms which allow you to filter which students to view, sort your "
+"of all the studentscurrently enrolled in this course.  The top of the page "
+"contains forms which allow you to filterwhich students to view, sort your "
 "students in a chosen order, edit student records, give new passwords to "
 "students, import/export student records from/to external files, or add/"
 "delete students. To use, please select the action you would like to perform, "
@@ -8746,6 +8747,7 @@ msgstr ""
 "Action!\" button at the bottom of the form.  The bottom of the page contains "
 "a table containing the student usernames and their information."
 
+#
 #. (CGI::strong($r->maketext($course)
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Login.pm:199
 msgid "_EXTERNAL_AUTH_MESSAGE"
@@ -8787,42 +8789,22 @@ msgstr ""
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/ProblemSetList2.pm:2761
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/ProblemSetList2.pm:2763
 msgid "_PROBLEM_SET_SUMMARY"
-msgstr ""
-"This is a table showing the current Homework sets for this class.  The "
-"fields from left to right are: Edit Set Data, Edit Problems, Edit Assigned "
-"Users, Visibility to students, Reduced Scoring Enabled, Date it was opened, "
-"Date it is due, and the Date during which the answers are posted.  The Edit "
-"Set Data field contains checkboxes for selection and a link to the set data "
-"editing page.  The cells in the Edit Problems fields contain links which "
-"take you to a page where you can edit the containing problems, and the cells "
-"in the edit assigned users field contains links which take you to a page "
-"where you can edit what students the set is assigned to."
+msgstr "_PROBLEM_SET_SUMMARY"
 
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator.pm:2094
 msgid "_REQUEST_ERROR"
 msgstr ""
-"WeBWorK has encountered a software error while attempting to process this "
-"problem. It is likely that there is an error in the problem itself. If you "
-"are a student, report this error message to your professor to have it "
+"WeBWorK has encountered a software error while attempting to process "
+"thisproblem. It is likely that there is an error in the problem itself. If "
+"you are a student, report this error message to your professor to have it "
 "corrected. If you are a professor, please consult the error output below for "
 "more information."
 
+
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:1871
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm:1873
 msgid "_USER_TABLE_SUMMARY"
-msgstr ""
-"A table showing all the current users along with several fields of user "
-"information. The fields from left to right are: Login Name, Login Status, "
-"Assigned Sets, First Name, Last Name, Email Address, Student ID, Enrollment "
-"Status, Section, Recitation, Comments, and Permission Level.  Clicking on "
-"the links in the column headers will sort the table by the field it "
-"corresponds to. The Login Name fields contain checkboxes for selecting the "
-"user.  Clicking the link of the name itself will allow you to act as the "
-"selected user.  There will also be an image link following the name which "
-"will take you to a page where you can edit the selected user's information.  "
-"Clicking the emails will allow you to email the corresponding user.  "
-"Clicking the links in the entries in the assigned sets columns will take you "
-"to a page where you can view and reassign the sets for the selected user."
+msgstr "_USER_TABLE_SUMMARY"
 
 # Context is "Create set ______ as a duplicate of the first selected set"
 #: /opt/webwork/webwork2/lib/WeBWorK/ContentGenerator/Instructor/AchievementList.pm:707
diff --git a/lib/WeBWorK/PG/Local.pm b/lib/WeBWorK/PG/Local.pm
index 20997a7e4d..209407ed8f 100644
--- a/lib/WeBWorK/PG/Local.pm
+++ b/lib/WeBWorK/PG/Local.pm
@@ -245,7 +245,7 @@ sub new_helper {
 	# Safe::Rootx:: and NOT to Safe::Root1::, which is the value of main::
 	# at compile time.
 	# 
-	# TO ENABLE CACHEING UNCOMMENT THE FOLLOWING:
+	# TO ENABLE CACHEING UNCOMMENT THE FOLLOWING:  -- this has not been used in some time
 # 	eval{$translator->pre_load_macro_files(
 # 		$WeBWorK::PG::Local::safeCache,
 # 		$ce->{pg}->{directories}->{macros}, 
diff --git a/lib/WeBWorK/Request.pm b/lib/WeBWorK/Request.pm
index 7a691b8b44..6353ab539f 100644
--- a/lib/WeBWorK/Request.pm
+++ b/lib/WeBWorK/Request.pm
@@ -28,7 +28,7 @@ use warnings;
 
 use mod_perl;
 use constant MP2 => ( exists $ENV{MOD_PERL_API_VERSION} and $ENV{MOD_PERL_API_VERSION} >= 2 );
-
+use Encode;
 
 use WeBWorK::Localize;
 
@@ -58,10 +58,14 @@ sub mutable_param {
 	my $self = shift;
 	
 	if (not defined $self->{paramcache}) {
-		my @names = $self->SUPER::param;
-		@{$self->{paramcache}}{@names} = map { [ $self->SUPER::param($_) ] } @names;
+	    my @names = $self->SUPER::param();
+	    foreach my $name (@names) {
+		my @params = $self->SUPER::param($name);
+		@params = map {Encode::decode_utf8($_)} @params;
+		$self->{paramcache}{$name} = [@params];
+	    }
 	}
-	
+
 	@_ or return keys %{$self->{paramcache}};
 	
 	my $name = shift;
diff --git a/lib/WeBWorK/Utils.pm b/lib/WeBWorK/Utils.pm
index 890ae5162d..2487822af3 100644
--- a/lib/WeBWorK/Utils.pm
+++ b/lib/WeBWorK/Utils.pm
@@ -31,15 +31,20 @@ use DateTime;
 use DateTime::TimeZone;
 use Date::Parse;
 use Date::Format;
+use Encode qw(encode_utf8 decode_utf8);
 use File::Copy;
 use File::Spec::Functions qw(canonpath);
 use Time::Zone;
-use MIME::Base64;
+use MIME::Base64 qw(encode_base64 decode_base64);
 use Errno;
 use File::Path qw(rmtree);
 use Storable;
 use Carp;
 #use Mail::Sender;
+use Storable qw(nfreeze thaw);
+
+
+use open IO => ':encoding(UTF-8)';
 
 use constant MKDIR_ATTEMPTS => 10;
 
@@ -68,8 +73,10 @@ our @EXPORT_OK = qw(
 	constituency_hash
 	cryptPassword
 	decodeAnswers
+        decode_utf8_base64
 	dequote
 	encodeAnswers
+        encode_utf8_base64
 	fisher_yates_shuffle
 	formatDateTime
 	has_aux_files
@@ -78,6 +85,7 @@ our @EXPORT_OK = qw(
 	listFilesRecursive
 	makeTempDirectory
 	max
+        nfreeze_base64
 	not_blank
 	parseDateTime
 	path_is_subdir
@@ -93,6 +101,7 @@ our @EXPORT_OK = qw(
 	textDateTime
 	timeToSec
 	trim_spaces
+        thaw_base64
 	undefstr
 	writeCourseLog
 	writeLog
@@ -179,11 +188,41 @@ sub force_eoln($) {
 
 sub readFile($) {
 	my $fileName = shift;
+	# debugging code: found error in CourseEnvironment.pm with this
+# 	if ($fileName =~ /___/ or $fileName =~ /the-course-should-be-determined-at-run-time/) {
+# 		print STDERR "File $fileName not found.\n Usually an unnecessary call to readFile from\n", 
+# 		join("\t ", caller()), "\n";
+# 		return();
+# 	}
 	local $/ = undef; # slurp the whole thing into one string
-	open my $dh, "<", $fileName
-		or croak "failed to read file $fileName: $!";
-	my $result = <$dh>;
-	close $dh;
+	my $result='';  # need this initialized because the file (e.g. simple.conf) may not exist
+	if (-r $fileName) {
+		eval{
+			# CODING WARNING:
+			# if (open my $dh, "<", $fileName){
+			# will cause a utf8 "\xA9" does not map to Unicode warning if © is in latin-1 file
+			# use the following instead
+			if (open my $dh, "<:raw", $fileName){
+				$result = <$dh>;
+				decode_utf8($result) or die "failed to decode $fileName";
+				close $dh;
+			} else {
+				print STDERR "File $fileName cannot be read."; # this is not a fatal error.
+			}
+		};
+		if ($@) {
+			print STDERR "reading $fileName:  error in Utils::readFile: $@\n";
+		}
+		utf8::decode($result) or  warn  "Non-fatal warning: file $fileName contains at least one character code which ". 
+		 "is not valid in UTF-8. (The copyright sign is often a culprit -- use '&amp;copy;' instead.)\n". 
+		 "While this is not fatal you should fix it\n";
+		# FIXME
+		# utf8::decode($result) raises an error about the copyright sign
+		# decode_utf8 and Encode::decode_utf8 do not -- which is doing the right thing?
+		# Done:: should direct this to warn instead of STDERR to debug files written with accents
+		# in latin-1 files /Done
+	}
+	# returns the empty string if the file cannot be read
 	return force_eoln($result);
 }
 
@@ -777,7 +816,7 @@ sub writeCourseLog($$@) {
 	my $logFile = $ce->{courseFiles}->{logs}->{$facility};
 	surePathToFile($ce->{courseDirs}->{root}, $logFile);
 	local *LOG;
-	if (open LOG, ">>", $logFile) {
+	if (open LOG, ">>:utf8", $logFile) {
 		print LOG "[", time2str("%a %b %d %H:%M:%S %Y", time), "] @message\n";
 		close LOG;
 	} else {
@@ -889,6 +928,10 @@ sub decodeAnswers($) {
 	}
 }
 
+sub decode_utf8_base64 {
+    return decode_utf8(decode_base64(shift));
+}
+
 sub encodeAnswers(\%\@) {
 	my %hash = %{shift()};
 	my @order = @{shift()};
@@ -900,8 +943,30 @@ sub encodeAnswers(\%\@) {
 
 }
 
+sub encode_utf8_base64 {
+    return encode_base64(encode_utf8(shift));
+}
+
+sub nfreeze_base64 {
+    return encode_base64(nfreeze(shift));
+}
+
+sub thaw_base64 {
+    my $string = shift;
+    my $result;
 
+    eval {
+	$result = thaw(decode_base64($string));
+    };
 
+    if ($@) {
+	warn("Deleting corrupted achievement data.");
+	return {};
+    } else {
+	return $result;
+    }
+
+}
 sub max(@) {
 	my $soFar;
 	foreach my $item (@_) {
diff --git a/lib/WeBWorK/Utils/CourseManagement.pm b/lib/WeBWorK/Utils/CourseManagement.pm
index bcfe97fcc8..46d262e07f 100644
--- a/lib/WeBWorK/Utils/CourseManagement.pm
+++ b/lib/WeBWorK/Utils/CourseManagement.pm
@@ -279,7 +279,7 @@ sub addCourse {
 	##### step 4: write course.conf file #####
 	
 	my $courseEnvFile = $ce->{courseFiles}->{environment};
-	open my $fh, ">", $courseEnvFile
+	open my $fh, ">:utf8", $courseEnvFile
 		or die "failed to open $courseEnvFile for writing.\n";
 	writeCourseConf($fh, $ce, %courseOptions);
 	close $fh;
@@ -1247,7 +1247,7 @@ sub writeCourseConf {
 #!perl
 ################################################################################
 # WeBWorK Online Homework Delivery System
-# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/
+# Copyright 2000-2016 The WeBWorK Project, http://openwebwork.sf.net/
 # 
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of either: (a) the GNU General Public License as published by the
diff --git a/lib/WeBWorK/Utils/DetermineProblemLangAndDirection.pm b/lib/WeBWorK/Utils/DetermineProblemLangAndDirection.pm
new file mode 100644
index 0000000000..c3c2fdeef1
--- /dev/null
+++ b/lib/WeBWorK/Utils/DetermineProblemLangAndDirection.pm
@@ -0,0 +1,207 @@
+################################################################################
+# WeBWorK Online Homework Delivery System
+# Copyright © 2000-2017 The WeBWorK Project, http://openwebwork.sf.net/
+# 
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of either: (a) the GNU General Public License as published by the
+# Free Software Foundation; either version 2, or (at your option) any later
+# version, or (b) the "Artistic License" which comes with this package.
+# 
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See either the GNU General Public License or the
+# Artistic License for more details.
+################################################################################
+
+package WeBWorK::Utils::DetermineProblemLangAndDirection;
+use base qw(Exporter);
+
+=head1 NAME
+
+WeBWorK::Utils::DetermineProblemLangAndDirection - utilities to determine
+the language and text direction of a problem based on settings from the
+PG flags, the course configuration variable $perProblemLangAndDirSettingMode,
+and the course language.
+
+=head1 SYNOPSIS
+
+ use WeBWorK::Utils::DetermineProblemLangAndDirection;
+
+=head1 DESCRIPTION
+
+This module provides s function which determines the "recommended"
+language and text direction of a problem based on settings from the                                                                         
+PG flags, the course configuration variable $perProblemLangAndDirSettingMode,                                                                  and the course language.
+
+=cut
+
+use strict;
+use warnings;
+use Carp;
+use WeBWorK::PG; 
+use WeBWorK::Debug;
+
+our @EXPORT    = qw(get_problem_lang_and_dir);
+our @EXPORT_OK = ();
+
+
+=head1 FUNCTIONS
+
+=over
+
+=item get_problem_lang_and_dir subroutine
+
+ @output = get_problem_lang_and_dir( $self, $pg );
+
+returns an array of tagname tagvalue pairs.
+
+In some cases, the result is empty.
+
+=cut
+
+# get_problem_lang_and_dir subroutine
+
+# used to determine the language and maybe also the dir setting for the 
+# DIV tag attributes, if needed by the PROBLEM language
+
+# Return an array of key-value pairs key1 val1 key2 val2
+
+sub get_problem_lang_and_dir {
+    my $self = shift;
+    my $pg   = shift;
+    
+    my @result = ();
+    
+    my $ce_requested_mode = $self->r->ce->{perProblemLangAndDirSettingMode}; # Mode requested
+    
+    if ( $ce_requested_mode eq "none" ) {
+	# Requested mode is "none" so no output should be made.
+	return( @result );
+    }
+    
+    # Get course-wide language setting
+    my $ce_lang = $self->r->ce->{language}; # Course wide setting
+    my $ce_dir = "ltr"; # default
+    
+    if ( $ce_lang =~ /^he/i ) { # supports also the current "heb" option
+	# Hebrew - requires RTL direction
+	$ce_lang = "he";  # Hebrew - standard form
+	$ce_dir  = "rtl"; # RTL
+    } elsif ( $ce_lang =~ /^ar/i ) {
+	# Arabic - requires RTL direction
+	$ce_lang = "ar";  # Arabic
+	$ce_dir  = "rtl"; # RTL
+    }
+
+    my @tmp1 = split(':',$ce_requested_mode);
+    my $reqMode = $tmp1[0];
+    my $reqLang = $tmp1[1];
+    my $reqDir  = $tmp1[2];
+    
+    $reqLang = "none" if ( ! defined( $reqMode ) );
+    $reqLang = ""     if ( ! defined( $reqLang ) );
+    $reqDir  = ""     if ( ! defined( $reqDir  ) );
+    
+    if ( $reqMode eq "force" ) {
+	# Requested mode is to force the LANG and DIR attributes regardless of lang data from problem PG code.
+	if ( $reqLang ne "" ) {
+	    push( @result, "lang", $reqLang ); # forced setting
+	}
+	push( @result, "dir", $reqDir ); # forced setting
+	return( @result );
+    }
+    
+    if ( $reqMode ne "auto" ) {
+	# The mode setting is not valid, treat like none
+	return( @result );
+    }
+    
+    # We are now handling an "auto" setting, so want to handle data from PG
+    
+    my $pg_lang = "en-US"; # system default
+    my $pg_dir  = "ltr";   # system default
+    
+    # Determine the language code to use
+    if ( defined( $pg->{flags}->{language} ) ) {
+	# Language set by PG
+	$pg_lang = $pg->{flags}->{language};
+    } else {
+	# Language not set by PG, use provided default language (if set) or fall back to the system default
+	if ( $reqLang ne "" ) {
+	    $pg_lang = $reqLang;
+	}
+    }
+    
+    # Determine the direction code to use
+    # we changed the order of precedence here.
+    if ( defined( $pg->{flags}->{textdirection} ) ) {
+	# Direction set by PG
+	$pg_dir =  $pg->{flags}->{textdirection};
+    } elsif ( defined( $pg->{flags}->{language} ) ) {
+	# Direction not set by PG, 
+	# but PG did set the language.
+	# Fallback is to use LTR, except for Hebrew and Arabic.
+	$pg_dir  = "ltr"; # correct for most languages
+	if ( ( $pg->{flags}->{language} =~ /^he/i ) ||
+	     ( $pg->{flags}->{language} =~ /^ar/i )    ) {
+	    $pg_dir  = "rtl"; # should be correct for these languages
+	}
+    } elsif ( $reqDir ne "" ) {
+	# We have a request for a direction when PG did not set it
+	$pg_dir = $reqDir;
+    } else {
+	# Direction not set by PG, nor was a default setting provided
+	# and PG did NOT set the language.
+	# For SetMaker, we are assuming that a problem without a PG direction
+	# setting should be in LTR mode.
+	$pg_dir  = "ltr"; # correct for most languages
+
+	# Even for Arabic and Hebrew do NOT change to RTL. 
+	# The teacher should add the language and direction setting to
+	# the PG file of the problem.
+    }
+    
+    # Make these string all lowercase (just in case)
+    $pg_lang = lc( $pg_lang );
+    $pg_dir  = lc( $pg_dir );
+    $ce_lang = lc( $ce_lang );
+    $ce_dir  = lc( $ce_dir );
+    
+    # We are ALWAYS setting this for this mode.
+    push( @result, "lang", $pg_lang ); # send the problem language that was selected
+    
+    if ( ( $ce_dir eq "rtl" ) && # Possible hack for RTL direction courses and OPL problems
+	 ( $reqDir eq "rtl" ) &&
+	 ! defined( $pg->{flags}->{textdirection} ) && 	   # problem does not set the language or
+	 ! defined( $pg->{flags}->{language} )         ) { #                      the text direction
+	# In a RTL language course, we may really want to force LTR use for unknown problems.
+	# that would best be handled by always including the language setting in RTL language
+	# problems, and using a setting which falls back to LTR when there is no setting from
+	# the problem (expected on OPL problems).
+	
+	# May want to issue a warning
+	
+	# Right now - we are not trying to do the following
+	# push( @result, "dir", "ltr" ); # override to problem textdirection or "expected" LTR textdirection
+    }
+
+    # We are ALWAYS setting this for this mode.
+    push( @result, "dir", $pg_dir ); # override to $pg_dir
+
+    # " ce_lang $ce_lang ce_dir $ce_dir reqMain $reqMain reqLang $reqLand reqDir $reqDir result_lang $pg_lang result_dir $pg_dir ";
+
+    return( @result );
+}
+
+
+=back
+
+=cut
+
+=head1 AUTHOR
+
+Written by Nathan Wallach, tani (at) mathnet.technion.ac.il
+
+=cut
+
+1;
diff --git a/lib/WeBWorK/Utils/ListingDB.pm b/lib/WeBWorK/Utils/ListingDB.pm
index 987c1e9d9b..b94c303a3c 100644
--- a/lib/WeBWorK/Utils/ListingDB.pm
+++ b/lib/WeBWorK/Utils/ListingDB.pm
@@ -439,6 +439,12 @@ sub getDBListings {
 	my $subj = $r->param('library_subjects') || "";
 	my $chap = $r->param('library_chapters') || "";
 	my $sec = $r->param('library_sections') || "";
+	
+	# Make sure these strings are internally encoded in UTF-8
+	utf8::upgrade($subj);
+	utf8::upgrade($chap);
+	utf8::upgrade($sec);
+
 	my $keywords = $r->param('library_keywords') || "";
 	# Next could be an array, an array reference, or nothing
 	my @levels = $r->param('level');
@@ -510,6 +516,7 @@ sub getDBListings {
 #               $kw2";
 
 	my $pg_id_ref;
+	$dbh->do(qq{SET NAMES 'utf8';});
 	if($haveTextInfo) {
 		my $query = "SELECT $selectwhat from `$tables{pgfile}` pgf, 
 			`$tables{dbsection}` dbsc, `$tables{dbchapter}` dbc, `$tables{dbsubject}` dbsj,
diff --git a/lib/WeBWorK/Utils/Tags.pm b/lib/WeBWorK/Utils/Tags.pm
index 6a1f7fdc5f..4333c7bb39 100644
--- a/lib/WeBWorK/Utils/Tags.pm
+++ b/lib/WeBWorK/Utils/Tags.pm
@@ -1,6 +1,6 @@
 ################################################################################
 # WeBWorK Online Homework Delivery System
-# Copyright � 2000-1307 The WeBWorK Project, http://openwebwork.sf.net/
+# Copyright © 2000-1307 The WeBWorK Project, http://openwebwork.sf.net/
 # 
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of either: (a) the GNU General Public License as published by the
@@ -203,7 +203,7 @@ sub new {
   my $textno;
   my $textinfo=[];
 
-  open(IN,"$name") or die "can not open $name: $!";
+  open(IN,'<:encoding(UTF-8)',"$name") or die "can not open $name: $!";
   if ($name !~ /pg$/ && $name !~ /\.pg\.[-a-zA-Z0-9_.@]*\.tmp$/) {
     warn "Not a pg file";  #print caused trouble with XMLRPC 
     $self->{file}= undef;
diff --git a/lib/WebworkClient.pm b/lib/WebworkClient.pm
index c01e03ef43..38650d8b1c 100755
--- a/lib/WebworkClient.pm
+++ b/lib/WebworkClient.pm
@@ -108,8 +108,7 @@ use Crypt::SSLeay;  # needed for https
 use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib";
 use lib "$WeBWorK::Constants::PG_DIRECTORY/lib";
 use XMLRPC::Lite;
-use MIME::Base64 qw( encode_base64 decode_base64);
-use WeBWorK::Utils qw( wwRound);
+use WeBWorK::Utils qw( wwRound encode_utf8_base64 decode_utf8_base64);
 use WeBWorK::Utils::AttemptsTable;
 use WeBWorK::CourseEnvironment;
 
@@ -323,12 +322,12 @@ sub xmlrpcCall {
 		  $self->fault(1); # set fault flag to true
 		  return $self;  
 	  } else {
-	  	  if (ref($result->result())=~/HASH/ and defined($result->result()->{text}) ) {
-	  		$result->result()->{text} = decode_base64($result->result()->{text});
-		}
-	  	if (ref($result->result())=~/HASH/ and defined($result->result()->{header_text}) ) {
-		    $result->result()->{header_text} = decode_base64($result->result()->{header_text});
-	  	}
+	      if (ref($result->result())=~/HASH/ and defined($result->result()->{text}) ) {
+		  $result->result()->{text} = decode_utf8_base64($result->result()->{text});
+	      }
+	      if (ref($result->result())=~/HASH/ and defined($result->result()->{header_text}) ) {
+		  $result->result()->{header_text} = decode_utf8_base64($result->result()->{header_text});
+	      }
 
 		$self->return_object($result->result());
 		# print "\n retrieve result ",  keys %{$self->return_object};
@@ -394,7 +393,7 @@ sub jsXmlrpcCall {
 sub encodeSource {
 	my $self = shift;
 	my $source = shift||'';
-	$self->{encoded_source} =encode_base64($source);
+	$self->{encoded_source} =encode_utf8_base64($source);
 }
 
 =head2  Accessor methods
@@ -630,7 +629,7 @@ sub formatRenderedProblem {
 
 	if ( defined ($rh_result->{WARNINGS}) and $rh_result->{WARNINGS} ){
 		$warnings = "<div style=\"background-color:pink\">
-		             <p >WARNINGS</p><p>".decode_base64($rh_result->{WARNINGS})."</p></div>";
+		             <p >WARNINGS</p><p>".decode_utf8_base64($rh_result->{WARNINGS})."</p></div>";
 	}
 	#warn "keys: ", join(" | ", sort keys %{$rh_result });
 	
diff --git a/lib/WebworkWebservice/CourseActions.pm b/lib/WebworkWebservice/CourseActions.pm
index ea65c244f3..ebc4f91ada 100644
--- a/lib/WebworkWebservice/CourseActions.pm
+++ b/lib/WebworkWebservice/CourseActions.pm
@@ -11,12 +11,11 @@ use WebworkWebservice;
 use base qw(WebworkWebservice); 
 use WeBWorK::DB;
 use WeBWorK::DB::Utils qw(initializeUserProblem);
-use WeBWorK::Utils qw(runtime_use cryptPassword formatDateTime parseDateTime);
+use WeBWorK::Utils qw(runtime_use cryptPassword formatDateTime parseDateTime  encode_utf8_base64 decode_utf8_base64);
 use WeBWorK::Utils::CourseManagement qw(addCourse);
 use WeBWorK::Debug;
 use WeBWorK::ContentGenerator::Instructor::SendMail;
 use JSON;
-use MIME::Base64 qw( encode_base64 decode_base64);
 
 use Time::HiRes qw/gettimeofday/; # for log timestamp
 use Date::Format; # for log timestamp
@@ -143,14 +142,14 @@ sub listUsers {
 
 
     $out->{ra_out} = \@userInfo;
-    $out->{text} = encode_base64("Users for course: ".$self->{courseName});
+    $out->{text} = encode_utf8_base64("Users for course: ".$self->{courseName});
     return $out;
 }
 
 sub addUser {
 	my ($self, $params) = @_;
 	my $out = {};
-	$out->{text} = encode_base64("");
+	$out->{text} = encode_utf8_base64("");
 	my $db = $self->db;
 	my $ce = $self->ce;
 
@@ -291,7 +290,7 @@ sub deleteUser {
 	my $out = {};
 	my $db = $self->db;
 	my $ce = $self->ce;
-	$out->{text} = encode_base64("");
+	$out->{text} = encode_utf8_base64("");
 	
 	my $user = $params->{'id'};
 	
@@ -326,12 +325,12 @@ sub deleteUser {
 		{
 			my $result;
 			$result->{delete} = "success";
-			$out->{text} .=encode_base64("User " . $user . " successfully deleted");
+			$out->{text} .=encode_utf8_base64("User " . $user . " successfully deleted");
 			$out->{ra_out} .= "delete: success";
 		}
 		else 
 		{
-			$out->{text}=encode_base64("User " . $user . " could not be deleted");
+			$out->{text}=encode_utf8_base64("User " . $user . " could not be deleted");
 			$out->{ra_out} .= "delete : failed";
 		}
 
@@ -348,7 +347,7 @@ sub editUser {
     my $ce = $self->ce;
     my $out = {};
     debug("Webservices edit user request.");
-    $out->{text} = encode_base64("");
+    $out->{text} = encode_utf8_base64("");
     # make sure course actions are enabled
     #if (!$ce->{webservices}{enableCourseActions}) {
     #	$out->{status} = "failure";
@@ -367,7 +366,7 @@ sub editUser {
     	}
     }
     if($params->{'id'} eq $params->{'user'}){
-        $out->{text} .= encode_base64("You cannot change your own permissions.");
+        $out->{text} .= encode_utf8_base64("You cannot change your own permissions.");
     } else {
         foreach my $field ($PermissionLevel->NONKEYFIELDS()) {
     	    my $param = "${field}";
@@ -395,7 +394,7 @@ sub editUser {
     
 
     $out->{ra_out} = $User;
-    $out->{text} .= encode_base64("Changes saved");
+    $out->{text} .= encode_utf8_base64("Changes saved");
 
 	return $out;
 }
@@ -410,7 +409,7 @@ sub changeUserPassword {
 	my $out = {};
 	my $db = $self->db;
 	my $ce = $self->ce;
-	$out->{text} = encode_base64("");
+	$out->{text} = encode_utf8_base64("");
 	
 	my $userid = $params->{'id'};
 	
@@ -438,7 +437,7 @@ sub changeUserPassword {
 
     #my $User = $db->getUser($params->{'id'}); # checked
     if(!(defined $User)){
-	$out->{text}=encode_base64("No record found for user: ". $params->{'id'});
+	$out->{text}=encode_utf8_base64("No record found for user: ". $params->{'id'});
 	return $out;
     }
     
@@ -453,7 +452,7 @@ sub changeUserPassword {
 	}
 
 	$self->{passwordMode} = 0;
-	$out->{text} = encode_base64("New passwords saved");
+	$out->{text} = encode_utf8_base64("New passwords saved");
 	$out->{ra_out}= "password_change: success";
 	return $out;
 }
@@ -579,7 +578,7 @@ sub getCourseSettings {
   	
 	my $out = {};
 	$out->{ra_out} = $ConfigValues;
-	$out->{text} = encode_base64("Successfully found the course settings");
+	$out->{text} = encode_utf8_base64("Successfully found the course settings");
     return $out;
 
 }
@@ -665,7 +664,7 @@ sub updateSetting {
 
 	my $out = {};
 	$out->{ra_out} = "";
-	$out->{text} = encode_base64("Successfully updated the course settings");
+	$out->{text} = encode_utf8_base64("Successfully updated the course settings");
     return $out;
 }
 
diff --git a/lib/WebworkWebservice/LibraryActions.pm b/lib/WebworkWebservice/LibraryActions.pm
index 70f5d767dc..80abce3de1 100644
--- a/lib/WebworkWebservice/LibraryActions.pm
+++ b/lib/WebworkWebservice/LibraryActions.pm
@@ -23,12 +23,11 @@ use sigtrap;
 use Carp;
 use WWSafe;
 #use Apache;
-use WeBWorK::Utils qw(readDirectory sortByName);
+use WeBWorK::Utils qw(readDirectory sortByName encode_utf8_base64 decode_utf8_base64);
 use WeBWorK::CourseEnvironment;
 use WeBWorK::PG::Translator;
 use WeBWorK::PG::IO;
 use Benchmark;
-use MIME::Base64 qw( encode_base64 decode_base64);
 
 ##############################################
 #   Obtain basic information about directories, course name and host 
@@ -87,7 +86,7 @@ sub listLibraries {  # list the problem libraries that are available.
 	my @outListLib = sort keys %libraries;
 	my $out = {};
 	$out->{ra_out} = \@outListLib;
-	$out->{text} = encode_base64("success");
+	$out->{text} = encode_utf8_base64("success");
 	return $out;
 }
 
@@ -120,7 +119,7 @@ sub readFile {
 		open IN, "<$filePath";
 		local($/)=undef;
 		my $text = <IN>;
-		$out->{text}= encode_base64($text);
+		$out->{text}= encode_utf8_base64($text);
 		my $sb=stat($filePath);
 		$out->{size}=$sb->size;
 		$out->{path}=$filePath;
@@ -208,7 +207,7 @@ sub listLib {
 									find({wanted=>$wanted,follow_fast=>1 }, $dirPath);
 									@outListLib = sort @outListLib;
 									$out->{ra_out} = \@outListLib;
-									$out->{text} = encode_base64( join("\n", @outListLib) );
+									$out->{text} = encode_utf8_base64( join("\n", @outListLib) );
 									return($out);
 		};
 		$command eq 'dirOnly' &&   do {
@@ -225,7 +224,7 @@ sub listLib {
 										warn "result: ", join(" ", %libDirectoryList);
 										delete $libDirectoryList{""};
 										$out->{ra_out} = \%libDirectoryList;
-										$out->{text} = encode_base64("Loaded libraries");
+										$out->{text} = encode_utf8_base64("Loaded libraries");
 										return($out);
 									} else {
 									   warn "Can't open directory  $dirPath2";
@@ -238,7 +237,7 @@ sub listLib {
 # 			$command eq 'dirOnly' && do {
 # 				my @subdirs = File::Find::Rule->directory->in( ($dirPath) );
 # 				$out->{ra_out} = \@subdirs;
-# 				$out->{text} = encode_base64("Loaded libraries".$dirPath);
+# 				$out->{text} = encode_utf8_base64("Loaded libraries".$dirPath);
 # 				return($out);			
 # 			};
 		$command eq 'buildtree' &&   do {
@@ -248,7 +247,7 @@ sub listLib {
 									#@outListLib = sort keys %libDirectoryList;
 									$out->{ra_out} = $tree;
 									warn "output of build_tree is ", %$tree;
-									$out->{text} = encode_base64("Loaded libraries");
+									$out->{text} = encode_utf8_base64("Loaded libraries");
 									return($out);
 		};
 		
@@ -260,8 +259,8 @@ sub listLib {
 									 if ( -e $dirPath2 and $dirPath2 !~ m|//| ) {
 										 find($wanted, $dirPath2);
 										 @outListLib = sort @outListLib;
-										 #$out ->{text} = encode_base64( join("", @outListLib ) );
-										 $out ->{text} = encode_base64( "Problems loaded" );
+										 #$out ->{text} = encode_utf8_base64( join("", @outListLib ) );
+										 $out ->{text} = encode_utf8_base64( "Problems loaded" );
 										 $out->{ra_out} = \@outListLib;
 									 } else {
 									   warn "Can't open directory  $dirPath2 in listLib files";
@@ -296,14 +295,14 @@ sub searchLib {    #API for searching the NPL database
 	'getAllDBsubjects' eq $subcommand && do {
 		my @subjects = WeBWorK::Utils::ListingDB::getAllDBsubjects($self);
 		$out->{ra_out} = \@subjects;
-		$out->{text} = encode_base64("Subjects loaded.");
+		$out->{text} = encode_utf8_base64("Subjects loaded.");
 		return($out);		
 	};
 	'getAllDBchapters' eq $subcommand && do {
 		$self->{library_subjects} = $rh->{library_subjects};
 		my @chaps = WeBWorK::Utils::ListingDB::getAllDBchapters($self);
 		$out->{ra_out} = \@chaps;
-        $out->{text} = encode_base64("Chapters loaded.");
+        $out->{text} = encode_utf8_base64("Chapters loaded.");
 
 		return($out);		
 	};
@@ -331,7 +330,7 @@ sub searchLib {    #API for searching the NPL database
 
 		my @section_listings = WeBWorK::Utils::ListingDB::getAllDBsections($self);
 		$out->{ra_out} = \@section_listings;
-        $out->{text} = encode_base64("Sections loaded.");
+        $out->{text} = encode_utf8_base64("Sections loaded.");
 
 		return($out);
 	};
@@ -345,7 +344,7 @@ sub searchLib {    #API for searching the NPL database
 		$self->{library_textchapter} = $rh->{library_textchapter};
 		$self->{library_textsection} = $rh->{library_textsection};
 		my $count = WeBWorK::Utils::ListingDB::countDBListings($self);
-					$out->{text} = encode_base64("Count done.");
+					$out->{text} = encode_utf8_base64("Count done.");
 		$out->{ra_out} = [$count];
 		return($out);
 	};
@@ -407,7 +406,7 @@ sub getProblemDirectories {
 	unshift @all_problem_directories, $main if($includetop);
 
 	$out->{ra_out} = \@all_problem_directories;
-    $out->{text} = encode_base64("Problem Directories loaded.");
+    $out->{text} = encode_utf8_base64("Problem Directories loaded.");
 
 	return($out);
 }
@@ -439,7 +438,7 @@ sub buildBrowseTree {
 		}
 	}
 	$out->{ra_out} = \@tree;
-	$out->{text} = encode_base64("Subjects, Chapters and Sections loaded.");
+	$out->{text} = encode_utf8_base64("Subjects, Chapters and Sections loaded.");
 	return($out);
 }
 
@@ -451,7 +450,7 @@ sub getProblemTags {
         # Get a pointer to a hash of DBchapter, ..., DBsection
 	my $tags = WeBWorK::Utils::ListingDB::getProblemTags($path);
 	$out->{ra_out} = $tags;
-	$out->{text} = encode_base64("Tags loaded.");
+	$out->{text} = encode_utf8_base64("Tags loaded.");
 	
 	return($out);
 }
@@ -468,7 +467,7 @@ sub setProblemTags {
 	# result is [success, message] with success = 0 or 1
 	my $result = WeBWorK::Utils::ListingDB::setProblemTags($path, $dbsubj, $dbchap, $dbsect, $level, $stat);
 	my $out = {};
-	$out->{text} = encode_base64($result->[1]);
+	$out->{text} = encode_utf8_base64($result->[1]);
 	return($out);
 }
 
diff --git a/lib/WebworkWebservice/RenderProblem.pm b/lib/WebworkWebservice/RenderProblem.pm
index ad8dc3370c..1c6254623c 100644
--- a/lib/WebworkWebservice/RenderProblem.pm
+++ b/lib/WebworkWebservice/RenderProblem.pm
@@ -33,13 +33,12 @@ use WeBWorK::PG::Translator;
 use WeBWorK::PG::Local;
 use WeBWorK::DB;
 use WeBWorK::Constants;
-use WeBWorK::Utils qw(runtime_use formatDateTime makeTempDirectory);
+use WeBWorK::Utils qw(runtime_use formatDateTime makeTempDirectory encode_utf8_base64 decode_utf8_base64);
 use WeBWorK::DB::Utils qw(global2user user2global);
 use WeBWorK::Utils::Tasks qw(fake_set fake_problem);
 use WeBWorK::PG::IO;
 use WeBWorK::PG::ImageGenerator;
 use Benchmark;
-use MIME::Base64 qw( encode_base64 decode_base64);
 
 #print "rereading Webwork\n";
 
@@ -366,7 +365,7 @@ sub renderProblem {
 	my $problem_source;
 	my $r_problem_source =undef;
  	if (defined($rh->{source}) and $rh->{source}) {
-  		$problem_source = decode_base64($rh->{source});
+  		$problem_source = decode_utf8_base64($rh->{source});
   		$problem_source =~ tr /\r/\n/;
 		$r_problem_source =\$problem_source;
 		# warn "source included in request";
@@ -476,11 +475,11 @@ sub renderProblem {
     }
 	# new version of output:
 	my $out2   = {
-		text 						=> encode_base64( $pg->{body_text}  ),
-		header_text 				=> encode_base64( $pg->{head_text} ),
+		text 						=> encode_utf8_base64( $pg->{body_text}  ),
+		header_text 				=> encode_utf8_base64( $pg->{head_text} ),
 		answers 					=> $pg->{answers},
 		errors         				=> $pg->{errors},
-		WARNINGS	   				=> encode_base64( 
+		WARNINGS	   				=> encode_utf8_base64( 
 		                                 "WARNINGS\n".$warning_messages."\n<br/>More<br/>\n".$pg->{warnings} 
 		                               ),
 		PG_ANSWERS_HASH             => $pg->{pgcore}->{PG_ANSWERS_HASH},
@@ -491,7 +490,7 @@ sub renderProblem {
 		debug_messages              => $pgdebug_messages,
 		internal_debug_messages     => $internal_debug_messages,
 	};
-	
+
 	# Filter out bad reference types
 	###################
 	# DEBUGGING CODE
@@ -505,6 +504,7 @@ sub renderProblem {
 	}
 	$out2 = xml_filter($out2); # check this -- it might not be working correctly
 	##################
+	print DEBUGCODE "\n\nStop xml encoding\n";
 	close(DEBUGCODE) if $debugXmlCode;
 	###################
 	
@@ -548,8 +548,10 @@ sub xml_filter {
 		$level++;
 		my $tmp = [];
 		foreach my $item (@{$input}) {
+		    # print DEBUGCODE "-----checking $item of type\n",ref($item) if $debugXmlCode;
 			$item = xml_filter($item,$level);
 			push @$tmp, $item;
+			# print DEBUGCODE "-----end checking $item\n" if $debugXmlCode;
 		}
 		$input = $tmp;
 		$level--;
diff --git a/lib/WebworkWebservice/SetActions.pm b/lib/WebworkWebservice/SetActions.pm
index 250866069c..42c48e8cbe 100644
--- a/lib/WebworkWebservice/SetActions.pm
+++ b/lib/WebworkWebservice/SetActions.pm
@@ -21,7 +21,7 @@ use sigtrap;
 use Carp;
 use WWSafe;
 #use Apache;
-use WeBWorK::Utils;
+use WeBWorK::Utils qw( encode_utf8_base64 decode_utf8_base64 );
 use WeBWorK::Debug qw(debug);
 use JSON;
 use WeBWorK::CourseEnvironment;
@@ -29,7 +29,6 @@ use WeBWorK::PG::Translator;
 use WeBWorK::DB::Utils qw(initializeUserProblem);
 use WeBWorK::PG::IO;
 use Benchmark;
-use MIME::Base64 qw( encode_base64 decode_base64);
 
 ##############################################
 #   Some of this may have to be moved, to allow for flexability
@@ -52,7 +51,7 @@ sub listLocalSets{
   @found_sets = $db->listGlobalSets;
   my $out = {};
   $out->{ra_out} = \@found_sets;
-  $out->{text} = encode_base64("Loaded sets for course: ".$self->{courseName});
+  $out->{text} = encode_utf8_base64("Loaded sets for course: ".$self->{courseName});
   return $out;
 }
 
@@ -96,7 +95,7 @@ sub listLocalSetProblems{
 	
   	my $out = {};
   	$out->{ra_out} = \@problems;
-  	$out->{text} = encode_base64("Loaded Problems for set: " . $setName);
+  	$out->{text} = encode_utf8_base64("Loaded Problems for set: " . $setName);
   	return $out;
 }
 
@@ -121,7 +120,7 @@ sub getSets{
   
   my $out = {};
   $out->{ra_out} = \@all_sets;
-  $out->{text} = encode_base64("Sets for course: ".$self->{courseName});
+  $out->{text} = encode_utf8_base64("Sets for course: ".$self->{courseName});
   return $out;
 }
 
@@ -148,7 +147,7 @@ sub getUserSets{
   
   my $out = {};
   $out->{ra_out} = \@userSets;
-  $out->{text} = encode_base64("Sets for course: ".$self->{courseName});
+  $out->{text} = encode_utf8_base64("Sets for course: ".$self->{courseName});
   return $out;
 }
 
@@ -170,7 +169,7 @@ sub getSet {
   
   my $out = {};
   $out->{ra_out} = $set;
-  $out->{text} = encode_base64("Sets for course: ".$self->{courseName});
+  $out->{text} = encode_utf8_base64("Sets for course: ".$self->{courseName});
   return $out;
   }
 
@@ -247,7 +246,7 @@ sub updateSetProperties {
 
 	my $out = {};
 	$out->{ra_out} = $set;
-	$out->{text} = encode_base64("Successfully updated set " . $params->{set_id});
+	$out->{text} = encode_utf8_base64("Successfully updated set " . $params->{set_id});
 	return $out;
 }
 
@@ -258,7 +257,7 @@ sub listSetUsers {
     my $out = {};
     my @users = $db->listSetUsers($params->{set_id});
     $out->{ra_out} = \@users;
-    $out->{text} = encode_base64("Successfully returned the users for set " . $params->{set_id});
+    $out->{text} = encode_utf8_base64("Successfully returned the users for set " . $params->{set_id});
     return $out;
 
 }
@@ -282,7 +281,7 @@ sub createNewSet{
 
 		my $newSetRecord = $db->getGlobalSet($newSetName);
 		if (defined($newSetRecord)) {
-            $out->{out}=encode_base64("Failed to create set, you may need to try another name."),
+            $out->{out}=encode_utf8_base64("Failed to create set, you may need to try another name."),
             $out->{ra_out} = {'success' => 'false'};
 		} else {			# Do it!
 			# DBFIXME use $db->newGlobalSet
@@ -322,7 +321,7 @@ sub createNewSet{
 			
 			$db->addGlobalSet($newSetRecord);
 			if ($@) {
-				$out->{text} = encode_base64("Failed to create set, you may need to try another name.");
+				$out->{text} = encode_utf8_base64("Failed to create set, you may need to try another name.");
 				#$self->addbadmessage("Problem creating set $newSetName<br> $@");
 			} else {
 				my $selfassign = $params->{selfassign};
@@ -380,7 +379,7 @@ sub assignSetToUsers {
 
 	my $out = {};
 	$out->{ra_out} = \@results;
-	$out->{text} = encode_base64("Successfully assigned users to set " . $params->{set_id});
+	$out->{text} = encode_utf8_base64("Successfully assigned users to set " . $params->{set_id});
 	return $out;
 }
 
@@ -421,7 +420,7 @@ sub deleteProblemSet {
 	debug("deleted set:  $setID");
 	debug($result);
 
-	my $out->{text} = encode_base64("Deleted Problem Set " . $setID);
+	my $out->{text} = encode_utf8_base64("Deleted Problem Set " . $setID);
 
 
 
@@ -475,7 +474,7 @@ sub reorderProblems {
 
 	my $out;
 
-	$out->{text} = encode_base64("Successfully reordered problems");
+	$out->{text} = encode_utf8_base64("Successfully reordered problems");
 	return $out;
 }
 
@@ -497,7 +496,7 @@ sub updateProblem{
 	}
 
 		
-	my $out->{text} = encode_base64("Updated Problem Set " . $setID);
+	my $out->{text} = encode_utf8_base64("Updated Problem Set " . $setID);
 
 
 
@@ -540,7 +539,7 @@ sub updateUserSet {
   
   my $out = {};
   #$out->{ra_out} = $set;
-  $out->{text} = encode_base64("Successfully updated set " . $params->{set_id} . " for users " . $params->{users});
+  $out->{text} = encode_utf8_base64("Successfully updated set " . $params->{set_id} . " for users " . $params->{users});
   return $out;
 }
 
@@ -564,7 +563,7 @@ sub getUserSets {
 
 	my $out = {};
 	$out->{ra_out} = \@userData;
-	$out->{text} = encode_base64("Returning all users sets for set " . $params->{set_id});
+	$out->{text} = encode_utf8_base64("Returning all users sets for set " . $params->{set_id});
 
 	return $out;
 }
@@ -586,7 +585,7 @@ sub saveUserSets {
 
 	my $out = {};
 	$out->{ra_out} = "";
-	$out->{text} = encode_base64("Updating the overrides for set " . $params->{set_id});
+	$out->{text} = encode_utf8_base64("Updating the overrides for set " . $params->{set_id});
 
 	return $out;
 }
@@ -606,7 +605,7 @@ sub unassignSetFromUsers {
 		my $result = $db->deleteUserSet($user, $params->{set_id});
 	}
 	my $out = {};
-	$out->{text} = encode_base64("Successfully unassigned users: " + $params->{users} + " from set " + $params->{set_id});
+	$out->{text} = encode_utf8_base64("Successfully unassigned users: " + $params->{users} + " from set " + $params->{set_id});
 }
 
 =item assignAllSetsToUser($userID)
@@ -727,7 +726,7 @@ sub addProblem {
 	
 
 	#assignProblemToAllSetUsers($self, $problemRecord);
-	my $out->{text} = encode_base64("Problem added to ".$setName);
+	my $out->{text} = encode_utf8_base64("Problem added to ".$setName);
 	return $out;
 }
 
@@ -749,7 +748,7 @@ sub deleteProblem {
 			$db->deleteGlobalProblem($setName, $problemRecord->problem_id);
 		}
 	}
-	my $out->{text} = encode_base64("Problem removed from ".$setName);
+	my $out->{text} = encode_utf8_base64("Problem removed from ".$setName);
 	return $out;
 }