UTF-8 changes

bin/importClassList.pl

@@ -31,8 +31,6 @@ BEGIN
 #use WeBWorK::Utils qw(readFile readDirectory cryptPassword x);
 use WeBWorK::Utils qw(cryptPassword);
 
-use Encode qw(decode_utf8 encode_utf8);
-
 use strict;
 use warnings;
 

lib/Apache/WeBWorK.pm

@@ -68,9 +68,9 @@ sub handler($) {
 	my $uri = $r->uri;
 
 
-	# We set the bimode for print to utf8 because some language options
+	# We set the binmode for print to utf8 because some language options
 	# use utf8 characters
-	binmode(STDOUT, "encoding(UTF-8)");
+	binmode(STDOUT, ":encoding(UTF-8)");
 	# the warning handler accumulates warnings in $r->notes("warnings") for
 	# later cumulative reporting
 	my $warning_handler;
@@ -79,11 +79,11 @@ sub handler($) {
 			my ($warning) = @_;
 			chomp $warning;
 			my $warnings = $r->notes->get("warnings");
-			$warnings = Encode::decode_utf8($warnings);
+			$warnings = Encode::decode("UTF-8",$warnings);
 			$warnings .= "$warning\n";
 			#my $backtrace = join("\n",backtrace());
 			#$warnings .= "$backtrace\n\n";
-			$warnings = Encode::encode_utf8($warnings);
+			$warnings = Encode::encode("UTF-8",$warnings);
 			$r->notes->set(warnings => $warnings);
 
 			$log->warn("[$uri] $warning");

lib/FormatRenderedProblem.pm

@@ -32,7 +32,6 @@ use WeBWorK::PG::ImageGenerator;
 use WeBWorK::Utils qw( wwRound encode_utf8_base64 decode_utf8_base64);
 use XML::Simple qw(XMLout);
 use WeBWorK::Utils::DetermineProblemLangAndDirection;
-use Encode qw(encode_utf8 decode_utf8);
 use JSON;
 
 our $UNIT_TESTS_ON  = 0; 

lib/WeBWorK/ContentGenerator.pm

@@ -1261,7 +1261,7 @@ sub warnings {
 	print CGI::p("Entering ContentGenerator::warnings") if $TRACE_WARNINGS;
 	print "\n<!-- BEGIN " . __PACKAGE__ . "::warnings -->\n";
 	my $warnings = MP2 ? $r->notes->get("warnings") : $r->notes("warnings");
-	$warnings = Encode::decode_utf8($warnings);
+	$warnings = Encode::decode("UTF-8",$warnings);
 	print $self->warningOutput($warnings) if $warnings;
 	print "<!-- END " . __PACKAGE__ . "::warnings -->\n";
 

lib/WeBWorK/ContentGenerator/Feedback.pm

@@ -29,7 +29,6 @@ WeBWorK::ContentGenerator::Feedback - Send mail to professors.
 use strict;
 use warnings;
 use utf8;
-use Encode qw(encode_utf8 encode);
 use Data::Dumper;
 use Data::Dump qw/dump/;
 use WeBWorK::Debug;
@@ -143,7 +142,7 @@ sub body {
 					# Encode the user name using "MIME-Header" encoding,
 					# (RFC 2047) which allows UTF-8 encoded names to be
 					# encoded inside the mail header using a special format.
-					$sender = encode("MIME-Header", $user->full_name) . " <$from>";
+					$sender = Encode::encode("MIME-Header", $user->full_name) . " <$from>";
 				} else {
 					$sender = $from;
 				}
@@ -180,7 +179,7 @@ sub body {
 
 		# If in the future any fields in the subject can contain non-ASCII characters
 		# then we will also need:
-		# $subject = encode("MIME-Header", $subject);
+		# $subject = Encode::encode("MIME-Header", $subject);
 		# at present, this does not seem to be necessary.
 
 		# get info about remote user (stolen from &WeBWorK::Authen::write_log_entry)
@@ -292,7 +291,7 @@ $emailableURL
 			$msg .= Dumper($ce). "\n\n";
 		}
 
-    $email->body_set(encode_utf8($msg));
+    $email->body_set(Encode::encode("UTF-8",$msg));
 
 		try {
 			sendmail($email,{transport => $transport});

lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm

@@ -23,7 +23,6 @@ use WeBWorK::Upload;
 use File::Path;
 use File::Copy;
 use File::Spec;
-use Encode qw(encode_utf8 decode_utf8);
 
 use String::ShellQuote;
 

lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm

@@ -34,7 +34,6 @@ use Email::Simple;
 use Email::Sender::Simple qw(sendmail);
 use Email::Sender::Transport::SMTP qw();
 use Try::Tiny;
-use Encode qw(encode_utf8 encode);
 use Data::Dump qw/dump/;
 use WeBWorK::Debug;
 
@@ -569,7 +568,7 @@ sub print_preview {
 		    # In a real mails we would UTF-8 encode the message
 		    # and give the Content-Type header, for the preview which
 		    # is displayed - just add the header, but do NOT use
-		    # encode_utf8($msg) as it will be done late.
+		    # Encode::encode("UTF-8",$msg) as it will be done late.
 		    "Content-Type: text/plain; charset=UTF-8\n\n",
 		    $msg, # will be in HTML output, and gets encoded to UTF-8 later on
 		    "\n"
@@ -897,7 +896,7 @@ sub mail_message_to_recipients {
 					From => $from,
 					Subject => $subject,
 					"Content-Type" => "text/plain; charset=UTF-8" ],
-				body => encode_utf8($msg)
+				body => Encode::encode("UTF-8",$msg)
 			);
 			$email->header_set("X-Remote-Host: ",$self->{remote_host});
 
@@ -954,7 +953,7 @@ sub email_notification {
 			Subject => $subject,
 			"Content-Type" => "text/plain; charset=UTF-8"
 		],
-		body => encode_utf8($result_message),
+		body => Encode::encode("UTF-8",$result_message),
 	);
 	$email->header_set("X-Remote-Host: ",$self->{remote_host});
 

lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm

@@ -405,7 +405,7 @@ sub browse_local_panel {
 	my $r = $self->r;	
 	my $library_selected = shift;
 	my $lib = shift || ''; $lib =~ s/^browse_//;
-	my $name = ($lib eq '')? $r->maketext('Local') : Encode::decode_utf8($problib{$lib});
+	my $name = ($lib eq '')? $r->maketext('Local') : Encode::decode("UTF-8",$problib{$lib});
 
 	my $list_of_prob_dirs= get_problem_directories($r,$lib);
 	if(scalar(@$list_of_prob_dirs) == 0) {
@@ -836,7 +836,7 @@ sub make_top_row {
 	##	Make buttons for additional problem libraries
 	my $libs = '';
 	foreach my $lib (sort(keys(%problib))) {
-		$libs .= ' '. CGI::submit(-name=>"browse_$lib", -value=>Encode::decode_utf8($problib{$lib}),
+		$libs .= ' '. CGI::submit(-name=>"browse_$lib", -value=>Encode::decode("UTF-8",$problib{$lib}),
 																 ($browse_which eq "browse_$lib")? (-disabled=>1): ())
 			if (-d "$ce->{courseDirs}{templates}/$lib");
 	}

lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm

@@ -77,7 +77,6 @@ use constant HIDE_USERS_THRESHHOLD => 200;
 use constant EDIT_FORMS => [qw(saveEdit cancelEdit)];
 use constant PASSWORD_FORMS => [qw(savePassword cancelPassword)];
 use constant VIEW_FORMS => [qw(filter sort edit password import export add delete)];
-use Encode qw(decode_utf8 encode_utf8);
 # permissions needed to perform a given action
 use constant FORM_PERMS => {
 		saveEdit => "modify_student_data",
@@ -1753,7 +1752,7 @@ sub recordEditHTML {
 		my $fieldValue = $User->$field;
 # FIXME  utf8, utf8mb4 debugging codes
 #		warn "user values ".join(" ", $User->user_id,$User->first_name, $User->last_name)."\n";
-#		warn "variants". join(" ",$User->user_id, decode_utf8($User->first_name), decode_utf8($User->last_name))."\n";
+#		warn "variants". join(" ",$User->user_id, Encode::decode("UTF-8",$User->first_name), Encode::decode("UTF-8",$User->last_name))."\n";
 #		warn "is_utf8 flag, first_name, last_name ".join(" ", utf8::is_utf8($User->first_name)?1:0, utf8::is_utf8($User->last_name)?1:0   )."\n";
 		my %properties = %{ FIELD_PROPERTIES()->{$field} };
 	        next if $properties{access} eq 'hidden';

lib/WeBWorK/ContentGenerator/Login.pm

@@ -196,7 +196,7 @@ sub body {
 	# us to yell at the user for doing that, since Authen isn't a content-
 	# generating module.
 	my $authen_error = MP2 ? $r->notes->get("authen_error") : $r->notes("authen_error");
-	$authen_error = Encode::decode_utf8($authen_error);
+	$authen_error = Encode::decode("UTF-8",$authen_error);
 
 	if ($authen_error) {
 		print CGI::div({class=>"ResultsWithError", tabindex=>'0'},

lib/WeBWorK/ContentGenerator/Problem.pm

@@ -48,7 +48,7 @@ use WeBWorK::Utils::AttemptsTable;
 
 use utf8;
 #use open ':encoding(utf8)';
-binmode(STDOUT, ":utf8");
+binmode(STDOUT, ":encoding(UTF-8)");
 ################################################################################
 # CGI param interface to this module (up-to-date as of v1.153)
 ################################################################################

lib/WeBWorK/ContentGenerator/ProblemUtil/ProblemUtil.pm

@@ -21,7 +21,6 @@
 package WeBWorK::ContentGenerator::ProblemUtil::ProblemUtil;
 use base qw(WeBWorK);
 use base qw(WeBWorK::ContentGenerator);
-use Encode qw(encode_utf8 encode);
 
 =head1 NAME
 
@@ -611,7 +610,7 @@ sub jitar_send_warning_email {
 	# Encode the user name using "MIME-Header" encoding, (RFC 2047) which
 	# allows UTF-8 encoded names to be encoded inside the mail header using
 	# a special format.
-	$sender = encode("MIME-Header", $user->full_name);
+	$sender = Encode::encode("MIME-Header", $user->full_name);
     } else {
 	$sender = $userID;
     }
@@ -635,7 +634,7 @@ sub jitar_send_warning_email {
 
     # If in the future any fields in the subject can contain non-ASCII characters
     # then we will also need:
-    # $subject = encode("MIME-Header", $subject);
+    # $subject = Encode::encode("MIME-Header", $subject);
     # at present, this does not seem to be necessary.
 
 
@@ -683,7 +682,7 @@ Comment:    $comment
 /;
 
 	# Encode the body in UTF-8 when adding it.
-	$email->body_set(encode_utf8($msg));
+	$email->body_set(Encode::encode("UTF-8",$msg));
 
 		## try to send the email
 

lib/WeBWorK/Request.pm

@@ -61,7 +61,7 @@ sub mutable_param {
 	    my @names = $self->SUPER::param();
 	    foreach my $name (@names) {
 		my @params = $self->SUPER::param($name);
-		@params = map {Encode::decode_utf8($_)} @params;
+		@params = map {Encode::decode("UTF-8",$_)} @params;
 		$self->{paramcache}{$name} = [@params];
 	    }
 	}

lib/WeBWorK/Utils.pm

@@ -31,7 +31,6 @@ use DateTime;
 use DateTime::TimeZone;
 use Date::Parse;
 use Date::Format;
-use Encode qw(encode_utf8 decode_utf8);
 use File::Copy;
 use File::Spec::Functions qw(canonpath);
 use Time::Zone;
@@ -206,7 +205,7 @@ sub readFile($) {
 			# use the following instead
 			if (open my $dh, "<:raw", $fileName){
 				$result = <$dh>;
-				decode_utf8($result) or die "failed to decode $fileName";
+				Encode::decode("UTF-8",$result) or die "failed to decode $fileName";
 				close $dh;
 			} else {
 				print STDERR "File $fileName cannot be read."; # this is not a fatal error.
@@ -215,9 +214,10 @@ sub readFile($) {
 		if ($@) {
 			print STDERR "reading $fileName:  error in Utils::readFile: $@\n";
 		}
-		my $prevent_error_message = utf8::decode($result) or  warn  "Non-fatal warning: file $fileName contains at least one character code which ". 
-		 "is not valid in UTF-8. (The copyright sign is often a culprit -- use '&amp;copy;' instead.)\n". 
-		 "While this is not fatal you should fix it\n";
+		my $prevent_error_message = utf8::decode($result) or warn join("",
+			"Non-fatal warning: file $fileName contains at least one character code which ",
+			"is not valid in UTF-8. (The copyright sign is often a culprit -- use '&amp;copy;' instead.)\n",
+			"While this is not fatal you should fix it\n");
 		# FIXME
 		# utf8::decode($result) raises an error about the copyright sign
 		# decode_utf8 and Encode::decode_utf8 do not -- which is doing the right thing?
@@ -942,7 +942,7 @@ sub decodeAnswers($) {
 }
 
 sub decode_utf8_base64 {
-    return decode_utf8(decode_base64(shift));
+    return Encode::decode("UTF-8",decode_base64(shift));
 }
 
 sub OLD_encodeAnswers(\%\@) {
@@ -965,7 +965,7 @@ sub encodeAnswers(\%\@) {
 }
 
 sub encode_utf8_base64 {
-    return encode_base64(encode_utf8(shift));
+    return encode_base64(Encode::encode("UTF-8",shift));
 }
 
 sub nfreeze_base64 {

lib/WebworkClient.pm

@@ -111,7 +111,6 @@ use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib";
 use lib "$WeBWorK::Constants::PG_DIRECTORY/lib";
 use XMLRPC::Lite;
 use WeBWorK::Utils qw( wwRound encode_utf8_base64 decode_utf8_base64);
-use Encode qw(encode_utf8 decode_utf8 decode);
 use WeBWorK::Utils::AttemptsTable;
 use WeBWorK::CourseEnvironment;
 use WeBWorK::Utils::DetermineProblemLangAndDirection;
@@ -442,7 +441,7 @@ sub xml_utf_decode { # Do UTF-8 decoding where xml_filter applied encoding
 					# Get the original name back
 					my $new_item = $item;
 					$new_item =~ s/^xmlrpc_UTF8_encoded_//;
-					$input->{$new_item} = decode("UTF-8", $filtered_value);
+					$input->{$new_item} = Encode::decode("UTF-8", $filtered_value);
 					delete( $input->{$item} ); # remove the temporary encoded value with the modified key
 				} else {
 					$input->{$item} = $filtered_value; # No decoding needed