From 0616de8f8a6110caf6b17b5581eee8e03653dde3 Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Mon, 2 Nov 2020 21:58:10 +0200 Subject: [PATCH] Replace encode_utf8($x) by Encode::encode("UTF-8",$x) and replace encode_utf8($x) by Encode::decode("UTF-8",$x) as the prior is not a proper conversion, and the documentation at https://perldoc.perl.org/Encode#encode_utf8 state that it should not be used for data exchange. In several places use Encode::encode() explicitly, instead of just encode() after a use Encode qw(encode). Replace binmode(STDOUT, ":utf8"); with binmode(STDOUT, ":encoding(UTF-8)"); as Perl 5.30 makes sysread and syswrite on ":utf8" filehandles fatal (it was deprecated in 5.24). See: https://perldoc.perl.org/perl5300delta#Previously-deprecated-sysread()/syswrite()-on-:utf8-handles-is-now-fatal --- bin/importClassList.pl | 2 -- lib/Apache/WeBWorK.pm | 8 ++++---- lib/FormatRenderedProblem.pm | 1 - lib/WeBWorK/ContentGenerator.pm | 2 +- lib/WeBWorK/ContentGenerator/Feedback.pm | 7 +++---- .../ContentGenerator/Instructor/FileManager.pm | 1 - .../ContentGenerator/Instructor/SendMail.pm | 7 +++---- .../ContentGenerator/Instructor/SetMaker.pm | 4 ++-- .../ContentGenerator/Instructor/UserList2.pm | 3 +-- lib/WeBWorK/ContentGenerator/Login.pm | 2 +- lib/WeBWorK/ContentGenerator/Problem.pm | 2 +- .../ContentGenerator/ProblemUtil/ProblemUtil.pm | 7 +++---- lib/WeBWorK/Request.pm | 2 +- lib/WeBWorK/Utils.pm | 14 +++++++------- lib/WebworkClient.pm | 3 +-- 15 files changed, 28 insertions(+), 37 deletions(-) diff --git a/bin/importClassList.pl b/bin/importClassList.pl index 6c6a466b07..c4b12caa41 100755 --- a/bin/importClassList.pl +++ b/bin/importClassList.pl @@ -31,8 +31,6 @@ BEGIN #use WeBWorK::Utils qw(readFile readDirectory cryptPassword x); use WeBWorK::Utils qw(cryptPassword); -use Encode qw(decode_utf8 encode_utf8); - use strict; use warnings; diff --git a/lib/Apache/WeBWorK.pm b/lib/Apache/WeBWorK.pm index 5753ca5889..205ab3a346 100644 --- a/lib/Apache/WeBWorK.pm +++ b/lib/Apache/WeBWorK.pm @@ -68,9 +68,9 @@ sub handler($) { my $uri = $r->uri; - # We set the bimode for print to utf8 because some language options + # We set the binmode for print to utf8 because some language options # use utf8 characters - binmode(STDOUT, "encoding(UTF-8)"); + binmode(STDOUT, ":encoding(UTF-8)"); # the warning handler accumulates warnings in $r->notes("warnings") for # later cumulative reporting my $warning_handler; @@ -79,11 +79,11 @@ sub handler($) { my ($warning) = @_; chomp $warning; my $warnings = $r->notes->get("warnings"); - $warnings = Encode::decode_utf8($warnings); + $warnings = Encode::decode("UTF-8",$warnings); $warnings .= "$warning\n"; #my $backtrace = join("\n",backtrace()); #$warnings .= "$backtrace\n\n"; - $warnings = Encode::encode_utf8($warnings); + $warnings = Encode::encode("UTF-8",$warnings); $r->notes->set(warnings => $warnings); $log->warn("[$uri] $warning"); diff --git a/lib/FormatRenderedProblem.pm b/lib/FormatRenderedProblem.pm index cbdc9e2e6a..5a80b1a0e3 100755 --- a/lib/FormatRenderedProblem.pm +++ b/lib/FormatRenderedProblem.pm @@ -32,7 +32,6 @@ use WeBWorK::PG::ImageGenerator; use WeBWorK::Utils qw( wwRound encode_utf8_base64 decode_utf8_base64); use XML::Simple qw(XMLout); use WeBWorK::Utils::DetermineProblemLangAndDirection; -use Encode qw(encode_utf8 decode_utf8); use JSON; our $UNIT_TESTS_ON = 0; diff --git a/lib/WeBWorK/ContentGenerator.pm b/lib/WeBWorK/ContentGenerator.pm index 5e29544824..1d2d16d014 100644 --- a/lib/WeBWorK/ContentGenerator.pm +++ b/lib/WeBWorK/ContentGenerator.pm @@ -1261,7 +1261,7 @@ sub warnings { print CGI::p("Entering ContentGenerator::warnings") if $TRACE_WARNINGS; print "\n\n"; my $warnings = MP2 ? $r->notes->get("warnings") : $r->notes("warnings"); - $warnings = Encode::decode_utf8($warnings); + $warnings = Encode::decode("UTF-8",$warnings); print $self->warningOutput($warnings) if $warnings; print "\n"; diff --git a/lib/WeBWorK/ContentGenerator/Feedback.pm b/lib/WeBWorK/ContentGenerator/Feedback.pm index 8037180ada..407aaab5ea 100644 --- a/lib/WeBWorK/ContentGenerator/Feedback.pm +++ b/lib/WeBWorK/ContentGenerator/Feedback.pm @@ -29,7 +29,6 @@ WeBWorK::ContentGenerator::Feedback - Send mail to professors. use strict; use warnings; use utf8; -use Encode qw(encode_utf8 encode); use Data::Dumper; use Data::Dump qw/dump/; use WeBWorK::Debug; @@ -143,7 +142,7 @@ sub body { # Encode the user name using "MIME-Header" encoding, # (RFC 2047) which allows UTF-8 encoded names to be # encoded inside the mail header using a special format. - $sender = encode("MIME-Header", $user->full_name) . " <$from>"; + $sender = Encode::encode("MIME-Header", $user->full_name) . " <$from>"; } else { $sender = $from; } @@ -180,7 +179,7 @@ sub body { # If in the future any fields in the subject can contain non-ASCII characters # then we will also need: - # $subject = encode("MIME-Header", $subject); + # $subject = Encode::encode("MIME-Header", $subject); # at present, this does not seem to be necessary. # get info about remote user (stolen from &WeBWorK::Authen::write_log_entry) @@ -292,7 +291,7 @@ $emailableURL $msg .= Dumper($ce). "\n\n"; } - $email->body_set(encode_utf8($msg)); + $email->body_set(Encode::encode("UTF-8",$msg)); try { sendmail($email,{transport => $transport}); diff --git a/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm b/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm index 36d8765da2..12672dfb13 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/FileManager.pm @@ -23,7 +23,6 @@ use WeBWorK::Upload; use File::Path; use File::Copy; use File::Spec; -use Encode qw(encode_utf8 decode_utf8); use String::ShellQuote; diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm b/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm index f5a09da3c2..9303472373 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SendMail.pm @@ -34,7 +34,6 @@ use Email::Simple; use Email::Sender::Simple qw(sendmail); use Email::Sender::Transport::SMTP qw(); use Try::Tiny; -use Encode qw(encode_utf8 encode); use Data::Dump qw/dump/; use WeBWorK::Debug; @@ -569,7 +568,7 @@ sub print_preview { # In a real mails we would UTF-8 encode the message # and give the Content-Type header, for the preview which # is displayed - just add the header, but do NOT use - # encode_utf8($msg) as it will be done late. + # Encode::encode("UTF-8",$msg) as it will be done late. "Content-Type: text/plain; charset=UTF-8\n\n", $msg, # will be in HTML output, and gets encoded to UTF-8 later on "\n" @@ -897,7 +896,7 @@ sub mail_message_to_recipients { From => $from, Subject => $subject, "Content-Type" => "text/plain; charset=UTF-8" ], - body => encode_utf8($msg) + body => Encode::encode("UTF-8",$msg) ); $email->header_set("X-Remote-Host: ",$self->{remote_host}); @@ -954,7 +953,7 @@ sub email_notification { Subject => $subject, "Content-Type" => "text/plain; charset=UTF-8" ], - body => encode_utf8($result_message), + body => Encode::encode("UTF-8",$result_message), ); $email->header_set("X-Remote-Host: ",$self->{remote_host}); diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 1b020c1a3d..d08775517a 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -405,7 +405,7 @@ sub browse_local_panel { my $r = $self->r; my $library_selected = shift; my $lib = shift || ''; $lib =~ s/^browse_//; - my $name = ($lib eq '')? $r->maketext('Local') : Encode::decode_utf8($problib{$lib}); + my $name = ($lib eq '')? $r->maketext('Local') : Encode::decode("UTF-8",$problib{$lib}); my $list_of_prob_dirs= get_problem_directories($r,$lib); if(scalar(@$list_of_prob_dirs) == 0) { @@ -836,7 +836,7 @@ sub make_top_row { ## Make buttons for additional problem libraries my $libs = ''; foreach my $lib (sort(keys(%problib))) { - $libs .= ' '. CGI::submit(-name=>"browse_$lib", -value=>Encode::decode_utf8($problib{$lib}), + $libs .= ' '. CGI::submit(-name=>"browse_$lib", -value=>Encode::decode("UTF-8",$problib{$lib}), ($browse_which eq "browse_$lib")? (-disabled=>1): ()) if (-d "$ce->{courseDirs}{templates}/$lib"); } diff --git a/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm b/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm index b216d2bf57..274c7198af 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm @@ -77,7 +77,6 @@ use constant HIDE_USERS_THRESHHOLD => 200; use constant EDIT_FORMS => [qw(saveEdit cancelEdit)]; use constant PASSWORD_FORMS => [qw(savePassword cancelPassword)]; use constant VIEW_FORMS => [qw(filter sort edit password import export add delete)]; -use Encode qw(decode_utf8 encode_utf8); # permissions needed to perform a given action use constant FORM_PERMS => { saveEdit => "modify_student_data", @@ -1753,7 +1752,7 @@ sub recordEditHTML { my $fieldValue = $User->$field; # FIXME utf8, utf8mb4 debugging codes # warn "user values ".join(" ", $User->user_id,$User->first_name, $User->last_name)."\n"; -# warn "variants". join(" ",$User->user_id, decode_utf8($User->first_name), decode_utf8($User->last_name))."\n"; +# warn "variants". join(" ",$User->user_id, Encode::decode("UTF-8",$User->first_name), Encode::decode("UTF-8",$User->last_name))."\n"; # warn "is_utf8 flag, first_name, last_name ".join(" ", utf8::is_utf8($User->first_name)?1:0, utf8::is_utf8($User->last_name)?1:0 )."\n"; my %properties = %{ FIELD_PROPERTIES()->{$field} }; next if $properties{access} eq 'hidden'; diff --git a/lib/WeBWorK/ContentGenerator/Login.pm b/lib/WeBWorK/ContentGenerator/Login.pm index 0409727256..0eb111008b 100644 --- a/lib/WeBWorK/ContentGenerator/Login.pm +++ b/lib/WeBWorK/ContentGenerator/Login.pm @@ -196,7 +196,7 @@ sub body { # us to yell at the user for doing that, since Authen isn't a content- # generating module. my $authen_error = MP2 ? $r->notes->get("authen_error") : $r->notes("authen_error"); - $authen_error = Encode::decode_utf8($authen_error); + $authen_error = Encode::decode("UTF-8",$authen_error); if ($authen_error) { print CGI::div({class=>"ResultsWithError", tabindex=>'0'}, diff --git a/lib/WeBWorK/ContentGenerator/Problem.pm b/lib/WeBWorK/ContentGenerator/Problem.pm index 2fbe1a4f5b..6ce3f9c8e2 100644 --- a/lib/WeBWorK/ContentGenerator/Problem.pm +++ b/lib/WeBWorK/ContentGenerator/Problem.pm @@ -48,7 +48,7 @@ use WeBWorK::Utils::AttemptsTable; use utf8; #use open ':encoding(utf8)'; -binmode(STDOUT, ":utf8"); +binmode(STDOUT, ":encoding(UTF-8)"); ################################################################################ # CGI param interface to this module (up-to-date as of v1.153) ################################################################################ diff --git a/lib/WeBWorK/ContentGenerator/ProblemUtil/ProblemUtil.pm b/lib/WeBWorK/ContentGenerator/ProblemUtil/ProblemUtil.pm index e0f8f63f6e..a7fd343859 100644 --- a/lib/WeBWorK/ContentGenerator/ProblemUtil/ProblemUtil.pm +++ b/lib/WeBWorK/ContentGenerator/ProblemUtil/ProblemUtil.pm @@ -21,7 +21,6 @@ package WeBWorK::ContentGenerator::ProblemUtil::ProblemUtil; use base qw(WeBWorK); use base qw(WeBWorK::ContentGenerator); -use Encode qw(encode_utf8 encode); =head1 NAME @@ -611,7 +610,7 @@ sub jitar_send_warning_email { # Encode the user name using "MIME-Header" encoding, (RFC 2047) which # allows UTF-8 encoded names to be encoded inside the mail header using # a special format. - $sender = encode("MIME-Header", $user->full_name); + $sender = Encode::encode("MIME-Header", $user->full_name); } else { $sender = $userID; } @@ -635,7 +634,7 @@ sub jitar_send_warning_email { # If in the future any fields in the subject can contain non-ASCII characters # then we will also need: - # $subject = encode("MIME-Header", $subject); + # $subject = Encode::encode("MIME-Header", $subject); # at present, this does not seem to be necessary. @@ -683,7 +682,7 @@ Comment: $comment /; # Encode the body in UTF-8 when adding it. - $email->body_set(encode_utf8($msg)); + $email->body_set(Encode::encode("UTF-8",$msg)); ## try to send the email diff --git a/lib/WeBWorK/Request.pm b/lib/WeBWorK/Request.pm index b6ce8e544b..3c0e7a4991 100644 --- a/lib/WeBWorK/Request.pm +++ b/lib/WeBWorK/Request.pm @@ -61,7 +61,7 @@ sub mutable_param { my @names = $self->SUPER::param(); foreach my $name (@names) { my @params = $self->SUPER::param($name); - @params = map {Encode::decode_utf8($_)} @params; + @params = map {Encode::decode("UTF-8",$_)} @params; $self->{paramcache}{$name} = [@params]; } } diff --git a/lib/WeBWorK/Utils.pm b/lib/WeBWorK/Utils.pm index 491d755ebc..0eeca64147 100644 --- a/lib/WeBWorK/Utils.pm +++ b/lib/WeBWorK/Utils.pm @@ -31,7 +31,6 @@ use DateTime; use DateTime::TimeZone; use Date::Parse; use Date::Format; -use Encode qw(encode_utf8 decode_utf8); use File::Copy; use File::Spec::Functions qw(canonpath); use Time::Zone; @@ -206,7 +205,7 @@ sub readFile($) { # use the following instead if (open my $dh, "<:raw", $fileName){ $result = <$dh>; - decode_utf8($result) or die "failed to decode $fileName"; + Encode::decode("UTF-8",$result) or die "failed to decode $fileName"; close $dh; } else { print STDERR "File $fileName cannot be read."; # this is not a fatal error. @@ -215,9 +214,10 @@ sub readFile($) { if ($@) { print STDERR "reading $fileName: error in Utils::readFile: $@\n"; } - my $prevent_error_message = utf8::decode($result) or warn "Non-fatal warning: file $fileName contains at least one character code which ". - "is not valid in UTF-8. (The copyright sign is often a culprit -- use '&copy;' instead.)\n". - "While this is not fatal you should fix it\n"; + my $prevent_error_message = utf8::decode($result) or warn join("", + "Non-fatal warning: file $fileName contains at least one character code which ", + "is not valid in UTF-8. (The copyright sign is often a culprit -- use '&copy;' instead.)\n", + "While this is not fatal you should fix it\n"); # FIXME # utf8::decode($result) raises an error about the copyright sign # decode_utf8 and Encode::decode_utf8 do not -- which is doing the right thing? @@ -942,7 +942,7 @@ sub decodeAnswers($) { } sub decode_utf8_base64 { - return decode_utf8(decode_base64(shift)); + return Encode::decode("UTF-8",decode_base64(shift)); } sub OLD_encodeAnswers(\%\@) { @@ -965,7 +965,7 @@ sub encodeAnswers(\%\@) { } sub encode_utf8_base64 { - return encode_base64(encode_utf8(shift)); + return encode_base64(Encode::encode("UTF-8",shift)); } sub nfreeze_base64 { diff --git a/lib/WebworkClient.pm b/lib/WebworkClient.pm index 9ff8486f50..fd21107186 100755 --- a/lib/WebworkClient.pm +++ b/lib/WebworkClient.pm @@ -111,7 +111,6 @@ use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib"; use lib "$WeBWorK::Constants::PG_DIRECTORY/lib"; use XMLRPC::Lite; use WeBWorK::Utils qw( wwRound encode_utf8_base64 decode_utf8_base64); -use Encode qw(encode_utf8 decode_utf8 decode); use WeBWorK::Utils::AttemptsTable; use WeBWorK::CourseEnvironment; use WeBWorK::Utils::DetermineProblemLangAndDirection; @@ -442,7 +441,7 @@ sub xml_utf_decode { # Do UTF-8 decoding where xml_filter applied encoding # Get the original name back my $new_item = $item; $new_item =~ s/^xmlrpc_UTF8_encoded_//; - $input->{$new_item} = decode("UTF-8", $filtered_value); + $input->{$new_item} = Encode::decode("UTF-8", $filtered_value); delete( $input->{$item} ); # remove the temporary encoded value with the modified key } else { $input->{$item} = $filtered_value; # No decoding needed