New develop candidate multilingual

lib/PGcore.pm

@@ -29,7 +29,9 @@ use Tie::IxHash;
 use WeBWorK::Debug;
 use MIME::Base64();
 use PGUtil();
-
+use Encode qw(encode_utf8 decode_utf8);
+use utf8;
+binmode(STDOUT, ":uft8");
 ##################################
 # PGcore object
 ##################################
@@ -565,13 +567,15 @@ sub PG_restricted_eval {
 sub decode_base64 ($) {
 	my $self = shift;
 	my $str = shift;
-	MIME::Base64::decode_base64($str);
+	$str = MIME::Base64::decode_base64($str);
+	decode_utf8($str);
 }
 
 sub encode_base64 ($;$) {
 	my $self = shift;
 	my $str  = shift;
 	my $option = shift;
+	$str = encode_utf8($str);
 	MIME::Base64::encode_base64($str);
 }
 

lib/PGloadfiles.pm

@@ -232,7 +232,7 @@ sub compile_file {
  	local($/);
  	$/ = undef;   # allows us to treat the file as a single line
 
- 	open(MACROFILE, "<$filePath") || die "Cannot open file: $filePath";
+ 	open(MACROFILE, "encoding(UTF-8)", $filePath) || die "Cannot open file: $filePath";
  	my $string = 'BEGIN {push @__eval__, __FILE__};' . "\n" . <MACROFILE>;
  	#warn "compiling $string";
  	my ($result,$error,$fullerror) = $self->PG_macro_file_eval($string);

lib/Value/String.pm

@@ -80,7 +80,7 @@ sub compare {
 #
 #  Mark a string to be display verbatim
 #
-sub verb {shift; return "\\verb".chr(0x85).(shift).chr(0x85)}
+sub verb {shift; return "\\verb".chr(0x1F).(shift).chr(0x1F)}
 
 #
 #  Put normal strings into \text{} and others into \verb

lib/WeBWorK/PG/IO.pm

@@ -4,12 +4,17 @@
 ################################################################################
 
 package WeBWorK::PG::IO;
+use warnings qw(FATAL utf8);
 use parent qw(Exporter);
+use Encode qw( encode decode);
 use JSON qw(decode_json);
 use PGUtil qw(not_null);
 use WeBWorK::Utils qw(path_is_subdir);
 use WeBWorK::CourseEnvironment;
-
+use utf8;
+#binmode(STDOUT,":encoding(UTF-8)");
+#binmode(STDIN,":encoding(UTF-8)");
+#binmode(INPUT,":encoding(UTF-8)");
 my $CE = new WeBWorK::CourseEnvironment({
     webwork_dir => $ENV{WEBWORK_ROOT},
 					});
@@ -39,6 +44,7 @@ BEGIN {
 	our @SHARED_FUNCTIONS = qw(
                 includePGtext
                 read_whole_problem_file
+                read_whole_file
                 convertPath
                 fileFromPath
                 directoryFromPath
@@ -139,13 +145,24 @@ sub read_whole_file {
 	    unless path_is_course_subdir($filePath);
 
 	local (*INPUT);
-	open(INPUT, "<$filePath") || die "$0: read_whole_file subroutine: <BR>Can't read file $filePath";
+	open(INPUT, "<:raw", $filePath) || die "$0: read_whole_file subroutine: <BR>Can't read file $filePath";
 	local($/)=undef;
-	my $string = <INPUT>;  # can't append spaces because this causes trouble with <<'EOF'   \nEOF construction
+	my $string = <INPUT>;
+	my $backup_string = $string;
+	# can't append spaces because this causes trouble with <<'EOF'   \nEOF construction
+	my $success = utf8::decode($string);
+	unless ($success) {
+		warn "There was an error decoding $filePath as UTF-8, will try to upgrade";
+		utf8:upgrade($backup_string);
+		$string = $backup_string;
+	}
 	close(INPUT);
 	\$string;
 }
-
+# <:utf8 is more relaxed on input, <:encoding(UTF-8) would be better, but 
+# perhaps it's not so horrible to have lax input. encoding(UTF-8) tries to use require
+# to import Encode, Encode::Alias::find_encoding and Safe raises an exception.
+# haven't figured a way around this yet. 
 =item convertPath($path)
 
 Currently a no-op. Returns $path unmodified.
@@ -201,7 +218,7 @@ sub createFile {
 
 	die 'Path is unsafe' unless path_is_course_subdir($fileName);
 
-	open(TEMPCREATEFILE, ">$fileName")
+	open(TEMPCREATEFILE, ">:encoding(UTF-8)",$fileName)
 		or die "Can't open $fileName: $!";
 	my @stat = stat TEMPCREATEFILE;
 	close(TEMPCREATEFILE);