diff --git a/.dockerignore b/.dockerignore index 5bc0ec3e32..1250384bf4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,10 @@ Dockerfile logs/* tmp/* +htdocs/tmp/* +htdocs/DATA/library*.json +htdocs/DATA/tagging-taxonomy.json +htdocs/DATA/textbooks-tree.json .git .data .idea diff --git a/.env b/.env new file mode 100644 index 0000000000..2b7fb7d447 --- /dev/null +++ b/.env @@ -0,0 +1,5 @@ +COURSES_DIRECTORY_ON_HOST=../ww-docker-data/courses +WEBWORK2_HTTP_PORT_ON_HOST=8080 +WEBWORK_DB_USER=webworkWrite +WEBWORK_DB_PASSWORD=passwordRWsetItBeforeFirstStartingTheDBcontainer +WEBWORK_MYSQL_ROOT_PASSWORD=sqlRootPasswordSetThisPasswordBEFOREfirstStartingTheDBcontainer diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000000..210c0555a2 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: [openwebwork] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: ["https://webwork.maa.org/wiki"] diff --git a/.gitignore b/.gitignore index 8dca4e4871..d5fd810081 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.conf *-config +.env applets tmp logs @@ -30,6 +31,9 @@ WeBWorK.sublime-workspace conf/*.apache-config math4-overrides.css math4-overrides.js +htdocs/themes/math4/images/* +htdocs/themes/math4-green/images/* +htdocs/themes/math4-red/images/* DATA/* *.swp .dump_past_answers_salt diff --git a/Dockerfile b/Dockerfile index ec2052c65a..5bb3124644 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,115 +1,288 @@ -FROM ubuntu:16.04 - -ENV WEBWORK_URL /webwork2 -ENV WEBWORK_ROOT_URL http://localhost -ENV WEBWORK_DB_HOST db -ENV WEBWORK_DB_PORT 3306 -ENV WEBWORK_DB_NAME webwork -ENV WEBWORK_DB_DSN DBI:mysql:${WEBWORK_DB_NAME}:${WEBWORK_DB_HOST}:${WEBWORK_DB_PORT} -ENV WEBWORK_DB_USER webworkWrite -ENV WEBWORK_DB_PASSWORD passwordRW -ENV WEBWORK_SMTP_SERVER localhost -ENV WEBWORK_SMTP_SENDER webwork@example.com -ENV WEBWORK_TIMEZONE America/New_York -ENV APACHE_RUN_USER www-data -ENV APACHE_RUN_GROUP www-data -# temporary state file location. This might be changed to /run in Wheezy+1 -ENV APACHE_PID_FILE /var/run/apache2/apache2.pid -ENV APACHE_RUN_DIR /var/run/apache2 -ENV APACHE_LOCK_DIR /var/lock/apache2 -# Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. -ENV APACHE_LOG_DIR /var/log/apache2 -ENV APP_ROOT /opt/webwork -ENV WEBWORK_ROOT $APP_ROOT/webwork2 -ENV PG_ROOT $APP_ROOT/pg -ENV DEV 0 +# Optional things to change/configure below: +# +# 1. Which branch of webwork2/ and pg/ to install. +# +# 2. Installing the OPL in the Docker image itself. +# (almost 850MB: 290+MB for the main OPL, 90+MB for Pending, 460+MB for Contrib) +# +# By default this is NOT done, and it will instead be installed in +# a named Docker storage volume when the container is first started. +# +# Note: For typical use, we recommend that the OPL be either mounted from +# a local directory on the source or from a separate named data volume. +# That approach precludes needing to download the OPL for each update to +# the Docker image, and allows it to be easily upgraded using git in its +# persistent location. +# +# 3. Some things should be handled by setting environment variables which +# take effect at container startup. They can usually be set in +# docker-compose.yml. +# +# SSL=1 +# will turn on SSL at startup +# ADD_LOCALES="locale1,locale2,locale3" +# will build these locales at startup +# PAPERSIZE=a4 +# will set the system papersize to A4 +# SYSTEM_TIMEZONE=zone/city +# will set the system timezone to zone/city +# Make sure to use a valid setting. +# "/usr/bin/timedatectl list-timezones" on Ubuntu will find valid values +# ADD_APT_PACKAGES="package1 package2 package3" +# will have these additional Ubuntu packages installed at startup. +# +# ================================================================== + +# Phase 1 - download some Git repos for later use: +# as suggested by Nelson Moller in https://gist.github.com/nmoller/81bd8e149e6aa2a7cf051e0bf248b2e2 + +FROM alpine/git AS base + +# build args specifying the branches for webwork2 and pg used to build the image +ARG WEBWORK2_GIT_URL +ARG WEBWORK2_BRANCH +ARG PG_GIT_URL +ARG PG_BRANCH + +WORKDIR /opt/base + +RUN echo Cloning branch $WEBWORK2_BRANCH from $WEBWORK2_GIT_URL \ + && echo git clone --single-branch --branch ${WEBWORK2_BRANCH} --depth 1 $WEBWORK2_GIT_URL \ + && git clone --single-branch --branch ${WEBWORK2_BRANCH} --depth 1 $WEBWORK2_GIT_URL \ + && rm -rf webwork2/.git webwork2/{*ignore,Dockerfile,docker-compose.yml,docker-config} + +RUN echo Cloning branch $PG_BRANCH branch from $PG_GIT_URL \ + && echo git clone --single-branch --branch ${PG_BRANCH} --depth 1 $PG_GIT_URL \ + && git clone --single-branch --branch ${PG_BRANCH} --depth 1 $PG_GIT_URL \ + && rm -rf pg/.git + +# Optional - include OPL (also need to uncomment further below when an included OPL is desired): +#RUN git clone --single-branch --branch master --depth 1 https://github.com/openwebwork/webwork-open-problem-library.git \ +# && rm -rf webwork-open-problem-library/.git + +# ================================================================== + +# Phase 2 - set ENV variables + +# we need to change FROM before setting the ENV variables + +FROM ubuntu:20.04 + +ENV WEBWORK_URL=/webwork2 \ + WEBWORK_ROOT_URL=http://localhost \ + WEBWORK_SMTP_SERVER=localhost \ + WEBWORK_SMTP_SENDER=webwork@example.com \ + WEBWORK_TIMEZONE=America/New_York \ + APACHE_RUN_USER=www-data \ + APACHE_RUN_GROUP=www-data \ + # temporary state file location. This might be changed to /run in Wheezy+1 \ + APACHE_PID_FILE=/var/run/apache2/apache2.pid \ + APACHE_RUN_DIR=/var/run/apache2 \ + APACHE_LOCK_DIR=/var/lock/apache2 \ + # Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. + APACHE_LOG_DIR=/var/log/apache2 \ + APP_ROOT=/opt/webwork \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + DEV=0 + +# Environment variables which depend on a prior environment variable must be set +# in an ENV call after the dependencies were defined. +ENV WEBWORK_ROOT=$APP_ROOT/webwork2 \ + PG_ROOT=$APP_ROOT/pg \ + PATH=$PATH:$APP_ROOT/webwork2/bin + +# ================================================================== + +# Phase 3 - Ubuntu 20.04 base image + required packages + +# Packages changes/added for ubuntu 20.04: +# libcgi-pm-perl (for CGI::Cookie), libdbd-mariadb-perl + +# Do NOT include "apt-get -y upgrade" +# see: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ - apache2 \ - curl \ - dvipng \ - gcc \ - libapache2-request-perl \ - libcrypt-ssleay-perl \ - libdatetime-perl \ - libdancer-perl \ - libdancer-plugin-database-perl \ - libdbd-mysql-perl \ - libemail-address-perl \ - libexception-class-perl \ - libextutils-xsbuilder-perl \ - libfile-find-rule-perl-perl \ - libgd-perl \ - libhtml-scrubber-perl \ - libjson-perl \ - liblocale-maketext-lexicon-perl \ - libmail-sender-perl \ - libmime-tools-perl \ - libnet-ip-perl \ - libnet-ldap-perl \ - libnet-oauth-perl \ - libossp-uuid-perl \ - libpadwalker-perl \ - libpath-class-perl \ - libphp-serialization-perl \ - libsoap-lite-perl \ - libsql-abstract-perl \ - libstring-shellquote-perl \ - libtemplate-perl \ - libtext-csv-perl \ - libtimedate-perl \ - libuuid-tiny-perl \ - libxml-parser-perl \ - libxml-writer-perl \ - libapache2-reload-perl \ - make \ - netpbm \ - preview-latex-style \ - texlive \ - texlive-latex-extra \ - libc6-dev \ - git \ - mysql-client \ - && curl -Lk https://cpanmin.us | perl - App::cpanminus \ - && cpanm install XML::Parser::EasyTree Iterator Iterator::Util Pod::WSDL Array::Utils HTML::Template XMLRPC::Lite Mail::Sender Email::Sender::Simple Data::Dump Statistics::R::IO \ - && rm -fr /var/lib/apt/lists/* ./cpanm /root/.cpanm /tmp/* - -RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/webwork2 - -COPY VERSION /tmp - -RUN WEBWORK_VERSION=`cat /tmp/VERSION|sed -n 's/.*\(develop\)'\'';/\1/p' && cat /tmp/VERSION|sed -n 's/.*\([0-9]\.[0-9]*\)'\'';/PG\-\1/p'` \ - && curl -fSL https://github.com/openwebwork/pg/archive/${WEBWORK_VERSION}.tar.gz -o /tmp/${WEBWORK_VERSION}.tar.gz \ - && tar xzf /tmp/${WEBWORK_VERSION}.tar.gz \ - && mv pg-${WEBWORK_VERSION} $APP_ROOT/pg \ - && rm /tmp/${WEBWORK_VERSION}.tar.gz \ - && curl -fSL https://github.com/openwebwork/webwork-open-problem-library/archive/master.tar.gz -o /tmp/opl.tar.gz \ - && tar xzf /tmp/opl.tar.gz \ - && mv webwork-open-problem-library-master $APP_ROOT/libraries/webwork-open-problem-library \ - && rm /tmp/opl.tar.gz \ - && curl -fSL https://github.com/mathjax/MathJax/archive/master.tar.gz -o /tmp/mathjax.tar.gz \ - && tar xzf /tmp/mathjax.tar.gz \ - && mv MathJax-master $APP_ROOT/MathJax \ - && rm /tmp/mathjax.tar.gz \ - && rm /tmp/VERSION - #curl -fSL https://github.com/openwebwork/webwork2/archive/WeBWorK-${WEBWORK_VERSION}.tar.gz -o /tmp/WeBWorK-${WEBWORK_VERSION}.tar.gz \ - #&& tar xzf /tmp/WeBWorK-${WEBWORK_VERSION}.tar.gz \ - #&& mv webwork2-WeBWorK-${WEBWORK_VERSION} $APP_ROOT/webwork2 \ - #&& rm /tmp/WeBWorK-${WEBWORK_VERSION}.tar.gz \ - -RUN echo "PATH=$PATH:$APP_ROOT/webwork2/bin" >> /root/.bashrc - -COPY . $APP_ROOT/webwork2 - -RUN cd $APP_ROOT/webwork2/courses.dist \ - && cp *.lst $APP_ROOT/courses/ \ - && cp -R modelCourse $APP_ROOT/courses/ \ - && cd $APP_ROOT/pg/lib/chromatic \ - && gcc color.c -o color - -# setup apache + apache2 \ + curl \ + dvipng \ + dvisvgm \ + gcc \ + libapache2-request-perl \ + libarchive-zip-perl \ + libcgi-pm-perl \ + libcrypt-ssleay-perl \ + libdatetime-perl \ + libdbd-mysql-perl \ + libdbd-mariadb-perl \ + libemail-address-xs-perl \ + libexception-class-perl \ + libextutils-xsbuilder-perl \ + libfile-find-rule-perl-perl \ + libgd-perl \ + libhtml-scrubber-perl \ + libjson-perl \ + liblocale-maketext-lexicon-perl \ + libmail-sender-perl \ + libmime-tools-perl \ + libnet-ip-perl \ + libnet-ldap-perl \ + libnet-oauth-perl \ + libossp-uuid-perl \ + libpadwalker-perl \ + libpath-class-perl \ + libphp-serialization-perl \ + libxml-simple-perl \ + libnet-https-nb-perl \ + libhttp-async-perl \ + libsoap-lite-perl \ + libsql-abstract-perl \ + libstring-shellquote-perl \ + libtemplate-perl \ + libtext-csv-perl \ + libtimedate-perl \ + libuuid-tiny-perl \ + libxml-parser-perl \ + libxml-writer-perl \ + libxmlrpc-lite-perl \ + libapache2-reload-perl \ + cpanminus \ + libxml-parser-easytree-perl \ + libiterator-perl \ + libiterator-util-perl \ + libpod-wsdl-perl \ + libtest-xml-perl \ + libmodule-build-perl \ + libxml-semanticdiff-perl \ + libxml-xpath-perl \ + libpath-tiny-perl \ + libarray-utils-perl \ + libhtml-template-perl \ + libtest-pod-perl \ + libemail-sender-perl \ + libmail-sender-perl \ + libmodule-pluggable-perl \ + libemail-date-format-perl \ + libcapture-tiny-perl \ + libthrowable-perl \ + libdata-dump-perl \ + libfile-sharedir-install-perl \ + libclass-tiny-perl \ + libtest-requires-perl \ + libtest-mockobject-perl \ + libtest-warn-perl \ + libsub-uplevel-perl \ + libtest-exception-perl \ + libuniversal-can-perl \ + libuniversal-isa-perl \ + libtest-fatal-perl \ + libjson-xs-perl \ + libjson-maybexs-perl \ + libcpanel-json-xs-perl \ + make \ + netpbm \ + patch \ + pdf2svg \ + preview-latex-style \ + texlive \ + texlive-latex-extra \ + texlive-plain-generic \ + texlive-xetex \ + texlive-latex-recommended \ + texlive-lang-other \ + texlive-lang-arabic \ + libc6-dev \ + git \ + mysql-client \ + tzdata \ + apt-utils \ + locales \ + debconf-utils \ + ssl-cert \ + ca-certificates \ + culmus \ + fonts-linuxlibertine \ + lmodern \ + zip \ + iputils-ping \ + imagemagick \ + jq \ + npm \ + && apt-get clean \ + && rm -fr /var/lib/apt/lists/* /tmp/* + +# Developers may want to add additional packages inside the image +# such as: telnet vim mc file + +# ================================================================== + +# Phase 4 - Install webwork2 and pg which were downloaded to /opt/base/ in phase 1 +# Option: Install the OPL in the image also (about 850 MB) + +RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/libraries/webwork-open-problem-library $APP_ROOT/webwork2 /www/www/html + +COPY --from=base /opt/base/webwork2 $APP_ROOT/webwork2 +COPY --from=base /opt/base/pg $APP_ROOT/pg + +# Optional - include OPL (also need to uncomment above to clone from GitHub when needed): +# ??? could/should this include the main OPL = /opt/base/webwork-open-problem-library/OpenProblemLibrary and not Contrib and Pending ??? +#COPY --from=base /opt/base/webwork-open-problem-library $APP_ROOT/libraries/webwork-open-problem-library + +# ================================================================== + +# Phase 5 - some configuration work + +# 1. Setup PATH. +# 2. Compiles color.c in the copy INSIDE the image, will also be done in docker-entrypoint.sh for externally mounted locations. +# 3. Some chown/chmod for material INSIDE the image. +# 4. Build some standard locales. +# 5. Set the default system timezone to be UTC. +# 6. Install third party javascript files. + +RUN echo "PATH=$PATH:$APP_ROOT/webwork2/bin" >> /root/.bashrc \ + && cd $APP_ROOT/pg/lib/chromatic && gcc color.c -o color \ + && cd $APP_ROOT/webwork2/ \ + && chown www-data DATA ../courses htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ + && chmod -R u+w DATA ../courses htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ + && echo "en_US ISO-8859-1\nen_US.UTF-8 UTF-8" > /etc/locale.gen \ + && /usr/sbin/locale-gen \ + && echo "locales locales/default_environment_locale select en_US.UTF-8\ndebconf debconf/frontend select Noninteractive" > /tmp/preseed.txt \ + && debconf-set-selections /tmp/preseed.txt \ + && rm /etc/localtime /etc/timezone && echo "Etc/UTC" > /etc/timezone \ + && dpkg-reconfigure -f noninteractive tzdata \ + && cd $WEBWORK_ROOT/htdocs \ + && npm install + +# These lines were moved into docker-entrypoint.sh so the bind mount of courses will be available +#RUN cd $APP_ROOT/webwork2/courses.dist \ +# && cp *.lst $APP_ROOT/courses/ \ +# && cp -R modelCourse $APP_ROOT/courses/ + +# ================================================================== + +# Phase 6 - install additional Perl modules from CPAN (not packaged for Ubuntu or outdated in Ubuntu) + +RUN cpanm install Statistics::R::IO \ + && rm -fr ./cpanm /root/.cpanm /tmp/* + +# ================================================================== + +# Phase 7 - setup apache + +# Note we always create the /etc/ssl/local directory in case it will be needed, as +# the SSL config can also be done via a modified docker-entrypoint.sh script. + +# Always provide the dummy default-ssl.conf file: +COPY docker-config/ssl/default-ssl.conf /etc/apache2/sites-available/default-ssl.conf + +# Patch files that are applied below +COPY docker-config/xmlrpc-lite-utf8-fix.patch /tmp +COPY docker-config/imagemagick-allow-pdf-read.patch /tmp + +# However SSL will only be enabled at container startup via docker-entrypoint.sh. + RUN cd $APP_ROOT/webwork2/conf \ && cp webwork.apache2.4-config.dist webwork.apache2.4-config \ && cp $APP_ROOT/webwork2/conf/webwork.apache2.4-config /etc/apache2/conf-enabled/webwork.conf \ @@ -117,30 +290,59 @@ RUN cd $APP_ROOT/webwork2/conf \ && a2enmod mpm_prefork \ && sed -i -e 's/Timeout 300/Timeout 1200/' /etc/apache2/apache2.conf \ && sed -i -e 's/MaxRequestWorkers 150/MaxRequestWorkers 20/' \ - -e 's/MaxConnectionsPerChild 0/MaxConnectionsPerChild 100/' \ - /etc/apache2/mods-available/mpm_prefork.conf \ + -e 's/MaxConnectionsPerChild 0/MaxConnectionsPerChild 100/' \ + /etc/apache2/mods-available/mpm_prefork.conf \ && cp $APP_ROOT/webwork2/htdocs/favicon.ico /var/www/html \ + && mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR \ + && mkdir /etc/ssl/local \ + && a2enmod rewrite \ && sed -i -e 's/^$/\ - PerlPassEnv WEBWORK_URL\n\ - PerlPassEnv WEBWORK_ROOT_URL\n\ - PerlPassEnv WEBWORK_DB_DSN\n\ - PerlPassEnv WEBWORK_DB_USER\n\ - PerlPassEnv WEBWORK_DB_PASSWORD\n\ - PerlPassEnv WEBWORK_SMTP_SERVER\n\ - PerlPassEnv WEBWORK_SMTP_SENDER\n\ - PerlPassEnv WEBWORK_TIMEZONE\n\ - \n/' /etc/apache2/conf-enabled/webwork.conf - -RUN cd $APP_ROOT/webwork2/ \ - && chown www-data DATA ../courses htdocs/tmp htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ - && chmod -R u+w DATA ../courses htdocs/tmp htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic - -COPY docker-entrypoint.sh /usr/local/bin/ + PerlPassEnv WEBWORK_URL\n\ + PerlPassEnv WEBWORK_ROOT_URL\n\ + PerlPassEnv WEBWORK_DB_DRIVER\n\ + PerlPassEnv WEBWORK_DB_NAME\n\ + PerlPassEnv WEBWORK_DB_HOST\n\ + PerlPassEnv WEBWORK_DB_PORT\n\ + PerlPassEnv WEBWORK_DB_USER\n\ + PerlPassEnv WEBWORK_DB_PASSWORD\n\ + PerlPassEnv WEBWORK_SMTP_SERVER\n\ + PerlPassEnv WEBWORK_SMTP_SENDER\n\ + PerlPassEnv WEBWORK_TIMEZONE\n\ + \n/' /etc/apache2/conf-enabled/webwork.conf \ + && patch -p1 -d / < /tmp/xmlrpc-lite-utf8-fix.patch \ + && rm /tmp/xmlrpc-lite-utf8-fix.patch \ + && patch -p1 -d / < /tmp/imagemagick-allow-pdf-read.patch \ + && rm /tmp/imagemagick-allow-pdf-read.patch + +EXPOSE 80 +WORKDIR $APP_ROOT + +# Enabling SSL is NOT done here. +# Instead it is done by docker-entrypoint.sh at container startup when SSL=1 +# is set in the environment, for example by docker-compose.yml. +#RUN a2enmod ssl && a2ensite default-ssl +#EXPOSE 443 + +# ================================================================== + +# Phase 8 - prepare docker-entrypoint.sh +# Done near the end, so that an update to docker-entrypoint.sh can be +# done without rebuilding the earlier layers of the Docker image. + +COPY docker-config/docker-entrypoint.sh /usr/local/bin/ ENTRYPOINT ["docker-entrypoint.sh"] -EXPOSE 80 +# ================================================================== -WORKDIR $APP_ROOT +# Add enviroment variables to control some things during container startup + +ENV SSL=0 \ + PAPERSIZE=letter \ + SYSTEM_TIMEZONE=UTC \ + ADD_LOCALES=0 \ + ADD_APT_PACKAGES=0 + +# ================================================ CMD ["apache2", "-DFOREGROUND"] diff --git a/DockerfileStage1 b/DockerfileStage1 new file mode 100644 index 0000000000..bca0b397dc --- /dev/null +++ b/DockerfileStage1 @@ -0,0 +1,146 @@ +# This is the Stage 1 Dockerfile, which builds a base OS image (webwork-base) +# on top of which the WeBWorK parts will be installed by the Stage 2 Dockerfile. + +FROM ubuntu:20.04 + +# ================================================================== + +# Phase 1 - set base OS image install stage ENV variables +# +# We only need install time ENV variables, not those needed by the WeBWorK system + +ENV DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true + +# ================================================================== + +# Phase 2 - Ubuntu 20.04 base image + required packages + +# Packages changes/added for ubuntu 20.04: +# libcgi-pm-perl (for CGI::Cookie), libdbd-mariadb-perl + +# Do NOT include "apt-get -y upgrade" +# see: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ + +RUN apt-get update \ + && apt-get install -y --no-install-recommends --no-install-suggests \ + apache2 \ + curl \ + dvipng \ + dvisvgm \ + gcc \ + libapache2-request-perl \ + libarchive-zip-perl \ + libcgi-pm-perl \ + libcrypt-ssleay-perl \ + libdatetime-perl \ + libdbd-mysql-perl \ + libdbd-mariadb-perl \ + libemail-address-xs-perl \ + libexception-class-perl \ + libextutils-xsbuilder-perl \ + libfile-find-rule-perl-perl \ + libgd-perl \ + libhtml-scrubber-perl \ + libjson-perl \ + liblocale-maketext-lexicon-perl \ + libmail-sender-perl \ + libmime-tools-perl \ + libnet-ip-perl \ + libnet-ldap-perl \ + libnet-oauth-perl \ + libossp-uuid-perl \ + libpadwalker-perl \ + libpath-class-perl \ + libphp-serialization-perl \ + libxml-simple-perl \ + libnet-https-nb-perl \ + libhttp-async-perl \ + libsoap-lite-perl \ + libsql-abstract-perl \ + libstring-shellquote-perl \ + libtemplate-perl \ + libtext-csv-perl \ + libtimedate-perl \ + libuuid-tiny-perl \ + libxml-parser-perl \ + libxml-writer-perl \ + libxmlrpc-lite-perl \ + libapache2-reload-perl \ + cpanminus \ + libxml-parser-easytree-perl \ + libiterator-perl \ + libiterator-util-perl \ + libpod-wsdl-perl \ + libtest-xml-perl \ + libmodule-build-perl \ + libxml-semanticdiff-perl \ + libxml-xpath-perl \ + libpath-tiny-perl \ + libarray-utils-perl \ + libhtml-template-perl \ + libtest-pod-perl \ + libemail-sender-perl \ + libmail-sender-perl \ + libmodule-pluggable-perl \ + libemail-date-format-perl \ + libcapture-tiny-perl \ + libthrowable-perl \ + libdata-dump-perl \ + libfile-sharedir-install-perl \ + libclass-tiny-perl \ + libtest-requires-perl \ + libtest-mockobject-perl \ + libtest-warn-perl \ + libsub-uplevel-perl \ + libtest-exception-perl \ + libuniversal-can-perl \ + libuniversal-isa-perl \ + libtest-fatal-perl \ + libjson-xs-perl \ + libjson-maybexs-perl \ + libcpanel-json-xs-perl \ + make \ + netpbm \ + patch \ + pdf2svg \ + preview-latex-style \ + texlive \ + texlive-latex-extra \ + texlive-plain-generic \ + texlive-xetex \ + texlive-latex-recommended \ + texlive-lang-other \ + texlive-lang-arabic \ + libc6-dev \ + git \ + mysql-client \ + tzdata \ + apt-utils \ + locales \ + debconf-utils \ + ssl-cert \ + ca-certificates \ + culmus \ + fonts-linuxlibertine \ + lmodern \ + zip \ + iputils-ping \ + imagemagick \ + jq \ + npm \ + && apt-get clean \ + && rm -fr /var/lib/apt/lists/* /tmp/* + +# Developers may want to add additional packages inside the image +# such as: telnet vim mc file + +# ================================================================== + +# Phase 3 - install additional Perl modules from CPAN (not packaged for Ubuntu or outdated in Ubuntu) + +RUN cpanm install Statistics::R::IO \ + && rm -fr ./cpanm /root/.cpanm /tmp/* + +# ================================================================== + diff --git a/DockerfileStage2 b/DockerfileStage2 new file mode 100644 index 0000000000..4d31300627 --- /dev/null +++ b/DockerfileStage2 @@ -0,0 +1,222 @@ +# This is the Stage 2 Dockerfile, which handles the WeBWorK part, building +# on top of the base OS image (webwork-base) created by the Stage 1 Dockerfile. +# +# ================================================================== +# +# Optional things to change/configure below: +# +# 1. Which branch of webwork2/ and pg/ to install. +# +# 2. Installing the OPL in the Docker image itself. +# (almost 850MB: 290+MB for the main OPL, 90+MB for Pending, 460+MB for Contrib) +# +# By default this is NOT done, and it will instead be installed in +# a named Docker storage volume when the container is first started. +# +# Note: For typical use, we recommend that the OPL be either mounted from +# a local directory on the source or from a separate named data volume. +# That approach precludes needing to download the OPL for each update to +# the Docker image, and allows it to be easily upgraded using git in its +# persistent location. +# +# 3. Some things should be handled by setting environment variables which +# take effect at container startup. They can usually be set in +# docker-compose.yml. +# +# SSL=1 +# will turn on SSL at startup +# ADD_LOCALES="locale1,locale2,locale3" +# will build these locales at startup +# PAPERSIZE=a4 +# will set the system papersize to A4 +# SYSTEM_TIMEZONE=zone/city +# will set the system timezone to zone/city +# Make sure to use a valid setting. +# "/usr/bin/timedatectl list-timezones" on Ubuntu will find valid values +# ADD_APT_PACKAGES="package1 package2 package3" +# will have these additional Ubuntu packages installed at startup. +# +# ================================================================== + +# Phase 1 - download some Git repos for later use: +# as suggested by Nelson Moller in https://gist.github.com/nmoller/81bd8e149e6aa2a7cf051e0bf248b2e2 + +FROM alpine/git AS base + +# build args specifying the branches for webwork2 and pg used to build the image +ARG WEBWORK2_GIT_URL +ARG WEBWORK2_BRANCH +ARG PG_GIT_URL +ARG PG_BRANCH + +WORKDIR /opt/base + +RUN echo Cloning branch $WEBWORK2_BRANCH from $WEBWORK2_GIT_URL \ + && echo git clone --single-branch --branch ${WEBWORK2_BRANCH} --depth 1 $WEBWORK2_GIT_URL \ + && git clone --single-branch --branch ${WEBWORK2_BRANCH} --depth 1 $WEBWORK2_GIT_URL \ + && rm -rf webwork2/.git webwork2/{*ignore,Dockerfile,docker-compose.yml,docker-config} + +RUN echo Cloning branch $PG_BRANCH branch from $PG_GIT_URL \ + && echo git clone --single-branch --branch ${PG_BRANCH} --depth 1 $PG_GIT_URL \ + && git clone --single-branch --branch ${PG_BRANCH} --depth 1 $PG_GIT_URL \ + && rm -rf pg/.git + +# Optional - include OPL (also need to uncomment further below when an included OPL is desired): +#RUN git clone --single-branch --branch master --depth 1 https://github.com/openwebwork/webwork-open-problem-library.git \ +# && rm -rf webwork-open-problem-library/.git + +# ================================================================== + +# Phase 2 - set ENV variables + +# we need to change FROM before setting the ENV variables + +FROM webwork-base:forWW216 + +ENV WEBWORK_URL=/webwork2 \ + WEBWORK_ROOT_URL=http://localhost \ + WEBWORK_SMTP_SERVER=localhost \ + WEBWORK_SMTP_SENDER=webwork@example.com \ + WEBWORK_TIMEZONE=America/New_York \ + APACHE_RUN_USER=www-data \ + APACHE_RUN_GROUP=www-data \ + # temporary state file location. This might be changed to /run in Wheezy+1 \ + APACHE_PID_FILE=/var/run/apache2/apache2.pid \ + APACHE_RUN_DIR=/var/run/apache2 \ + APACHE_LOCK_DIR=/var/lock/apache2 \ + # Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. + APACHE_LOG_DIR=/var/log/apache2 \ + APP_ROOT=/opt/webwork \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + DEV=0 + +# Environment variables which depend on a prior environment variable must be set +# in an ENV call after the dependencies were defined. +ENV WEBWORK_ROOT=$APP_ROOT/webwork2 \ + PG_ROOT=$APP_ROOT/pg \ + PATH=$PATH:$APP_ROOT/webwork2/bin + +# ================================================================== + +# Phase 3 - Install webwork2 and pg which were downloaded to /opt/base/ in phase 1 +# Option: Install the OPL in the image also (about 850 MB) + +RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/libraries/webwork-open-problem-library $APP_ROOT/webwork2 /www/www/html + +COPY --from=base /opt/base/webwork2 $APP_ROOT/webwork2 +COPY --from=base /opt/base/pg $APP_ROOT/pg + +# Optional - include OPL (also need to uncomment above to clone from GitHub when needed): +# ??? could/should this include the main OPL = /opt/base/webwork-open-problem-library/OpenProblemLibrary and not Contrib and Pending ??? +#COPY --from=base /opt/base/webwork-open-problem-library $APP_ROOT/libraries/webwork-open-problem-library + +# ================================================================== + +# Phase 4 - some configuration work + +# 1. Setup PATH. +# 2. Compiles color.c in the copy INSIDE the image, will also be done in docker-entrypoint.sh for externally mounted locations. +# 3. Some chown/chmod for material INSIDE the image. +# 4. Build some standard locales. +# 5. Set the default system timezone to be UTC. +# 6. Install third party javascript files. + +RUN echo "PATH=$PATH:$APP_ROOT/webwork2/bin" >> /root/.bashrc \ + && cd $APP_ROOT/pg/lib/chromatic && gcc color.c -o color \ + && cd $APP_ROOT/webwork2/ \ + && chown www-data DATA ../courses htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ + && chmod -R u+w DATA ../courses htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ + && echo "en_US ISO-8859-1\nen_US.UTF-8 UTF-8" > /etc/locale.gen \ + && /usr/sbin/locale-gen \ + && echo "locales locales/default_environment_locale select en_US.UTF-8\ndebconf debconf/frontend select Noninteractive" > /tmp/preseed.txt \ + && debconf-set-selections /tmp/preseed.txt \ + && rm /etc/localtime /etc/timezone && echo "Etc/UTC" > /etc/timezone \ + && dpkg-reconfigure -f noninteractive tzdata \ + && cd $WEBWORK_ROOT/htdocs \ + && npm install + +# These lines were moved into docker-entrypoint.sh so the bind mount of courses will be available +#RUN cd $APP_ROOT/webwork2/courses.dist \ +# && cp *.lst $APP_ROOT/courses/ \ +# && cp -R modelCourse $APP_ROOT/courses/ + +# ================================================================== + +# Phase 5 - setup apache + +# Note we always create the /etc/ssl/local directory in case it will be needed, as +# the SSL config can also be done via a modified docker-entrypoint.sh script. + +# Always provide the dummy default-ssl.conf file: +COPY docker-config/ssl/default-ssl.conf /etc/apache2/sites-available/default-ssl.conf + +# Patch files that are applied below +COPY docker-config/xmlrpc-lite-utf8-fix.patch /tmp +COPY docker-config/imagemagick-allow-pdf-read.patch /tmp + +# However SSL will only be enabled at container startup via docker-entrypoint.sh. + +RUN cd $APP_ROOT/webwork2/conf \ + && cp webwork.apache2.4-config.dist webwork.apache2.4-config \ + && cp $APP_ROOT/webwork2/conf/webwork.apache2.4-config /etc/apache2/conf-enabled/webwork.conf \ + && a2dismod mpm_event \ + && a2enmod mpm_prefork \ + && sed -i -e 's/Timeout 300/Timeout 1200/' /etc/apache2/apache2.conf \ + && sed -i -e 's/MaxRequestWorkers 150/MaxRequestWorkers 20/' \ + -e 's/MaxConnectionsPerChild 0/MaxConnectionsPerChild 100/' \ + /etc/apache2/mods-available/mpm_prefork.conf \ + && cp $APP_ROOT/webwork2/htdocs/favicon.ico /var/www/html \ + && mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR \ + && mkdir /etc/ssl/local \ + && a2enmod rewrite \ + && sed -i -e 's/^$/\ + PerlPassEnv WEBWORK_URL\n\ + PerlPassEnv WEBWORK_ROOT_URL\n\ + PerlPassEnv WEBWORK_DB_DRIVER\n\ + PerlPassEnv WEBWORK_DB_NAME\n\ + PerlPassEnv WEBWORK_DB_HOST\n\ + PerlPassEnv WEBWORK_DB_PORT\n\ + PerlPassEnv WEBWORK_DB_USER\n\ + PerlPassEnv WEBWORK_DB_PASSWORD\n\ + PerlPassEnv WEBWORK_SMTP_SERVER\n\ + PerlPassEnv WEBWORK_SMTP_SENDER\n\ + PerlPassEnv WEBWORK_TIMEZONE\n\ + \n/' /etc/apache2/conf-enabled/webwork.conf \ + && patch -p1 -d / < /tmp/xmlrpc-lite-utf8-fix.patch \ + && rm /tmp/xmlrpc-lite-utf8-fix.patch \ + && patch -p1 -d / < /tmp/imagemagick-allow-pdf-read.patch \ + && rm /tmp/imagemagick-allow-pdf-read.patch + +EXPOSE 80 +WORKDIR $APP_ROOT + +# Enabling SSL is NOT done here. +# Instead it is done by docker-entrypoint.sh at container startup when SSL=1 +# is set in the environment, for example by docker-compose.yml. +#RUN a2enmod ssl && a2ensite default-ssl +#EXPOSE 443 + +# ================================================================== + +# Phase 6 - prepare docker-entrypoint.sh +# Done near the end, so that an update to docker-entrypoint.sh can be +# done without rebuilding the earlier layers of the Docker image. + +COPY docker-config/docker-entrypoint.sh /usr/local/bin/ + +ENTRYPOINT ["docker-entrypoint.sh"] + +# ================================================================== + +# Add enviroment variables to control some things during container startup + +ENV SSL=0 \ + PAPERSIZE=letter \ + SYSTEM_TIMEZONE=UTC \ + ADD_LOCALES=0 \ + ADD_APT_PACKAGES=0 + +# ================================================ + +CMD ["apache2", "-DFOREGROUND"] diff --git a/LICENSE b/LICENSE index 98bc5d4468..5b1ce94e24 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,8 @@ Online Homework Delivery System Version 2.* - Copyright 2000-2017, The WeBWorK Project + Copyright 2000-2019, The WeBWorK Project + All rights reserved. This program is free software; you can redistribute it and/or modify @@ -20,11 +21,11 @@ the GNU General Public License or the Artistic License for more details. You should have received a copy of the Artistic License with this - package, in the file named "Artistic". If not, we'll be glad to provide + package, in the file named "Artistic" inside the `doc` folder. If not, we'll be glad to provide one. You should also have received a copy of the GNU General Public License - along with this program in the file named "Copying". If not, write to the + along with this program in the file named "Copying" inside the `doc` folder. If not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA or visit their web page on the internet at http://www.gnu.org/copyleft/gpl.html. diff --git a/PG_VERSION b/PG_VERSION deleted file mode 120000 index adae9e58c6..0000000000 --- a/PG_VERSION +++ /dev/null @@ -1 +0,0 @@ -../pg/VERSION \ No newline at end of file diff --git a/README b/README index ca428fb85d..e5f5f5c736 100644 --- a/README +++ b/README @@ -3,8 +3,9 @@ Version 2.* Branch: github.com/openwebwork - http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.13 - Copyright 2000-2017, The WeBWorK Project + http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.16 + Copyright 2000-2021, The WeBWorK Project + http://webwork.maa.org All rights reserved. diff --git a/README.md b/README.md index 0040a125ad..bfaf37f3fa 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ + WeBWorK + Online Homework Delivery System + Version 2.* + Branch: github.com/openwebwork + + https://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.16 + Copyright 2000-2021, The WeBWorK Project + https://openwebwork.org/ + http://webwork.maa.org + All rights reserved. + # Welcome to WeBWorK WeBWorK is an open-source online homework system for math and sciences courses. WeBWorK is supported by the MAA and the NSF and comes with an Open Problem Library (OPL) of over 30,000 homework problems. Problems in the OPL target most lower division undergraduate math courses and some advanced courses. Supported courses include college algebra, discrete mathematics, probability and statistics, single and multivariable calculus, differential equations, linear algebra and complex analysis. Find out more at the main WeBWorK [webpage](http://webwork.maa.org). @@ -5,13 +16,41 @@ WeBWorK is an open-source online homework system for math and sciences courses. ## Information for Users New users interested in getting started with their own WeBWorK server, or instructors looking to learn more about how to use WeBWorK in their classes, should take a look at one of the following resources: +* The [WeBWorK project home page](https://openwebwork.org/) - General information and resources including announcements of events and important project news * [WeBWorK wiki](http://webwork.maa.org/wiki/Main_Page) - The main WeBWorK wiki - * [Instructors](http://webwork.maa.org/wiki/Instructors) - Information for Instructors - * [Problem Authors](http://webwork.maa.org/wiki/Authors) - Information for Problem Authors +* [Instructors](http://webwork.maa.org/wiki/Instructors) - Information for Instructors +* [Problem Authors](http://webwork.maa.org/wiki/Authors) - Information for Problem Authors * [WW_Install](http://github.com/aubreyja/ww_install) - Information for using the WW_install script * [Forum](http://webwork.maa.org/moodle/mod/forum/index.php?id=3) - The WeBWorK Forum * [Frequently Asked Questions](https://github.com/openwebwork/webwork2/wiki/Frequently-Asked-Questions) - A list of frequently asked questions. +## Information for downloading + +* The current version is WeBWorK-2.16 and its companion PG-2.16 + +* Installation manuals can be found at https://webwork.maa.org/wiki/Category:Installation_Manuals + +* If you would prefer to download a previous release, say WeBWorK 2.14, then run the following commands: + +``` +cd /opt/webwork/webwork2 +git checkout -b WeBWorK2.14+ WeBWorK2.14+ +``` +* If you want to pull the PG-2.14 branch of pg then run: + +``` +cd /opt/webwork/pg +git checkout -b PG-2.14+ PG-2.14+ +``` +* If you also need an earlier branch of MathJax then run: + +``` +cd /opt/webwork/MathJax +git checkout legacy-v2 +``` + +* A tab to the upper right lists the releases that are available. + ## Information For Developers People interested in developing new features for WeBWorK should take a look at the following resources. People interested in developing new problems for WeBWorK should visit [Problem Authors](http://webwork.maa.org/wiki/Authors). diff --git a/VERSION b/VERSION index f0c0b6a886..e7fc183991 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ -$WW_VERSION = 'develop'; -$WW_COPYRIGHT_YEARS = '1996-2017'; +$WW_VERSION = '2.16+develop'; +$WW_COPYRIGHT_YEARS = '1996-2021'; 1; diff --git a/bin/OPL-update b/bin/OPL-update index 9425a89527..806c90589f 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -13,6 +13,7 @@ # correctly defined (as with other scripts here). # 3) Configuration for the OPL in site.conf needs to be # done (basically just setting the path to the OPL files). +# #use strict; use File::Find; @@ -21,16 +22,11 @@ use File::Basename; use Cwd; use DBI; -# Command line arguments -# for some reason the first command line argument is disappearing before it gets here - my $myLib = "OPL" ; # default value my $clearAll = 1 ; # default value - drop ALL old tables if ( ( 0 + @ARGV ) > 0 ) { - #print join(" , ", @ARGV ); - $myLib = shift; print "myLib = $myLib \n"; @@ -46,8 +42,6 @@ if ( ( 0 + @ARGV ) > 0 ) { } } - - #(maximum varchar length is 255 for mysql version < 5.0.3. #You can increase path length to 4096 for mysql > 5.0.3) @@ -138,21 +132,51 @@ use lib "$ENV{WEBWORK_ROOT}/lib"; use lib "$ENV{WEBWORK_ROOT}/bin"; use WeBWorK::CourseEnvironment; use WeBWorK::Utils::Tags; -use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/; +use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile/; my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0; +print "using utf8mb4 \n\n" if $ENABLE_UTF8MB4; + +# The DBD::MariaDB driver should not get the +# mysql_enable_utf8mb4 or mysql_enable_utf8 settings, +# but DBD::mysql should. +my %utf8_parameters = (); + +if ( $ce->{database_driver} =~ /^mysql$/i ) { + # Only needed for older DBI:mysql driver + if ( $ENABLE_UTF8MB4 ) { + $utf8_parameters{mysql_enable_utf8mb4} = 1; + } else { + $utf8_parameters{mysql_enable_utf8} = 1; + } +} + my $dbh = DBI->connect( - $ce->{problemLibrary_db}->{dbsource}, - $ce->{problemLibrary_db}->{user}, - $ce->{problemLibrary_db}->{passwd}, - { - PrintError => 0, - RaiseError => 1, - }, + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + PrintError => 0, + RaiseError => 1, + %utf8_parameters, + }, ); +my $character_set=''; +$character_set=($ENABLE_UTF8MB4)?"utf8mb4":"utf8"; +$dbh->prepare("SET NAMES '$character_set'")->execute(); + +print "using character set $character_set to build OPL database\n"; + my $libraryRoot = $ce->{problemLibrary}->{$myLib}->{root}; $libraryRoot =~ s|/+$||; +print "using libraryRoot $libraryRoot\n"; +print "WEBWORK_ROOT $ENV{WEBWORK_ROOT}\n"; my $libraryVersion = $ce->{problemLibrary}->{$myLib}->{version}; my $db_storage_engine = $ce->{problemLibrary_db}->{storage_engine}; @@ -204,13 +228,13 @@ foreach $tmp1 ( @special_tables ) { @create_tables = ( ["dbsubject",$tables{dbsubject}, ' DBsubject_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, KEY DBsubject (name), PRIMARY KEY (DBsubject_id) '], ["dbchapter",$tables{dbchapter}, ' DBchapter_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, DBsubject_id int(15) DEFAULT 0 NOT NULL, KEY DBchapter (name), KEY (DBsubject_id), @@ -218,7 +242,7 @@ foreach $tmp1 ( @special_tables ) { '], ["dbsection",$tables{dbsection}, ' DBsection_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, DBchapter_id int(15) DEFAULT 0 NOT NULL, KEY DBsection (name), KEY (DBchapter_id), @@ -235,7 +259,7 @@ foreach $tmp1 ( @special_tables ) { '], ["path",$tables{path}, ' path_id int(15) NOT NULL auto_increment, - path varchar(255) NOT NULL, + path varchar(245) NOT NULL, machine varchar(255), user varchar(255), KEY (path), @@ -257,7 +281,7 @@ foreach $tmp1 ( @special_tables ) { '], ["keyword",$tables{keyword}, ' keyword_id int(15) NOT NULL auto_increment, - keyword varchar(256) NOT NULL, + keyword varchar(245) NOT NULL, KEY (keyword), PRIMARY KEY (keyword_id) '], @@ -281,7 +305,7 @@ foreach $tmp1 ( @special_tables ) { chapter_id int (15) NOT NULL auto_increment, textbook_id int (15), number int(3), - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, page int(4), KEY (textbook_id, name), KEY (number), @@ -291,7 +315,7 @@ foreach $tmp1 ( @special_tables ) { section_id int(15) NOT NULL auto_increment, chapter_id int (15), number int(3), - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, page int(4), KEY (chapter_id, name), KEY (number), @@ -308,7 +332,7 @@ foreach $tmp1 ( @special_tables ) { '], ["morelt",$tables{morelt}, ' morelt_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, DBsection_id int(15), leader int(15), # pgfile_id of the MLT leader KEY (name), @@ -368,7 +392,7 @@ for my $tableinfo (@create_tables) { my $query = "DROP TABLE IF EXISTS `$tabname`"; $dbh->do($query); - $query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine"; + $query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine CHARACTER SET $character_set"; $dbh->do($query); if($lib eq 'OPL') { $old_tabname = $tabname; @@ -452,7 +476,7 @@ my ($name,$pgfile,$pgpath); #### First read in textbook information -if(open(IN, "$libraryRoot/Textbooks")) { +if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Textbooks")) { print "Reading in textbook data from Textbooks in the library $libraryRoot.\n"; my %textinfo = ( TitleText => '', EditionText =>'', AuthorText=>''); my $bookid = undef; @@ -551,10 +575,10 @@ my $tagtaxo = []; my ($chaplist, $seclist) = ([],[]); my $canopenfile = 0; -if(open(IN, "$libraryRoot/Taxonomy2")) { +if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy2")) { print "Reading in OPL taxonomy from Taxonomy2 in the library $libraryRoot.\n"; $canopenfile = 1; -} elsif(open(IN, "$libraryRoot/Taxonomy")) { +} elsif(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy")) { print "Reading in OPL taxonomy from Taxonomy in the library $libraryRoot.\n"; $canopenfile = 1; } else { @@ -618,6 +642,8 @@ if($canopenfile) { } #### End of taxonomy/taxonomy2 +use JSON; + #### Save the official taxonomy in json format my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; @@ -626,9 +652,9 @@ my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; my $taxo_file = $ce->{problemLibrary}->{$myLib}->{taxo}; my $file = "$webwork_htdocs/DATA/${taxo_file}"; -open(OUTF, ">$file") or die "Cannot open $file"; -print OUTF to_json($tagtaxo,{pretty=>1}) or die "Cannot write to $file"; -close(OUTF); +#my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json"; + +writeJSONtoFile($tagtaxo,$file); print "Saved taxonomy to $file.\n"; #### Now deal with cross-listed sections @@ -691,11 +717,6 @@ sub pgfiles { my @textproblems = (-1); #print "\n$name"; - if ($name =~ /swf$/) { # Found a flash applet - my $applet_file = basename($name); - symlink($name,$ce->{webworkDirs}->{htdocs}."/applets/".$applet_file); - } - if ($name =~ /\.pg$/) { $pgfile = basename($name); $pgpath = dirname($name); @@ -1079,6 +1100,10 @@ build_library_directory_tree($myLib,$ce); build_library_subject_tree($myLib,$ce,$dbh); build_library_textbook_tree($myLib,$ce,$dbh); +# Note: this used to build some JSON versions of the textbooks, subjects and directory trees +# that could be used in the library browswer. It's functionality is now +# in the updateOPLextras.pl script. + $dbh->disconnect; if ( $myLib eq "OPL" ) { @@ -1088,8 +1113,10 @@ if ( $myLib eq "OPL" ) { $ce->{problemLibrary}{$myLib}{showLibraryGlobalStats}) { print "\nUpdating Library Statistics.\n"; do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics'; + + print "\nLoading global statistics (if possible).\n"; + do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl'; } } - print "\nDone.\n"; diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm index 62ceb25650..d7a3332760 100644 --- a/bin/OPLUtils.pm +++ b/bin/OPLUtils.pm @@ -3,33 +3,28 @@ package OPLUtils; use base qw(Exporter); -# This file contains the subroutines that build JSON files from the database to help speed up the client side. +# This file contains the subroutines that build JSON files from the database to help speed up the client side. # # The following files are created: # 1. $webwork_htdocs/DATA/LIBNAME-library-directory-tree.json (the directory structure of the library) # 2. $webwork_htdocs/DATA/LIBNAME-library-subject-tree.json (the subject/chapter/section struture of the library) -# 3. $webwork_htdocs/DATA/LIBNAME-textbook-tree.json (textbook data) +# 3. $webwork_htdocs/DATA/LIBNAME-textbook-tree.json (the subject/chapter/section struture of the library) # The filenames are set via hash values for the current library in conf/defaults.config and/or conf/localOverrides.conf # This is used to create the file LIBNAME-library-directory-tree.json which can be used to load in -# directory information for the OPL. It writes the file as a JSON of directories to be easily loaded. +# directory information for the OPL. +# The above JSON files can be used to load and more quickly lookup OPL information use strict; use warnings; use File::Find::Rule; use File::Basename; use open qw/:std :utf8/; -# use Cwd; -# use DBI; use JSON; our @EXPORT = (); -our @EXPORT_OK = qw(build_library_directory_tree build_library_subject_tree build_library_textbook_tree); - -# what library are we handling, now is first argument of the function -my $myLib = shift; - +our @EXPORT_OK = qw(build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile); ### Data for creating the database tables @@ -73,11 +68,9 @@ my %NPLtables = ( sub build_library_directory_tree { # what library are we handling, now is first argument of the function - my $myLib = shift; + my ($myLib,$ce,$verbose) = @_; - my $ce = shift; - - print "Creating the Directory Tree\n"; + print "Creating the Directory Tree\n" if $verbose; my $libraryRoot = $ce->{problemLibrary}->{$myLib}->{root}; $libraryRoot =~ s|/+$||; @@ -96,22 +89,9 @@ sub build_library_directory_tree { my $file = "$webwork_htdocs/DATA/${jsonFile}"; - # use a variable for the file handle - my $OUTFILE; - - # use the three arguments version of open - # and check for errors - open $OUTFILE, '>', $file or die "Cannot open $file"; - - # you can check for errors (e.g., if after opening the disk gets full) - print { $OUTFILE } to_json(\@dirArray) or die "Cannot write to $file"; - - # check for errors - close $OUTFILE or die "Cannot close $file"; - - - print "Wrote Library Directory Tree to $file\n"; + writeJSONtoFile(\@dirArray,$file); + print "Wrote Library Directory Tree to $file\n" if $verbose; } sub buildTree { @@ -135,10 +115,9 @@ sub buildTree { } else { $b = {}; $b->{name} = $dir; - + my @files = File::Find::Rule->file()->name("*.pg")->in($absoluteDir . "/" . $dir); - - #print $absoluteDir . "/" . $dir . " " . $b->{num_files} . "\n"; + if (scalar(@files)>0){ $b->{num_files} = scalar(@files); push(@branches,$b); @@ -150,20 +129,18 @@ sub buildTree { my @files = File::Find::Rule->file()->name("*.pg")->in($absoluteDir); $branch->{num_files} = scalar(@files); - return $branch; } sub build_library_subject_tree { # what library are we handling, now is first argument of the function - my ($myLib,$ce,$dbh) = @_; + my ($myLib,$ce,$dbh,$verbose) = @_; my $libraryRoot = $ce->{problemLibrary}->{$myLib}->{root}; $libraryRoot =~ s|/+$||; my $libraryVersion = $ce->{problemLibrary}->{$myLib}->{version}; - my %tables = ($libraryVersion eq '2.5')? %OPLtables : %NPLtables; # Modify table names for per-library tables @@ -181,42 +158,37 @@ sub build_library_subject_tree { $tables{$tmp1} = $tblName; } - my $selectClause = "select subj.name, ch.name, sect.name, path.path,pg.filename from `$tables{dbsection}` AS sect " - ."JOIN `$tables{dbchapter}` AS ch ON ch.DBchapter_id = sect.DBchapter_id " - ."JOIN `$tables{dbsubject}` AS subj ON subj.DBsubject_id = ch.DBsubject_id " - ."JOIN `$tables{pgfile}` AS pg ON sect.DBsection_id = pg.DBsection_id " - ."JOIN `$tables{path}` AS path ON pg.path_id = path.path_id "; - - my $tree; # the library subject tree will be stored as arrays of objects. - - my $results = $dbh->selectall_arrayref("select subj.name from `$tables{dbsubject}` AS subj"); + # query the database for all of the subject names + my $cmd = qq/select name from $tables{dbsubject};/; + my @subject_names = map { $_->[0]} $dbh->selectall_array($cmd); - my @subject_names = map { $_->[0]} @{$results}; + my $tree; # the library subject tree will be stored as arrays of objects. - my $i=0; # counter to print to the screen. + print "Building the subject-tree. There are " . scalar(@subject_names) . " subjects\n" if $verbose; - print "Building the subject-tree. There are " . scalar(@subject_names) . " subjects\n"; + my @subject_tree; # array to store the individual library tree for each subject - my @subject_tree; # array to store the individual library tree for each subject + my $selectClause = ""; for my $subj_name (@subject_names){ - + my $subj = $subj_name; - $subj =~ s/'/\'/g; + $subj =~ s/'/\'/g; # escape any single quotes; + print "subject: $subj_name is being processed.\n" if $verbose; - my $results = $dbh->selectall_arrayref("select ch.name from `$tables{dbsubject}` AS subj JOIN `$tables{dbchapter}` AS ch " - . " ON subj.DBsubject_id = ch.DBsubject_id WHERE subj.name='$subj';"); - my @chapter_names = map {$_->[0]} @{$results}; + my $cmd = qq/SELECT ch.name from $tables{dbchapter} AS ch + JOIN $tables{dbsubject} AS subj ON ch.DBsubject_id=subj.DBsubject_id + WHERE subj.name='$subj';/; + my @chapter_names = map { $_->[0] } $dbh->selectall_array($cmd); + my @chapter_tree; # array to store the individual library tree for each chapter - #print Dumper(\@chapter_names); - for my $ch_name (@chapter_names){ - + my $ch = $ch_name; - $ch =~ s/'/\'/g; + $ch =~ s/'/\'/g; # escape any single quotes; my $results = $dbh->selectall_arrayref("SELECT sect.name from `$tables{dbsubject}` AS subj " ."JOIN `$tables{dbchapter}` AS ch ON subj.DBsubject_id = ch.DBsubject_id " @@ -226,78 +198,60 @@ sub build_library_subject_tree { my @section_names = map { $_->[0]} @{$results}; my @subfields = (); - - for my $sect_name (@section_names){ - my $section_tree = {}; - $section_tree->{name} = $sect_name; - ## Determine the number of files that falls into each - my $sect = $section_tree->{name}; - $sect =~ s/'/\\'/g; + for my $sect_name (@section_names){ + my $section_tree = {name => $sect_name}; - my $whereClause ="WHERE sect.name='$sect' AND ch.name='$ch' AND subj.name='$subj'"; + ## Determine the number of files that falls into each - my $sth = $dbh->prepare($selectClause.$whereClause); - $sth->execute; - my $numFiles= scalar @{$sth->fetchall_arrayref()}; + my $sect = $sect_name; + $sect =~ s/'/\\'/g; # escape any single quotes - $section_tree->{num_files} = $numFiles; + my $cmd = qq/SELECT COUNT(*) from $tables{dbsection} AS sect + JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id + JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id + JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id + where subj.name = '$subj' AND ch.name='$ch' AND sect.name='$sect';/; + $section_tree->{num_files} = $dbh->selectrow_array($cmd); my $clone = { %{ $section_tree } }; # need to clone it before pushing into the @subfield array. - push(@subfields,$clone); - } + push(@subfields,$clone); + } - my $chapter_tree; - $chapter_tree->{name} = $ch_name; - $chapter_tree->{subfields} = \@subfields; + my $chapter_tree = {name => $ch_name, subfields => \@subfields}; ## determine the number of files in each chapter - my $whereClause ="WHERE subj.name='$subj' AND ch.name='$ch'"; + my $cmd = qq/select COUNT(*) from $tables{dbsection} AS sect + JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id + JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id + JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id + JOIN $tables{path} AS path ON pg.path_id = path.path_id + where ch.name = '$ch' AND subj.name = '$subj_name';/; - - my $sth = $dbh->prepare($selectClause.$whereClause); - $sth->execute; - my $numFiles = scalar @{$sth->fetchall_arrayref()}; - # my $allFiles = $sth->fetchall_arrayref; - $chapter_tree->{num_files} = $numFiles; + $chapter_tree->{num_files} = $dbh->selectrow_array($cmd); my $clone = { %{ $chapter_tree } }; # need to clone it before pushing into the @chapter_tree array. push(@chapter_tree,$clone); - - - } - my $subject_tree; - $subject_tree->{name} = $subj_name; - $subject_tree->{subfields} = \@chapter_tree; + my $subject_tree = {name => $subj_name, subfields => \@chapter_tree}; ## find the number of files on the subject level - my $whereClause ="WHERE subj.name='$subj'"; - - - my $sth = $dbh->prepare($selectClause.$whereClause); - $sth->execute; - my $numFiles = scalar @{$sth->fetchall_arrayref()}; - $subject_tree->{num_files} = $numFiles; - - $i++; - - print sprintf("%3d", $i); + $cmd = qq/select COUNT(*) from $tables{dbsection} AS sect + JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id + JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id + JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id + JOIN $tables{path} AS path ON pg.path_id = path.path_id + where subj.name = '$subj_name';/; - if ($i%10 == 0) { - print "\n"; - } + $subject_tree->{num_files} = $dbh->selectrow_array($cmd); my $clone = { % {$subject_tree}}; push (@subject_tree, $clone); } - - print "\n"; - my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; # Determine the proper json file names to use for THIS library via the @@ -306,26 +260,14 @@ sub build_library_subject_tree { my $file = "$webwork_htdocs/DATA/${jsonFile}"; - # use a variable for the file handle - my $OUTFILE; - - # use the three arguments version of open - # and check for errors - open $OUTFILE, '>', $file or die "Cannot open $file"; + writeJSONtoFile(\@subject_tree,$file); - # you can check for errors (e.g., if after opening the disk gets full) - print { $OUTFILE } to_json(\@subject_tree,{pretty=>1}) or die "Cannot write to $file"; - - # check for errors - close $OUTFILE or die "Cannot close $file"; - - - print "Wrote Library Subject Tree to $file\n"; + print "Wrote Library Subject Tree to $file\n" if $verbose; } sub build_library_textbook_tree { # what library are we handling, now is first argument of the function - my ($myLib,$ce,$dbh) = @_; + my ($myLib,$ce,$dbh,$verbose) = @_; my $libraryRoot = $ce->{problemLibrary}->{$myLib}->{root}; $libraryRoot =~ s|/+$||; @@ -348,28 +290,30 @@ sub build_library_textbook_tree { $tables{$tmp1} = $tblName; } - my $selectClause = "SELECT pg.pgfile_id from `$tables{path}` as path " - ."LEFT JOIN `$tables{pgfile}` AS pg ON pg.path_id=path.path_id " - ."LEFT JOIN `$tables{pgfile_problem}` AS pgprob ON pgprob.pgfile_id=pg.pgfile_id " - ."LEFT JOIN `$tables{problem}` AS prob ON prob.problem_id=pgprob.problem_id " - ."LEFT JOIN `$tables{section}` AS sect ON sect.section_id=prob.section_id " - ."LEFT JOIN `$tables{chapter}` AS ch ON ch.chapter_id=sect.chapter_id " - ."LEFT JOIN `$tables{textbook}` AS text ON text.textbook_id=ch.textbook_id "; + my $selectClause = "SELECT pg.pgfile_id from $tables{path} as path " + ."LEFT JOIN $tables{pgfile} AS pg ON pg.path_id=path.path_id " + ."LEFT JOIN $tables{pgfile_problem} AS pgprob ON pgprob.pgfile_id=pg.pgfile_id " + ."LEFT JOIN $tables{problem} AS prob ON prob.problem_id=pgprob.problem_id " + ."LEFT JOIN $tables{section} AS sect ON sect.section_id=prob.section_id " + ."LEFT JOIN $tables{chapter} AS ch ON ch.chapter_id=sect.chapter_id " + ."LEFT JOIN $tables{textbook} AS text ON text.textbook_id=ch.textbook_id "; my $results = $dbh->selectall_arrayref("select * from `$tables{textbook}` ORDER BY title;"); my @textbooks=map { {textbook_id=>$_->[0],title=>$_->[1],edition=>$_->[2], author=>$_->[3],publisher=>$_->[4],isbn=>$_->[5],pubdate=>$_->[6]}} @{$results}; + my @output = (); + my $i =0; ## index to alert user the length of the build - print "Building the Textbook Library Tree\n"; - print "There are ". $#textbooks ." textbooks to process.\n"; + print "Building the Textbook Library Tree\n" if $verbose; + print "There are ". $#textbooks ." textbooks to process.\n" if $verbose; for my $textbook (@textbooks){ $i++; - printf("%4d",$i); - print("\n") if ($i %10==0); + printf("%4d",$i) if $verbose; + print("\n") if ($i % 10==0 && $verbose); my $results = $dbh->selectall_arrayref("select ch.chapter_id,ch.name,ch.number " . " from `$tables{chapter}` AS ch JOIN `$tables{textbook}` AS text ON ch.textbook_id=text.textbook_id " @@ -377,6 +321,8 @@ sub build_library_textbook_tree { my @chapters=map { {chapter_id=>$_->[0],name=>$_->[1],number=>$_->[2]}} @{$results}; + my @chs = (); + for my $chapter (@chapters){ my $results = $dbh->selectall_arrayref("select sect.section_id,sect.name,sect.number " @@ -391,24 +337,36 @@ sub build_library_textbook_tree { for my $section (@sections){ - my $whereClause ="WHERE sect.section_id='". $section->{section_id} + my $whereClause ="WHERE sect.section_id='". $section->{section_id} ."' AND ch.chapter_id='". $chapter->{chapter_id}."' AND " ."text.textbook_id='".$textbook->{textbook_id}."'"; my $sth = $dbh->prepare($selectClause.$whereClause); $sth->execute; $section->{num_probs}=scalar @{$sth->fetchall_arrayref()}; - } my $whereClause ="WHERE ch.chapter_id='". $chapter->{chapter_id}."' AND " - ."text.textbook_id='".$textbook->{textbook_id}."'"; + ."text.textbook_id='".$textbook->{textbook_id}."'"; my $sth = $dbh->prepare($selectClause.$whereClause); $sth->execute; $chapter->{num_probs}=scalar @{$sth->fetchall_arrayref()}; $chapter->{sections}=\@sections; - + + my @sects = map {{ + name=>$_->{name}, + section_id => $_->{section_id}, + num_files=>$_->{num_probs} + }} @sections; + + push(@chs,{ + name=>$chapter->{name}, + chapter_id => $chapter->{chapter_id}, + num_files=>$chapter->{num_probs}, + subfields=>\@sects + }); + } my $whereClause ="WHERE text.textbook_id='".$textbook->{textbook_id}."'"; @@ -417,6 +375,13 @@ sub build_library_textbook_tree { $textbook->{num_probs}=scalar @{$sth->fetchall_arrayref()}; $textbook->{chapters}=\@chapters; + + push(@output,{ + name=>$textbook->{title}. " - " . $textbook->{author}, + textbook_id => $textbook->{textbook_id}, + subfields=>\@chs, + num_files=>$sth->rows + }); } print "\n"; @@ -429,23 +394,20 @@ sub build_library_textbook_tree { my $file = "$webwork_htdocs/DATA/${jsonFile}"; - # use a variable for the file handle - my $OUTFILE; - - # use the three arguments version of open - # and check for errors - open $OUTFILE, '>', $file or die "Cannot open $file"; + writeJSONtoFile(\@output,$file); - # you can check for errors (e.g., if after opening the disk gets full) - print { $OUTFILE } to_json(\@textbooks,{pretty=>1}) or die "Cannot write to $file"; - - # check for errors - close $OUTFILE or die "Cannot close $file"; + print "\n\nWrote Library Textbook Tree to $file\n" if $verbose; +} - print "Wrote Library Textbook Tree to $file\n"; +# this takes a hash created in the other subroutines and write the result to a file +sub writeJSONtoFile { + my ($data,$filename) = @_; + my $json = JSON->new->utf8->encode($data); + open my $fh, ">", $filename or die "Cannot open $filename"; + print $fh $json; + close $fh; } - 1; diff --git a/bin/OPL_releases/Makefile b/bin/OPL_releases/Makefile new file mode 100644 index 0000000000..8e96915083 --- /dev/null +++ b/bin/OPL_releases/Makefile @@ -0,0 +1,21 @@ +RELEASE_FILENAME=webwork-open-problem-library-METADATA.tar.gz +GITHUB_USERNAME=heiderich +GITHUB_REPO=webwork-open-problem-library +JSON_FILENAME=latest_release.json +RELEASE_TAG=latest_release.tag + +${JSON_FILENAME}: + @echo "Download a JSON containing information on the latest release" + curl -s https://api.github.com/repos/${GITHUB_USERNAME}/${GITHUB_REPO}/releases/latest > ${JSON_FILENAME} + +${RELEASE_TAG}: ${JSON_FILENAME} + jq -r '.tag_name' ${JSON_FILENAME} > ${RELEASE_TAG} + +${RELEASE_FILENAME}: ${JSON_FILENAME} + curl -L `jq -r '[.assets | .[] | select(.name == "${RELEASE_FILENAME}" ) | .browser_download_url][0]' ${JSON_FILENAME}` -o ${RELEASE_FILENAME} + +extract: ${RELEASE_FILENAME} + tar xfz ${RELEASE_FILENAME} + +clean: + rm -f ${RELEASE_FILENAME} ${RELEASE_TAG} ${JSON_FILENAME} diff --git a/bin/OPL_releases/release.sh b/bin/OPL_releases/release.sh new file mode 100755 index 0000000000..24aca8b197 --- /dev/null +++ b/bin/OPL_releases/release.sh @@ -0,0 +1,369 @@ +#!/bin/bash +# +# Shell script to make releases of webwork problem libraries, heavily inspired +# by https://github.com/gap-system/ReleaseTools/blob/master/release + +set -e + +###################################################################### +# +# Usage information +# +help() { +cat < directory containing the library [Default: current directory] + --tmpdir to a temporary directory [Default: tmp subdirectory of current directory] + --libraryname library name + +Custom settings + -t, --tag git tag for the release + -r, --repository set GitHub repository (as USERNAME/REPONAME) + --token GitHub access token + +Notes: +* To learn how to create a GitHub access token, please consult + https://help.github.com/articles/creating-an-access-token-for-command-line-use/ +EOF + exit 0 +} + +###################################################################### +# +# Various little helper functions + + +# print notices in green +notice() { + printf '\033[32m%s\033[0m\n' "$*" +} + +# print warnings in yellow +warning() { + printf '\033[33mWARNING: %s\033[0m\n' "$*" +} + +# print error in red and exit +error() { + printf '\033[31mERROR: %s\033[0m\n' "$*" + exit 1 +} + +# check for uncommitted changes +verify_git_clean() { + git update-index --refresh + git diff-index --quiet HEAD -- || + error "uncommitted changes detected" +} + +# helper function for parsing GitHub's JSON output. Right now, +# we only extra the value of a single key at a time. This means +# we may end up parsing the same JSON data two times, but that +# doesn't really matter as it is tiny. +json_get_key() { + echo "$response" | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get("'"$1"'",""))' +} + +# On Mac OS X, tar stores extended attributes in ._FOO files inside archives. +# Setting COPYFILE_DISABLE prevents that. See +export COPYFILE_DISABLE=1 + + +###################################################################### +# +# Command line processing +# +CONTENT="METADATA" +LIBRARY_DIR="$PWD" +TMP_DIR="$PWD/tmp" + +FORCE=no +while [ x"$1" != x ]; do + option="$1" ; shift + case "$option" in + -h | --help ) help ;; + + --librarydir ) LIBRARY_DIR="$1"; shift ;; + --libraryname ) LIBRARY_NAME="$1"; shift ;; + --tmpdir ) TMP_DIR="$1"; shift ;; + + -t | --tag ) TAG="$1"; shift ;; + -r | --repository ) REPO="$1"; shift ;; + --token ) TOKEN="$1"; shift ;; + + -f | --force ) FORCE=yes ;; + --no-force ) FORCE=no ;; + + -- ) break ;; + * ) error "unknown option '$option'" ;; + esac +done + + +###################################################################### +# +# Some initial sanity checks +# + +command -v curl >/dev/null 2>&1 || + error "the 'curl' command was not found, please install it" + +command -v git >/dev/null 2>&1 || + error "the 'git' command was not found, please install it" + +command -v python >/dev/null 2>&1 || + error "the 'python' command was not found, please install it" + +cd $LIBRARY_DIR +#verify_git_clean + + +###################################################################### +# +# Determine the basename for the package archives +# +# + +notice "Library path: " $LIBRARY_DIR + +notice "Updating OPL tables" +OPL-update + +notice "Dumping OPL tables" +dump-OPL-tables.pl + +###################################################################### +# +# Fetch GitHub oauth token, used to authenticate the following commands. +# See https://help.github.com/articles/git-automation-with-oauth-tokens/ +# +if [ "x$TOKEN" = x ] ; then + TOKEN=$(git config --get github.token || echo) +fi +if [ "x$TOKEN" = x ] && [ -r ~/.github_shell_token ] ; then + TOKEN=$(cat ~/.github_shell_token) +fi +if [ "x$TOKEN" = x ] ; then + error "could not determine GitHub access token" +fi + + +###################################################################### +# +# Determine GitHub repository and username, and the current branch +# +if [ x"$REPO" = "x" ] ; then + error "could not guess GitHub repository" +fi +notice "Using GitHub repository $REPO" + +GITHUB_USER=$(dirname "$REPO") +notice "Using GitHub username $GITHUB_USER" + + +###################################################################### +# +# Derive API urls +# +API_URL=https://api.github.com/repos/$REPO/releases +UPLOAD_URL=https://uploads.github.com/repos/$REPO/releases + + +###################################################################### +# +# Determine the tag, and validate it +# +#verify_git_clean + +cd $LIBRARY_DIR + +if git show-ref -q "$TAG" ; then + notice "Using git tag $TAG" +else + notice "Creating git tag $TAG" + git tag "$TAG" +fi; + +HEAD_REF=$(git rev-parse --verify HEAD) +TAG_REF=$(git rev-parse --verify "$TAG^{}") + +if [ "x$TAG_REF" != "x$HEAD_REF" ] ; then + error "tag $TAG is not the HEAD commit -- did you tag the right commit?" +fi + + +echo "" + + +###################################################################### +# +# Get fresh (unmodified) copies of the files, and generate some stuff +# + +# Clean any remains of previous export attempts +mkdir -p "$TMP_DIR" +rm -rf "${TMP_DIR:?}/$LIBRARY_NAME"* + +# Set umask to ensure the file permissions in the release +# archives are sane. +umask 0022 + +notice "Preparing content" +mkdir "$TMP_DIR/$LIBRARY_NAME" +cp -r $LIBRARY_DIR/TABLE-DUMP "$TMP_DIR/$LIBRARY_NAME/TABLE-DUMP" +cp -r $LIBRARY_DIR/JSON-SAVED "$TMP_DIR/$LIBRARY_NAME/JSON-SAVED" + +#notice "Removing unnecessary files" +#rm -f .git* .hg* .cvs* +#rm -f .appveyor.yml .codecov.yml .travis.yml + + +###################################################################### +# +# Push commits to GitHub +# + +cd "$LIBRARY_DIR" + +# construct GitHub URL for pusing +REMOTE="https://$GITHUB_USER:$TOKEN@github.com/$REPO" + +# Make sure the branch is on the server +notice "Pushing your branch to GitHub" +notice "Remote: $REMOTE" + +# Make sure the tag is on the server +notice "Pushing your tag to GitHub" +if [ "x$FORCE" = xyes ] ; then + git push --force "$REMOTE" "$TAG" +else + git push "$REMOTE" "$TAG" +fi + +###################################################################### +# +# Create the GitHub release +# + +# check if release already exists +response=$(curl -s -S -X GET "$API_URL/tags/$TAG?access_token=$TOKEN") +MESSAGE=$(json_get_key message) +RELEASE_ID=$(json_get_key id) + +if [ "$MESSAGE" = "Not Found" ] ; then + MESSAGE= # release does not yet exist -> that's how we like it +elif [ x"$RELEASE_ID" != x ] ; then + # release already exists -> error out or delete it + if [ "x$FORCE" = xyes ] ; then + notice "Deleting existing release $TAG from GitHub" + response=$(curl --fail -s -S -X DELETE "$API_URL/$RELEASE_ID?access_token=$TOKEN") + MESSAGE= + else + error "release $TAG already exists on GitHub, aborting (use --force to override this)" + fi +fi + +if [ x"$MESSAGE" != x ] ; then + error "accessing GitHub failed: $MESSAGE" +fi + +# Create the release by sending suitable JSON +DATA=$(cat < "$ARCHIVENAME" + MIMETYPE="application/x-gzip" + ;; + .tar.bz2) + tar cf - "$LIBRARY_NAME" | bzip2 -9c > "$ARCHIVENAME" + MIMETYPE="application/x-bzip2" + ;; + .zip) + zip -r9 --quiet "$ARCHIVENAME" "$LIBRARY_NAME" + MIMETYPE="application/zip" + ;; + *) + warning "unsupported archive format $EXT" + continue + ;; + esac + if [ ! -f "$FULLNAME" ] ; then + error "failed creating $FULLNAME" + fi + notice "Uploading $ARCHIVENAME with mime type $MIMETYPE" + response=$(curl --fail --progress-bar -o "$TMP_DIR/upload.log" \ + -X POST "$UPLOAD_URL/$RELEASE_ID/assets?name=$ARCHIVENAME" \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: $MIMETYPE" \ + --data-binary @"$FULLNAME") +done + + + +exit 0 diff --git a/bin/addcourse b/bin/addcourse index c360808c47..3fbd9a56d8 100755 --- a/bin/addcourse +++ b/bin/addcourse @@ -1,8 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/bin/addcourse,v 1.20 2006/12/09 03:29:56 sh002i Exp $ +# Copyright © 2000-2018 The WeBWorK Project, https://github.com/openwebwork # # This program is free software; you can redistribute it and/or modify it under # the terms of either: (a) the GNU General Public License as published by the @@ -190,6 +189,11 @@ $ce = WeBWorK::CourseEnvironment->new({ courseName => $courseID }); +# Do not attempt to create a course whose name is too long. +if ( length($courseID) > $ce->{maxCourseIdLength} ) { + die "Aborting addcourse: Course ID cannot exceed " . $ce->{maxCourseIdLength} . " characters."; +} + if ($dbLayout) { die "Database layout $dbLayout does not exist in the course environment.", " (It must be defined in defaults.config.)\n" diff --git a/bin/change_user_id b/bin/change_user_id index 3cd0bde055..49fc8b57b2 100755 --- a/bin/change_user_id +++ b/bin/change_user_id @@ -27,7 +27,7 @@ BEGIN { use lib "$ENV{WEBWORK_ROOT}/lib"; use WeBWorK::CourseEnvironment; use WeBWorK::DB; -use WeBWorK::Utils qw(runtime_use readFile cryptPassword); +use WeBWorK::Utils qw(runtime_use readFile); use Data::Dumper; if((scalar(@ARGV) != 3)) { diff --git a/bin/check_database_charsets.pl b/bin/check_database_charsets.pl new file mode 100755 index 0000000000..c6e0f8eeef --- /dev/null +++ b/bin/check_database_charsets.pl @@ -0,0 +1,10 @@ +#!/usr/bin/perl -w + +my $host = $ENV{WEBWORK_DB_HOST}; +my $port = $ENV{WEBWORK_DB_PORT}; +my $database_name = $ENV{WEBWORK_DB_NAME}; +my $database_user = $ENV{WEBWORK_DB_USER}; +my $database_password = $ENV{WEBWORK_DB_PASSWORD}; + + +print `mysql -u $database_user -p$database_password $database_name -h $host -e "SHOW VARIABLES WHERE Variable_name LIKE \'character\_set\_%\' OR Variable_name LIKE \'collation%\' or Variable_name LIKE \'init_connect\' "`; \ No newline at end of file diff --git a/bin/check_latex.tex b/bin/check_latex.tex index b75cb28180..0238df7a53 100644 --- a/bin/check_latex.tex +++ b/bin/check_latex.tex @@ -1,6 +1,6 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % WeBWorK Online Homework Delivery System -% Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +% Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ % $CVSHeader: webwork2/bin/check_latex.tex,v 1.2 2004/10/06 21:09:33 gage Exp $ % % This program is free software; you can redistribute it and/or modify it under diff --git a/bin/check_modules.pl b/bin/check_modules.pl index ab69cc9c78..a8307dd00f 100755 --- a/bin/check_modules.pl +++ b/bin/check_modules.pl @@ -1,18 +1,58 @@ #!/usr/bin/env perl # + +################################################################################ +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2021 The WeBWorK Project, https://github.com/openwebwork +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +################################################################################ + +=head1 NAME + +check_modules.pl - check to ensure that all applications and perl modules are installed. + +=head1 SYNOPSIS + +check_modules.pl [options] + + Options: + -a|--apache-version Which apache version to use. Defaults to 2. + -m|--modules Lists the perl modules needed to be installed. + -p|--programs Lists the programs/applications that are needed. + -A|--all checks both programs and modules (Default if -m or -p is not selected) + +=head1 DESCRIPTION + +Lists all needed applications for webwork as well as a perl modules. + +=cut use strict; use warnings; use version; +use Getopt::Long qw(:config bundling); +use Pod::Usage; my @applicationsList = qw( - curl + convert + curl + dvisvgm mkdir mv mysql tar - git + git gzip latex + pdf2svg pdflatex dvipng giftopnm @@ -32,20 +72,20 @@ my @apache2ModulesList = qw( Apache2::Request - Apache2::Cookie Apache2::ServerRec Apache2::ServerUtil ); + + my @modulesList = qw( + Archive::Zip Array::Utils Benchmark Carp CGI + CGI::Cookie Class::Accessor - #Crypt::SSLeay - Dancer - Dancer::Plugin::Database Data::Dump Data::Dumper Data::UUID @@ -56,10 +96,10 @@ DBI Digest::MD5 Digest::SHA - Email::Address - Email::Simple + Email::Address::XS Email::Sender::Simple Email::Sender::Transport::SMTP + Email::Simple Errno Exception::Class File::Copy @@ -76,11 +116,13 @@ HTML::Scrubber HTML::Tagset HTML::Template + HTTP::Async IO::File IO::Socket::SSL Iterator Iterator::Util JSON + JSON::MaybeXS Locale::Maketext::Lexicon Locale::Maketext::Simple LWP::Protocol::https @@ -100,7 +142,6 @@ Scalar::Util SOAP::Lite Socket - SQL::Abstract Statistics::R::IO String::ShellQuote Template @@ -113,6 +154,7 @@ UUID::Tiny XML::Parser XML::Parser::EasyTree + XML::Simple XML::Writer XMLRPC::Lite YAML @@ -124,35 +166,39 @@ 'IO::Socket::SSL' => 2.007 ); -# modules used by disabled code -# RQP::Render (RQP) - -#main +my ($test_programs,$test_modules,$show_help); +my $test_all = 1; +my $apache_version = "2"; -my $apache_version = shift @ARGV; -unless (defined $apache_version and $apache_version =~ /^apache[12]$/) { - warn "invalid apache version specified -- assuming apache2\n"; - warn "usage: $0 { apache1 | apache2 }\n"; - sleep 1; - $apache_version = "apache2"; -} +GetOptions( + 'a|apache-version=s' => \$apache_version, + 'm|modules' => \$test_modules, + 'p|programs' => \$test_programs, + 'A|all' => \$test_all, + 'h|help' => \$show_help, +); +pod2usage(2) if $show_help; -if ($apache_version eq "apache1") { +if ($apache_version eq "1") { push @modulesList, @apache1ModulesList; -} elsif ($apache_version eq "apache2") { +} elsif ($apache_version eq "2") { push @modulesList, @apache2ModulesList; } my @PATH = split(/:/, $ENV{PATH}); -check_apps(@applicationsList); -check_modules(@modulesList); +if ($test_all or $test_programs) { + check_apps(@applicationsList); +} + +if ($test_all or $test_modules) { + check_modules(@modulesList); +} sub check_apps { my @applicationsList = @_; print "\nChecking your \$PATH for executables required by WeBWorK...\n"; -# print "\$PATH=", shift @PATH, "\n"; # this throws away the first item -- usually /bin - print "\$PATH="; + print "\$PATH="; print join ("\n", map(" $_", @PATH)), "\n\n"; foreach my $app (@applicationsList) { @@ -201,4 +247,29 @@ sub check_modules { print " $module found and loaded\n"; } } + checkSQLabstract(); +} + +## this is specialized code to check for either SQL::Abstract or SQL::Abstract::Classic + +sub checkSQLabstract { + print "\n checking for SQL::Abstract\n\n"; + eval "use SQL::Abstract"; + my $sql_abstract = not($@); + my $sql_abstract_version = $SQL::Abstract::VERSION if $sql_abstract; + + eval "use SQL::Abstract::Classic"; + my $sql_abstract_classic = not($@); + + if($sql_abstract_classic) { + print qq/ You have SQL::Abstract::Classic installed. This package will be used if either + the installed version of SQL::Abstract is version > 1.87 or if that package is not installed.\n/; + } elsif ($sql_abstract && $sql_abstract_version <= 1.87) { + print "You have version $sql_abstract_version of SQL::Abstract installed. This will be used\n"; + } else { + print qq/You need either SQL::Abstract version <= 1.87 or need to install SQL::Abstract::Classic. + If you are using cpan or cpanm, it is recommended to install SQL::Abstract::Classic.\n/; + } } + +1; diff --git a/bin/delcourse b/bin/delcourse index eb49d0c2c3..8dbb119a60 100755 --- a/bin/delcourse +++ b/bin/delcourse @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/delcourse,v 1.4 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under @@ -96,7 +96,7 @@ use lib "$ENV{PG_ROOT}/lib"; use WeBWorK::CourseEnvironment; use WeBWorK::DB; -use WeBWorK::Utils qw(runtime_use readFile cryptPassword); +use WeBWorK::Utils qw(runtime_use readFile); use WeBWorK::Utils::CourseManagement qw(addCourse deleteCourse listCourses); sub usage { diff --git a/bin/dev_scripts/PODParser.pm b/bin/dev_scripts/PODParser.pm new file mode 100644 index 0000000000..bddb2286af --- /dev/null +++ b/bin/dev_scripts/PODParser.pm @@ -0,0 +1,93 @@ +#!/usr/bin/perl +################################################################################ +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2021 The WeBWorK Project, https://github.com/openwebwork +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +################################################################################ + +use strict; +use warnings; + +use Pod::Simple::XHTML; + +package PODParser; +our @ISA = qw(Pod::Simple::XHTML); + +use File::Find; + +sub new { + my $invocant = shift; + my $class = ref $invocant || $invocant; + my $self = $class->SUPER::new(@_); + $self->perldoc_url_prefix("https://metacpan.org/pod/"); + $self->index(1); + $self->backlink(1); + return bless $self, $class; +} + +# Attempt to resolve links to local files. If a local file is not found, then +# let Pod::Simple::XHTML resolve to a cpan link. +sub resolve_pod_page_link { + my ($self, $to, $section) = @_; + + # This ignores $to if $section is set. It would probably be better + # to check for "$to/$section" if both are set. + $self->{pod_search} = $section // $to; + find({ wanted => $self->pod_wanted }, $self->{source_root}); + undef $self->{pod_search}; + + if ($self->{pod_found}) { + my $pod_url = $self->{pod_found} =~ s/^$self->{source_root}/$self->{base_url}/r; + undef $self->{pod_found}; + print "Resolved local pod link $to" . ($section ? "/$section" : "") . " to $pod_url\n" if $self->{verbose} > 2; + return $self->encode_entities($pod_url); + } + + print "Using cpan pod link for $to" . ($section ? "/$section" : "") . "\n" if $self->{verbose} > 2; + return $self->SUPER::resolve_pod_page_link($to, $section); +} + +# This takes the first file found that matches. +sub pod_wanted { + my $self = shift; + return sub { + my $filename = $_; + my $path = $File::Find::name; + my $dir = $File::Find::dir; + + $File::Find::prune = 1, return if ($self->{pod_found}); + + if (-d $path && $filename =~ /^(\.git|\.github|t|htdocs)$/) { + $File::Find::prune = 1; + return; + } + + if ($filename eq $self->{pod_search}) { + $self->{pod_found} = $path =~ s/\.(pm|pl)$/.html/r; + $File::Find::prune = 1; + return; + } + }; +} + +# Don't output anything for a malformed =head[1-4] with no header text. +# This is done in pg/lib/Matrix.pm, for example. +sub _end_head { + my $self = shift; + if ($self->{scratch}) { + return $self->SUPER::_end_head(@_); + } + delete $self->{in_head}; + return ""; +} + +1; diff --git a/bin/dev_scripts/PODtoHTML.pm b/bin/dev_scripts/PODtoHTML.pm new file mode 100644 index 0000000000..e6dee7e6c2 --- /dev/null +++ b/bin/dev_scripts/PODtoHTML.pm @@ -0,0 +1,247 @@ +#!/usr/bin/perl +################################################################################ +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2021 The WeBWorK Project, https://github.com/openwebwork +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +################################################################################ + +use strict; +use warnings; + +package PODtoHTML; + +use File::Find qw(find); +use File::Path qw(make_path); +use IO::File; +use Pod::Find qw(contains_pod); +use POSIX qw(strftime); +use PODParser; + +our @sections = ( + '/' => "(root)", + bin => "Scripts", + conf => "Config Files", + doc => "Documentation", + lib => "Libraries", + macros => "Macros", + clients => "Clients" +); + +sub new { + my ($invocant, %o) = @_; + my $class = ref $invocant || $invocant; + + my @section_list = ref($o{sections}) eq 'ARRAY' ? @{$o{sections}} : @sections; + my $section_hash = {@section_list}; + my $section_order = [ map { $section_list[2 * $_] } 0 .. $#section_list / 2 ]; + delete $o{sections}; + + my $self = { + %o, + idx => {}, + section_hash => $section_hash, + section_order => $section_order, + }; + return bless $self, $class; +} + +sub convert_pods { + my $self = shift; + my $source_root = $self->{source_root}; + my $dest_root = $self->{dest_root}; + + find({ wanted => $self->gen_pod_wanted, no_chdir => 1 }, $source_root); + $self->write_index("$dest_root/index.html"); +} + +sub gen_pod_wanted { + my $self = shift; + return sub { + my $path = $File::Find::name; + my $dir = $File::Find::dir; + my ($name) = $path =~ m|^$dir(?:/(.*))?$|; + $name = '' unless defined $name; + + if ($name =~ /^\./) { + $File::Find::prune = 1; + return; + } + unless (-f $path or -d $path) { + $File::Find::prune = 1; + return; + } + if (-d _ and $name =~ /^(\.git|\.github|t|htdocs)$/) { + $File::Find::prune = 1; + return; + } + + return if -d _; + return unless contains_pod($path); + + print "Processing file: $path\n" if $self->{verbose} > 1; + + $self->process_pod($path); + }; +} + +sub process_pod { + my ($self, $pod_path) = @_; + + my $pod_name; + + my ($subdir, $filename) = $pod_path =~ m|^$self->{source_root}/(?:(.*)/)?(.*)$|; + + my ($subdir_first, $subdir_rest) = ('', ''); + + if (defined $subdir) { + if ($subdir =~ m|/|) { + ($subdir_first, $subdir_rest) = $subdir =~ m|^([^/]*)/(.*)|; + } else { + $subdir_first = $subdir; + } + } + + $pod_name = (defined $subdir_rest ? "$subdir_rest/" : "") . $filename; + if ($filename =~ /\.pl$/) { + $filename =~ s/\.pl$/.html/; + } elsif ($filename =~ /\.pod$/) { + $pod_name =~ s/\.pod$//; + $filename =~ s/\.pod$/.html/; + } elsif ($filename =~ /\.pm$/) { + $pod_name =~ s/\.pm$//; + $pod_name =~ s|/+|::|g; + $filename =~ s/\.pm$/.html/; + } elsif ($filename !~ /\.html$/) { + $filename .= ".html"; + } + + $pod_name =~ s/^(\/|::)//; + + my $html_dir = $self->{dest_root} . (defined $subdir ? "/$subdir" : ""); + my $html_path = "$html_dir/$filename"; + my $html_rel_path = defined $subdir ? "$subdir/$filename" : $filename; + + $self->update_index($subdir, $html_rel_path, $pod_name); + make_path($html_dir); + my $html = $self->do_pod2html( + pod_path => $pod_path, + pod_name => $pod_name + ); + my $fh = new IO::File($html_path, '>') + or die "Failed to open file '$html_path' for writing: $!\n"; + print $fh $html; +} + +sub update_index { + my ($self, $subdir, $html_rel_path, $pod_name) = @_; + $subdir =~ s|/.*$||; + my $idx = $self->{idx}; + my $sections = $self->{section_hash}; + if (exists $sections->{$subdir}) { + push @{$idx->{$subdir}}, [ $html_rel_path, $pod_name ]; + } else { + warn "no section for subdir '$subdir'\n"; + } +} + +sub write_index { + my ($self, $out_path) = @_; + my $idx = $self->{idx}; + my $sections = $self->{section_hash}; + my $section_order = $self->{section_order}; + my $source_root = $self->{source_root}; + $source_root =~ s|^.*/||; + + my $title = "Index for $source_root"; + my $content_start = "
    "; + my $content = ""; + + for my $section (@$section_order) { + next unless defined $idx->{$section}; + my $section_name = $sections->{$section}; + $content_start .= qq{
  • $section_name
    • }; + my @files = sort @{$idx->{$section}}; + $content .= qq{}; + $content .= qq{

    $section_name

      }; + for my $file (sort { $a->[1] cmp $b->[1] } @files) { + my ($path, $name) = @$file; + $content .= qq{
    • $name
      • }; + } + $content .= "
    "; + } + + $content_start .= "
"; + my $date = strftime "%a %b %e %H:%M:%S %Z %Y", localtime; + + my $fh = new IO::File($out_path, '>') or die "Failed to open index '$out_path' for writing: $!\n"; + print $fh ( + get_header($title), + $content_start, + $content, + "

Generated $date

", + get_footer() + ); +} + +sub do_pod2html { + my $self = shift; + my %o = @_; + my $psx = new PODParser; + $psx->{source_root} = $self->{source_root}; + $psx->{verbose} = $self->{verbose}; + $psx->{base_url} = ($self->{dest_url} // "") . "/" . (($self->{source_root} // "") =~ s|^.*/||r); + $psx->output_string(\my $html); + $psx->html_header(get_header($o{pod_name})); + $psx->html_footer(get_footer()); + $psx->parse_file($o{pod_path}); + return $html; +} + +sub get_header { + my $title = shift; + return < + + + + +$title + + +

$title

+ +
+
+EOF +} + +sub get_footer { + return <<'EOF'; +
+
+
+
Site Navigation
+ +
+ + +EOF +} + +1; + + diff --git a/bin/dev_scripts/generate-ww-pg-pod.pl b/bin/dev_scripts/generate-ww-pg-pod.pl new file mode 100755 index 0000000000..1b864cdc04 --- /dev/null +++ b/bin/dev_scripts/generate-ww-pg-pod.pl @@ -0,0 +1,129 @@ +#!/usr/bin/perl +################################################################################ +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2021 The WeBWorK Project, https://github.com/openwebwork +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +################################################################################ + +=head1 NAME + +generate-ww-pg-pod.pl - Convert WeBWorK and PG POD into HTML form. + +=head1 SYNOPSIS + +generate-ww-pg-pod.pl [options] + + Options: + -w|--webwork-root Directory containing a git clone of webwork2. + If this option is not set, then the environment + variable $WEBWORK_ROOT will be used if it is set. + -p|--pg-root Directory containing a git clone of pg. + If this option is not set, then the environment + variable $PG_ROOT will be used if it is set. + -o|--output-dir Directory to save the output files to. (required) + -b|--base-url Base url location used on server. (default: /) + This is needed for internal POD links to work correctly. + -v|--verbose Increase the verbosity of the output. + (Use multiple times for more verbosity.) + +Note that at least one of the options --webwork-root or --pg-root must be provided +(or there is nothing to do!). + +=head1 DESCRIPTION + +Convert WeBWorK and PG POD into HTML form. + +=cut + +use strict; +use warnings; +use Getopt::Long qw(:config bundling); +use Pod::Usage; + +my ($webwork_root, $pg_root, $output_dir, $base_url); +my $verbose = 0; +GetOptions( + 'w|webwork-root=s' => \$webwork_root, + 'p|pg-root=s' => \$pg_root, + 'o|output-dir=s' => \$output_dir, + 'b|base-url=s' => \$base_url, + 'v|verbose+' => \$verbose +); + +$webwork_root = $ENV{WEBWORK_ROOT} if !$webwork_root; +$pg_root = $ENV{PG_ROOT} if !$pg_root; + +pod2usage(2) unless (($webwork_root || $pg_root) && $output_dir); + +$base_url = "/" if !$base_url; + +use IO::File; +use File::Path qw(make_path remove_tree); +use File::Basename qw(dirname); + +use lib dirname(__FILE__); +use PODtoHTML; + +for my $dir ($webwork_root, $pg_root) { + next unless $dir && -d $dir; + print "Reading: $dir\n" if $verbose; + process_dir($dir); +} + +my $index_fh = new IO::File("$output_dir/index.html", '>') + or die "failed to open '$output_dir/index.html' for writing: $!\n"; +write_index($index_fh); + +sub process_dir { + my $source_dir = shift; + return unless $source_dir =~ /\/webwork2$/ || $source_dir =~ /\/pg$/; + + my $dest_dir = $source_dir; + $dest_dir =~ s/^$webwork_root/$output_dir\/webwork2/ if ($source_dir =~ /\/webwork2$/); + $dest_dir =~ s/^$pg_root/$output_dir\/pg/ if ($source_dir =~ /\/pg$/); + + remove_tree($dest_dir); + make_path($dest_dir); + + my $htmldocs = new PODtoHTML( + source_root => $source_dir, + dest_root => $dest_dir, + dest_url => $base_url, + verbose => $verbose + ); + $htmldocs->convert_pods; +} + +sub write_index { + my $fh = shift; + print $fh < + + + + +WeBWorK/PG POD + + +

WeBWorK/PG POD

+

(Plain Old Documentation)

+
+
    +EOF + + print $fh q{
  • PG
    • } if $pg_root; + print $fh q{
  • WeBWorK
    • } if $webwork_root; + + print $fh "
"; +} + +1; diff --git a/bin/dump-OPL-tables.pl b/bin/dump-OPL-tables.pl new file mode 100755 index 0000000000..997be3d8c4 --- /dev/null +++ b/bin/dump-OPL-tables.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script dumps the OPL library tables to a dump file. +use strict; + +# Get the necessary packages, including adding webwork to our path. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use String::ShellQuote; +use DBI; + +# get course environment and configured OPL path + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +my $configured_OPL_path = $ce->{problemLibrary}{root}; + + +# Drop the "OpenProblemLibrary" from the end of the path + +$configured_OPL_path =~ s+OpenProblemLibrary++; + +# Check that it exists + +if ( -d "$configured_OPL_path" ) { + print "OPL path seems to be $configured_OPL_path\n"; +} else { + print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n"; + exit; +} + +# Set TABLE-DUMP path and make directory if necessary + +my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP"; +if ( ! -d "$prepared_OPL_tables_dir" ) { + `mkdir -p $prepared_OPL_tables_dir`; +} + +# Set dump file name + +my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql"; + +# Get DB connection settings + +my $db = $ce->{database_name}; +my $host = $ce->{database_host}; +my $port = $ce->{database_port}; +my $dbuser = $ce->{database_username}; +my $dbpass = $ce->{database_password}; + +$dbuser = shell_quote($dbuser); +$db = shell_quote($db); + +$ENV{'MYSQL_PWD'}=$dbpass; + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + +# Get mysqldump_command + +my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; + +# The tables to dump are: + +my $OPL_tables_to_dump = "OPL_DBsubject OPL_DBchapter OPL_DBsection OPL_author OPL_path OPL_pgfile OPL_keyword OPL_pgfile_keyword OPL_textbook OPL_chapter OPL_section OPL_problem OPL_morelt OPL_pgfile_problem"; + +# Tables NOT dumped: +# OPL_problem_user - is created by bin/update-OPL-statistics and need not be archived +# OPL_global_statistics - loaded from a special file provide by the OPL +# OPL_local_statistics - locally generated + +print "Dumping OPL tables\n"; + +# Conditionally add --column-statistics=0 as MariaDB databases do not support it +# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109 +# https://github.com/drush-ops/drush/issues/4410 + +my $column_statistics_off = ""; +my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`; +if ( $test_for_column_statistics ) { + $column_statistics_off = " --column-statistics=0 "; +} + +`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $column_statistics_off $db $OPL_tables_to_dump > $prepared_OPL_tables_file`; + +print "OPL database dump created: $prepared_OPL_tables_file\n"; + +1; diff --git a/bin/dump_past_answers b/bin/dump_past_answers index 6b45ae45a5..ace64481ee 100755 --- a/bin/dump_past_answers +++ b/bin/dump_past_answers @@ -2,7 +2,7 @@ ############################################################################## # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/fix_copyright.sh b/bin/fix_copyright.sh new file mode 100755 index 0000000000..a44a16034e --- /dev/null +++ b/bin/fix_copyright.sh @@ -0,0 +1,14 @@ +#!/bin/sh +sed -i .bak '/Copyright/c\ +# Copyright © 2000-2019. The WeBWorK Project. https://github.com/openwebwork/webwork2\ +' $1 + + + +#obtained by trial and error after much toil -- mostly error. +# this version works on a mac +# the space after -i might need to be removed for linux. +# produces $1.bak file + +# use with the find command: +# find . -name course.conf -exec /opt/webwork/webwork2/bin/fix_copyright.sh {} ';' \ No newline at end of file diff --git a/bin/importClassList.pl b/bin/importClassList.pl new file mode 100755 index 0000000000..8fc9ae4eaa --- /dev/null +++ b/bin/importClassList.pl @@ -0,0 +1,153 @@ +#!/usr/bin/env perl +################################################################################ +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +################################################################################ + +BEGIN { + die "WEBWORK_ROOT not found in environment.\n" + unless exists $ENV{WEBWORK_ROOT}; + my $webwork_dir = $ENV{WEBWORK_ROOT}; + print "importClassList.pl: WeBWorK root directory set to $webwork_dir\n"; + + # link to WeBWorK code libraries + eval "use lib '$webwork_dir/lib'"; die $@ if $@; + eval "use WeBWorK::CourseEnvironment"; die $@ if $@; +} + +use WeBWorK::DB qw(check_user_id); +use WeBWorK::File::Classlist; + +#use WeBWorK::Utils qw(readFile readDirectory cryptPassword x); +use WeBWorK::Utils qw(cryptPassword); + +use strict; +use warnings; + +if((scalar(@ARGV) != 2)) { + print "\nSyntax is: importClassList.pl course_id path_to_classlist_file.lst\n\n"; + exit(); +} + +my $courseID = shift; + +my $fileName = shift; +die "Not able to read from file $fileName : does it exist? is it readable?" unless ( -r "$fileName" ); + +my $ce = WeBWorK::CourseEnvironment->new({ + webwork_dir => $ENV{WEBWORK_ROOT}, + courseName => $courseID +}); + +my $db = new WeBWorK::DB($ce->{dbLayout}); + +my $createNew = 1; # Always set to true, so add new users +my $replaceExisting = "none"; # Always set to "none" so no existing accounts are changed +my @replaceList =(); # Empty list +my (@replaced, @added, @skipped); + +# This was copied with MINOR changes from lib/WeBWorK/ContentGenerator/Instructor/UserList2.pm +# FIXME REFACTOR this belongs in a utility class so that addcourse can use it! +# (we need a whole suite of higher-level import/export functions somewhere) +sub importUsersFromCSV { + my ($fileName, $createNew, $replaceExisting, @replaceList) = @_; + + my @allUserIDs = grep {$_ !~ /^set_id:/} $db->listUsers; + my %allUserIDs = map { $_ => 1 } @allUserIDs; + + my %replaceOK; + if ($replaceExisting eq "none") { + %replaceOK = (); + } elsif ($replaceExisting eq "listed") { + %replaceOK = map { $_ => 1 } @replaceList; + } elsif ($replaceExisting eq "any") { + %replaceOK = %allUserIDs; + } + + my $default_permission_level = $ce->{default_permission_level}; + + my (@replaced, @added, @skipped); + + # get list of hashrefs representing lines in classlist file + my @classlist = parse_classlist("$fileName"); + + # Default status is enrolled -- fetch abbreviation for enrolled + my $default_status_abbrev = $ce->{statuses}->{Enrolled}->{abbrevs}->[0]; + + foreach my $record (@classlist) { + my %record = %$record; + my $user_id = $record{user_id}; + + print "Saw user_id = $user_id\n"; + + unless (WeBWorK::DB::check_user_id($user_id) ) { # try to catch lines with bad characters + push @skipped, $user_id; + next; + } + + if (exists $allUserIDs{$user_id} and not exists $replaceOK{$user_id}) { + push @skipped, $user_id; + next; + } + + if (not exists $allUserIDs{$user_id} and not $createNew) { + push @skipped, $user_id; + next; + } + + # set default status is status field is "empty" + $record{status} = $default_status_abbrev + unless defined $record{status} and $record{status} ne ""; + + # set password from student ID if password field is "empty" + if (not defined $record{password} or $record{password} eq "") { + if (defined $record{student_id} and $record{student_id} =~ /\S/) { + # crypt the student ID and use that + $record{password} = cryptPassword($record{student_id}); + } else { + # an empty password field in the database disables password login + $record{password} = ""; + } + } + + # set default permission level if permission level is "empty" + $record{permission} = $default_permission_level + unless defined $record{permission} and $record{permission} ne ""; + + my $User = $db->newUser(%record); + my $PermissionLevel = $db->newPermissionLevel(user_id => $user_id, permission => $record{permission}); + my $Password = $db->newPassword(user_id => $user_id, password => $record{password}); + + # DBFIXME use REPLACE + if (exists $allUserIDs{$user_id}) { + $db->putUser($User); + $db->putPermissionLevel($PermissionLevel); + $db->putPassword($Password); + push @replaced, $user_id; + } else { + $db->addUser($User); + $db->addPermissionLevel($PermissionLevel); + $db->addPassword($Password); + push @added, $user_id; + } + } + + + print( "Added:\n\t", join("\n\t", @added), "\n\n" ); + print( "Skipped:\n\t", join("\n\t", @skipped), "\n\n" ); + print( "Replaced:\n\t", join("\n\t", @replaced), "\n\n" ); + +} + +importUsersFromCSV($fileName, $createNew, $replaceExisting, @replaceList); + diff --git a/bin/integrity_check.pl b/bin/integrity_check.pl index 69bf341075..523e513029 100644 --- a/bin/integrity_check.pl +++ b/bin/integrity_check.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_upgrade,v 1.17 2007/08/13 22:59:50 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/load-OPL-global-statistics.pl b/bin/load-OPL-global-statistics.pl new file mode 100755 index 0000000000..8c8d909d08 --- /dev/null +++ b/bin/load-OPL-global-statistics.pl @@ -0,0 +1,77 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script loads the OPL global statistics, which is often done by bin/update-OPL-statistics but may need to be done outside of that setting. +use strict; + +# Get the necessary packages, including adding webwork to our path. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use String::ShellQuote; +use DBI; + +# get course environment and configured OPL path + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +my $dbh = DBI->connect( + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + AutoCommit => 0, + PrintError => 0, + RaiseError => 1, + }, +); + +# check to see if the global statistics file exists and if it does, upload it. + +my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql'; + +if (-e $global_sql_file) { + + my $db = $ce->{database_name}; + my $host = $ce->{database_host}; + my $port = $ce->{database_port}; + my $dbuser = $ce->{database_username}; + my $dbpass = $ce->{database_password}; + + $dbh->do(<commit(); + + $dbuser = shell_quote($dbuser); + $db = shell_quote($db); + + $ENV{'MYSQL_PWD'}=$dbpass; + + my $mysql_command = $ce->{externalPrograms}->{mysql}; + + `$mysql_command --host=$host --port=$port --user=$dbuser $db < $global_sql_file`; + +} + +1; diff --git a/bin/newpassword b/bin/newpassword index 60c40818db..180a8d6f9a 100755 --- a/bin/newpassword +++ b/bin/newpassword @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/newpassword,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under @@ -83,7 +83,7 @@ sub dopasswd { my ($db, $user, $newpass) = @_; my $passwordRecord = eval {$db->getPassword($user)}; die "Can't get password for user |$user| $@" if $@ or not defined($passwordRecord); - my $cryptedPassword = cryptPassword($newP); + my $cryptedPassword = cryptPassword($newpass); $passwordRecord->password($cryptedPassword); eval {$db->putPassword($passwordRecord) }; if ($@) { diff --git a/bin/old_scripts/timing b/bin/old_scripts/timing index b44e004f55..64db42617f 100755 --- a/bin/old_scripts/timing +++ b/bin/old_scripts/timing @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/timing,v 1.5 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwaddindexing b/bin/old_scripts/wwaddindexing index a258d7edfa..7dfa7418a8 100755 --- a/bin/old_scripts/wwaddindexing +++ b/bin/old_scripts/wwaddindexing @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwaddindexing,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwdb_addgw b/bin/old_scripts/wwdb_addgw index 939bbf7c94..3ca4c12c1e 100755 --- a/bin/old_scripts/wwdb_addgw +++ b/bin/old_scripts/wwdb_addgw @@ -1,7 +1,7 @@ #!/usr/bin/perl -w ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_addgw,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwdb_check b/bin/old_scripts/wwdb_check index a624523058..2b4e00207a 100755 --- a/bin/old_scripts/wwdb_check +++ b/bin/old_scripts/wwdb_check @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_check,v 1.7 2006/08/14 17:23:56 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwdb_upgrade b/bin/old_scripts/wwdb_upgrade index 9a4b6ac859..208f33239b 100755 --- a/bin/old_scripts/wwdb_upgrade +++ b/bin/old_scripts/wwdb_upgrade @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_upgrade,v 1.16 2007/07/22 05:24:20 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/pg-append-textbook-tags b/bin/pg-append-textbook-tags index 009ab087c9..952d8ff567 100755 --- a/bin/pg-append-textbook-tags +++ b/bin/pg-append-textbook-tags @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/pg-append-textbook-tags,v 1.1 2007/10/17 16:56:16 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under @@ -43,7 +43,7 @@ sub main { sub add_tags_to_file { my ($new_tags, $file) = @_; - my $pgfile = new IO::File($file, '+<:utf8') or do { + my $pgfile = new IO::File($file, '+ $ENV{WEBWORK_ROOT}, + }); + +my $configured_OPL_path = $ce->{problemLibrary}{root}; + + +# Drop the "OpenProblemLibrary" from the end of the path + +$configured_OPL_path =~ s+OpenProblemLibrary++; + +# Check that it exists + +if ( -d "$configured_OPL_path" ) { + print "OPL path seems to be $configured_OPL_path\n"; +} else { + print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n"; + exit; +} + +# Set TABLE-DUMP path and make directory if necessary + +my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP"; +if ( ! -d "$prepared_OPL_tables_dir" ) { + `mkdir -p $prepared_OPL_tables_dir`; +} + +# Set dump file name + +my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql"; + +# Get DB connection settings + +my $db = $ce->{database_name}; +my $host = $ce->{database_host}; +my $port = $ce->{database_port}; +my $dbuser = $ce->{database_username}; +my $dbpass = $ce->{database_password}; + +$dbuser = shell_quote($dbuser); +$db = shell_quote($db); + +$ENV{'MYSQL_PWD'}=$dbpass; + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + +my $mysql_command = $ce->{externalPrograms}->{mysql}; + +# check to see if the prepared_OPL_tables_file exists and if it does load it in + +if (-e $prepared_OPL_tables_file) { + `$mysql_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db < $prepared_OPL_tables_file`; +} + +1; diff --git a/bin/setfilepermissions b/bin/setfilepermissions index 00854c9d69..2f7a4d9b7c 100755 --- a/bin/setfilepermissions +++ b/bin/setfilepermissions @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/newpassword,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under @@ -50,6 +50,22 @@ chomp($me); #my $serverid = $ce->{server_userID}; my $servergroup = $ce->{server_groupID}; +if ( ! $me ) { + # In a bash shell in a Docker container running webwork, the login name is usually not available. + if ( `/usr/bin/grep -c www-data /etc/passwd`) { + warn "No login name available, using www-data as the user name for the chown calls."; + $me = "www-data"; + } elsif ( `/usr/bin/grep -c apache /etc/passwd`) { + warn "No login name available, using apache as the user name for the chown calls."; + $me = "apache"; + } else { + die "Aborting: No login name available, and cannot fall back to www-data or apache for the chown calls."; + } +} +if ( ! $servergroup ) { + die "Aborting: server_groupID not set - so cannot do the chgrp calls. Check your conf/site.conf file."; +} + my $wwroot = $ENV{WEBWORK_ROOT}; # Course directories diff --git a/bin/update-OPL-statistics b/bin/update-OPL-statistics deleted file mode 100755 index 35efd8cdaa..0000000000 --- a/bin/update-OPL-statistics +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/perl - -############################################################################## -# WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of either: (a) the GNU General Public License as published by the -# Free Software Foundation; either version 2, or (at your option) any later -# version, or (b) the "Artistic License" which comes with this package. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the -# Artistic License for more details. -############################################################################## - -use strict; - -# Get the necessary packages, including adding -# webwork and pg library to our path. -BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') - unless($ENV{WEBWORK_ROOT});} -use lib "$ENV{WEBWORK_ROOT}/lib"; -use WeBWorK::CourseEnvironment; -use String::ShellQuote; - -# hack to set version so that the script runs without warnings in -# earlier versions of WeBWorK, e.g. WW 2.7 - -BEGIN { $main::VERSION = "2.4"; } - -BEGIN{ - my $ce = new WeBWorK::CourseEnvironment({ - webwork_dir => $ENV{WEBWORK_ROOT}, - }); - - my $pg_dir = $ce->{pg_dir}; - eval "use lib '$pg_dir/lib'"; - die $@ if $@; -} - -use DBI; -use WeBWorK::Utils::CourseIntegrityCheck; -use WeBWorK::Utils::CourseManagement qw/listCourses/; - -my $time = time(); - -# get course environment and open up database -my $ce = new WeBWorK::CourseEnvironment({ - webwork_dir => $ENV{WEBWORK_ROOT}, - }); - -my $dbh = DBI->connect( - $ce->{problemLibrary_db}->{dbsource}, - $ce->{problemLibrary_db}->{user}, - $ce->{problemLibrary_db}->{passwd}, - { - AutoCommit => 0, - PrintError => 0, - RaiseError => 1, - }, -); - -# get course list -my @courses = listCourses($ce); - -# create tables. We always redo the statistics table. -$dbh->do(<do(<do(<commit(); - -# for each course get the data from the user problem table into the -# opl user problem table. - -print "Importing statistics for ".scalar(@courses)." courses.\n"; - -my $counter = 0; - -foreach my $courseID (@courses) { - $counter++; - print sprintf("%*d",4,$counter); - if ($counter % 10 == 0) { - print "\n"; - } - - next if $courseID eq 'admin' || $courseID eq 'modelCourse'; - - # we extract the identifying information of the problem, - # the status, attempted flag, number of attempts. - # and the source_file - # we strip of the local in front of the source file - # (assuming that these are mostly the same as their library counterparts - $dbh->do(<commit(); - -# compile desired statistics from opl problem user table. -$dbh->do(<commit(); - -# check to see if the global statistics file exists and if it does, upload it. - -# FIXME - currently hardwired for OPL -my $global_sql_file = $ce->{problemLibrary}{OPL}{root}.'/OPL_global_statistics.sql'; - -if (-e $global_sql_file) { - - my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); - - $host = 'localhost' unless $host; - - $port = 3306 unless $port; - - my $dbuser = $ce->{database_username}; - my $dbpass = $ce->{database_password}; - - - $dbh->do(<commit(); - - # Added NSW - $dbh->disconnect(); - - $dbuser = shell_quote($dbuser); - $dbpass = shell_quote($dbpass); - $db = shell_quote($db); - - my $mysql_command = $ce->{externalPrograms}->{mysql}; - - `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`; - -} else { - $dbh->disconnect(); -} - -1; diff --git a/bin/update-OPL-statistics.pl b/bin/update-OPL-statistics.pl new file mode 100755 index 0000000000..480558118c --- /dev/null +++ b/bin/update-OPL-statistics.pl @@ -0,0 +1,237 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ +# $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +use strict; + +# Get the necessary packages, including adding +# webwork and pg library to our path. +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; +use WeBWorK::CourseEnvironment; +use String::ShellQuote; + +# hack to set version so that the script runs without warnings in +# earlier versions of WeBWorK, e.g. WW 2.7 + +BEGIN { $main::VERSION = "2.4"; } + +BEGIN{ + my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + + my $pg_dir = $ce->{pg_dir}; + eval "use lib '$pg_dir/lib'"; + die $@ if $@; +} + +use DBI; +use WeBWorK::Utils::CourseIntegrityCheck; +use WeBWorK::Utils::CourseManagement qw/listCourses/; + +my $time = time(); + +# get course environment and open up database +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + + +my $dbh = DBI->connect( + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + AutoCommit => 0, + PrintError => 0, + RaiseError => 1, + }, +); + +# get course list +my @courses = listCourses($ce); + +# create tables. We always redo the statistics table. + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + +$dbh->do(<do(<do(<commit(); + +# for each course get the data from the user problem table into the +# opl user problem table. + +print "Importing statistics for ".scalar(@courses)." courses.\n"; + +my $counter = 0; + +foreach my $courseID (@courses) { + $counter++; + print sprintf("%*d",4,$counter); + if ($counter % 10 == 0) { + print "\n"; + } + + next if $courseID eq 'admin' || $courseID eq 'modelCourse'; + + # we extract the identifying information of the problem, + # the status, attempted flag, number of attempts. + # and the source_file + # we strip of the local in front of the source file + # (assuming that these are mostly the same as their library counterparts + $dbh->do(<commit(); + +# compile desired statistics from opl problem user table. +$dbh->do(<commit(); + +## <<<<<<< HEAD:bin/update-OPL-statistics +## # check to see if the global statistics file exists and if it does, upload it. +## +## # FIXME - currently hardwired for OPL +## my $global_sql_file = $ce->{problemLibrary}{OPL}{root}.'/OPL_global_statistics.sql'; +## +## if (-e $global_sql_file) { +## +## my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); +## +## $host = 'localhost' unless $host; +## +## $port = 3306 unless $port; +## +## my $dbuser = $ce->{database_username}; +## my $dbpass = $ce->{database_password}; +## +## +## $dbh->do(<commit(); +## +## # Added NSW +## $dbh->disconnect(); +## +## $dbuser = shell_quote($dbuser); +## $dbpass = shell_quote($dbpass); +## $db = shell_quote($db); +## +## my $mysql_command = $ce->{externalPrograms}->{mysql}; +## +## `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`; +## +## } else { +## $dbh->disconnect(); +## } +## ======= +# We no longer automatically load the global statistics data here. +print( "You may want to run load-OPL-global-statistics.pl to update the global statistics data.\n", + "If this is being run by OPL-update, that will be done automatically.\n"); + +1; diff --git a/bin/updateOPLextras.pl b/bin/updateOPLextras.pl new file mode 100755 index 0000000000..108061f7a5 --- /dev/null +++ b/bin/updateOPLextras.pl @@ -0,0 +1,119 @@ +#!/usr/bin/perl + +=head1 NAME + +updateOPLextras - re-build library trees + +=head1 SYNOPSIS + +updateOPLextras [options] + + Options: + -t --textbooks (rebuild textbook tree) + -s --subjects (rebuild subject tree) + -d --directories (rebuild directory tree) + -a --all (rebuild all trees) + -h --help (display this text) + -v --verbose (turn on verbosity mode) + +=head1 OPTIONS + +=over 8 + +=item B<-t> I<--textbooks> + +Rebuild the textbook tree and write to a JSON file. + +=item B<-s> I<--subjects> + +Rebuild the subject tree and write to a JSON file. + +=item B<-d> I<--directories> + +Rebuild the directory tree and write to a JSON file. + +=item B<-v> I<--verbosity> + +Turn on verbosity mode. + +=back + +=head1 DESCRIPTION + +B will rebuild the specified library trees +from the existing library contents in the database. + +=cut + +use strict; +use warnings; +use DBI; +use Getopt::Long; +use Pod::Usage; +Getopt::Long::Configure ("bundling"); + +my ($textbooks, $directories, $subjects, $verbose, $all); +GetOptions ( + 't|textbooks' => \$textbooks, + 'd|directories' => \$directories, + 's|subjects' => \$subjects, + 'a|all' => \$all, + 'v|verbose' => \$verbose +); +pod2usage(2) unless ($textbooks || $directories || $subjects || $all); + +##### +# +# This script allows to rerun a few scripts related to the OPL but doesn't require +# the entire OPLupdate script to be run. +# +#### + + +BEGIN { + die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT}; +} + +use lib "$ENV{WEBWORK_ROOT}/bin"; +use lib "$ENV{WEBWORK_ROOT}/lib"; +use WeBWorK::CourseEnvironment; +use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/; + +my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0; +print "using utf8mb4 \n\n" if $ENABLE_UTF8MB4; + +# The DBD::MariaDB driver should not get the +# mysql_enable_utf8mb4 or mysql_enable_utf8 settings, +# but DBD::mysql should. +my %utf8_parameters = (); + +if ( $ce->{database_driver} =~ /^mysql$/i ) { + # Only needed for older DBI:mysql driver + if ( $ENABLE_UTF8MB4 ) { + $utf8_parameters{mysql_enable_utf8mb4} = 1; + } else { + $utf8_parameters{mysql_enable_utf8} = 1; + } +} + +my $dbh = DBI->connect( + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + PrintError => 0, + RaiseError => 1, + %utf8_parameters, + }, +); + +build_library_textbook_tree($ce,$dbh,$verbose) if ($all || $textbooks); +build_library_directory_tree($ce,$verbose) if ($all || $directories); +build_library_subject_tree($ce,$dbh,$verbose) if ($all || $subjects); + +1; diff --git a/bin/update_localization_files b/bin/update_localization_files index 5ade104627..572c1d33d2 100755 --- a/bin/update_localization_files +++ b/bin/update_localization_files @@ -18,4 +18,4 @@ echo "Updating $WEBWORK_ROOT/webwork2.pot" xgettext.pl -o webwork2.pot -D $WEBWORK_ROOT/lib -D $PG_ROOT/lib -D $PG_ROOT/macros -find $LOCDIR -name '*.po' -exec bash -c "echo \"Updating {}\"; msgmerge -qU {} webwork2.pot" \; +find $LOCDIR -name '*.po' -exec bash -c "echo \"Updating {}\"; msgmerge -qUN {} webwork2.pot" \; diff --git a/bin/upgrade-database-to-utf8mb4.pl b/bin/upgrade-database-to-utf8mb4.pl new file mode 100755 index 0000000000..2feaca0984 --- /dev/null +++ b/bin/upgrade-database-to-utf8mb4.pl @@ -0,0 +1,310 @@ +#!/usr/bin/perl +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2021 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +=head1 NAME + +upgrade-database-to-utf8mb4.pl -- Upgrade webwork course database tables from +latin1 to utf8mb4. + +=head1 SYNOPSIS + + upgrade-database-to-utf8mb4.pl [options] + + Options: + -c|--course-id [course] Course id to upgrade the database for. + (This option may be given multiple times.) + -a|--all Upgrade the database for all existing courses + including the admin course. + (Preempts the previous option.) + -2|--run-second-pass Run a second pass to change column text types to + be the defaults for webwork. This pass is not run + by default. + -n|--upgrade-non-native Upgrade the non-native tables + (locations, location_addresses, depths) + --no-backup Do not backup the database before making changes + to the database. (Not recommended) + -b|--backup-file [file] Filename for the database backup file. + Default: ./webwork.sql + -v|--verbose Show progress output. + -h|--help Show full help for this script. + +=head1 DESCRIPTION + +Upgrade webwork course database tables from latin1 to utf8mb4. + +This script assumes that you have already properly configured the database to +work with the utf8mb4 character set. See L. + +Also, make sure to upgrade the course via webwork2/admin "Upgrade Courses" +before running this script for the course. + +If you are upgrading a WeBWorK installation from a version prior to version 2.15 +use + + upgrade-database-to-utf8mb4.pl -na + +If you are upgrading a single course that was created with a version of WeBWorK +prior to version 2.15, use + + upgrade-database-to-utf8mb4.pl -c courseId + +If there are errors when running this script, then restore the database using +the backup created by the script (unless you used --no-backup) by running + + mysql -u webworkWrite -p webwork < webwork.sql + +This is where C is the C<$database_username> set in site.conf. +You may need to change C if you used a different name for the +database backup file. You will be prompted to enter the password, which should +be the value of C<$database_password> in site.conf. + +=head1 OPTIONS + +=over + +=item -c|--course-id [course] + +Course id or list of course ids to upgrade the database tables for. Use this +option multiple times to upgrade the database tables for multiple courses at one +time. + +=item -a|--all + +Ignore the previous option and upgrade the database tables for all existing +courses, including the admin course. + +=item -2|--run-second-pass + +On the first pass this script will change the datatypes of all columns that are +different from the datatype defined in the webwork database schema to that in +the schema. Then it will convert the table to use the utf8mb4 charset. When +this conversion is done the database automatically enlarges text datatypes. If +this option is enabled then the second pass will change those back to the +smaller text datatypes as defined in the webwork database schema. + +This second pass is not strictly neccessary. The larger text datatypes should +still work with WeBWorK. + +This pass is not run by default. Note that running this script again will also +perform this second pass, if desired. + +=item --no-backup + +Do not dump the entire webwork database to a backup sql file before performing +changes. It is recommended that you make a backup before any of the other +changes that this script makes. If you have already created a database backup, +then you can use this option to prevent the creation of another backup file. + +=item -b|--backup-file [filename] + +Filename for the database backup file. By default the database is dumped to the +file C<./webwork.sql> in the directory the script is run from. + +=item -v|--verbose + +Make this script show output for the things that it is doing. + +=back + +=cut + +use strict; +use warnings; + +BEGIN { + die "WEBWORK_ROOT not found in environment.\n" unless $ENV{WEBWORK_ROOT}; + die "PG_ROOT not found in environment.\n" unless $ENV{PG_ROOT}; +} + +use Getopt::Long qw(:config bundling); +use Pod::Usage; +use DBI; +use String::ShellQuote; + +my (@courses, $all, $second_pass, $upgrade_non_native, $no_backup, $dump_file, + $verbose, $show_help); +GetOptions( + 'c|course-id=s@' => \@courses, + 'a|all' => \$all, + '2|run-second-pass' => \$second_pass, + 'n|upgrade-non-native' => \$upgrade_non_native, + 'no-backup' => \$no_backup, + 'b|backup-file=s' => \$dump_file, + 'v|verbose' => \$verbose, + 'h|help' => \$show_help +); +pod2usage(-verbose => $show_help ? 2 : 0) if $show_help || !(@courses || $all || $upgrade_non_native); + +use lib "$ENV{WEBWORK_ROOT}/lib"; +use lib "$ENV{PG_ROOT}/lib"; +use WeBWorK::CourseEnvironment; +use WeBWorK::DB; +use WeBWorK::Utils::CourseManagement qw{listCourses}; + +# Load a minimal course environment. +my $ce = new WeBWorK::CourseEnvironment({ webwork_dir => $ENV{WEBWORK_ROOT} }); + +# Get DB connection settings. +my $dbname = $ce->{database_name}; +my $host = $ce->{database_host}; +my $port = $ce->{database_port}; +my $dbuser = shell_quote($ce->{database_username}); +my $dbpass = $ce->{database_password}; + +$ENV{'MYSQL_PWD'} = $dbpass; + +if (!$no_backup) { + # Backup the database + $dump_file = "./webwork.sql" if !$dump_file || $dump_file eq ""; + + my $replace = 'Y'; + if (-e $dump_file) { + $replace = 'n'; + print "The file '$dump_file' already exists. Do you want to overwrite it? [Yn] "; + $replace = <>; + chomp($replace); + print "Overwriting '$dump_file' with new database dump.\n" if $replace eq 'Y'; + print "Not creating new database dump.\n" if $replace ne 'Y'; + + if ($replace ne 'Y') { + my $proceed = 'n'; + print "Do you want to proceed with the script anyway? [Yn] "; + $proceed = <>; + chomp($proceed); + exit if $proceed ne 'Y'; + } + } + + if ($replace eq 'Y') { + print "Backing up database to '$dump_file'.\n" if $verbose; + `$ce->{externalPrograms}{mysqldump} --host=$host --port=$port --user=$dbuser $dbname > $dump_file`; + die("There was an error creating a database backup.\n" . + "Please make a manual backup if needed before proceeding.") if $?; + } +} + +# Get a list of courses. +my @server_courses = listCourses($ce); +@courses = @server_courses if $all; + +my $dbh = DBI->connect( + $ce->{database_dsn}, + $ce->{database_username}, + $ce->{database_password}, + { + PrintError => 0, + RaiseError => 1, + }, +); + +my $db = new WeBWorK::DB($ce->{dbLayouts}{$ce->{dbLayoutName}}); +my @table_types = sort(grep { !$db->{$_}{params}{non_native} } keys %$db); + +sub checkAndUpdateTableColumnTypes { + my $table = shift; + my $table_type = shift; + my $pass = shift // 1; + + print "\tChecking '$table' (pass $pass)\n" if $verbose; + my $schema_field_data = $db->{$table_type}{record}->FIELD_DATA; + for my $field (keys %$schema_field_data) { + my $field_name = $db->{$table_type}{params}{fieldOverride}{$field} || $field; + my @name_type = @{$dbh->selectall_arrayref("SELECT COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS " . + "WHERE TABLE_SCHEMA='$dbname' AND TABLE_NAME='$table' AND COLUMN_NAME='$field_name';")}; + + print("\t\tThe '$field_name' column is missing from '$table'.\n" . + "\t\tYou should upgrade the course via course administration to fix this.\n" . + "\t\tYou may need to run this script again after doing that.\n"), + next if !exists($name_type[0][0]); + + my $data_type = $name_type[0][0]; + next if !$data_type; + $data_type =~ s/\(\d*\)$// if $data_type =~ /^(big|small)?int\(\d*\)$/; + $data_type = lc($data_type); + my $schema_data_type = lc($schema_field_data->{$field}{type} =~ s/ .*$//r); + if ($data_type ne $schema_data_type) { + print "\t\tUpdating data type for column '$field_name' in table '$table'\n" if $verbose; + print "\t\t\t$data_type -> $schema_data_type\n" if $verbose; + eval { + $dbh->do("ALTER TABLE `$table` MODIFY $field_name $schema_field_data->{$field}{type};"); + }; + my $indent = $verbose ? "\t\t" : ""; + die("${indent}Failed to modify '$field_name' in '$table' from '$data_type' to '$schema_data_type.\n" . + "${indent}It is recommended that you restore a database backup. Make note of the\n" . + "${indent}error output below as it may help in diagnosing the problem. Note that\n" . + "${indent}the most common reason for this error is the existence of a data value\n" . + "${indent}in a column that does not fit into the smaller size data type that was\n" . + "${indent}needed for the utf8mb4 change.\n$@") + if $@; + } + } + return 0; +} + +sub checkAndChangeTableCharacterSet { + my $table = shift; + + print "\tChecking character set for '$table'\n" if $verbose; + my @table_data = @{$dbh->selectall_arrayref("SELECT CCSA.character_set_name FROM information_schema.TABLES T, " . + "information_schema.COLLATION_CHARACTER_SET_APPLICABILITY CCSA " . + "WHERE CCSA.collation_name = T.table_collation AND T.table_schema = '$dbname' AND T.table_name = '$table'")}; + for (@table_data) { + if ($_->[0] ne 'utf8mb4') { + print "\t\tConverting '$table' character set to utf8mb4\n" if $verbose; + eval { + $dbh->do("ALTER TABLE `$table` CONVERT TO CHARACTER SET utf8mb4;"); + }; + my $indent = $verbose ? "\t\t" : ""; + die("${indent}Failed to alter charset of '$table' to utf8mb4:\n" . + "${indent}It is recommended that you restore a database backup. Make note of the\n" . + "${indent}error output below as it may help in diagnosing the problem.\n$@") if $@; + } + } + return 0; +} + +my $error = 0; + +for my $course (@courses) { + print("The course '$course' does not exist on the server\n"), next + if !grep($course eq $_, @server_courses); + + print "Checking tables for '$course'\n" if $verbose; + for my $table_type (@table_types) { + my $table = "${course}_$table_type"; + next unless @{$dbh->selectall_arrayref("SELECT * FROM INFORMATION_SCHEMA.TABLES " . + "WHERE TABLE_SCHEMA = '$dbname' AND TABLE_NAME='$table';")}; + + checkAndUpdateTableColumnTypes($table, $table_type); + checkAndChangeTableCharacterSet($table); + checkAndUpdateTableColumnTypes($table, $table_type, 2) if ($second_pass); + } +} + +if ($upgrade_non_native) { + print "Checking native tables\n" if $verbose; + + my @native_tables = grep { $db->{$_}{params}{non_native} } keys %$db; + for my $native_table (@native_tables) { + # Skip the fake tables + next unless @{$dbh->selectall_arrayref("SELECT * FROM INFORMATION_SCHEMA.TABLES " . + "WHERE TABLE_SCHEMA = '$dbname' AND TABLE_NAME='$native_table';")}; + + checkAndUpdateTableColumnTypes($native_table, $native_table); + checkAndChangeTableCharacterSet($native_table); + checkAndUpdateTableColumnTypes($native_table, $native_table, 2) if ($second_pass); + } +} diff --git a/bin/upgrade_admin_db.pl b/bin/upgrade_admin_db.pl index 21b45ee6ed..73cc8166d7 100755 --- a/bin/upgrade_admin_db.pl +++ b/bin/upgrade_admin_db.pl @@ -2,7 +2,7 @@ ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/upload-OPL-statistics b/bin/upload-OPL-statistics deleted file mode 100755 index 6f26e85ea9..0000000000 --- a/bin/upload-OPL-statistics +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/perl - -############################################################################## -# WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of either: (a) the GNU General Public License as published by the -# Free Software Foundation; either version 2, or (at your option) any later -# version, or (b) the "Artistic License" which comes with this package. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the -# Artistic License for more details. -############################################################################## - -# This script dumps the local OPL statistics table and uploads it. - -BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') - unless($ENV{WEBWORK_ROOT});} -use lib "$ENV{WEBWORK_ROOT}/lib"; - -use WeBWorK::CourseEnvironment; - -use Net::Domain; -use String::ShellQuote; - -my $ce = new WeBWorK::CourseEnvironment({ - webwork_dir => $ENV{WEBWORK_ROOT},}); - -my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); - -$host = 'localhost' unless $host; - -$port = 3306 unless $port; - -my $dbuser = $ce->{database_username}; -my $dbpass = $ce->{database_password}; - -my $domainname = Net::Domain::domainname; -my $time = time(); -my $output_file = "$domainname-$time-opl.sql"; - -print "Dumping local OPL statistics\n"; - -$dbuser = shell_quote($dbuser); -$dbpass = shell_quote($dbpass); -$db = shell_quote($db); - -my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; - -`$mysqldump_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db OPL_local_statistics > $output_file`; - -print "Database File Created\n"; - -my $done; -my $desc; -my $input; - -do { - - print "\nWe would appreciate it if you could provide \nsome basic information to help us \nkeep track of the data we receive.\n\n"; - - $desc = "File:\n$output_file\n"; - - print "What university is this data for?\n"; - - $desc .= "University:\n"; - $input = ; - $desc .= $input; - - print "What department is this data for?\n"; - - $desc .= "Department:\n"; - $input = ; - $desc .= $input; - - print "What is your name?\n"; - - $desc .= "Name:\n"; - $input = ; - $desc .= $input; - - print "What is your email address?\n"; - - $desc .= "Email:\n"; - $input = ; - $desc .= $input; - - print "Have you uploaded data from this server before?\n"; - - $desc .= "Uploaded Previously:\n"; - $input = ; - $desc .= $input; - - print "Approximately what years does this data span?\n"; - - $desc .= "Years:\n"; - $input = ; - $desc .= $input; - - print "Approximately how many classes are included?\n"; - - $desc .= "Number of Classes:\n"; - $input = ; - $desc .= $input; - - print "Additional Comments?\n"; - - $desc .= "Additional Comments:\n"; - $input = ; - $desc .= $input; - - print "The data you just entered is below:\n\n"; - - print $desc."\n"; - - my $answered; - - do { - print "Please choose one of the following:\n"; - print "1. Upload Data\n"; - print "2. Reenter above information.\n"; - print "3. Cancel.\n"; - print "[1/2/3]? "; - - $input = ; - chomp $input; - - if ($input eq '3') { - exit; - } elsif ($input eq '2') { - $done = 0; - $answered = 1; - } elsif ($input eq '1') { - $done = 1; - $answered = 1; - } else { - $answered = 0; - } - - } while (!$answered); - - -} while (!$done); - -my $desc_file = "$domainname-$time-desc.txt"; - -open(my $fh, ">", $desc_file) - or die "Couldn't open file for saving description."; - -print $fh $desc; - -close($fh); - -my $tar_file = "$domainname-$time-data.tar.gz"; - -print "Zipping files\n"; -`tar -czf $tar_file $output_file $desc_file`; - -print "Uploading file\n"; - -`echo "put $tar_file" | sftp -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oPort=57281 wwdata\@52.88.32.79`; - - -1; diff --git a/bin/upload-OPL-statistics.pl b/bin/upload-OPL-statistics.pl new file mode 100755 index 0000000000..3cee5efedc --- /dev/null +++ b/bin/upload-OPL-statistics.pl @@ -0,0 +1,179 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ +# $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script dumps the local OPL statistics table and uploads it. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use Net::Domain; +use String::ShellQuote; + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +# Get DB connection settings + +my $db = $ce->{database_name}; +my $host = $ce->{database_host}; +my $port = $ce->{database_port}; +my $dbuser = $ce->{database_username}; +my $dbpass = $ce->{database_password}; + +my $domainname = Net::Domain::domainname; +my $time = time(); +my $output_file = "$domainname-$time-opl.sql"; + +print "Dumping local OPL statistics\n"; + +$dbuser = shell_quote($dbuser); +$db = shell_quote($db); + +$ENV{'MYSQL_PWD'}=$dbpass; + +my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; + +# Conditionally add --column-statistics=0 as MariaDB databases do not support it +# see: https://serverfault.com/questions/912162/mysqldump-throws-unknown-table-column-statistics-in-information-schema-1109 +# https://github.com/drush-ops/drush/issues/4410 + +my $column_statistics_off = ""; +my $test_for_column_statistics = `$mysqldump_command --help | grep 'column-statistics'`; +if ( $test_for_column_statistics ) { + $column_statistics_off = " --column-statistics=0 "; +} + +`$mysqldump_command --host=$host --port=$port --user=$dbuser $column_statistics_off $db OPL_local_statistics > $output_file`; + +print "Database File Created\n"; + +my $done; +my $desc; +my $input; + +do { + + print "\nWe would appreciate it if you could provide \nsome basic information to help us \nkeep track of the data we receive.\n\n"; + + $desc = "File:\n$output_file\n"; + + print "What university is this data for?\n"; + + $desc .= "University:\n"; + $input = ; + $desc .= $input; + + print "What department is this data for?\n"; + + $desc .= "Department:\n"; + $input = ; + $desc .= $input; + + print "What is your name?\n"; + + $desc .= "Name:\n"; + $input = ; + $desc .= $input; + + print "What is your email address?\n"; + + $desc .= "Email:\n"; + $input = ; + $desc .= $input; + + print "Have you uploaded data from this server before?\n"; + + $desc .= "Uploaded Previously:\n"; + $input = ; + $desc .= $input; + + print "Approximately what years does this data span?\n"; + + $desc .= "Years:\n"; + $input = ; + $desc .= $input; + + print "Approximately how many classes are included?\n"; + + $desc .= "Number of Classes:\n"; + $input = ; + $desc .= $input; + + print "Additional Comments?\n"; + + $desc .= "Additional Comments:\n"; + $input = ; + $desc .= $input; + + print "The data you just entered is below:\n\n"; + + print $desc."\n"; + + my $answered; + + do { + print "Please choose one of the following:\n"; + print "1. Upload Data\n"; + print "2. Reenter above information.\n"; + print "3. Cancel.\n"; + print "[1/2/3]? "; + + $input = ; + chomp $input; + + if ($input eq '3') { + exit; + } elsif ($input eq '2') { + $done = 0; + $answered = 1; + } elsif ($input eq '1') { + $done = 1; + $answered = 1; + } else { + $answered = 0; + } + + } while (!$answered); + + +} while (!$done); + +my $desc_file = "$domainname-$time-desc.txt"; + +open(my $fh, ">", $desc_file) + or die "Couldn't open file for saving description."; + +print $fh $desc; + +close($fh); + +my $tar_file = "$domainname-$time-data.tar.gz"; + +print "Zipping files\n"; +`tar -czf $tar_file $output_file $desc_file`; + +print "Uploading file\n"; + +`echo "put $tar_file" | sftp -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oPort=57281 wwdata\@52.88.32.79`; + + +1; diff --git a/bin/ww_purge_old_nonces b/bin/ww_purge_old_nonces index a21f046f2b..0ff43a88b7 100755 --- a/bin/ww_purge_old_nonces +++ b/bin/ww_purge_old_nonces @@ -32,11 +32,12 @@ are more than 10 seconds old. =head1 OPTIONS +=over + =item course Course for which old nonces should be deleted. - =back =cut diff --git a/bin/wwapache2ctl.dist b/bin/wwapache2ctl.dist index 4b263b25f0..4a3e310b7e 100755 --- a/bin/wwapache2ctl.dist +++ b/bin/wwapache2ctl.dist @@ -1,7 +1,7 @@ #!/bin/sh ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwapache2ctl.dist,v 1.3 2007/08/13 22:59:50 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/wwapachectl.dist b/bin/wwapachectl.dist index 3a6b8a6cc6..83363e82d8 100755 --- a/bin/wwapachectl.dist +++ b/bin/wwapachectl.dist @@ -1,7 +1,7 @@ #!/bin/sh ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwapachectl.dist,v 1.17 2006/05/31 18:25:18 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/wwdb b/bin/wwdb index 834fadf86d..44aa435c00 100755 --- a/bin/wwdb +++ b/bin/wwdb @@ -1,7 +1,7 @@ #!/usr/bin/perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/wwsh b/bin/wwsh index 75c3ecfd7f..9219fec6b9 100755 --- a/bin/wwsh +++ b/bin/wwsh @@ -1,7 +1,7 @@ #!/usr/bin/perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwsh,v 1.10 2006/05/31 01:07:25 gage Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/clients/hello_world_apps/webwork_xmlrpc_client.pl b/clients/hello_world_apps/webwork_xmlrpc_client.pl index 8c30320c7c..1d838471ff 100755 --- a/clients/hello_world_apps/webwork_xmlrpc_client.pl +++ b/clients/hello_world_apps/webwork_xmlrpc_client.pl @@ -29,7 +29,7 @@ use constant REQUEST_URI => 'mod_xmlrpc'; use constant TEMPOUTPUTFILE => '/Users/gage/Desktop/renderProblemOutput.html'; -our $XML_URL = 'http://localhost:80'; +our $SITE_URL = 'http://localhost:80'; our $FORM_ACTION_URL = 'http://localhost:80/webwork2/html2xml'; our $XML_PASSWORD = 'xmlwebwork'; our $XML_COURSE = 'gage_course'; @@ -98,7 +98,7 @@ # Build client ############################################ our $xmlrpc_client = new WebworkClient ( - url => $XML_URL, + site_url => $SITE_URL, form_action_url => $FORM_ACTION_URL, displayMode => DISPLAYMODE(), diff --git a/clients/old/check2_problems_in_dir.sh b/clients/old/check2_problems_in_dir.sh deleted file mode 100755 index 0d3f342172..0000000000 --- a/clients/old/check2_problems_in_dir.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -#usage -# check_problems_in_dir.s directory_name - -echo "Checking problems in directory $1" -echo '' > "$WEBWORK_ROOT/DATA/bad_problems.txt" -echo "Results sent to" -echo "$WEBWORK_ROOT/DATA/bad_problems.txt" -time find $1 -name "*.pg" -exec /usr/bin/perl $WEBWORK_ROOT/clients/sendXMLRPC.pl -C -B {} ';' #tail -f /Volumes/WW_test/opt/webwork/webwork2/DATA/bad_problems.txt; -echo "start search" -#tail -f "$WEBWORK_ROOT/DATA/bad_problems.txt" -echo "Time for $1"; -echo "" -echo "" -exit - diff --git a/clients/old/checkProblem.pl b/clients/old/checkProblem.pl deleted file mode 100755 index 3c7f700e43..0000000000 --- a/clients/old/checkProblem.pl +++ /dev/null @@ -1,426 +0,0 @@ -#!/usr/bin/perl -w - -################################################################################ -# WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/clients/renderProblem.pl,v 1.4 2010/05/11 15:44:05 gage Exp $ -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of either: (a) the GNU General Public License as published by the -# Free Software Foundation; either version 2, or (at your option) any later -# version, or (b) the "Artistic License" which comes with this package. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the -# Artistic License for more details. -################################################################################ - -=head1 NAME - -webwork2/clients/checkProblem.pl - -This script will take a file and send it to a WeBWorK daemon webservice -to have it rendered. The result is split into the basic HTML rendering -and evaluation of answers and then passed to a browser for printing. - -The formatting allows the browser presentation to be interactive with the -daemon running the script webwork2/lib/renderViaXMLRPC.pm - -Rembember to configure the local output file and display command !!!!!!!! - -=cut - -use strict; -use warnings; - - -####################################################### -# Find the webwork2 root directory -####################################################### -BEGIN { - die "WEBWORK_ROOT not found in environment. \n - WEBWORK_ROOT can be defined in your .cshrc or .bashrc file\n - It should be set to the webwork2 directory (e.g. /opt/webwork/webwork2)" - unless exists $ENV{WEBWORK_ROOT}; - # Unused variable, but define it twice to avoid an error message. - $WeBWorK::Constants::WEBWORK_DIRECTORY = $ENV{WEBWORK_ROOT}; - $WeBWorK::Constants::PG_DIRECTORY = "$ENV{WEBWORK_ROOT}/../pg/"; - unless (-r $WeBWorK::Constants::WEBWORK_DIRECTORY ) { - die "Cannot read webwork root directory at $WeBWorK::Constants::WEBWORK_DIRECTORY"; - } - unless (-r $WeBWorK::Constants::PG_DIRECTORY ) { - die "Cannot read webwork pg directory at $WeBWorK::Constants::PG_DIRECTORY"; - } -} - -use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib"; -use lib "$WeBWorK::Constants::PG_DIRECTORY/lib"; -use Crypt::SSLeay; # needed for https -use WebworkClient; -use Time::HiRes qw/time/; -use MIME::Base64 qw( encode_base64 decode_base64); - -############################################# -# Configure -############################################# - - -### verbose output when UNIT_TESTS_ON =1; - our $UNIT_TESTS_ON = 0; - -### Command line for displaying the temporary file in a browser. - #use constant DISPLAY_COMMAND => 'open -a firefox '; #browser opens tempoutputfile above - use constant DISPLAY_COMMAND => "open -a 'Google Chrome' "; - #use constant DISPLAY_COMMAND => " less "; # display tempoutputfile with less - -### Path to a temporary file for storing the output of renderProblem.pl -use constant LOG_FILE => "$ENV{WEBWORK_ROOT}/DATA/bad_problems.txt"; -die "You must first create an output file at ".LOG_FILE(). - " with permissions 777 " unless -w LOG_FILE(); - -### set display mode -use constant DISPLAYMODE => 'MathJax'; -use constant PROBLEMSEED => '32145'; - - -### select a rendering site -my $use_site; - #$use_site = 'test_webwork'; # select a rendering site - #$use_site = 'local'; # select a rendering site - #$use_site = 'hosted2'; # select a rendering site - $use_site ="credentials"; - -# credentials file location -- search for one of these files -my $credential_path; -my @path_list = ( "$ENV{HOME}/.ww_credentials", "$ENV{HOME}/ww_session_credentials", 'ww_credentials',); - -=head2 credentials file - - # Place a credential file containing the following information at one of the locations above. - # %credentials = ( - # userID => "my login name for the webwork course", - # course_password => "my password ", - # courseID => "the name of the webwork course", - # XML_URL => "url of rendering site - # XML_PASSWORD => "site password" # preliminary access to site - # $FORM_ACTION_URL = 'http://localhost:80/webwork2/html2xml'; #action url for form - # ); - -=cut - - ############################################################ - # End configure - ############################################################ - - - ############################################################ - -=head2 URLs - - # To configure a new target webwork server - # two URLs are required - # 1. $XML_URL http://test.webwork.maa.org/mod_xmlrpc - # points to the Webservice.pm and Webservice/RenderProblem modules - # Is used by the client to send the original XML request to the webservice - # - # 2. $FORM_ACTION_URL http://test.webwork.maa.org/webwork2/html2xml - # points to the renderViaXMLRPC.pm module. - # - # This url is placed as form action url when the rendered HTML from the original - # request is returned to the client from Webservice/RenderProblem. The client - # reorganizes the XML it receives into an HTML page (with a WeBWorK form) and - # pipes it through a local browser. - # - # The browser uses this url to resubmit the problem (with answers) via the standard - # HTML webform used by WeBWorK to the renderViaXMLRPC.pm handler. - # - # This renderViaXMLRPC.pm handler acts as an intermediary between the browser - # and the webservice. It interprets the HTML form sent by the browser, - # rewrites the form data in XML format, submits it to the WebworkWebservice.pm - # which processes it and sends the the resulting HTML back to renderViaXMLRPC.pm - # which in turn passes it back to the browser. - # 3. The second time a problem is submitted renderViaXMLRPC.pm receives the WeBWorK form - # submitted directly by the browser. - # The renderViaXMLRPC.pm translates the WeBWorK form, has it processes by the webservice - # and returns the result to the browser. - # The The client renderProblem.pl script is no longer involved. - # 4. Summary: renderProblem.pl is only involved in the first round trip - # of the submitted problem. After that the communication is between the browser and - # renderViaXMLRPC using HTML forms and between renderViaXMLRPC and the WebworkWebservice.pm - # module using XML_RPC. - # 5. The XML_PASSWORD is defined on the site. In future versions a more secure password method - # may be implemented. This is sufficient to keep out robots. - # 6. The course "daemon_course" must be a course that has been created on the server or an error will - # result. A different name can be used but the course must exist on the server. - # 7. More secure authentication is achieved using courseID, userID and course_password. - # The course_password must be the password for userID in the course courseID and that - # user must have sufficient permissions in the course. - # The permission level is set in the WebworkWebservice code. - - -=cut - - -#################################################### -# get credentials -#################################################### -my $credentials_string = < "my login name for the webwork course", - course_password => "my password ", - courseID => "the name of the webwork course", - XML_URL => "url of rendering site", - XML_PASSWORD => "site password", # preliminary access to site - FORM_ACTION_URL => 'http://localhost:80/webwork2/html2xml', #action url for form - ); -1; -EOF - -foreach my $path (@path_list) { - if (-r "$path" ) { - $credential_path = $path; - last; - } -} -if ( $credential_path ) { - print "Credentials taken from file $credential_path\n" if $UNIT_TESTS_ON; -} else { - die <; #slurp standard input -}; -die "Something is wrong with the contents of $fileName\n" if $@; - -### adjust fileName so that it is relative to the rendering course directory - #$fileName =~ s|/opt/webwork/libraries/NationalProblemLibrary|Library|; - $fileName =~ s|^.*?/webwork-open-problem-library/OpenProblemLibrary|Library|; - print "fileName changed to $fileName\n" if $UNIT_TESTS_ON; - #print "source $source\n" if $UNIT_TESTS_ON; - print $source if $UNIT_TESTS_ON; # return input to BBedit - -############################################ -# Build client -############################################ - - -our $xmlrpc_client = new WebworkClient ( - url => $XML_URL, - form_action_url => $FORM_ACTION_URL, -# displayMode => DISPLAYMODE(), - site_password => $XML_PASSWORD//'', - courseID => $credentials{courseID}, - userID => $credentials{userID}, - session_key => $credentials{session_key}//'', - sourceFilePath => $fileName, - inputs_ref => {displayMode => DISPLAYMODE(), problemSeed => PROBLEMSEED(),}, -); - - $xmlrpc_client->encodeSource($source); - - my $input = { - userID => $credentials{userID}//'', - session_key => $credentials{session_key}//'', - courseID => $credentials{courseID}//'', - courseName => $credentials{courseID}//'', - course_password => $credentials{course_password}//'', - site_password => $XML_PASSWORD//'', - envir => $xmlrpc_client->environment( - fileName => $fileName, - sourceFilePath => $fileName - ), - }; -$input->{envir}->{inputs_ref} = { displayMode => DISPLAYMODE(), - problemSeed => PROBLEMSEED(), -}; - -$xmlrpc_client->{sourceFilePath} = $fileName; - -############################################ -# Call server via xmlrpc_client -# Format the returned values -############################################ -our($output, $return_string, $result); - - if ( $result = $xmlrpc_client->xmlrpcCall('renderProblem', $input) ) { - print "\n\n Result of renderProblem \n\n" if $UNIT_TESTS_ON; - #print pretty_print_rh($result) if $UNIT_TESTS_ON; - #$output = $xmlrpc_client->{output}; - if (not defined $result) { #FIXME make sure this is the right error message if site is unavailable - $return_string = "Could not connect to rendering site $XML_URL\n"; - } elsif (defined($result->{flags}->{error_flag}) and $output->{flags}->{error_flag} ) { - $return_string = "0\t $fileName has errors\n"; - } elsif (defined($result->{errors}) and $output->{errors} ){ - $return_string = "0\t $fileName has syntax errors\n"; - } else { - # - if (defined($result->{flags}->{DEBUG_messages}) ) { - my @debug_messages = @{$output->{flags}->{DEBUG_messages}}; - $return_string .= (pop @debug_messages ) ||'' ; #avoid error if array was empty - if (@debug_messages) { - $return_string .= join(" ", @debug_messages); - } else { - $return_string = ""; - } - } - if (defined($result->{errors}) ) { - $return_string= $result->{errors}; - } - if (defined($result->{flags}->{WARNING_messages}) ) { - my @warning_messages = @{$output->{flags}->{WARNING_messages}}; - $return_string .= (pop @warning_messages)||''; #avoid error if array was empty - $@=undef; - if (@warning_messages) { - $return_string .= join(" ", @warning_messages); - } else { - $return_string = ""; - } - } - - unless ($return_string) { - $return_string = "1\t $fileName is ok\n"; - } else { - $return_string = "0\t $fileName has errors\n"; - } - } - local(*FH); - open(FH, '>>',LOG_FILE()) or die "Can't open file ".LOG_FILE()." for writing"; - print FH $return_string; - close(FH); -} else { - print "0 $fileName something went wrong -- could not render file\n"; - print STDERR "Useage: ./checkProblem.pl [file_name]\n"; - print STDERR "For example: ./checkProblem.pl input.txt\n"; - print STDERR "Output is sent to the log file: ",LOG_FILE(); - -} - -################################################## -# log elapsed time -################################################## -my $scriptName = 'checkProblem'; -my $cg_end = time; -my $cg_duration = $cg_end - $cg_start; -WebworkClient::writeRenderLogEntry("", "{script:$scriptName; file:$fileName; ". sprintf("duration:%.3f sec;", $cg_duration)." url: $XML_URL; }",''); - - -################################################## -# utilities -################################################## - -sub pretty_print_rh { - shift if UNIVERSAL::isa($_[0] => __PACKAGE__); - my $rh = shift; - my $indent = shift || 0; - my $out = ""; - my $type = ref($rh); - - if (defined($type) and $type) { - $out .= " type = $type; "; - } elsif (! defined($rh )) { - $out .= " type = UNDEFINED; "; - } - return $out." " unless defined($rh); - - if ( ref($rh) =~/HASH/ or "$rh" =~/HASH/ ) { - $out .= "{\n"; - $indent++; - foreach my $key (sort keys %{$rh}) { - $out .= " "x$indent."$key => " . pretty_print_rh( $rh->{$key}, $indent ) . "\n"; - } - $indent--; - $out .= "\n"." "x$indent."}\n"; - - } elsif (ref($rh) =~ /ARRAY/ or "$rh" =~/ARRAY/) { - $out .= " ( "; - foreach my $elem ( @{$rh} ) { - $out .= pretty_print_rh($elem, $indent); - - } - $out .= " ) \n"; - } elsif ( ref($rh) =~ /SCALAR/ ) { - $out .= "scalar reference ". ${$rh}; - } elsif ( ref($rh) =~/Base64/ ) { - $out .= "base64 reference " .$$rh; - } else { - $out .= $rh; - } - - return $out." "; -} - - -1; diff --git a/clients/old/check_problems_in_dir.sh b/clients/old/check_problems_in_dir.sh deleted file mode 100755 index 5b8a00ce1d..0000000000 --- a/clients/old/check_problems_in_dir.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -#usage -# check_problems_in_dir.s directory_name -# this file is no longer necessary sendXMLRPC accepts directories and walks through them. - -echo "Checking problems in directory $1" -echo '' > "$WEBWORK_ROOT/DATA/bad_problems.txt" -echo "Results sent to" -echo "$WEBWORK_ROOT/DATA/bad_problems.txt" -time find $1 -name "*.pg" -exec /usr/bin/perl $WEBWORK_ROOT/clients/checkProblem.pl -b {} ';' #tail -f /Volumes/WW_test/opt/webwork/webwork2/DATA/bad_problems.txt; -echo "start search" -#tail -f "$WEBWORK_ROOT/DATA/bad_problems.txt" -echo "Time for $1"; -echo "" -echo "" -exit - diff --git a/clients/old/renderProblem.pl b/clients/old/renderProblem.pl deleted file mode 100755 index 678cfad995..0000000000 --- a/clients/old/renderProblem.pl +++ /dev/null @@ -1,393 +0,0 @@ -#!/usr/bin/perl -w - -################################################################################ -# WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/clients/renderProblem.pl,v 1.4 2010/05/11 15:44:05 gage Exp $ -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of either: (a) the GNU General Public License as published by the -# Free Software Foundation; either version 2, or (at your option) any later -# version, or (b) the "Artistic License" which comes with this package. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the -# Artistic License for more details. -################################################################################ - -=head1 NAME - -webwork2/clients/renderProblem.pl - -This script will take a file and send it to a WeBWorK daemon webservice -to have it rendered. The result is split into the basic HTML rendering -and evaluation of answers and then passed to a browser for printing. - -The formatting allows the browser presentation to be interactive with the -daemon running the script webwork2/lib/renderViaXMLRPC.pm - -Rembember to configure the local output file and display command !!!!!!!! - -=cut - -use strict; -use warnings; - - -####################################################### -# Find the webwork2 root directory -####################################################### -BEGIN { - die "WEBWORK_ROOT not found in environment. \n - WEBWORK_ROOT can be defined in your .cshrc or .bashrc file\n - It should be set to the webwork2 directory (e.g. /opt/webwork/webwork2)" - unless exists $ENV{WEBWORK_ROOT}; - # Unused variable, but define it twice to avoid an error message. - $WeBWorK::Constants::WEBWORK_DIRECTORY = $ENV{WEBWORK_ROOT}; - $WeBWorK::Constants::PG_DIRECTORY = "$ENV{WEBWORK_ROOT}/../pg/"; - unless (-r $WeBWorK::Constants::WEBWORK_DIRECTORY ) { - die "Cannot read webwork root directory at $WeBWorK::Constants::WEBWORK_DIRECTORY"; - } - unless (-r $WeBWorK::Constants::PG_DIRECTORY ) { - die "Cannot read webwork pg directory at $WeBWorK::Constants::PG_DIRECTORY"; - } -} - -use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib"; -use lib "$WeBWorK::Constants::PG_DIRECTORY/lib"; -use Crypt::SSLeay; # needed for https -use WebworkClient; -use Time::HiRes qw/time/; -use MIME::Base64 qw( encode_base64 decode_base64); - -############################################# -# Configure -############################################# - - -### verbose output when UNIT_TESTS_ON =1; - our $UNIT_TESTS_ON = 0; - -### Command line for displaying the temporary file in a browser. - #use constant DISPLAY_COMMAND => 'open -a firefox '; #browser opens tempoutputfile above - use constant DISPLAY_COMMAND => "open -a 'Google Chrome' "; - #use constant DISPLAY_COMMAND => " less "; # display tempoutputfile with less - -### Path to a temporary file for storing the output of renderProblem.pl - use constant TEMPOUTPUTFILE => "$ENV{WEBWORK_ROOT}/DATA/renderProblemOutput.html"; - die "You must first create an output file at ".TEMPOUTPUTFILE(). - " with permissions 777 " unless -w TEMPOUTPUTFILE(); - -### set display mode -use constant DISPLAYMODE => 'MathJax'; -use constant PROBLEMSEED => '32145'; - - -### select a rendering site -my $use_site; - #$use_site = 'test_webwork'; # select a rendering site - #$use_site = 'local'; # select a rendering site - #$use_site = 'hosted2'; # select a rendering site - $use_site ="credentials"; - -# credentials file location -- search for one of these files -my $credential_path; -my @path_list = ( "$ENV{HOME}/.ww_credentials", "$ENV{HOME}/ww_session_credentials", 'ww_credentials',); - -=head2 credentials file - - # Place a credential file containing the following information at one of the locations above. - # %credentials = ( - # userID => "my login name for the webwork course", - # course_password => "my password ", - # courseID => "the name of the webwork course", - # XML_URL => "url of rendering site - # XML_PASSWORD => "site password" # preliminary access to site - # $FORM_ACTION_URL = 'http://localhost:80/webwork2/html2xml'; #action url for form - # ); - -=cut - - ############################################################ - # End configure - ############################################################ - - - ############################################################ - -=head2 URLs - - # To configure a new target webwork server - # two URLs are required - # 1. $XML_URL http://test.webwork.maa.org/mod_xmlrpc - # points to the Webservice.pm and Webservice/RenderProblem modules - # Is used by the client to send the original XML request to the webservice - # - # 2. $FORM_ACTION_URL http://test.webwork.maa.org/webwork2/html2xml - # points to the renderViaXMLRPC.pm module. - # - # This url is placed as form action url when the rendered HTML from the original - # request is returned to the client from Webservice/RenderProblem. The client - # reorganizes the XML it receives into an HTML page (with a WeBWorK form) and - # pipes it through a local browser. - # - # The browser uses this url to resubmit the problem (with answers) via the standard - # HTML webform used by WeBWorK to the renderViaXMLRPC.pm handler. - # - # This renderViaXMLRPC.pm handler acts as an intermediary between the browser - # and the webservice. It interprets the HTML form sent by the browser, - # rewrites the form data in XML format, submits it to the WebworkWebservice.pm - # which processes it and sends the the resulting HTML back to renderViaXMLRPC.pm - # which in turn passes it back to the browser. - # 3. The second time a problem is submitted renderViaXMLRPC.pm receives the WeBWorK form - # submitted directly by the browser. - # The renderViaXMLRPC.pm translates the WeBWorK form, has it processes by the webservice - # and returns the result to the browser. - # The The client renderProblem.pl script is no longer involved. - # 4. Summary: renderProblem.pl is only involved in the first round trip - # of the submitted problem. After that the communication is between the browser and - # renderViaXMLRPC using HTML forms and between renderViaXMLRPC and the WebworkWebservice.pm - # module using XML_RPC. - # 5. The XML_PASSWORD is defined on the site. In future versions a more secure password method - # may be implemented. This is sufficient to keep out robots. - # 6. The course "daemon_course" must be a course that has been created on the server or an error will - # result. A different name can be used but the course must exist on the server. - # 7. More secure authentication is achieved using courseID, userID and course_password. - # The course_password must be the password for userID in the course courseID and that - # user must have sufficient permissions in the course. - # The permission level is set in the WebworkWebservice code. - - -=cut - - -#################################################### -# get credentials -#################################################### -my $credentials_string = < "my login name for the webwork course", - course_password => "my password ", - courseID => "the name of the webwork course", - XML_URL => "url of rendering site", - XML_PASSWORD => "site password", # preliminary access to site - FORM_ACTION_URL => 'http://localhost:80/webwork2/html2xml', #action url for form - ); -1; -EOF - -foreach my $path (@path_list) { - if (-r "$path" ) { - $credential_path = $path; - last; - } -} -if ( $credential_path ) { - print "Credentials taken from file $credential_path\n" if $UNIT_TESTS_ON; -} else { - die <; #slurp standard input -}; -die "Something is wrong with the contents of $fileName\n" if $@; - -### adjust fileName so that it is relative to the rendering course directory - #$fileName =~ s|/opt/webwork/libraries/NationalProblemLibrary|Library|; - $fileName =~ s|^.*?/webwork-open-problem-library/OpenProblemLibrary|Library|; - print "fileName changed to $fileName\n" if $UNIT_TESTS_ON; - #print "source $source\n" if $UNIT_TESTS_ON; - print $source if $UNIT_TESTS_ON; # return input to BBedit - -############################################ -# Build client -############################################ - - -our $xmlrpc_client = new WebworkClient ( - url => $XML_URL, - form_action_url => $FORM_ACTION_URL, -# displayMode => DISPLAYMODE(), - site_password => $XML_PASSWORD//'', - courseID => $credentials{courseID}, - userID => $credentials{userID}, - session_key => $credentials{session_key}//'', - sourceFilePath => $fileName, - inputs_ref => {displayMode => DISPLAYMODE(), problemSeed => PROBLEMSEED(),}, -); - - $xmlrpc_client->encodeSource($source); - - my $input = { - userID => $credentials{userID}//'', - session_key => $credentials{session_key}//'', - courseID => $credentials{courseID}//'', - courseName => $credentials{courseID}//'', - course_password => $credentials{course_password}//'', - site_password => $XML_PASSWORD//'', - envir => $xmlrpc_client->environment( - fileName => $fileName, - sourceFilePath => $fileName - ), - }; -$input->{envir}->{inputs_ref} = { displayMode => DISPLAYMODE(), - problemSeed => PROBLEMSEED(), -}; - -$xmlrpc_client->{sourceFilePath} = $fileName; - -############################################ -# Call server via xmlrpc_client -# Format the returned values -############################################ -our($output, $return_string, $result); - -if ( $result = $xmlrpc_client->xmlrpcCall('renderProblem', $input) ) { - print "\n\n Result of renderProblem \n\n" if $UNIT_TESTS_ON; - print pretty_print_rh($result) if $UNIT_TESTS_ON; - $output = $xmlrpc_client->formatRenderedProblem; -} else { - print "\n\n ERRORS in renderProblem \n\n"; - $output = $xmlrpc_client->return_object; # error report -} - -################################################## -# print the output and display -################################################## - -local(*FH); -open(FH, '>'.TEMPOUTPUTFILE) or die "Can't open file ".TEMPOUTPUTFILE()." for writing"; -print FH $output; -close(FH); - -system(DISPLAY_COMMAND().TEMPOUTPUTFILE()); - -################################################## -# log elapsed time -################################################## -my $scriptName = 'renderProblem'; -my $cg_end = time; -my $cg_duration = $cg_end - $cg_start; -WebworkClient::writeRenderLogEntry("", "{script:$scriptName; file:$fileName; ". sprintf("duration: %.3f sec;", $cg_duration)." url: $XML_URL; }",''); - - -################################################## -# utilities -################################################## - -sub pretty_print_rh { - shift if UNIVERSAL::isa($_[0] => __PACKAGE__); - my $rh = shift; - my $indent = shift || 0; - my $out = ""; - my $type = ref($rh); - - if (defined($type) and $type) { - $out .= " type = $type; "; - } elsif (! defined($rh )) { - $out .= " type = UNDEFINED; "; - } - return $out." " unless defined($rh); - - if ( ref($rh) =~/HASH/ ) { - $out .= "{\n"; - $indent++; - foreach my $key (sort keys %{$rh}) { - $out .= " "x$indent."$key => " . pretty_print_rh( $rh->{$key}, $indent ) . "\n"; - } - $indent--; - $out .= "\n"." "x$indent."}\n"; - - } elsif (ref($rh) =~ /ARRAY/ or "$rh" =~/ARRAY/) { - $out .= " ( "; - foreach my $elem ( @{$rh} ) { - $out .= pretty_print_rh($elem, $indent); - - } - $out .= " ) \n"; - } elsif ( ref($rh) =~ /SCALAR/ ) { - $out .= "scalar reference ". ${$rh}; - } elsif ( ref($rh) =~/Base64/ ) { - $out .= "base64 reference " .$$rh; - } else { - $out .= $rh; - } - - return $out." "; -} - - -1; diff --git a/clients/old/renderProblem_rawoutput.pl b/clients/old/renderProblem_rawoutput.pl deleted file mode 100755 index 72f8d0a645..0000000000 --- a/clients/old/renderProblem_rawoutput.pl +++ /dev/null @@ -1,394 +0,0 @@ -#!/usr/bin/perl -w - -################################################################################ -# WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/clients/renderProblem.pl,v 1.4 2010/05/11 15:44:05 gage Exp $ -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of either: (a) the GNU General Public License as published by the -# Free Software Foundation; either version 2, or (at your option) any later -# version, or (b) the "Artistic License" which comes with this package. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the -# Artistic License for more details. -################################################################################ - -=head1 NAME - -webwork2/clients/renderProblem.pl - -This script will take a file and send it to a WeBWorK daemon webservice -to have it rendered. The result is split into the basic HTML rendering -and evaluation of answers and then passed to a browser for printing. - -The formatting allows the browser presentation to be interactive with the -daemon running the script webwork2/lib/renderViaXMLRPC.pm - -Rembember to configure the local output file and display command !!!!!!!! - -=cut - -use strict; -use warnings; - - -####################################################### -# Find the webwork2 root directory -####################################################### -BEGIN { - die "WEBWORK_ROOT not found in environment. \n - WEBWORK_ROOT can be defined in your .cshrc or .bashrc file\n - It should be set to the webwork2 directory (e.g. /opt/webwork/webwork2)" - unless exists $ENV{WEBWORK_ROOT}; - # Unused variable, but define it twice to avoid an error message. - $WeBWorK::Constants::WEBWORK_DIRECTORY = $ENV{WEBWORK_ROOT}; - $WeBWorK::Constants::PG_DIRECTORY = "$ENV{WEBWORK_ROOT}/../pg/"; - unless (-r $WeBWorK::Constants::WEBWORK_DIRECTORY ) { - die "Cannot read webwork root directory at $WeBWorK::Constants::WEBWORK_DIRECTORY"; - } - unless (-r $WeBWorK::Constants::PG_DIRECTORY ) { - die "Cannot read webwork pg directory at $WeBWorK::Constants::PG_DIRECTORY"; - } -} - -use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib"; -use lib "$WeBWorK::Constants::PG_DIRECTORY/lib"; -use Crypt::SSLeay; # needed for https -use WebworkClient; -use Time::HiRes qw/time/; -use MIME::Base64 qw( encode_base64 decode_base64); - -############################################# -# Configure -############################################# - - -### verbose output when UNIT_TESTS_ON =1; - our $UNIT_TESTS_ON = 0; - -### Command line for displaying the temporary file in a browser. - #use constant DISPLAY_COMMAND => 'open -a firefox '; #browser opens tempoutputfile above - #use constant DISPLAY_COMMAND => "open -a 'Google Chrome' "; - use constant DISPLAY_COMMAND => " less "; # display tempoutputfile with less - -### Path to a temporary file for storing the output of renderProblem.pl - use constant TEMPOUTPUTFILE => "$ENV{WEBWORK_ROOT}/DATA/renderProblemOutput.html"; - die "You must first create an output file at ".TEMPOUTPUTFILE(). - " with permissions 777 " unless -w TEMPOUTPUTFILE(); - -### set display mode -use constant DISPLAYMODE => 'MathJax'; -use constant PROBLEMSEED => '32145'; - - -### select a rendering site -my $use_site; - #$use_site = 'test_webwork'; # select a rendering site - #$use_site = 'local'; # select a rendering site - #$use_site = 'hosted2'; # select a rendering site - $use_site ="credentials"; - -# credentials file location -- search for one of these files -my $credential_path; -my @path_list = ( "$ENV{HOME}/.ww_credentials", "$ENV{HOME}/ww_session_credentials", 'ww_credentials',); - -=head2 credentials file - - # Place a credential file containing the following information at one of the locations above. - # %credentials = ( - # userID => "my login name for the webwork course", - # course_password => "my password ", - # courseID => "the name of the webwork course", - # XML_URL => "url of rendering site - # XML_PASSWORD => "site password" # preliminary access to site - # $FORM_ACTION_URL = 'http://localhost:80/webwork2/html2xml'; #action url for form - # ); - -=cut - - ############################################################ - # End configure - ############################################################ - - - ############################################################ - -=head2 URLs - - # To configure a new target webwork server - # two URLs are required - # 1. $XML_URL http://test.webwork.maa.org/mod_xmlrpc - # points to the Webservice.pm and Webservice/RenderProblem modules - # Is used by the client to send the original XML request to the webservice - # - # 2. $FORM_ACTION_URL http://test.webwork.maa.org/webwork2/html2xml - # points to the renderViaXMLRPC.pm module. - # - # This url is placed as form action url when the rendered HTML from the original - # request is returned to the client from Webservice/RenderProblem. The client - # reorganizes the XML it receives into an HTML page (with a WeBWorK form) and - # pipes it through a local browser. - # - # The browser uses this url to resubmit the problem (with answers) via the standard - # HTML webform used by WeBWorK to the renderViaXMLRPC.pm handler. - # - # This renderViaXMLRPC.pm handler acts as an intermediary between the browser - # and the webservice. It interprets the HTML form sent by the browser, - # rewrites the form data in XML format, submits it to the WebworkWebservice.pm - # which processes it and sends the the resulting HTML back to renderViaXMLRPC.pm - # which in turn passes it back to the browser. - # 3. The second time a problem is submitted renderViaXMLRPC.pm receives the WeBWorK form - # submitted directly by the browser. - # The renderViaXMLRPC.pm translates the WeBWorK form, has it processes by the webservice - # and returns the result to the browser. - # The The client renderProblem.pl script is no longer involved. - # 4. Summary: renderProblem.pl is only involved in the first round trip - # of the submitted problem. After that the communication is between the browser and - # renderViaXMLRPC using HTML forms and between renderViaXMLRPC and the WebworkWebservice.pm - # module using XML_RPC. - # 5. The XML_PASSWORD is defined on the site. In future versions a more secure password method - # may be implemented. This is sufficient to keep out robots. - # 6. The course "daemon_course" must be a course that has been created on the server or an error will - # result. A different name can be used but the course must exist on the server. - # 7. More secure authentication is achieved using courseID, userID and course_password. - # The course_password must be the password for userID in the course courseID and that - # user must have sufficient permissions in the course. - # The permission level is set in the WebworkWebservice code. - - -=cut - - -#################################################### -# get credentials -#################################################### -my $credentials_string = < "my login name for the webwork course", - course_password => "my password ", - courseID => "the name of the webwork course", - XML_URL => "url of rendering site", - XML_PASSWORD => "site password", # preliminary access to site - FORM_ACTION_URL => 'http://localhost:80/webwork2/html2xml', #action url for form - ); -1; -EOF - -foreach my $path (@path_list) { - if (-r "$path" ) { - $credential_path = $path; - last; - } -} -if ( $credential_path ) { - print "Credentials taken from file $credential_path\n" if $UNIT_TESTS_ON; -} else { - die <; #slurp standard input -}; -die "Something is wrong with the contents of $fileName\n" if $@; - -### adjust fileName so that it is relative to the rendering course directory - #$fileName =~ s|/opt/webwork/libraries/NationalProblemLibrary|Library|; - $fileName =~ s|^.*?/webwork-open-problem-library/OpenProblemLibrary|Library|; - print "fileName changed to $fileName\n" if $UNIT_TESTS_ON; - #print "source $source\n" if $UNIT_TESTS_ON; - print $source if $UNIT_TESTS_ON; # return input to BBedit - -############################################ -# Build client -############################################ - - -our $xmlrpc_client = new WebworkClient ( - url => $XML_URL, - form_action_url => $FORM_ACTION_URL, -# displayMode => DISPLAYMODE(), - site_password => $XML_PASSWORD//'', - courseID => $credentials{courseID}, - userID => $credentials{userID}, - session_key => $credentials{session_key}//'', - sourceFilePath => $fileName, - inputs_ref => {displayMode => DISPLAYMODE(), problemSeed => PROBLEMSEED(),}, -); - - $xmlrpc_client->encodeSource($source); - - my $input = { - userID => $credentials{userID}//'', - session_key => $credentials{session_key}//'', - courseID => $credentials{courseID}//'', - courseName => $credentials{courseID}//'', - course_password => $credentials{course_password}//'', - site_password => $XML_PASSWORD//'', - envir => $xmlrpc_client->environment( - fileName => $fileName, - sourceFilePath => $fileName - ), - }; -$input->{envir}->{inputs_ref} = { displayMode => DISPLAYMODE(), - problemSeed => PROBLEMSEED(), -}; - -$xmlrpc_client->{sourceFilePath} = $fileName; - -############################################ -# Call server via xmlrpc_client -# Format the returned values -############################################ -our($output, $return_string, $result); - -if ( $result = $xmlrpc_client->xmlrpcCall('renderProblem', $input) ) { - print "\n\n Result of renderProblem \n\n" if $UNIT_TESTS_ON; - $output = "1\n"; - $output = pretty_print_rh( $result ); -} else { - $output = "0\n"; - print "\n\n ERRORS in renderProblem \n\n"; - $output = $xmlrpc_client->return_object; # error report -} - -################################################## -# print the output and display -################################################## - -local(*FH); -open(FH, '>'.TEMPOUTPUTFILE) or die "Can't open file ".TEMPOUTPUTFILE()." for writing"; -print FH $output; -close(FH); - -system(DISPLAY_COMMAND().TEMPOUTPUTFILE()); - -################################################## -# log elapsed time -################################################## -my $scriptName = 'renderProblem_rawoutput'; -my $cg_end = time; -my $cg_duration = $cg_end - $cg_start; -WebworkClient::writeRenderLogEntry("", "{$scriptName: $fileName; ". sprintf("duration: %.3f sec;", $cg_duration)." url: $XML_URL; }",''); - - -################################################## -# utilities -################################################## - -sub pretty_print_rh { - shift if UNIVERSAL::isa($_[0] => __PACKAGE__); - my $rh = shift; - my $indent = shift || 0; - my $out = ""; - my $type = ref($rh); - - if (defined($type) and $type) { - $out .= " type = $type; "; - } elsif (! defined($rh )) { - $out .= " type = UNDEFINED; "; - } - return $out." " unless defined($rh); - - if ( ref($rh) =~/HASH/ ) { - $out .= "{\n"; - $indent++; - foreach my $key (sort keys %{$rh}) { - $out .= " "x$indent."$key => " . pretty_print_rh( $rh->{$key}, $indent ) . "\n"; - } - $indent--; - $out .= "\n"." "x$indent."}\n"; - - } elsif (ref($rh) =~ /ARRAY/ or "$rh" =~/ARRAY/) { - $out .= " ( "; - foreach my $elem ( @{$rh} ) { - $out .= pretty_print_rh($elem, $indent); - - } - $out .= " ) \n"; - } elsif ( ref($rh) =~ /SCALAR/ ) { - $out .= "scalar reference ". ${$rh}; - } elsif ( ref($rh) =~/Base64/ ) { - $out .= "base64 reference " .$$rh; - } else { - $out .= $rh; - } - - return $out." "; -} - - -1; diff --git a/clients/pr b/clients/pr new file mode 100755 index 0000000000..20036afe17 --- /dev/null +++ b/clients/pr @@ -0,0 +1,16 @@ +#!/bin/bash + +# inside the docker container +# this can be called from vim with :w ! ./pr + +# it sends the contents of the buffer to /var/www/html/xmlrpc_out.html aka localhost:8080/xmlrpc_output.html + +./sendXMLRPC.pl -h - >/var/www/html/xmlrpc_out.html + +# process STDIN with sendXMLRPC.p and send it to /var/www/html/xmlrpc_out.html +# where it can be read at localhost:8080/xmlrpc_out.html + + +# Not sure how to get rid of the ./ + +# This can also be called from the command line with cat t/input.pg |./pr diff --git a/clients/sendXMLRPC.pl b/clients/sendXMLRPC.pl index f740004b08..16559e97ca 100755 --- a/clients/sendXMLRPC.pl +++ b/clients/sendXMLRPC.pl @@ -93,6 +93,7 @@ =head2 credentials file # site_url => 'http://localhost:80', # form_action_url => 'http://localhost:80/webwork2/html2xml', # site_password => 'xmlrpc', + # forcePortNumber => '80', # A port number to be forced, when needed. # Set the identification credential used by the "daemon_course" on the remote site courseID => "daemon_course", @@ -173,7 +174,7 @@ =head2 Options Same as -c but the question is rendered with the correct answers submitted. This succeeds only if the correct answers, as determined from the answer hash, all succeed. -=item f=s +=item -f formatName Specify the format used by the browser in displaying the question. Choices for s are @@ -182,6 +183,9 @@ =head2 Options debug simple +=item -l lang + + Set a language for the HTML rendering to use. Should use a value which would be valid for a course. =item -v @@ -190,19 +194,26 @@ =head2 Options which are (will be) submitted to the question. =item -e + Open the source file in an editor. =item --tex + Process question in TeX mode and output to the command line =item --pdf + Process question in TeX mode, convert to PDF and display. +=item --json + + Process question in JSON mode and save to file =item The single letter options can be "bundled" e.g. -vcCbB =item --list pg_list + Read and process a list of .pg files contained in the file C. C consists of a sequence of lines each of which contains the full path to a pg file that should be processed. (For example this might be the output from an @@ -240,8 +251,19 @@ =head2 Options Prints help information. =item --log + Sets path to log file +=item --seed=s + + Sets problemSeed to the number contained in string s + +=item --psvn=s + + Sets psvn to the number contained in string s + + + =back =cut @@ -258,6 +280,7 @@ BEGIN } $ENV{MOD_PERL_API_VERSION} = 2; use lib "$main::dirname"; +print "home directory ".$main::dirname."\n"; #use lib "."; # is this needed? # some files such as FormatRenderedProblem.pm may need to be in the same directory @@ -279,13 +302,10 @@ BEGIN } } - - use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib"; -use lib "$WeBWorK::Constants::PG_DIRECTORY/lib"; + use Carp; -#use Crypt::SSLeay; # needed for https use LWP::Protocol::https; use Time::HiRes qw/time/; use MIME::Base64 qw( encode_base64 decode_base64); @@ -303,49 +323,10 @@ BEGIN use 5.10.0; $Carp::Verbose = 1; -############################################# -# CONFIGURE -# -# Configure displays for local operating system -# This section will differ from on operating system to another -# The default code is for the macOS and applications commonly available on macOS. -############################################# - - -#Default display commands. -use constant HTML_DISPLAY_COMMAND => "open -a 'Google Chrome' "; # (MacOS command) -use constant HASH_DISPLAY_COMMAND => ""; # display tempoutputfile to STDOUT - -### Path to a temporary file for storing the output of sendXMLRPC.pl - use constant TEMPOUTPUTDIR => "$ENV{WEBWORK_ROOT}/DATA/"; - die "You must make the directory ".TEMPOUTPUTDIR(). - " writeable " unless -w TEMPOUTPUTDIR(); - use constant TEMPOUTPUTFILE => TEMPOUTPUTDIR()."temporary_output.html"; - -### Default path to a temporary file for storing the output -### of sendXMLRPC.pl -use constant LOG_FILE => "$ENV{WEBWORK_ROOT}/DATA/xmlrpc_results.log"; - -### Command for editing the pg source file in the browswer -use constant EDIT_COMMAND =>"bbedit"; # for Mac BBedit editor (used as `EDIT_COMMAND() . " $file_path") - -### Command for editing and viewing the tex output of the pg question. -use constant TEX_DISPLAY_COMMAND =>"open -a 'TeXShop'"; - -### Command for editing and viewing the tex output of the pg question. -use constant PDF_DISPLAY_COMMAND =>"open -a 'Preview'"; - -### set display mode -use constant DISPLAYMODE => 'MathJax'; -use constant PROBLEMSEED => '987654321'; ### verbose output when UNIT_TESTS_ON =1; - our $UNIT_TESTS_ON = 0; +our $UNIT_TESTS_ON = 0; -############################################################ -# End configure displays for local operating system -############################################################ - ############################################################ # Read command line options ############################################################ @@ -361,9 +342,11 @@ BEGIN my $verbose = ''; my $credentials_path; my $format = 'standard'; +my $lang = 'en'; my $edit_source_file = ''; my $display_tex_output=''; my $display_pdf_output=''; +my $display_json_output=''; my $print_answer_hash; my $print_answer_group; my $print_pg_hash; @@ -371,37 +354,57 @@ BEGIN my $print_help_message; my $read_list_from_this_file; my $path_to_log_file; +my $problemSeed; +my $psvn; + +our %credentials; +our @path_list; +my $credentials_string; + + GetOptions( - 'a' => \$display_ans_output1, - 'A' => \$display_ans_output2, - 'b' => \$display_html_output1, - 'B' => \$display_html_output2, - 'h' => \$display_hash_output1, - 'H' => \$display_hash_output2, - 'c' => \$record_ok1, # record_problem_ok1 needs to be written - 'C' => \$record_ok2, - 'v' => \$verbose, - 'e' => \$edit_source_file, - 'tex' => \$display_tex_output, - 'pdf' => \$display_pdf_output, - 'list=s' =>\$read_list_from_this_file, # read file containing list of full file paths - 'pg' => \$print_pg_hash, - 'anshash' => \$print_answer_hash, - 'ansgrp' => \$print_answer_group, + 'a' => \$display_ans_output1, + 'A' => \$display_ans_output2, + 'b' => \$display_html_output1, + 'B' => \$display_html_output2, + 'h' => \$display_hash_output1, + 'H' => \$display_hash_output2, + 'c' => \$record_ok1, # record_problem_ok1 needs to be written + 'C' => \$record_ok2, + 'v' => \$verbose, + 'e' => \$edit_source_file, + 'tex' => \$display_tex_output, + 'pdf' => \$display_pdf_output, + 'json' => \$display_json_output, + 'list=s' =>\$read_list_from_this_file, # read file containing list of full file paths + 'pg' => \$print_pg_hash, + 'anshash' => \$print_answer_hash, + 'ansgrp' => \$print_answer_group, 'resource' => \$print_resource_hash, - 'f=s' => \$format, + 'f=s' => \$format, + 'l=s' => \$lang, 'credentials=s' => \$credentials_path, 'help' => \$print_help_message, 'log=s' => \$path_to_log_file, + 'seed=s' => \$problemSeed, + 'psvn=s' => \$psvn, ); - print_help_message() if $print_help_message; ############################################################ # End Read command line options ############################################################ + +################################################################################ + +# Move up the reading of credential files to here in order to get +# WEBWORK_URL defined before it is needed. (For Docker installs when +# called from outside Docker, it may not be in the environment variables.) + + + #################################################### # get credentials #################################################### @@ -409,9 +412,11 @@ BEGIN # credentials are needed # credentials file location -- search for one of these files -our @path_list = ("$ENV{HOME}/.ww_credentials", "$ENV{HOME}/ww_session_credentials", 'ww_credentials', 'ww_credentials.dist'); -my $credentials_string = < 'http://localhost:80', # form_action_url => 'http://localhost:80/webwork2/html2xml', # site_password => 'xmlrpc', + # forcePortNumber => '80', # A port number to be forced, when needed. + # Set the identification credential used by the "daemon_course" on the remote site courseID => "daemon_course", @@ -440,7 +447,9 @@ BEGIN # running sendXMLRPC.pl # Sample settings for Mac: - # html_display_command => "open -a 'Google Chrome' ", # A web browser + + # html_display_command => "open -a 'Google Chrome' ", # A web browser + # html_display_command => "open -a Firefox ", # tex_display_command => "open -a 'TeXShop'", # Editor or TeX editor # pdf_display_command => "open -a 'Preview'", # PDF viewer @@ -461,6 +470,12 @@ BEGIN ); EOF +if (defined $credentials_path and (-r $credentials_path) ) { +# we're all set +} elsif(defined $credentials_path) { #can't find credentials + die "Can't find credentials file $credentials_path searching\n"; +} + if (defined $credentials_path and (-r $credentials_path) ) { # we're all set @@ -481,17 +496,17 @@ BEGIN } # verify that a credentials file has been found -if ( $credentials_path ) { - print "Credentials taken from file $credentials_path\n" if $verbose; + +if ( $credentials_path ) { + print STDERR "Credentials taken from file $credentials_path\n" if $verbose; } else { #failed to find credentials file die <$credentials{$_} \n";} + foreach (sort keys %credentials){print STDERR "$_ =>$credentials{$_} \n";} } + + +################################################################################ + + +use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib"; +use lib "$WeBWorK::Constants::PG_DIRECTORY/lib"; + +use WebworkClient; +use FormatRenderedProblem; +#use Proc::ProcessTable; # use in standalonePGproblemRenderer + +use 5.10.0; +$Carp::Verbose = 1; + +############################################# +# CONFIGURE +# +# Configure displays for local operating system +# This section will differ from on operating system to another +# The default code is for the macOS and applications commonly available on macOS. +############################################# + + +#Default display commands. +use constant HTML_DISPLAY_COMMAND => "open -a 'Google Chrome' "; # (MacOS command) +use constant HASH_DISPLAY_COMMAND => ""; # display tempoutputfile to STDOUT + +### Path to a temporary file for storing the output of sendXMLRPC.pl + use constant TEMPOUTPUTDIR => "$ENV{WEBWORK_ROOT}/DATA/"; + die "You must make the directory ".TEMPOUTPUTDIR(). + " writeable " unless -w TEMPOUTPUTDIR(); + use constant TEMPOUTPUTFILE => TEMPOUTPUTDIR()."temporary_output.html"; + +### Default path to a temporary file for storing the output +### of sendXMLRPC.pl +use constant LOG_FILE => "$ENV{WEBWORK_ROOT}/DATA/xmlrpc_results.log"; + +### Command for editing the pg source file in the browswer +use constant EDIT_COMMAND =>"bbedit"; # for Mac BBedit editor (used as `EDIT_COMMAND() . " $file_path") + +### Command for editing and viewing the tex output of the pg question. +use constant TEX_DISPLAY_COMMAND =>"open -a 'TeXShop'"; + +### Command for editing and viewing the tex output of the pg question. +use constant PDF_DISPLAY_COMMAND =>"open -a 'Preview'"; + +### set display mode +use constant DISPLAYMODE => 'MathJax'; +use constant PROBLEMSEED => '987654321'; + +############################################################ +# End configure displays for local operating system +############################################################ + + #allow credentials to overrride the default displayMode #and the browser display our $HTML_DISPLAY_COMMAND = $credentials{html_display_command}//HTML_DISPLAY_COMMAND(); @@ -536,7 +608,7 @@ BEGIN eval { # attempt to create log file local(*FH); - open(FH, '>>',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; + open(FH, '>>:encoding(UTF-8)',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; close(FH); }; @@ -552,17 +624,20 @@ BEGIN ############################################ my $default_input = { - userID => $credentials{userID}//'', - session_key => $credentials{session_key}//'', - courseID => $credentials{courseID}//'', - courseName => $credentials{courseID}//'', - course_password => $credentials{course_password}//'', - }; + userID => $credentials{userID}//'', + session_key => $credentials{session_key}//'', + courseID => $credentials{courseID}//'', + courseName => $credentials{courseID}//'', + course_password => $credentials{course_password}//'', +}; my $default_form_data = { - displayMode => $DISPLAYMODE, - outputformat => $format//'standard', - problemSeed => PROBLEMSEED(), + displayMode => $DISPLAYMODE, + outputformat => $format//'standard', + problemSeed => $problemSeed//PROBLEMSEED(), + psvn => $psvn//'23456', + forcePortNumber => $credentials{forcePortNumber}//'', + language => $lang//'en', }; ################################################## @@ -601,11 +676,13 @@ BEGIN if (-d $item) { # if the item is a directory traverse the tree my $dir = abs_path($item); find(\&wanted, ($dir)); + } elsif ($item eq "-") { + process_pg_file($item); # process STDIN } elsif (-f $item) { # if the item is a file process it. my $file_path = abs_path($item); - next unless $file_path =~ /\.pg$/; - next if $file_path =~ /\-text\.pg$/; - next if $file_path =~ /header/i; + next unless $file_path =~ /\.pg$/; # only process pg files + next if $file_path =~ /\-text\.pg$/; # don't process auxiliary include files + next if $file_path =~ /header/i; # don't process header files process_pg_file($file_path); } else { print "$item cannot be found or read\n"; @@ -636,9 +713,9 @@ sub process_pg_file { my $file_path = shift; my $NO_ERRORS = ""; my $ALL_CORRECT = ""; - my $problemSeed1 = 1112; my $form_data1 = { %$default_form_data, - problemSeed => $problemSeed1}; + + }; if ($display_tex_output or $display_pdf_output) { my $form_data2 = { @@ -660,6 +737,19 @@ sub process_pg_file { system($PDF_DISPLAY_COMMAND." ".$pdf_path); } } + if ($display_json_output) { + my $form_data2 = { + %$form_data1, + outputformat => 'json', + displayMode =>'MathJax', + }; + print "Creating json\n" if $UNIT_TESTS_ON; + my ($error_flag, $formatter, $error_string) = + process_problem($file_path, $default_input, $form_data2); + my $json_file_name = create_json_output($file_path, $formatter); + print( "Created JSON data in file ", TEMPOUTPUTDIR(), $json_file_name, "\n"); + exit; + } my ($error_flag, $formatter, $error_string) = process_problem($file_path, $default_input, $form_data1); # extract and display result @@ -744,7 +834,6 @@ sub process_pg_file { } #end loop collecting correct answers. # adjust input and reinitialize form_data my $form_data2 = { %$default_form_data, - problemSeed => $problemSeed1, answersSubmitted => 1, WWsubmit => 1, # grade answers WWcorrectAns => 1, # show correct answers @@ -788,12 +877,12 @@ sub process_problem { ### build client my $xmlrpc_client = new WebworkClient ( - url => $credentials{site_url}, + site_url => $credentials{site_url}, form_action_url => $credentials{form_action_url}, - site_password => $credentials{site_password}//'', - courseID => $credentials{courseID}, - userID => $credentials{userID}, - session_key => $credentials{session_key}//'', + site_password => $credentials{site_password}//'', + courseID => $credentials{courseID}, + userID => $credentials{userID}, + session_key => $credentials{session_key}//'', sourceFilePath => $adj_file_path, ); @@ -805,10 +894,13 @@ sub process_problem { my $problemSeed = $form_data->{problemSeed}; die "problem seed not defined in sendXMLRPC::process_problem" unless $problemSeed; + + my $local_psvn = $form_data->{psvn}//34567; my $updated_input = {%$input, envir => $xmlrpc_client->environment( fileName => $adj_file_path, sourceFilePath => $adj_file_path, + psvn => $local_psvn, problemSeed => $problemSeed,), }; @@ -871,7 +963,7 @@ sub process_problem { WebworkClient::writeRenderLogEntry("", "{script:$scriptName; file:$file_path; ". sprintf("duration: %.3f sec;", $cg_duration). - " url: $credentials{site_url}; }",''); + " site_url: $credentials{site_url}; }",''); ####################################################################### # End processing of the pg file @@ -1046,7 +1138,7 @@ sub create_tex_output { $file_name =~ s/\.\w+$/\.tex/; # replace extension with tex my $output_file = TEMPOUTPUTDIR().$file_name; local(*FH); - open(FH, '>', $output_file) or die "Can't open file $output_file for writing"; + open(FH, '>:encoding(UTF-8)', $output_file) or die "Can't open file $output_file for writing"; print FH $output_text; close(FH); print "tex result sent to $output_file\n" if $UNIT_TESTS_ON; @@ -1055,6 +1147,25 @@ sub create_tex_output { return $file_name; } +sub create_json_output { + my $file_path = shift; + my $formatter = shift; + my $output_text = $formatter->formatRenderedProblem; + $file_path =~s|/$||; # remove final / + $file_path =~ m|/?([^/]+)$|; + my $file_name = $1; + $file_name =~ s/\.\w+$/\.json/; # replace extension with json + my $output_file = TEMPOUTPUTDIR().$file_name; + local(*FH); + open(FH, '>:encoding(UTF-8)', $output_file) or die "Can't open file $output_file for writing"; + print FH $output_text; + close(FH); + print "json result sent to $output_file\n" if $UNIT_TESTS_ON; +# sleep 5; #wait 5 seconds +# unlink($output_file); + return $file_name; +} + sub display_html_output { #display the problem in a browser my $file_path = shift; my $formatter = shift; @@ -1065,7 +1176,7 @@ sub create_tex_output { $file_name =~ s/\.\w+$/\.html/; # replace extension with html my $output_file = TEMPOUTPUTDIR().$file_name; local(*FH); - open(FH, '>', $output_file) or die "Can't open file $output_file for writing"; + open(FH, '>:encoding(UTF-8)', $output_file) or die "Can't open file $output_file for writing"; print FH $output_text; close(FH); @@ -1154,7 +1265,7 @@ sub record_problem_ok1 { } local(*FH); - open(FH, '>>',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; + open(FH, '>>:encoding(UTF-8)',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; print FH $return_string; close(FH); return $SHORT_RETURN_STRING; @@ -1177,7 +1288,7 @@ sub record_problem_ok2 { $all_correct = ".5" if $some_correct_answers_not_specified; $ALL_CORRECT = ($all_correct == 1)?'All answers are correct':'Some answers are incorrect'; local(*FH); - open(FH, '>>',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; + open(FH, '>>:encoding(UTF-8)',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; print FH "$all_correct $file_path\n"; # do we need this? compile_errors=$error_flag\n"; close(FH); return $ALL_CORRECT; @@ -1212,12 +1323,24 @@ sub edit_source_file { sub get_source { my $file_path = shift; my $source; - die "Unable to read file $file_path \n" unless -r $file_path; + die "Unable to read file $file_path \n" unless $file_path eq '-' or -r $file_path; eval { #File::Slurp would be faster (see perl monks) local $/=undef; - open(FH, '<',$file_path) or die "Couldn't open file $file_path: $!"; - $source = ; #slurp input - close FH; + if ($file_path eq '-') { + $source = ; + } else { + # To support proper behavior with UTF-8 files, we need to open them with "<:encoding(UTF-8)" + # as otherwise, the first HTML file will render properly, but when "Preview" "Submit answer" + # or "Show correct answer" is used it will make problems, as in process_problem() the + # encodeSource() method is called on a data which is still UTF-8 encoded, and leads to double + # encoding and gibberish. + # NEW: + open(FH, "<:encoding(UTF-8)" ,$file_path) or die "Couldn't open file $file_path: $!"; + # OLD: + #open(FH, "<" ,$file_path) or die "Couldn't open file $file_path: $!"; + $source = ; #slurp input + close FH; + } }; die "Something is wrong with the contents of $file_path\n" if $@; ### adjust file_path so that it is relative to the rendering course directory @@ -1316,12 +1439,12 @@ sub print_help_message { or create a file with this information and specify it with the --credentials option. %credentials = ( - userID => "my login name for the webwork course", - course_password => "my password ", - courseID => "the name of the webwork course", - XML_URL => "url of rendering site - XML_PASSWORD => "site password" # preliminary access to site - $FORM_ACTION_URL = 'http://localhost:80/webwork2/html2xml'; #action url for form + userID => "my login name for the webwork course", + course_password => "my password ", + courseID => "the name of the webwork course", + SITE_URL => "url of rendering site", + XML_PASSWORD => "site password", # preliminary access to site + form_action_url => 'http://localhost:80/webwork2/html2xml' #action url for form ); Options @@ -1361,32 +1484,29 @@ sub print_help_message { simple -v - Verbose output. Used mostly for debugging. - In particular it displays explicitly the correct answers which are (will be) submitted to the question. + Verbose output. Used mostly for debugging. + In particular it displays explicitly the correct answers which are (will be) + submitted to the question and it specifies which credential file is used. -e - Open the source file in an editor. - - The single letter options can be "bundled" e.g. -vcCbB - - --tex - Process question in TeX mode and output to the command line - + Open the source file in an editor. + The single letter options can be "bundled" e.g. -vcCbB + --tex + Process question in TeX mode and output to the command line --pdf - Process question in TeX mode, then by pdflatex and output - to the command line + Process question in TeX mode, then by pdflatex and output + to the command line - --list pg_list - Read and process a list of .pg files contained in the file C. C - consists of a sequence of lines each of which contains the full path to a pg - file that should be processed. (For example this might be the output from an - earlier run of sendXMLRPC using the -c flag. ) + --list pg_list + Read and process a list of .pg files contained in the file C. C + consists of a sequence of lines each of which contains the full path to a pg + file that should be processed. (For example this might be the output from an + earlier run of sendXMLRPC using the -c flag. ) --pg Triggers the printing of the all of the variables available to the PG question. The table appears within the question content. Use in conjunction with -b or -B. - --anshash Prints the answer hash for each answer in the PG_debug output which appears below the question content. Use in conjunction with -b or -B. @@ -1397,20 +1517,25 @@ sub print_help_message { Prints the PGanswergroup for each answer evaluator. The information appears in the PG_debug output which follows the question content. Use in conjunction with -b or -B. This contains more information than printing the answer hash. (perhaps too much). - - --resource - Prints the resources used by the question. The information appears in - the PG_debug output which follows the question content. Use in conjunction with -b or -B. + --resource + + Prints the resources used by the question. The information appears in + the PG_debug output which follows the question content. Use in conjunction with -b or -B. + --credentials=s - Specifies a file s where the credential information can be found. + + Specifies a file s where the credential information can be found. - --help - Prints help information. - - --log - Sets path to log file + --help + Prints help information. + + --log + Sets path to log file + + --seed=s + Sets problemSeed to the number contained in string s EOT diff --git a/clients/t/Second_semester_calculus_topics.html b/clients/t/Second_semester_calculus_topics.html deleted file mode 100644 index 218f6843f0..0000000000 --- a/clients/t/Second_semester_calculus_topics.html +++ /dev/null @@ -1,327 +0,0 @@ - - - - - - mth162_overview - - - - - - - - -
-

Math 162, Second semester Calculus

- -

Here are some remarks on the topics we will cover this semester with examples
-of the kinds of homework questions we’ll ask you to work on in each homework set.

- -

Review: 0. Fundamental theorem of Calculus, Integration by Substitution, Indefinite integration

-

-Be patient while the entire page of problems loads for the first time. -

-

-Additional information about WeBWorK can be found at https://webwork.maa.org/wiki and https://webwork.maa.org/planet -

- -

- -

Play with the Geogebra applet above to get a feeling for the relationship between \(f\), the height
-of the upper boundary of the region at \(x\), and \(F\), the area of the region to the left of \(x\).

- -

\(f\) is the derivative of \(F\) and \(F\) is an anti-derivative of \(f\). A visual image of this
-relationship is useful. (Why did I write an antiderivative?)

- -

- -

- -

1. Topics: , Areas Between Curves

- -

- -

The example above is simple once you draw the picture! We’ll see how
-this can be reduced to integrating \(f(x)-g(x)\). To make this visually
-obvious imagine that the region is composed of vertical tooth picks and
-then slide them so their butts all rest on the \(x\)-axis. The tooth picks
-haven’t changed length or thickness so the area hasn’t changed either.
-(look up Cavalieri and his principle !)

- -

2. Topics: Volumes, Work

- -

- -

Visualizing and describing the region whose area or volume you want to find is often the hardest part of the problem. You’ll get lots of practice visualizing as with the example above and the one below.

- -

- -

This problem is harder (at first). You first have to visualize the region, then understand the volume of each of the thin vertical cylinders it’s made up of, (cylinders are what you get if you rotate vertical toothpicks around the \(y\) axis), and then add these
-volumes together (the Riemann sum).

- -

The Riemann integral is simply the ‘one and only’ value (mathematicians say ‘unique’ value) you get when the toothpicks are thin enough. BUT the integral has the enormous advantage that you can calculate it using anti-derivatives instead of doing lots and lots of addition of volumes.

- -

3. Topics: Integration by Parts, Trig Integrals

- -

In some ways this section is more technical – a bunch of challenges, like solving cross-word puzzles – some people find this really fun.
-As an analyst and a geometer I can tell you that integration by parts also has REALLY important theoretical uses as well.
-I can’t explain the theoretical importance to you yet however, maybe next semester if we get to Stoke’s theorem in multivariable calculus. :-)

- -

4. Topics: Trig Substitution, Partial Fractions

- -

More technical challenges. For me, when I took calculus, these challenges were one of the most fun parts of the course.

- -

But I have admit, particularly now-a-days, that Mathematica or Maple or Sage (or you guys) can solve these faster than I can.

- -

These challenges do illustrate one larger concept however. These antidifferentiation techniques are the simplest example of Inverse Problems. We can differentiate most functions \(F\) mechanically –it’s much harder to do the reverse – to guess the function \(F\) given the function \(f\) (and then check your guess by differentiating \(F\)). And yes guessing (then checking) plays a big role in Inverse Problems.

- -

“Integration techniques” are just advanced, efficient methods of guessing of anti-derivatives. (And usually it is worth your while to validate your answer by differentiating.

- -

Another inverse problem is solving differential equations, and inverse problems just get more and more interesting from there.

- -

Given the shape of a drum one can calculate what it will sound like relatively easily. If you hear a drum can you predict its shape from the sound alone? Well?? What do you think?

- -

5 Topics: Improper Integrals, Arc Length

- -

These two topics don’t fit together that well. We’ll talk about about improper integrals more later. The arc length problem is simply calculating the length of a rope when it is not straight. You do it by approximating by straight lines and then adding everything up of course. Here is an example.

- -

6. Topics: Surface Area, Curves defined by parametric equations

- -

7. Topics: Calculus of parametric curves

- -

This is the beginning of a topic which is really important to physicists and engineers – how to describe things that don’t go in straight lines. It’s also important for video game designers (oh yes and mathematicians like them as well.) Here is a sample problem – don’t try to calculate the area yet just see why the formula makes sense. (You can try visualizing as the sum of two vectors that change with time.)

- -

- -

8. Topics: Polar coordinates: Areas and lengths, Sequences

- -

Physicists and differential geometers have a magic weapon. They choose the right coordinate system to describe the problem. If you get the coordinate system right the problem becomes much simpler. Here is an example.
-

- -

9 Topics: Series, Integral test

- -

Now we get to series. A topic that most students think is impossible at first and then up wondering what they thought was so hard about it. The hidden trick here – which is not immediately apparent from the cookbook methods – is the art of approximation – finding inequalities that give you useful information.
-This is not widely taught in high schools but its a very creative and interesting skill to acquire – usually through lots of practice. The squeeze principle which you used in the first semester to find limits is an example of this type of thinking. Here is an example of a series question.

- -

Below is an example (one from the power series chapter) where you can glimpse it’s importance. A power series defines a function which is just a really long polynomial. If the first 10 decimal points of \(f\)
-are completely determined by say the first 20 terms of the series then in that case you might as well just deal with the polynomial, which you understand concretely (and computers can calculate with). “Functions are just long polynomials.” This isn’t strictly true but it’s 90% true and thinking this way gets you started in the right direction.

- -

To get started solving the problem below just plug in some values. You can use the convention that \(0^0=1\) –it’s defined that way because it makes these sigma summation formulas easy to write.

- -

Don’t be worried by the sigma notation – get someone to show you how to write it out in long string … or look it up on the web.
-

- -

10 Topics: Comparison tests, Alternating series

- -

Positive series (where each term is positive) have two choices. Each term is like a derivative (the speed) and the sum is how far you have traveled. Since the speed is positive either you travel all the way to infinity along the \(x\) axis or you travel up to some finite bound (Think \(1/2 + 1/4 +1/8 +...\) can you ever get beyond \(1\) when each term is slowing down that fast? Once you realize this, the trick is to find a known series that goes only a finite distance but is larger than your series (which proves your series converges) or find a known series that is smaller and travels off to infinity, pushing your series ahead of it. Everything else is just details of finding one or the other of these known series.

- -

11 Topics: Ratio and Root tests, Absolute convergence

- -

These are the details – we’ll talk about them when the time comes.

- -

12 Topics: Power series

- -

More details – specialized for power series “really long polynomials”.

- -

13 Topics: Representation of functions as power series, Taylor series

- -

As I said above there is a sense in which every reasonable function is just a long polynomials. Talyor series come at the end of this course but they are the foundation of most applications and a lot of theory. Just ask anyone taking MTH280 (numerical analysis) or MTH285(applied mathematics) or MTH282 (Complex analysis). They model reality, they solve differential equations. Of course there are a few functions which are much weirder and don’t behave much like long polynomials and mathematicians (and some physicists) get excited about understanding those if for no other reason that they are a challenge to figure out.

- -

Topics: Application of power series

- -

Power series are very useful. Here is an example.
-

- -

For the JMM presentation

- -

Here is the magic code for embedding these problems.

-
&lt; iframe  width="800" height="400" 
-src="https://hosted2.webwork.rochester.edu/webwork2/html2xml?
-	&answersSubmitted=0&
-	&sourceFilePath=Library/Union/setSeriesTaylor/ur_sr_9_6.pg&
-	&problemSeed=123567&
-	&courseID=daemon_course&
-	&userID=daemon&
-	&course_password=daemon&
-	&showSummary=1&
-	&displayMode=MathJax&
-	&problemIdentifierPrefix=102&
-	&language=en&
-	&outputformat=sticky">
-&lt; /iframe>
-

-
-
-
-
- - -
- - \ No newline at end of file diff --git a/clients/t/Second_semester_calculus_topics2.html b/clients/t/Second_semester_calculus_topics2.html deleted file mode 100644 index 218f6843f0..0000000000 --- a/clients/t/Second_semester_calculus_topics2.html +++ /dev/null @@ -1,327 +0,0 @@ - - - - - - mth162_overview - - - - - - - - -
-

Math 162, Second semester Calculus

- -

Here are some remarks on the topics we will cover this semester with examples
-of the kinds of homework questions we’ll ask you to work on in each homework set.

- -

Review: 0. Fundamental theorem of Calculus, Integration by Substitution, Indefinite integration

-

-Be patient while the entire page of problems loads for the first time. -

-

-Additional information about WeBWorK can be found at https://webwork.maa.org/wiki and https://webwork.maa.org/planet -

- -

- -

Play with the Geogebra applet above to get a feeling for the relationship between \(f\), the height
-of the upper boundary of the region at \(x\), and \(F\), the area of the region to the left of \(x\).

- -

\(f\) is the derivative of \(F\) and \(F\) is an anti-derivative of \(f\). A visual image of this
-relationship is useful. (Why did I write an antiderivative?)

- -

- -

- -

1. Topics: , Areas Between Curves

- -

- -

The example above is simple once you draw the picture! We’ll see how
-this can be reduced to integrating \(f(x)-g(x)\). To make this visually
-obvious imagine that the region is composed of vertical tooth picks and
-then slide them so their butts all rest on the \(x\)-axis. The tooth picks
-haven’t changed length or thickness so the area hasn’t changed either.
-(look up Cavalieri and his principle !)

- -

2. Topics: Volumes, Work

- -

- -

Visualizing and describing the region whose area or volume you want to find is often the hardest part of the problem. You’ll get lots of practice visualizing as with the example above and the one below.

- -

- -

This problem is harder (at first). You first have to visualize the region, then understand the volume of each of the thin vertical cylinders it’s made up of, (cylinders are what you get if you rotate vertical toothpicks around the \(y\) axis), and then add these
-volumes together (the Riemann sum).

- -

The Riemann integral is simply the ‘one and only’ value (mathematicians say ‘unique’ value) you get when the toothpicks are thin enough. BUT the integral has the enormous advantage that you can calculate it using anti-derivatives instead of doing lots and lots of addition of volumes.

- -

3. Topics: Integration by Parts, Trig Integrals

- -

In some ways this section is more technical – a bunch of challenges, like solving cross-word puzzles – some people find this really fun.
-As an analyst and a geometer I can tell you that integration by parts also has REALLY important theoretical uses as well.
-I can’t explain the theoretical importance to you yet however, maybe next semester if we get to Stoke’s theorem in multivariable calculus. :-)

- -

4. Topics: Trig Substitution, Partial Fractions

- -

More technical challenges. For me, when I took calculus, these challenges were one of the most fun parts of the course.

- -

But I have admit, particularly now-a-days, that Mathematica or Maple or Sage (or you guys) can solve these faster than I can.

- -

These challenges do illustrate one larger concept however. These antidifferentiation techniques are the simplest example of Inverse Problems. We can differentiate most functions \(F\) mechanically –it’s much harder to do the reverse – to guess the function \(F\) given the function \(f\) (and then check your guess by differentiating \(F\)). And yes guessing (then checking) plays a big role in Inverse Problems.

- -

“Integration techniques” are just advanced, efficient methods of guessing of anti-derivatives. (And usually it is worth your while to validate your answer by differentiating.

- -

Another inverse problem is solving differential equations, and inverse problems just get more and more interesting from there.

- -

Given the shape of a drum one can calculate what it will sound like relatively easily. If you hear a drum can you predict its shape from the sound alone? Well?? What do you think?

- -

5 Topics: Improper Integrals, Arc Length

- -

These two topics don’t fit together that well. We’ll talk about about improper integrals more later. The arc length problem is simply calculating the length of a rope when it is not straight. You do it by approximating by straight lines and then adding everything up of course. Here is an example.

- -

6. Topics: Surface Area, Curves defined by parametric equations

- -

7. Topics: Calculus of parametric curves

- -

This is the beginning of a topic which is really important to physicists and engineers – how to describe things that don’t go in straight lines. It’s also important for video game designers (oh yes and mathematicians like them as well.) Here is a sample problem – don’t try to calculate the area yet just see why the formula makes sense. (You can try visualizing as the sum of two vectors that change with time.)

- -

- -

8. Topics: Polar coordinates: Areas and lengths, Sequences

- -

Physicists and differential geometers have a magic weapon. They choose the right coordinate system to describe the problem. If you get the coordinate system right the problem becomes much simpler. Here is an example.
-

- -

9 Topics: Series, Integral test

- -

Now we get to series. A topic that most students think is impossible at first and then up wondering what they thought was so hard about it. The hidden trick here – which is not immediately apparent from the cookbook methods – is the art of approximation – finding inequalities that give you useful information.
-This is not widely taught in high schools but its a very creative and interesting skill to acquire – usually through lots of practice. The squeeze principle which you used in the first semester to find limits is an example of this type of thinking. Here is an example of a series question.

- -

Below is an example (one from the power series chapter) where you can glimpse it’s importance. A power series defines a function which is just a really long polynomial. If the first 10 decimal points of \(f\)
-are completely determined by say the first 20 terms of the series then in that case you might as well just deal with the polynomial, which you understand concretely (and computers can calculate with). “Functions are just long polynomials.” This isn’t strictly true but it’s 90% true and thinking this way gets you started in the right direction.

- -

To get started solving the problem below just plug in some values. You can use the convention that \(0^0=1\) –it’s defined that way because it makes these sigma summation formulas easy to write.

- -

Don’t be worried by the sigma notation – get someone to show you how to write it out in long string … or look it up on the web.
-

- -

10 Topics: Comparison tests, Alternating series

- -

Positive series (where each term is positive) have two choices. Each term is like a derivative (the speed) and the sum is how far you have traveled. Since the speed is positive either you travel all the way to infinity along the \(x\) axis or you travel up to some finite bound (Think \(1/2 + 1/4 +1/8 +...\) can you ever get beyond \(1\) when each term is slowing down that fast? Once you realize this, the trick is to find a known series that goes only a finite distance but is larger than your series (which proves your series converges) or find a known series that is smaller and travels off to infinity, pushing your series ahead of it. Everything else is just details of finding one or the other of these known series.

- -

11 Topics: Ratio and Root tests, Absolute convergence

- -

These are the details – we’ll talk about them when the time comes.

- -

12 Topics: Power series

- -

More details – specialized for power series “really long polynomials”.

- -

13 Topics: Representation of functions as power series, Taylor series

- -

As I said above there is a sense in which every reasonable function is just a long polynomials. Talyor series come at the end of this course but they are the foundation of most applications and a lot of theory. Just ask anyone taking MTH280 (numerical analysis) or MTH285(applied mathematics) or MTH282 (Complex analysis). They model reality, they solve differential equations. Of course there are a few functions which are much weirder and don’t behave much like long polynomials and mathematicians (and some physicists) get excited about understanding those if for no other reason that they are a challenge to figure out.

- -

Topics: Application of power series

- -

Power series are very useful. Here is an example.
-

- -

For the JMM presentation

- -

Here is the magic code for embedding these problems.

-
&lt; iframe  width="800" height="400" 
-src="https://hosted2.webwork.rochester.edu/webwork2/html2xml?
-	&answersSubmitted=0&
-	&sourceFilePath=Library/Union/setSeriesTaylor/ur_sr_9_6.pg&
-	&problemSeed=123567&
-	&courseID=daemon_course&
-	&userID=daemon&
-	&course_password=daemon&
-	&showSummary=1&
-	&displayMode=MathJax&
-	&problemIdentifierPrefix=102&
-	&language=en&
-	&outputformat=sticky">
-&lt; /iframe>
-

-
-
-
-
- - -
- - \ No newline at end of file diff --git a/clients/t/TestWW.html b/clients/t/TestWW.html index 4de7de11af..393a04a816 100644 --- a/clients/t/TestWW.html +++ b/clients/t/TestWW.html @@ -4,8 +4,8 @@ Embed WW in HTML page - - + +
+
+ + + + + + \ No newline at end of file diff --git a/clients/t/test_formats_WWproblems.html b/clients/t/test_formats_WWproblems.html new file mode 100644 index 0000000000..1f37c23dbf --- /dev/null +++ b/clients/t/test_formats_WWproblems.html @@ -0,0 +1,106 @@ + + + + + test formats for embedded problems + + + + +

Format: sticky

+

+

Format: standard (has error section -- but there should be no errors)

+

+ +

Format: simple

+

+ +

Format: body_text

+

+ +

Format: PTX

+

+ +

Format: JSON

+

+ + + diff --git a/clients/t/test_simpleFormat_WWproblem.html b/clients/t/test_simpleFormat_WWproblem.html new file mode 100644 index 0000000000..079cd45992 --- /dev/null +++ b/clients/t/test_simpleFormat_WWproblem.html @@ -0,0 +1,64 @@ + + + + + Embed WW in HTML page + + + + + + + + + + +

Problem 1-- xmlrpc version of a simple template

+

+Problem is rendered at daemon_course -- no student information is retained. +

+
+ + +
+

Problem 2 -- webwork2 version of a simple template

+

+Direct access using RESTful url. Sign in is necessary and score is recorded. +

+
+ +
+ + diff --git a/clients/t/test_toggled_embedded_WWproblems.html b/clients/t/test_toggled_embedded_WWproblems.html new file mode 100644 index 0000000000..2587e2726f --- /dev/null +++ b/clients/t/test_toggled_embedded_WWproblems.html @@ -0,0 +1,202 @@ + + + + + Embedded WeBWorK Problems + + + + + + + + + + + + +

Embedded WeBWorK Problems

+

+Problem 1 -- interval notation +

+ +
+ Click here to toggle problem display +
+ +

+Problem 2 -- integration. Contains a Geogebra applet +

+ +
+ Click here to toggle problem display +
+ + +

+Problem 3 -- the pg code for this problem is embedded in the html text using base64 encoding +

+ +
+ Click here to toggle problem display +
+ + +

+Problem 4 -- PG lab. Enter PG code to test out here. +

+ +
+ Click here to toggle problem display +
+ + + + +

+Problem 5 -- debug -- This problem is useful for debugging and shows all the details of the hash returned by PG. +

+ +
+ Click here to toggle problem display +
+ + +

+Problem 6 -- A logic problem contributed from Montana State University. There +is a link "contribLibrary" in the daemon course to the "Contrib" library. See +http://github.com/openwebwork/webwork-open-problem-library +to see which problems are available in the OpenProblemLibrary and in Contrib. You can +also use the library browser to find the problems available in the OPL. + +

+ +
+ Click here to toggle problem display +
+ +
+
+ Click here to toggle problem display: (PGinfo example) +
+