diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index a4ccac98..f14dc9dd 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -20,13 +20,12 @@
- [ ] Refactoring / documentation update (non-breaking change which does not change functionality)
- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
-- [ ] Breaking change (fix, feature, or deprecation that would cause existing functionality to change)
+- [ ] Breaking change (fix, feature, or removal that would cause existing functionality to change)
- [ ] Release preparation
## Checklist:
-- [ ] `xdmod_data/__version__.py` has been updated to the next development version
- [ ] The milestone is set correctly on the pull request
- [ ] The appropriate labels have been added to the pull request
- [ ] Updates have been made to the `xdmod-notebooks` repository as necessary, and the notebooks all run successfully
diff --git a/docs/developing.md b/docs/developing.md
index db3647bc..47a076cb 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -116,8 +116,6 @@ git+https://github.com/username/xdmod-data.git@branch-name
1. In the `README.md`, update the Open XDMoD compatibility matrix.
1. Get the PR approved and merged.
1. In a PR to the same branch you just released:
- 1. In `xdmod_data/__version__.py`, make sure the version number is updated
- to a development pre-release of the next version, e.g., `3.0.0.dev01`.
1. In `README.md`:
1. If you just released a new major version:
1. Under the main heading, in the sentence that begins `This
@@ -141,5 +139,6 @@ git+https://github.com/username/xdmod-data.git@branch-name
1. Add the expected compatibility with Open XDMoD versions.
1. In `README.md`, add the new version under development to the Open XDMoD
compatibility matrix.
- 1. In `xdmod_data/__version__.py`, change the version number to the new
- version under development.
+ 1. In `xdmod_data/__version__.py`, change the version number to a
+ development release of the new version under development, e.g.,
+ `3.0.0.dev1`.
diff --git a/tests/ci/scripts/run-tests.sh b/tests/ci/scripts/run-tests.sh
index c9d03a04..4a591b06 100755
--- a/tests/ci/scripts/run-tests.sh
+++ b/tests/ci/scripts/run-tests.sh
@@ -20,103 +20,112 @@ declare -a xdmod_containers=$(yq '.services | keys | .[] | select(. == "xdmod-*"
# with Flake 8, and install the package and its testing
# dependencies.
for python_container in $python_containers; do
- docker cp $PROJECT_DIR/. $python_container:/home/circleci/project
- docker exec $python_container bash -c 'sudo chown -R circleci:circleci /home/circleci/project'
- docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade pip'
- docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade flake8 flake8-commas flake8-quotes'
- docker exec -w /home/circleci/project $python_container bash -c 'python3 -m flake8 . --max-complexity=10 --show-source --exclude __init__.py'
- docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install -e .'
- docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade python-dotenv pytest coverage'
- # The minimum version of each dependency should be tested in the
- # container with the minimum Python version.
- if [ "$python_container" = "python-min" ]; then
- min_dependency_versions=$(awk \
- '/install_requires/ {flag=1} flag && !/install_requires/ && NF {print $0} flag && /^\[.*\]$/ {flag=0}' \
- setup.cfg | tr -d '\n' | sed 's/ >= /==/g'
- )
- docker exec -w /home/circleci/project $python_container bash -c "python3 -m pip install --force-reinstall $min_dependency_versions"
- fi
+ docker cp $PROJECT_DIR/. $python_container:/home/circleci/project
+ docker exec $python_container bash -c 'sudo chown -R circleci:circleci /home/circleci/project'
+ docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade pip'
+ docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade flake8 flake8-commas flake8-quotes'
+ docker exec -w /home/circleci/project $python_container bash -c 'python3 -m flake8 . --max-complexity=10 --show-source --exclude __init__.py'
+ docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install -e .'
+ docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade python-dotenv pytest coverage'
+ # The minimum version of each dependency should be tested in the
+ # container with the minimum Python version.
+ if [ "$python_container" = "python-min" ]; then
+ min_dependency_versions=$(awk \
+ '/install_requires/ {flag=1} flag && !/install_requires/ && NF {print $0} flag && /^\[.*\]$/ {flag=0}' \
+ setup.cfg | tr -d '\n' | sed 's/ >= /==/g'
+ )
+ docker exec -w /home/circleci/project $python_container bash -c "python3 -m pip install --force-reinstall $min_dependency_versions"
+ fi
done
# Set up XDMoD web server containers.
for xdmod_container in $xdmod_containers; do
- # Generate OpenSSL key and certificate.
- docker exec $xdmod_container bash -c "openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/$xdmod_container.key"
- docker exec $xdmod_container bash -c "openssl req -new -key /etc/pki/tls/private/$xdmod_container.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/$xdmod_container.crt -subj '/C=XX/L=Default City/O=Default Company Ltd/CN=$xdmod_container' -addext 'subjectAltName=DNS:$xdmod_container'"
- # Update the server hostnames and certificates so the Python
- # containers can make requests to them.
- docker exec $xdmod_container bash -c "sed -i \"s/localhost/$xdmod_container/g\" /etc/httpd/conf.d/xdmod.conf"
- if [[ "$xdmod_container" =~ 'xdmod-*-dev' ]]; then
- if [ "$xdmod_container" = 'xdmod-main-dev' ]; then
- branch='main'
- else
- branch="xdmod$(echo $xdmod_container | sed 's/xdmod-\(.*\)-dev/\1/' | sed 's/-/./')"
+ # Generate OpenSSL key and certificate.
+ docker exec $xdmod_container bash -c "openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/$xdmod_container.key"
+ docker exec $xdmod_container bash -c "openssl req -new -key /etc/pki/tls/private/$xdmod_container.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/$xdmod_container.crt -subj '/C=XX/L=Default City/O=Default Company Ltd/CN=$xdmod_container' -addext 'subjectAltName=DNS:$xdmod_container'"
+ # For the containers with the development versions of the XDMoD web server,
+ if [[ "$xdmod_container" =~ xdmod-.+-dev ]]; then
+ # Install and run the latest development version of the XDMoD
+ # web server.
+ if [ "$xdmod_container" = 'xdmod-main-dev' ]; then
+ branch='main'
+ else
+ branch="xdmod$(echo $xdmod_container | sed 's/xdmod-\(.*\)-dev/\1/' | sed 's/-/./')"
+ fi
+ docker exec $xdmod_container bash -c "git clone --depth=1 --branch=$branch https://github.com/ubccr/xdmod.git /root/xdmod"
+ docker exec -w /root/xdmod $xdmod_container bash -c 'composer install'
+ docker exec -w /root/xdmod $xdmod_container bash -c '/root/bin/buildrpm xdmod'
+ # Work around the fact that the 11.0 version of bootstrap.sh contains
+ # an extra prompt in xdmod-upgrade.tcl for upgrading from 10.5 to 11.0,
+ # and in this case we are upgrading from an earlier version of 11.0 to
+ # the latest development version of 11.0, so that prompt should not be
+ # expected.
+ if [ "$xdmod_container" = 'xdmod-11-0-dev' ]; then
+ docker exec -w /root/xdmod $xdmod_container bash -c 'sed -i "/^confirmResourceSpecs/d" tests/ci/scripts/xdmod-upgrade.tcl'
+ fi
+ docker exec -w /root/xdmod $xdmod_container bash -c 'XDMOD_TEST_MODE=upgrade bash -x ./tests/ci/bootstrap.sh'
+ docker exec -w /root/xdmod $xdmod_container bash -c './tests/ci/validate.sh'
fi
- # Install and run the latest development version of the XDMoD
- # web server.
- docker exec $xdmod_container bash -c 'git clone --depth=1 --branch=$branch https://github.com/ubccr/xdmod.git /root/xdmod'
- docker exec -w /root/xdmod $xdmod_container bash -c 'composer install'
- docker exec -w /root/xdmod $xdmod_container bash -c '/root/bin/buildrpm xdmod'
- docker exec -w /root/xdmod $xdmod_container bash -c 'XDMOD_TEST_MODE=upgrade ./tests/ci/bootstrap.sh'
- docker exec -w /root/xdmod $xdmod_container bash -c './tests/ci/validate.sh'
- elif [[ "$xdmod_container" =~ xdmod-* ]]; then
- # Run the XDMoD web server.
- docker exec $xdmod_container bash -c '/root/bin/services start'
- fi
- # Copy the 10,000 users file into the container and shred it.
- # We use this file so we can test filters with more than 10,000
- # values and date ranges that span multiple quarters.
- docker cp $PROJECT_DIR/tests/ci/artifacts/10000users.log $xdmod_container:.
- docker exec $xdmod_container xdmod-shredder -r frearson -f slurm -i 10000users.log
- # Ingest and aggregate.
- date=$(date --utc +%Y-%m-%d)
- docker exec $xdmod_container xdmod-ingestor --ingest
- docker exec $xdmod_container xdmod-ingestor --aggregate=job --last-modified-start-date $date
- # Copy certificate (for doing requests) from the XDMoD container.
- docker cp $xdmod_container:/etc/pki/tls/certs/$xdmod_container.crt $PROJECT_DIR
- # Copy certificate to one of the Python containers and get an
- # XDMoD API token for the XDMoD container.
- docker cp $xdmod_container.crt $python_container:/home/circleci/project
- rest_token=$(docker exec \
- -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
- $python_container \
- bash -c "curl \
- -sS \
- -X POST \
- -c xdmod.cookie \
- -d 'username=normaluser&password=normaluser' \
- https://$xdmod_container/rest/auth/login \
- | jq -r '.results.token'"
- )
- docker exec $python_container bash -c 'echo -n "XDMOD_API_TOKEN="' > ${xdmod_container}-token
- docker exec \
- -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
- $python_container \
- bash -c "curl \
- -sS \
- -X POST \
- -b xdmod.cookie \
- https://$xdmod_container/rest/users/current/api/token?token=$rest_token \
- | jq -r '.data.token'" \
- >> ${xdmod_container}-token
+ # Update the server hostnames and certificates so the Python containers can
+ # make requests to them.
+ docker exec $xdmod_container bash -c "sed -i 's/localhost/$xdmod_container/g' /etc/httpd/conf.d/xdmod.conf"
+ # (Re)start the XDMoD-related services.
+ docker exec $xdmod_container bash -c '/root/bin/services restart'
+ # Copy the 10,000 users file into the container and shred it.
+ # We use this file so we can test filters with more than 10,000
+ # values and date ranges that span multiple quarters.
+ docker cp $PROJECT_DIR/tests/ci/artifacts/10000users.log $xdmod_container:.
+ docker exec $xdmod_container xdmod-shredder -r frearson -f slurm -i 10000users.log
+ # Ingest and aggregate.
+ date=$(date --utc +%Y-%m-%d)
+ docker exec $xdmod_container xdmod-ingestor --ingest
+ docker exec $xdmod_container xdmod-ingestor --aggregate=job --last-modified-start-date $date
+ # Copy certificate (for doing requests) from the XDMoD container.
+ docker cp $xdmod_container:/etc/pki/tls/certs/$xdmod_container.crt $PROJECT_DIR
+ # For each of the python containers,
+ for python_container in $python_containers; do
+ # Copy the certificate to the container.
+ docker cp $xdmod_container.crt $python_container:/home/circleci/project
+ done
+ # Use one of the Python containers to make requests to the XDMoD container
+ # to obtain an API token that will be used by all the Python containers.
+ rest_token=$(docker exec \
+ -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
+ $python_container \
+ bash -c "curl \
+ -sS \
+ -X POST \
+ -c xdmod.cookie \
+ -d 'username=normaluser&password=normaluser' \
+ https://$xdmod_container/rest/auth/login \
+ | jq -r '.results.token'"
+ )
+ api_token=$(docker exec \
+ -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
+ $python_container \
+ bash -c "curl \
+ -sS \
+ -X POST \
+ -b xdmod.cookie \
+ https://$xdmod_container/rest/users/current/api/token?token=$rest_token \
+ | jq -r '.data.token'"
+ )
+ echo "XDMOD_API_TOKEN=$api_token" > ${xdmod_container}-token
done
# Run the tests against each XDMoD web server.
for python_container in $python_containers; do
- for xdmod_container in $xdmod_containers; do
- # Copy certificate (for doing requests) to the Python
- # container.
- docker cp $xdmod_container.crt $python_container:/home/circleci/project
- # Copy XDMoD API token to the Python container.
- docker cp $PROJECT_DIR/${xdmod_container}-token $python_container:/home/circleci/.xdmod-data-token
- # Run tests in the Python container.
- docker exec \
- -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
- -e XDMOD_HOST="https://$xdmod_container" \
- -e XDMOD_VERSION="$xdmod_container" \
- $python_container \
- bash -c 'python3 -m coverage run --branch --append -m pytest -vvs -o log_cli=true tests/'
- done
- # Make sure 100% test coverage.
- docker exec $python_container bash -c 'python3 -m coverage report -m --fail-under=100'
+ for xdmod_container in $xdmod_containers; do
+ # Copy XDMoD API token to the Python container.
+ docker cp $PROJECT_DIR/${xdmod_container}-token $python_container:/home/circleci/.xdmod-data-token
+ # Run tests in the Python container.
+ docker exec \
+ -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
+ -e XDMOD_HOST="https://$xdmod_container" \
+ -e XDMOD_VERSION="$xdmod_container" \
+ $python_container \
+ bash -c 'python3 -m coverage run --branch --append -m pytest -vvs -o log_cli=true tests/'
+ done
+ # Make sure 100% test coverage.
+ docker exec $python_container bash -c 'python3 -m coverage report -m --fail-under=100'
done
diff --git a/tests/regression/data/default/jobs-dimensions.csv b/tests/regression/data/default/jobs-dimensions.csv
index 2f2acd90..5657e069 100644
--- a/tests/regression/data/default/jobs-dimensions.csv
+++ b/tests/regression/data/default/jobs-dimensions.csv
@@ -9,9 +9,12 @@ jobwalltime,Job Wall Time,A categorization of jobs into discrete groups based on
nodecount,Node Count,A categorization of jobs into discrete groups based on node count.
pi,PI,The principal investigator of a project.
fieldofscience,PI Group,PI Group
+pi_institution,PI Institution,Organizations that have PIs with allocations.
qos,QOS,The quality of service of the job
queue,Queue,Queue pertains to the low level job queues on each resource.
resource,Resource,A resource is a remote computer that can run jobs.
resource_type,Resource Type,A categorization of resources into by their general capabilities.
+provider,Service Provider,A service provider is an institution that hosts resources.
username,System Username,The specific system username of the users who ran jobs.
person,User,"A person who is on a PIs allocation, hence able to run jobs on resources."
+institution,User Institution,Organizations that have users with allocations.
diff --git a/tests/regression/data/default/jobs-metrics.csv b/tests/regression/data/default/jobs-metrics.csv
index 0e3e28ca..1016375e 100644
--- a/tests/regression/data/default/jobs-metrics.csv
+++ b/tests/regression/data/default/jobs-metrics.csv
@@ -6,7 +6,6 @@ avg_gpu_hours,GPU Hours: Per Job,"The average GPU hours (number of GPU cores x w
total_gpu_hours,GPU Hours: Total,"The total GPU hours (number of GPUs x wall time hours) used by Screwdriver jobs.
For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours."
max_processors,Job Size: Max (Core Count),The maximum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job.
min_processors,Job Size: Min (Core Count),The minimum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job.
-normalized_avg_processors,Job Size: Normalized (% of Total Cores),"The percentage average size Screwdriver job divided by the total number of cores in the resource where the job ran. The job normalization calculation assumes that the resource size is constant. This statistic should not be used with a time range where the resource size changes, because the statistic will be incorrect.
Normalized Job Size: The ratio of the total number of processor cores used by a (parallel) job over the total number of cores on the resource."
avg_processors,Job Size: Per Job (Core Count),The average job size per Screwdriver job.
Job Size: The number of processor cores used by a (parallel) job.
avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average Screwdriver job size weighted by CPU Hours. Defined as
Average Job Size Weighted By CPU Hours: sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i = 0 to n){job i cpu hours}
avg_job_size_weighted_by_gpu_hours,Job Size: Weighted By GPU Hours (GPU Count),The average Screwdriver job size weighted by GPU Hours. Defined as
Average Job Size Weighted By GPU Hours: sum(i = 0 to n){ job i core count * job i gpu hours}/sum(i = 0 to n){job i gpu hours}
diff --git a/tests/regression/data/default/raw-data-every-1000-no-fields-no-filters.csv b/tests/regression/data/default/raw-data-every-1000-no-fields-no-filters.csv
index 42a56249..23245476 100644
--- a/tests/regression/data/default/raw-data-every-1000-no-fields-no-filters.csv
+++ b/tests/regression/data/default/raw-data-every-1000-no-fields-no-filters.csv
@@ -1,56 +1,56 @@
-,Local Job Id,Resource,Timezone,System Username (Deidentified),User,Organization,Quality of Service,Submit Time (Timestamp),Start Time (Timestamp),End Time (Timestamp),Eligible Time (Timestamp),Nodes,Cores,GPUs,Memory Used,Wall Time,Wait Time,Core Time,GPU Time,Exit Code,Exit State,Requested Cores,Requested memory,Requested Wall Time,Queue,Decanal Unit,Department,PI Group
-0,6110386,Robertson,UTC,4003ebd9b2de239734970b4cf32cc3d6183d2c6b,"Harrier, Hen",Screwdriver,curry,1483118438,1483118438,1483118443,1483118438,1,1,0,-1,5,0,5,0,0:0,COMPLETED,1,48000Mn,172800,white,Computer and Information Science and Engineering,Computer and Computation Research,Computer and Computation Theory
-1000,970339,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483064122,1483064124,1483064151,1483064122,1,8,0,-1,27,2,216,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-2000,981731,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483102441,1483102444,1483102466,1483102441,1,8,0,-1,22,3,176,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-3000,6112283[436],Robertson,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,curry,1483135648,1483139913,1483139937,1483135648,1,1,0,-1,24,4265,24,0,0:0,COMPLETED,1,48000Mn,10800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
-4000,178502,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483116701,1483130204,1483130230,1483116701,1,8,0,-1,26,13503,208,0,1:0,FAILED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-5000,974487,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483077974,1483077974,1483078015,1483077974,1,8,0,-1,41,0,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-6000,982727,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483106505,1483106505,1483106556,1483106505,1,8,0,-1,51,0,408,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-7000,2281895,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483115132,1483115132,1483116616,1483115132,1,12,0,-1,1484,0,17808,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-8000,972196,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483070299,1483070301,1483070337,1483070299,1,8,0,-1,36,2,288,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-9000,978066,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483088977,1483088978,1483089027,1483088977,1,8,0,-1,49,1,392,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-10000,983035,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483107862,1483107863,1483108007,1483107862,1,8,0,-1,144,1,1152,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-11000,1009935,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483130897,1483130908,1483131130,1483130897,1,8,0,-1,222,11,1776,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
-12000,2284391,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483137557,1483137558,1483139202,1483137557,1,12,0,-1,1644,1,19728,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-13000,1005349,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483071422,1483071534,1483071778,1483071422,1,8,0,-1,244,112,1952,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
-14000,1007824,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483105333,1483105429,1483105948,1483105333,1,8,0,-1,519,96,4152,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
-15000,1010629,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483140307,1483140400,1483140550,1483140307,1,8,0,-1,150,93,1200,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
-16000,6111037[473],Robertson,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,curry,1483125912,1483129603,1483129747,1483125913,1,1,0,-1,144,3691,144,0,0:0,COMPLETED,1,2147487648Mn,900,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
-17000,988528,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483119249,1483125280,1483125321,1483119249,1,8,0,-1,41,6031,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-18000,991993,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483131196,1483136908,1483136994,1483131196,1,8,0,-1,86,5712,688,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-19000,1008740,Posidriv,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483117402,1483124540,1483126240,1483117402,1,12,0,-1,1700,7138,20400,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-20000,2276090,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483060299,1483060299,1483062513,1483060299,1,12,0,-1,2214,0,26568,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-21000,967122,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483053110,1483053111,1483056506,1483053110,1,8,0,-1,3395,1,27160,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-22000,2283293,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483126817,1483126818,1483128722,1483126817,1,12,0,-1,1904,1,22848,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-23000,1008638,Posidriv,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483116706,1483122377,1483124429,1483116706,1,12,0,-1,2052,5671,24624,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-24000,972040,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483069980,1483069981,1483074019,1483069980,1,8,0,-1,4038,1,32304,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-25000,991884,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483130901,1483136404,1483141877,1483130901,1,8,0,-1,5473,5503,43784,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-26000,6105349,Robertson,UTC,c1af1c2854199beceb6efb1041d32af91fa30b6a,Moorhen,Screwdriver,curry,1482948669,1482948735,1483066377,1482948669,1,1,0,-1,117642,66,117642,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication
-27000,1005309,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483208573,1483209389,1483209414,1483208573,1,8,0,-1,25,816,200,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-28000,6114243[425],Robertson,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,curry,1483202456,1483205646,1483205659,1483202456,1,1,0,-1,13,3190,13,0,0:0,COMPLETED,1,2147531648Mn,1800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
-29000,998101,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483159238,1483159264,1483154003,1,8,0,-1,26,5235,208,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-30000,184276,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483158935,1483191164,1483191182,1483158935,1,8,0,-1,18,32229,144,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-31000,2292895,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483183969,1483183969,1483185436,1483183969,1,12,0,-1,1467,0,17604,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-32000,2302043,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483211208,1483211208,1483212911,1483211208,1,12,0,-1,1703,0,20436,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-33000,2286102,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483154004,1483155247,1483154003,1,12,0,-1,1243,1,14916,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-34000,2295330,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483191135,1483191136,1483192490,1483191135,1,12,0,-1,1354,1,16248,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-35000,2304491,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483218813,1483218814,1483219999,1483218813,1,12,0,-1,1185,1,14220,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-36000,1004395,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483206031,1483207038,1483207076,1483206031,1,8,0,-1,38,1007,304,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-37000,1007905,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483216837,1483217890,1483217928,1483216837,1,8,0,-1,38,1053,304,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-38000,189492,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483219052,1483220591,1483220646,1483219052,1,8,0,-1,55,1539,440,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-39000,1014185,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483177983,1483178402,1483178494,1483177983,1,8,0,-1,92,419,736,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
-40000,6112860,Robertson,UTC,218266b45cc856b3bd485c25a25271f77c7108dc,Dunlin,Screwdriver,curry,1483142479,1483143257,1483143695,1483142479,1,12,0,-1,438,778,5256,0,0:0,COMPLETED,12,2147486448Mn,540,white,Biological Sciences,Environmental Biology,Systematic and Population Biology
-41000,993900,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483137554,1483144161,1483144201,1483137554,1,8,0,-1,40,6607,320,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-42000,997133,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483150115,1483156319,1483156355,1483150115,1,8,0,-1,36,6204,288,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-43000,1000379,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483160618,1483167881,1483167946,1483160618,1,8,0,-1,65,7263,520,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-44000,1003108,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483172599,1483182242,1483182311,1483172599,1,8,0,-1,69,9643,552,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-45000,183442,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483151833,1483184199,1483184231,1483151833,1,8,0,-1,32,32366,256,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-46000,2288079,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483166466,1483166466,1483168335,1483166466,1,12,0,-1,1869,0,22428,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-47000,2300772,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483207320,1483207320,1483209262,1483207320,1,12,0,-1,1942,0,23304,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-48000,2290078,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483174731,1483174734,1483177171,1483174731,1,12,0,-1,2437,3,29244,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-49000,2304088,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483217556,1483217557,1483219647,1483217556,1,12,0,-1,2090,1,25080,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-50000,1011153,Posidriv,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483146809,1483157125,1483159016,1483146809,1,12,0,-1,1891,10316,22692,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-51000,1006854,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483213460,1483214511,1483219230,1483213460,1,8,0,-1,4719,1051,37752,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-52000,6106621,Robertson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,curry,1483007306,1483150677,1483157807,1483007306,1,8,0,-1,7130,143371,57040,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
-53000,6110124,Robertson,UTC,56762592f0cc50c30d2379380f23c53c4e56fa1c,"Grey, Lesser",Screwdriver,curry,1483117470,1483124196,1483154281,1483117470,1,1,0,-1,30085,6726,30085,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Geosciences,Earth Sciences,Tectonics
-54000,6107522,Robertson,UTC,c1af1c2854199beceb6efb1041d32af91fa30b6a,Moorhen,Screwdriver,curry,1483033412,1483033544,1483158212,1483033412,1,1,0,-1,124668,132,124668,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication
+,Local Job Id,Resource,Service Provider,Timezone,System Username (Deidentified),User,User Institution,Principal Investigator,PI Institution,Quality of Service,Submit Time (Timestamp),Start Time (Timestamp),End Time (Timestamp),Eligible Time (Timestamp),Nodes,Cores,GPUs,Memory Used,Wall Time,Wait Time,Core Time,GPU Time,Exit Code,Exit State,Requested Cores,Requested memory,Requested Wall Time,Queue,Decanal Unit,Department,PI Group
+0,6110386,Robertson,screw - Screwdriver,UTC,4003ebd9b2de239734970b4cf32cc3d6183d2c6b,"Harrier, Hen",Screwdriver,"Harrier, Hen",Screwdriver,curry,1483118438,1483118438,1483118443,1483118438,1,1,0,-1,5,0,5,0,0:0,COMPLETED,1,48000Mn,172800,white,Computer and Information Science and Engineering,Computer and Computation Research,Computer and Computation Theory
+1000,970372,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483064234,1483064235,1483064263,1483064234,1,8,0,-1,28,1,224,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+2000,981747,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483102506,1483102507,1483102531,1483102506,1,8,0,-1,24,1,192,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+3000,989754,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483123544,1483129714,1483129736,1483123544,1,8,0,-1,22,6170,176,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+4000,968652,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483057618,1483057618,1483057667,1483057618,1,8,0,-1,49,0,392,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+5000,977607,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483087687,1483087687,1483087737,1483087687,1,8,0,-1,50,0,400,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+6000,984933,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483115155,1483115155,1483115189,1483115155,1,8,0,-1,34,0,272,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+7000,968469,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483056956,1483056957,1483057004,1483056956,1,8,0,-1,47,1,376,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+8000,974078,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483076500,1483076502,1483076639,1483076500,1,8,0,-1,137,2,1096,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+9000,979865,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483095044,1483095045,1483095077,1483095044,1,8,0,-1,32,1,256,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+10000,984613,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483114100,1483114101,1483114133,1483114100,1,8,0,-1,32,1,256,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+11000,2283982,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483133520,1483133521,1483135128,1483133520,1,12,0,-1,1607,1,19284,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+12000,6111392[227],Robertson,screw - Screwdriver,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,"Warbler, Cetti's",Screwdriver,curry,1483128560,1483131997,1483132033,1483128563,1,1,0,-1,36,3437,36,0,0:0,COMPLETED,1,48000Mn,10800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+13000,986946,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483116574,1483120362,1483120436,1483116574,1,8,0,-1,74,3788,592,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+14000,990388,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483125679,1483131567,1483131640,1483125679,1,8,0,-1,73,5888,584,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+15000,178141,Phillips,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483116629,1483126608,1483126663,1483116629,1,8,0,-1,55,9979,440,0,1:0,FAILED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+16000,6083230,Robertson,screw - Screwdriver,UTC,6e9b7834f396f086ab7cd2c87c522366595ef533,"Thrush, Swainson's",Screwdriver,"Tern, Caspian",Screwdriver,curry,1482264287,1483094066,1483095846,1482264287,1,4,0,-1,1780,829779,7120,0,0:0,COMPLETED,4,125Gn,7200,white,Mathematical and Physical Sciences,Astronomical Sciences,Galactic Astronomy
+17000,2280888,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483105941,1483105941,1483108128,1483105941,1,12,0,-1,2187,0,26244,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+18000,2279619,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483093758,1483093759,1483095795,1483093758,1,12,0,-1,2036,1,24432,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+19000,1006397,Posidriv,wrench - Wrench,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483085465,1483091366,1483093292,1483085465,1,12,0,-1,1926,5901,23112,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+20000,981123,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483100046,1483100046,1483103781,1483100046,1,8,0,-1,3735,0,29880,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+21000,988448,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483119066,1483124997,1483132142,1483119066,1,8,0,-1,7145,5931,57160,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+22000,6107766,Robertson,screw - Screwdriver,UTC,f7a636b57f1731bd9e0847e0bdde87f14758830d,"Sandpiper, Spotted",Screwdriver,Jackdaw,Screwdriver,curry,1483038579,1483075619,1483127457,1483038579,1,12,0,-1,51838,37040,622056,0,0:0,COMPLETED,12,2147486448Mn,259200,white,Humanities/Arts,Arts,Arts
+23000,1008539,Posidriv,wrench - Wrench,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,"Lark, Calandra",Wrench,banana-cream,1483115569,1483115647,1483115665,1483115569,1,8,0,-1,18,78,144,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+24000,1004844,Posidriv,wrench - Wrench,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,"Lark, Calandra",Wrench,banana-cream,1483064586,1483064674,1483064820,1483064586,1,8,0,-1,146,88,1168,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+25000,1007303,Posidriv,wrench - Wrench,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,"Lark, Calandra",Wrench,banana-cream,1483097873,1483097976,1483098058,1483097873,1,8,0,-1,82,103,656,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+26000,1010091,Posidriv,wrench - Wrench,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,"Lark, Calandra",Wrench,banana-cream,1483133111,1483133191,1483133297,1483133111,1,8,0,-1,106,80,848,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+27000,1005761,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483209904,1483210833,1483210856,1483209904,1,8,0,-1,23,929,184,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+28000,189006,Phillips,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483216774,1483218785,1483218805,1483216774,1,8,0,-1,20,2011,160,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+29000,1000886,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483162403,1483170225,1483170253,1483162403,1,8,0,-1,28,7822,224,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+30000,2286050,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483153190,1483153190,1483154632,1483153190,1,12,0,-1,1442,0,17304,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+31000,2294895,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483189873,1483189873,1483190980,1483189873,1,12,0,-1,1107,0,13284,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+32000,2303984,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483217125,1483217125,1483218715,1483217125,1,12,0,-1,1590,0,19080,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+33000,2290288,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483175735,1483175736,1483177474,1483175735,1,12,0,-1,1738,1,20856,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+34000,2300097,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483205387,1483205388,1483206656,1483205387,1,12,0,-1,1268,1,15216,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+35000,6113756[106],Robertson,screw - Screwdriver,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,"Warbler, Cetti's",Screwdriver,curry,1483199295,1483200296,1483200339,1483199297,1,1,0,-1,43,1001,43,0,0:0,COMPLETED,1,2147531648Mn,1800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+36000,1006184,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483211202,1483212335,1483212404,1483211202,1,8,0,-1,69,1133,552,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+37000,1009872,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483222462,1483223579,1483223674,1483222462,1,8,0,-1,95,1117,760,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+38000,186744,Phillips,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483205965,1483208237,1483208334,1483205965,1,8,0,-1,97,2272,776,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+39000,995160,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483142212,1483149731,1483149780,1483142212,1,8,0,-1,49,7519,392,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+40000,998889,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483156158,1483161700,1483161757,1483156158,1,8,0,-1,57,5542,456,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+41000,1001676,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483165800,1483175494,1483175821,1483165800,1,8,0,-1,327,9694,2616,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+42000,180971,Phillips,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483127595,1483161658,1483161692,1483127595,1,8,0,-1,34,34063,272,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+43000,1015515,Posidriv,wrench - Wrench,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483183323,1483207629,1483209365,1483183323,1,12,0,-1,1736,24306,20832,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+44000,2293190,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483184866,1483184866,1483186868,1483184866,1,12,0,-1,2002,0,24024,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+45000,2284699,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483140415,1483140416,1483143007,1483140415,1,12,0,-1,2591,1,31092,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+46000,2296556,Frearson,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483194872,1483194873,1483196868,1483194872,1,12,0,-1,1995,1,23940,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+47000,999141,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483156865,1483162665,1483166053,1483156865,1,8,0,-1,3388,5800,27104,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+48000,1003483,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483204264,1483204264,1483209282,1483204264,1,8,0,-1,5018,0,40144,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+49000,999645,Mortorq,screw - Screwdriver,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Wrench,"Flycatcher, Taiga",Screwdriver,banana-cream,1483158407,1483164819,1483168838,1483158407,1,8,0,-1,4019,6412,32152,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+50000,6109863,Robertson,screw - Screwdriver,UTC,56762592f0cc50c30d2379380f23c53c4e56fa1c,"Grey, Lesser",Screwdriver,"Shearwater, Macaronesian",Screwdriver,curry,1483116924,1483117165,1483144449,1483116924,1,1,0,-1,27284,241,27284,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Geosciences,Earth Sciences,Tectonics
+51000,6108458,Robertson,screw - Screwdriver,UTC,a185509f21643be72687169abd244222cbef7a70,"Thrush, Grey-cheeked",Screwdriver,"Chiffchaff, Iberian",Screwdriver,curry,1483063044,1483161307,1483184753,1483063044,1,2,0,-1,23446,98263,46892,0,0:0,COMPLETED,2,5000Mn,252000,white,Engineering,Electrical and Communication Systems,Solid-State and Microstructures
+52000,6071525,Robertson,screw - Screwdriver,UTC,ccbcfc90da52264ce0b4c7fed7c762fdd49f6be9,"Treecreeper, Short-toed",Screwdriver,Fulmar,Screwdriver,curry,1481918385,1482929004,1483184660,1482783437,1,8,0,-1,255656,1010619,2045248,0,0:0,COMPLETED,8,32000Mn,259200,white,Biological Sciences,Molecular Biosciences,Cell Biology
+53000,1012297,Posidriv,wrench - Wrench,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,"Lark, Calandra",Wrench,banana-cream,1483160435,1483160491,1483160861,1483160435,1,8,0,-1,370,56,2960,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+54000,1017369,Posidriv,wrench - Wrench,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,"Lark, Calandra",Wrench,banana-cream,1483196806,1483196920,1483197065,1483196806,1,8,0,-1,145,114,1160,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
diff --git a/tests/regression/data/default/raw-data-every-1000-with-fields-and-filters.csv b/tests/regression/data/default/raw-data-every-1000-with-fields-and-filters.csv
index 8beba5fe..7e4e847c 100644
--- a/tests/regression/data/default/raw-data-every-1000-with-fields-and-filters.csv
+++ b/tests/regression/data/default/raw-data-every-1000-with-fields-and-filters.csv
@@ -15,21 +15,21 @@
13000,2279980,banana-cream,1483097026,0,Social and Economic Science
14000,2278883,banana-cream,1483086983,0,Social and Economic Science
15000,979689,banana-cream,1483094414,0,Social and Economic Science
-16000,988604,banana-cream,1483125536,0,Social and Economic Science
-17000,1008884,banana-cream,1483220332,0,Social and Economic Science
-18000,999879,banana-cream,1483165819,0,Social and Economic Science
-19000,2292962,banana-cream,1483184182,0,Social and Economic Science
-20000,2302093,banana-cream,1483211322,0,Social and Economic Science
-21000,2288974,banana-cream,1483170354,0,Social and Economic Science
-22000,2298407,banana-cream,1483200279,0,Social and Economic Science
-23000,2306930,banana-cream,1483226400,0,Social and Economic Science
-24000,1006971,banana-cream,1483214983,0,Social and Economic Science
-25000,1010645,banana-cream,1483226141,0,Social and Economic Science
-26000,995601,banana-cream,1483151138,0,Social and Economic Science
-27000,999283,banana-cream,1483163221,0,Social and Economic Science
-28000,1002040,banana-cream,1483177317,0,Social and Economic Science
-29000,2289840,banana-cream,1483173994,0,Social and Economic Science
-30000,2303857,banana-cream,1483216696,0,Social and Economic Science
-31000,2293260,banana-cream,1483185052,0,Social and Economic Science
-32000,1005259,banana-cream,1483209237,0,Social and Economic Science
-33000,998935,banana-cream,1483161891,0,Social and Economic Science
+16000,988606,banana-cream,1483125543,0,Social and Economic Science
+17000,1009043,banana-cream,1483220909,0,Social and Economic Science
+18000,1000081,banana-cream,1483166659,0,Social and Economic Science
+19000,2293040,banana-cream,1483184407,0,Social and Economic Science
+20000,2302119,banana-cream,1483211385,0,Social and Economic Science
+21000,2289009,banana-cream,1483170508,0,Social and Economic Science
+22000,2298506,banana-cream,1483200588,0,Social and Economic Science
+23000,2306982,banana-cream,1483226560,0,Social and Economic Science
+24000,1007009,banana-cream,1483215106,0,Social and Economic Science
+25000,1010684,banana-cream,1483226302,0,Social and Economic Science
+26000,995681,banana-cream,1483151260,0,Social and Economic Science
+27000,999323,banana-cream,1483163355,0,Social and Economic Science
+28000,1002072,banana-cream,1483177450,0,Social and Economic Science
+29000,2290068,banana-cream,1483174694,0,Social and Economic Science
+30000,2303931,banana-cream,1483216905,0,Social and Economic Science
+31000,2293508,banana-cream,1483185774,0,Social and Economic Science
+32000,1007706,banana-cream,1483217272,0,Social and Economic Science
+33000,999535,banana-cream,1483164369,0,Social and Economic Science
diff --git a/tests/regression/data/xdmod-11-0-dev b/tests/regression/data/xdmod-11-0-dev
new file mode 120000
index 00000000..e8feaf79
--- /dev/null
+++ b/tests/regression/data/xdmod-11-0-dev
@@ -0,0 +1 @@
+xdmod-11-0
\ No newline at end of file
diff --git a/tests/regression/data/xdmod-11-0/jobs-dimensions.csv b/tests/regression/data/xdmod-11-0/jobs-dimensions.csv
new file mode 100644
index 00000000..2f2acd90
--- /dev/null
+++ b/tests/regression/data/xdmod-11-0/jobs-dimensions.csv
@@ -0,0 +1,17 @@
+id,label,description
+none,None,Summarizes Jobs data reported to the Screwdriver database.
+nsfdirectorate,Decanal Unit,Decanal Unit
+parentscience,Department,Department
+gpucount,GPU Count,A categorization of jobs into discrete groups based on the number of GPUs used by each job.
+jobsize,Job Size,A categorization of jobs into discrete groups based on the number of cores used by each job.
+jobwaittime,Job Wait Time,A categorization of jobs into discrete groups based on the total linear time each job waited.
+jobwalltime,Job Wall Time,A categorization of jobs into discrete groups based on the total linear time each job took to execute.
+nodecount,Node Count,A categorization of jobs into discrete groups based on node count.
+pi,PI,The principal investigator of a project.
+fieldofscience,PI Group,PI Group
+qos,QOS,The quality of service of the job
+queue,Queue,Queue pertains to the low level job queues on each resource.
+resource,Resource,A resource is a remote computer that can run jobs.
+resource_type,Resource Type,A categorization of resources into by their general capabilities.
+username,System Username,The specific system username of the users who ran jobs.
+person,User,"A person who is on a PIs allocation, hence able to run jobs on resources."
diff --git a/tests/regression/data/xdmod-11-0/jobs-metrics.csv b/tests/regression/data/xdmod-11-0/jobs-metrics.csv
new file mode 100644
index 00000000..0e3e28ca
--- /dev/null
+++ b/tests/regression/data/xdmod-11-0/jobs-metrics.csv
@@ -0,0 +1,27 @@
+id,label,description
+avg_cpu_hours,CPU Hours: Per Job,"The average CPU hours (number of CPU cores x wall time hours) per Screwdriver job.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
+total_cpu_hours,CPU Hours: Total,"The total CPU hours (number of CPU cores x wall time hours) used by Screwdriver jobs.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
+avg_gpus,GPU Count: Per Job,The average job size per Screwdriver job.
Job Size: The number of GPUs used by a (parallel) job.
+avg_gpu_hours,GPU Hours: Per Job,"The average GPU hours (number of GPU cores x wall time hours) per Screwdriver job.
For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours."
+total_gpu_hours,GPU Hours: Total,"The total GPU hours (number of GPUs x wall time hours) used by Screwdriver jobs.
For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours."
+max_processors,Job Size: Max (Core Count),The maximum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job.
+min_processors,Job Size: Min (Core Count),The minimum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job.
+normalized_avg_processors,Job Size: Normalized (% of Total Cores),"The percentage average size Screwdriver job divided by the total number of cores in the resource where the job ran. The job normalization calculation assumes that the resource size is constant. This statistic should not be used with a time range where the resource size changes, because the statistic will be incorrect.
Normalized Job Size: The ratio of the total number of processor cores used by a (parallel) job over the total number of cores on the resource."
+avg_processors,Job Size: Per Job (Core Count),The average job size per Screwdriver job.
Job Size: The number of processor cores used by a (parallel) job.
+avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average Screwdriver job size weighted by CPU Hours. Defined as
Average Job Size Weighted By CPU Hours: sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i = 0 to n){job i cpu hours}
+avg_job_size_weighted_by_gpu_hours,Job Size: Weighted By GPU Hours (GPU Count),The average Screwdriver job size weighted by GPU Hours. Defined as
Average Job Size Weighted By GPU Hours: sum(i = 0 to n){ job i core count * job i gpu hours}/sum(i = 0 to n){job i gpu hours}
+avg_node_hours,Node Hours: Per Job,The average node hours (number of nodes x wall time hours) per Screwdriver job.
+total_node_hours,Node Hours: Total,The total node hours (number of nodes x wall time hours) used by Screwdriver jobs.
+job_count,Number of Jobs Ended,The total number of Screwdriver jobs that ended within the selected duration.
+running_job_count,Number of Jobs Running,The total number of Screwdriver jobs that are running.
+started_job_count,Number of Jobs Started,The total number of Screwdriver jobs that started executing within the selected duration.
+submitted_job_count,Number of Jobs Submitted,The total number of Screwdriver jobs that submitted/queued within the selected duration.
+active_pi_count,Number of PIs: Active,The total number of PIs that used Screwdriver resources.
+active_resource_count,Number of Resources: Active,The total number of active Screwdriver resources.
+active_person_count,Number of Users: Active,The total number of users that used Screwdriver resources.
+utilization,Screwdriver CPU Utilization (%),The percentage of core time that a resource has been running jobs.
Screwdriver CPU Utilization: The ratio of the total CPU hours consumed by jobs over a given time period divided by the maximum CPU hours that the system could deliver (based on the number of cores present on the resource and the percent of the resource allocated through Screwdriver). This value does not take into account downtimes or outages. It is just calculated based on the number of cores in the resource specifications.
+expansion_factor,User Expansion Factor,The average job size per Screwdriver job.
Job Size: The number of processor cores used by a (parallel) job.
+avg_waitduration_hours,Wait Hours: Per Job,"The average time, in hours, a Screwdriver job waits before execution on the designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
+total_waitduration_hours,Wait Hours: Total,"The total time, in hours, Screwdriver jobs waited before execution on their designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
+avg_wallduration_hours,Wall Hours: Per Job,"The average time, in hours, a job takes to execute.
In timeseries view mode, the statistic shows the average wall time per job per time period. In aggregate view mode the statistic only includes the job wall hours between the defined time range. The wall hours outside the time range are not included in the calculation.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job."
+total_wallduration_hours,Wall Hours: Total,"The total time, in hours, Screwdriver jobs took to execute.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job."
diff --git a/tests/regression/data/xdmod-11-0/raw-data-every-1000-no-fields-no-filters.csv b/tests/regression/data/xdmod-11-0/raw-data-every-1000-no-fields-no-filters.csv
new file mode 100644
index 00000000..42a56249
--- /dev/null
+++ b/tests/regression/data/xdmod-11-0/raw-data-every-1000-no-fields-no-filters.csv
@@ -0,0 +1,56 @@
+,Local Job Id,Resource,Timezone,System Username (Deidentified),User,Organization,Quality of Service,Submit Time (Timestamp),Start Time (Timestamp),End Time (Timestamp),Eligible Time (Timestamp),Nodes,Cores,GPUs,Memory Used,Wall Time,Wait Time,Core Time,GPU Time,Exit Code,Exit State,Requested Cores,Requested memory,Requested Wall Time,Queue,Decanal Unit,Department,PI Group
+0,6110386,Robertson,UTC,4003ebd9b2de239734970b4cf32cc3d6183d2c6b,"Harrier, Hen",Screwdriver,curry,1483118438,1483118438,1483118443,1483118438,1,1,0,-1,5,0,5,0,0:0,COMPLETED,1,48000Mn,172800,white,Computer and Information Science and Engineering,Computer and Computation Research,Computer and Computation Theory
+1000,970339,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483064122,1483064124,1483064151,1483064122,1,8,0,-1,27,2,216,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+2000,981731,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483102441,1483102444,1483102466,1483102441,1,8,0,-1,22,3,176,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+3000,6112283[436],Robertson,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,curry,1483135648,1483139913,1483139937,1483135648,1,1,0,-1,24,4265,24,0,0:0,COMPLETED,1,48000Mn,10800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+4000,178502,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483116701,1483130204,1483130230,1483116701,1,8,0,-1,26,13503,208,0,1:0,FAILED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+5000,974487,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483077974,1483077974,1483078015,1483077974,1,8,0,-1,41,0,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+6000,982727,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483106505,1483106505,1483106556,1483106505,1,8,0,-1,51,0,408,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+7000,2281895,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483115132,1483115132,1483116616,1483115132,1,12,0,-1,1484,0,17808,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+8000,972196,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483070299,1483070301,1483070337,1483070299,1,8,0,-1,36,2,288,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+9000,978066,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483088977,1483088978,1483089027,1483088977,1,8,0,-1,49,1,392,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+10000,983035,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483107862,1483107863,1483108007,1483107862,1,8,0,-1,144,1,1152,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+11000,1009935,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483130897,1483130908,1483131130,1483130897,1,8,0,-1,222,11,1776,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+12000,2284391,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483137557,1483137558,1483139202,1483137557,1,12,0,-1,1644,1,19728,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+13000,1005349,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483071422,1483071534,1483071778,1483071422,1,8,0,-1,244,112,1952,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+14000,1007824,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483105333,1483105429,1483105948,1483105333,1,8,0,-1,519,96,4152,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+15000,1010629,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483140307,1483140400,1483140550,1483140307,1,8,0,-1,150,93,1200,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+16000,6111037[473],Robertson,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,curry,1483125912,1483129603,1483129747,1483125913,1,1,0,-1,144,3691,144,0,0:0,COMPLETED,1,2147487648Mn,900,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+17000,988528,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483119249,1483125280,1483125321,1483119249,1,8,0,-1,41,6031,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+18000,991993,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483131196,1483136908,1483136994,1483131196,1,8,0,-1,86,5712,688,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+19000,1008740,Posidriv,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483117402,1483124540,1483126240,1483117402,1,12,0,-1,1700,7138,20400,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+20000,2276090,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483060299,1483060299,1483062513,1483060299,1,12,0,-1,2214,0,26568,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+21000,967122,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483053110,1483053111,1483056506,1483053110,1,8,0,-1,3395,1,27160,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+22000,2283293,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483126817,1483126818,1483128722,1483126817,1,12,0,-1,1904,1,22848,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+23000,1008638,Posidriv,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483116706,1483122377,1483124429,1483116706,1,12,0,-1,2052,5671,24624,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+24000,972040,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483069980,1483069981,1483074019,1483069980,1,8,0,-1,4038,1,32304,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+25000,991884,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483130901,1483136404,1483141877,1483130901,1,8,0,-1,5473,5503,43784,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+26000,6105349,Robertson,UTC,c1af1c2854199beceb6efb1041d32af91fa30b6a,Moorhen,Screwdriver,curry,1482948669,1482948735,1483066377,1482948669,1,1,0,-1,117642,66,117642,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication
+27000,1005309,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483208573,1483209389,1483209414,1483208573,1,8,0,-1,25,816,200,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+28000,6114243[425],Robertson,UTC,eeb0dadecf619ec58074ea212b52634f9f974b80,"Grey, Great",Screwdriver,curry,1483202456,1483205646,1483205659,1483202456,1,1,0,-1,13,3190,13,0,0:0,COMPLETED,1,2147531648Mn,1800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+29000,998101,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483159238,1483159264,1483154003,1,8,0,-1,26,5235,208,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+30000,184276,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483158935,1483191164,1483191182,1483158935,1,8,0,-1,18,32229,144,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+31000,2292895,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483183969,1483183969,1483185436,1483183969,1,12,0,-1,1467,0,17604,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+32000,2302043,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483211208,1483211208,1483212911,1483211208,1,12,0,-1,1703,0,20436,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+33000,2286102,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483154004,1483155247,1483154003,1,12,0,-1,1243,1,14916,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+34000,2295330,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483191135,1483191136,1483192490,1483191135,1,12,0,-1,1354,1,16248,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+35000,2304491,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483218813,1483218814,1483219999,1483218813,1,12,0,-1,1185,1,14220,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+36000,1004395,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483206031,1483207038,1483207076,1483206031,1,8,0,-1,38,1007,304,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+37000,1007905,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483216837,1483217890,1483217928,1483216837,1,8,0,-1,38,1053,304,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+38000,189492,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483219052,1483220591,1483220646,1483219052,1,8,0,-1,55,1539,440,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+39000,1014185,Posidriv,UTC,04e627941b577c77164d3fc3a7aac7ec0fc9ad55,"Bunting, Reed",Screwdriver,banana-cream,1483177983,1483178402,1483178494,1483177983,1,8,0,-1,92,419,736,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+40000,6112860,Robertson,UTC,218266b45cc856b3bd485c25a25271f77c7108dc,Dunlin,Screwdriver,curry,1483142479,1483143257,1483143695,1483142479,1,12,0,-1,438,778,5256,0,0:0,COMPLETED,12,2147486448Mn,540,white,Biological Sciences,Environmental Biology,Systematic and Population Biology
+41000,993900,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483137554,1483144161,1483144201,1483137554,1,8,0,-1,40,6607,320,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+42000,997133,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483150115,1483156319,1483156355,1483150115,1,8,0,-1,36,6204,288,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+43000,1000379,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483160618,1483167881,1483167946,1483160618,1,8,0,-1,65,7263,520,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+44000,1003108,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483172599,1483182242,1483182311,1483172599,1,8,0,-1,69,9643,552,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+45000,183442,Phillips,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483151833,1483184199,1483184231,1483151833,1,8,0,-1,32,32366,256,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+46000,2288079,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483166466,1483166466,1483168335,1483166466,1,12,0,-1,1869,0,22428,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+47000,2300772,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483207320,1483207320,1483209262,1483207320,1,12,0,-1,1942,0,23304,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+48000,2290078,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483174731,1483174734,1483177171,1483174731,1,12,0,-1,2437,3,29244,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+49000,2304088,Frearson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483217556,1483217557,1483219647,1483217556,1,12,0,-1,2090,1,25080,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+50000,1011153,Posidriv,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483146809,1483157125,1483159016,1483146809,1,12,0,-1,1891,10316,22692,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+51000,1006854,Mortorq,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,banana-cream,1483213460,1483214511,1483219230,1483213460,1,8,0,-1,4719,1051,37752,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+52000,6106621,Robertson,UTC,0b888f86869ffa092ddb8e069c6bb9ec195964ee,Honey-buzzard,Screwdriver,curry,1483007306,1483150677,1483157807,1483007306,1,8,0,-1,7130,143371,57040,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+53000,6110124,Robertson,UTC,56762592f0cc50c30d2379380f23c53c4e56fa1c,"Grey, Lesser",Screwdriver,curry,1483117470,1483124196,1483154281,1483117470,1,1,0,-1,30085,6726,30085,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Geosciences,Earth Sciences,Tectonics
+54000,6107522,Robertson,UTC,c1af1c2854199beceb6efb1041d32af91fa30b6a,Moorhen,Screwdriver,curry,1483033412,1483033544,1483158212,1483033412,1,1,0,-1,124668,132,124668,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication
diff --git a/tests/regression/data/xdmod-11-0/raw-data-every-1000-with-fields-and-filters.csv b/tests/regression/data/xdmod-11-0/raw-data-every-1000-with-fields-and-filters.csv
new file mode 100644
index 00000000..8beba5fe
--- /dev/null
+++ b/tests/regression/data/xdmod-11-0/raw-data-every-1000-with-fields-and-filters.csv
@@ -0,0 +1,35 @@
+,Local Job Id,Quality of Service,Start Time (Timestamp),GPUs,Department
+0,968193,banana-cream,1483056132,0,Social and Economic Science
+1000,971425,banana-cream,1483068133,0,Social and Economic Science
+2000,984171,banana-cream,1483112446,0,Social and Economic Science
+3000,993014,banana-cream,1483140821,0,Social and Economic Science
+4000,976375,banana-cream,1483084157,0,Social and Economic Science
+5000,983957,banana-cream,1483111812,0,Social and Economic Science
+6000,2284603,banana-cream,1483139473,0,Social and Economic Science
+7000,973762,banana-cream,1483075149,0,Social and Economic Science
+8000,979555,banana-cream,1483093851,0,Social and Economic Science
+9000,984311,banana-cream,1483112900,0,Social and Economic Science
+10000,2283845,banana-cream,1483132302,0,Social and Economic Science
+11000,988946,banana-cream,1483126657,0,Social and Economic Science
+12000,992368,banana-cream,1483138292,0,Social and Economic Science
+13000,2279980,banana-cream,1483097026,0,Social and Economic Science
+14000,2278883,banana-cream,1483086983,0,Social and Economic Science
+15000,979689,banana-cream,1483094414,0,Social and Economic Science
+16000,988604,banana-cream,1483125536,0,Social and Economic Science
+17000,1008884,banana-cream,1483220332,0,Social and Economic Science
+18000,999879,banana-cream,1483165819,0,Social and Economic Science
+19000,2292962,banana-cream,1483184182,0,Social and Economic Science
+20000,2302093,banana-cream,1483211322,0,Social and Economic Science
+21000,2288974,banana-cream,1483170354,0,Social and Economic Science
+22000,2298407,banana-cream,1483200279,0,Social and Economic Science
+23000,2306930,banana-cream,1483226400,0,Social and Economic Science
+24000,1006971,banana-cream,1483214983,0,Social and Economic Science
+25000,1010645,banana-cream,1483226141,0,Social and Economic Science
+26000,995601,banana-cream,1483151138,0,Social and Economic Science
+27000,999283,banana-cream,1483163221,0,Social and Economic Science
+28000,1002040,banana-cream,1483177317,0,Social and Economic Science
+29000,2289840,banana-cream,1483173994,0,Social and Economic Science
+30000,2303857,banana-cream,1483216696,0,Social and Economic Science
+31000,2293260,banana-cream,1483185052,0,Social and Economic Science
+32000,1005259,banana-cream,1483209237,0,Social and Economic Science
+33000,998935,banana-cream,1483161891,0,Social and Economic Science
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index fd434213..91a6810e 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -108,6 +108,9 @@ def test_get_raw_data(valid_dw, capsys, additional_params, number, csv_title):
data,
dtype='string',
index_col=0,
+ override_default_data=(
+ XDMOD_VERSION in ['xdmod-11-0', 'xdmod-11-0-dev']
+ ),
)
assert 'Got ' + number + ' rows...DONE' in capsys.readouterr().out
@@ -136,6 +139,9 @@ def test_describe_metrics(valid_dw):
__assert_descriptor_dfs_equal(
'jobs-metrics.csv',
valid_dw.describe_metrics('Jobs'),
+ override_default_data=(
+ XDMOD_VERSION in ['xdmod-11-0', 'xdmod-11-0-dev']
+ ),
)
@@ -143,6 +149,9 @@ def test_describe_dimensions(valid_dw):
__assert_descriptor_dfs_equal(
'jobs-dimensions.csv',
valid_dw.describe_dimensions('Jobs'),
+ override_default_data=(
+ XDMOD_VERSION in ['xdmod-11-0', 'xdmod-11-0-dev']
+ ),
)
diff --git a/xdmod_data/__version__.py b/xdmod_data/__version__.py
index dd057c7d..c5191d85 100644
--- a/xdmod_data/__version__.py
+++ b/xdmod_data/__version__.py
@@ -1,2 +1,2 @@
__title__ = 'xdmod-data'
-__version__ = '1.1.0.dev09'
+__version__ = '1.1.0.dev1'
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index 708eba01..e7fb6e4c 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -44,26 +44,65 @@ def _request_data(self, params):
def _request_raw_data(self, params):
url_params = self.__get_raw_data_url_params(params)
data = []
- response_iter_lines = self.__request(
+ fields = None
+ response = self.__request(
path='/rest/v1/warehouse/raw-data?' + url_params,
post_fields=None,
stream=True,
)
- i = 0
- for line in response_iter_lines:
- line_text = line.decode('utf-8').replace('\x1e', '')
- line_json = json.loads(line_text)
- if i == 0:
- response = {'fields': line_json}
- else:
- data.append(line_json)
- # Only print every 10,000 rows to avoid I/O rate errors.
- if params['show_progress'] and i % 10000 == 0:
- self.__print_progress_msg(i, '\r')
- i += 1
- if params['show_progress']:
- self.__print_progress_msg(i, 'DONE\n')
- return (data, response['fields'])
+ num_rows_read = 0
+ # Once XDMoD 11.0 is no longer supported, only the else branch will
+ # be needed:
+ if response.headers['Content-Type'] == 'application/json-seq':
+ for line in response.iter_lines():
+ line_text = line.decode('utf-8').replace('\x1e', '')
+ (data, fields) = self.__process_raw_data_response_row(
+ line_text,
+ num_rows_read,
+ params['show_progress'],
+ data,
+ fields,
+ )
+ num_rows_read += 1
+ if params['show_progress']:
+ self.__print_progress_msg(num_rows_read, 'DONE\n')
+ else:
+ # The data stream consists of pairs of lines where the second
+ # line contains the row we care about and the first line
+ # contains the hex size of the second line.
+ is_first_line_in_pair = True
+ for line in response.iter_lines():
+ # There is a bug in Requests (see
+ # https://github.com/psf/requests/issues/5540) such that empty
+ # lines are occasionally sent via iter_lines(); ignore these.
+ if line == b'':
+ continue
+ line_text = line.decode('utf-8')
+ if is_first_line_in_pair:
+ last_line_size = line_text
+ # The last line will be of size 0 and should not be
+ # processed.
+ elif last_line_size != '0': # pragma: no branch
+ (data, fields) = self.__process_raw_data_response_row(
+ line_text,
+ num_rows_read,
+ params['show_progress'],
+ data,
+ fields,
+ )
+ num_rows_read += 1
+ is_first_line_in_pair = not is_first_line_in_pair
+ if params['show_progress']:
+ self.__print_progress_msg(num_rows_read, 'DONE\n')
+ if last_line_size != '0': # pragma: no cover
+ raise RuntimeError(
+ 'Connection closed before all data were received!'
+ + ' You may need to break your request into smaller'
+ + ' chunks by running `get_raw_data()` multiple times with'
+ + ' fewer days specified for `duration` and then piecing'
+ + ' the resulting data frames back together.',
+ )
+ return (data, fields)
def _request_filter_values(self, realm_id, dimension_id):
limit = 10000
@@ -88,7 +127,7 @@ def _request_filter_values(self, realm_id, dimension_id):
def _request_json(self, path, post_fields=None):
response = self.__request(path, post_fields)
- return json.loads(response)
+ return json.loads(response.text)
def __request(self, path='', post_fields=None, stream=False):
_validator._assert_runtime_context(self.__in_runtime_context)
@@ -99,11 +138,16 @@ def __request(self, path='', post_fields=None, stream=False):
url,
headers=self.__headers,
data=post_fields,
+ stream=stream,
)
else:
url += '&' if '?' in url else '?'
url += 'Bearer=' + self.__api_token
- response = self.__requests_session.get(url, headers=self.__headers)
+ response = self.__requests_session.get(
+ url,
+ headers=self.__headers,
+ stream=stream,
+ )
if response.status_code != 200:
msg = ''
try:
@@ -118,10 +162,7 @@ def __request(self, path='', post_fields=None, stream=False):
raise RuntimeError(
'Error ' + str(response.status_code) + msg,
) from None
- if stream:
- return response.iter_lines()
- else:
- return response.text
+ return response
def __get_data_post_fields(self, params):
post_fields = {
@@ -156,6 +197,25 @@ def __get_raw_data_url_params(self, params):
)
return urlencode(results)
+ def __process_raw_data_response_row(
+ self,
+ line_text,
+ num_rows_read,
+ show_progress,
+ data,
+ fields,
+ ):
+ line_json = json.loads(line_text)
+ if num_rows_read == 0:
+ fields = line_json
+ else:
+ data.append(line_json)
+ # Only print every 10,000 rows to avoid I/O rate
+ # errors.
+ if show_progress and num_rows_read % 10000 == 0:
+ self.__print_progress_msg(num_rows_read, '\r')
+ return (data, fields)
+
def __print_progress_msg(self, num_rows, end='\n'):
progress_msg = (
'Got ' + str(num_rows) + ' row' + ('' if num_rows == 1 else 's')
diff --git a/xdmod_data/warehouse.py b/xdmod_data/warehouse.py
index 1218d710..deffd5be 100644
--- a/xdmod_data/warehouse.py
+++ b/xdmod_data/warehouse.py
@@ -134,7 +134,7 @@ def get_data(
return _response_processor._process_get_data_response(
self,
params,
- response,
+ response.text,
)
def get_raw_data(