From 71f2eb90177b0ad5730fa87b01d5110ba3fa2db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 13:16:29 +0200 Subject: [PATCH 01/23] added utils/helper/export.js file in the backend --- backend/utils/helper/export.js | 610 +++++++++++++++++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 backend/utils/helper/export.js diff --git a/backend/utils/helper/export.js b/backend/utils/helper/export.js new file mode 100644 index 000000000..a41b1ff81 --- /dev/null +++ b/backend/utils/helper/export.js @@ -0,0 +1,610 @@ +const fs = require('fs'); +const { faker } = require('@faker-js/faker'); +const JSZip = require('jszip'); +const { deriveUserSeed } = require('../../webserver/auth/utils'); +const path = require('path'); +const storageDir = path.join(__dirname, "..", "..", "..", "files"); +const Papa = require('papaparse'); + +const SUPPORTED_EXPORT_TYPES = new Set(["submissions", "grades", "documents", "studies"]); +const { calculateAssessmentScore, buildScoresFromState } = require('assessment-score'); +const ASSESSMENT_RESULT_KEY = "assessment_result"; + + +/** + * Validates an export request and loads the project/users it targets. + * @param {Object} server - The server instance providing database models and Sequelize operators. + * @param {Object} params - Validation inputs. + * @param {number} params.parsedProjectId - The numeric project id. + * @param {string} params.exportType - The requested export type. + * @param {string} params.normalizedGradeFormat - The requested grade format, lowercased. + * @param {Array} params.userIds - Parsed user ids to export. + * @param {*} params.workflowIds - Raw workflowIds value from the request body, parsed here. + * @returns {Promise<{success: boolean, status?: number, message?: string, users?: Array, workflowIds?: Array}>} + */ +async function loadExportRequestContext(server, { parsedProjectId, exportType, normalizedGradeFormat, userIds, workflowIds }) { + if (!Number.isInteger(parsedProjectId)) { + return { success: false, status: 400, message: "Missing projectId." }; + } + if (!SUPPORTED_EXPORT_TYPES.has(exportType)) { + return { success: false, status: 400, message: "Unsupported export type." }; + } + workflowIds = typeof workflowIds === 'string' ? JSON.parse(workflowIds) : workflowIds; + if (!Array.isArray(workflowIds)) workflowIds = []; + if (exportType === "grades" && !["json", "csv"].includes(normalizedGradeFormat)) { + return { success: false, status: 400, message: "Unsupported grade format. Use json or csv." }; + } + if (userIds.length === 0) { + console.warn("Export aborted: No valid users selected."); + return { success: false, status: 400, message: "No valid users selected." }; + } + + const project = await server.db.models.project.findOne({ where: { id: parsedProjectId } }); + if (!project) { + console.warn(`${parsedProjectId} does not exist.`); + return { success: false, status: 403, message: "The selected project does not exist." }; + } + + const { Op } = server.db.Sequelize; + const users = await server.db.models.user.findAll({ where: { id: { [Op.in]: userIds } } }); + if (users.length === 0) { + console.warn("Export aborted: No existing users to export."); + return { success: false, status: 400, message: "No authorized users to export." }; + } + + return { success: true, users, workflowIds }; +} + +/** + * Opens a zip file, replaces the student's real name with a fake name in all .tex files, + * and returns the modified zip as a Buffer. + * @param {string} filePath - Path to the original zip file on disk + * @param {string} realName - The student's real name to search for + * @param {string} fakeName - The generated fake name to insert + * @returns {Promise} - The newly generated zip file buffer + */ +async function replaceAuthorInZip(filePath, realName, fakeName) { + const fileData = fs.readFileSync(filePath); + const zip = await JSZip.loadAsync(fileData); + const getFirstAndLastNameTokens = (name) => { + const parts = String(name || "").trim().split(/\s+/).filter(Boolean); + if (parts.length === 0) return ["", ""]; + if (parts.length === 1) return [parts[0], ""]; + return [parts[0], parts[parts.length - 1]]; + }; + const [realFirstName, realLastName] = getFirstAndLastNameTokens(realName); + const [fakeFirstName, fakeLastName] = getFirstAndLastNameTokens(fakeName); + + const authorRegex = /\\author\s*\{[^}]*\}/g; + + for (const [relativePath, zipEntry] of Object.entries(zip.files)) { + if (!zipEntry.dir && relativePath.toLowerCase().endsWith('.tex')) { + let text = await zipEntry.async("string"); + text = text.replace(authorRegex, `\\author{${fakeName}}`); + if (realFirstName && fakeFirstName) text = text.replace(realFirstName, fakeFirstName); + if (realLastName && fakeLastName) text = text.replace(realLastName, fakeLastName); + + zip.file(relativePath, text); + } + } + + return await zip.generateAsync({ + type: "nodebuffer", + compression: "DEFLATE" + }); +} + +/** + * Constructs a mapping of user IDs to aliases and generates a + * corresponding CSV string. + * @param {Array} users - Array of user objects from the database. + * @param {boolean} shouldGenerateAliases - Whether the export should use fake names. + * @param {boolean} hasPrivateInfoRight - Whether the current user is allowed to see/export full names. + * @param {number|string} fakerSeed - The base integer seed (from the form input). + * @param {string} salt - The hex-encoded salt string from the user's database record. + * @returns {Object} An object containing: + * - userMapping: An object mapping user IDs to their generated fake names. + * - mappingCsv: A CSV-formatted string containing the mapping (conditionally includes real names). + */ +function buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, salt) { + let userMapping = {}; + let csvRows = []; + + if (shouldGenerateAliases) { + if (fakerSeed && !isNaN(parseInt(fakerSeed, 10))) { + const derivedFakerSeed = deriveUserSeed(parseInt(fakerSeed, 10), salt); + faker.seed(derivedFakerSeed); + } + + const sortedUsers = [...users].sort((a, b) => Number(a.id) - Number(b.id)); + sortedUsers.forEach(u => { + const realUsername = u.userName; + const realName = `${u.firstName} ${u.lastName}`; + const fakeName = `${faker.person.firstName()} ${faker.person.lastName()}`; + + userMapping[u.id] = fakeName; + + let rowData = { + "Username": realUsername + }; + if (hasPrivateInfoRight) { + rowData["Real Name"] = realName; + } + + rowData["Generated Alias"] = fakeName; + + csvRows.push(rowData); + }); + } + const mappingCsv = csvRows.length > 0 ? Papa.unparse(csvRows) : ""; + return { userMapping, mappingCsv }; +} + +/** + * Normalizes a folder name so it can be used as a ZIP path segment without + * accidentally introducing invalid filename characters or nested paths. + * + * @param {string|number|null|undefined} value - The raw folder name. + * @returns {string} A sanitized folder name with reserved characters replaced. + */ +function sanitizeFolderName(value) { + return String(value || "unknown") + .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Returns a user's display name based on private info permissions. + * + * @param {Object|null} user - The user record. + * @param {boolean} hasPrivateInfoRight - Whether real names are allowed. + * @returns {string|null} Full name or username depending on permissions. + */ +function getPrivateAwareName(user, hasPrivateInfoRight) { + if (!user) return null; + if (hasPrivateInfoRight) return `${user.firstName} ${user.lastName}`.trim(); + // Usernames are considered anonymous-enough for exports when real names are restricted. + return user.userName ?? null; +} + +/** + * Resolves the display name for a user based on the current export settings. + * This wraps getPrivateAwareName with alias support for anonymized exports. + * + * @param {Object} user - The user record to display. + * @param {boolean} shouldGenerateAliases - Whether aliases should replace real names. + * @param {boolean} hasPrivateInfoRight - Whether the current user may export real names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @returns {string} The display name to write into the export. + */ +function getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping) { + if (shouldGenerateAliases) return userMapping[user.id]; + return getPrivateAwareName(user, hasPrivateInfoRight); +} + +/** + * Calculates the version number of a submission by traversing backwards + * through the chain of previous submissions. + * @param {Object} submission - The current submission object to start from. + * @param {Map} submissionMap - A Map containing all related + * submissions for quick lookup by ID. + * @returns {number} - The calculated version number (starting at 1 for the original). + */ +function calculateSubmissionVersion(submission, submissionMap) { + let version = 1; + let currentSub = submission; + while (currentSub && currentSub.previousSubmissionId) { + const prevSub = submissionMap.get(currentSub.previousSubmissionId); + if (!prevSub) break; + version++; + currentSub = prevSub; + } + return version; +} + +/** + * Parses an assessment state payload when it is stored as JSON text. + * + * @param {string} rawAssessmentState - The raw JSON string from document_data. + * @returns {Object} The parsed assessment state or an empty object on failure. + */ +function parseAssessmentState(rawAssessmentState) { + try { + const parsed = JSON.parse(rawAssessmentState); + return parsed && typeof parsed === "object" ? parsed : {}; + } catch (error) { + console.warn("Failed to parse assessment state:", error.message); + return {}; + } +} + +/** + * Reads the rubric configuration id from a study step configuration payload. + * + * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration object. + * @returns {number|null} The referenced configuration id or null when unavailable. + */ +function getAssessmentConfigurationId(studyStepConfiguration) { + if (!studyStepConfiguration || typeof studyStepConfiguration !== "object") return null; + const rawId = + studyStepConfiguration.settings?.configurationId ?? + studyStepConfiguration.configurationId ?? + null; + const parsedId = Number(rawId); + return Number.isInteger(parsedId) ? parsedId : null; +} + +/** + * Resolves the assessment rubric configuration referenced by a study step. + * Study steps are expected to store only a configurationId; rubric content + * is loaded from the configuration table. + * + * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration JSON. + * @param {Map} configurationsById - Loaded configuration records by id. + * @returns {Object|null} Assessment config content (with rubrics) or null. + */ +function resolveAssessmentConfigurationContent(studyStepConfiguration, configurationsById) { + const configurationId = getAssessmentConfigurationId(studyStepConfiguration); + if (configurationId === null) return null; + + const configuration = configurationsById.get(configurationId); + return configuration?.content ?? null; +} + +/** + * Records the assessment configuration content for a given configuration id, + * so each distinct configuration used across a grade export gets its own + * criteria reference file (a grade export may now span multiple configurations). + * + * @param {Map} referencesByConfigId - Mutable map of configurationId -> reference content. + * @param {number|null} configurationId - Resolved persisted configuration id. + * @param {Object|null} assessmentConfig - Resolved assessment configuration content. + * @returns {void} + */ +function addCriteriaReferenceEntry(referencesByConfigId, configurationId, assessmentConfig) { + if (!assessmentConfig || typeof assessmentConfig !== "object") return; + if (!Number.isInteger(configurationId)) return; + if (referencesByConfigId.has(configurationId)) return; + + referencesByConfigId.set(configurationId, { + configurationId, + ...assessmentConfig + }); +} + +/** + * Builds a flat CSV row for a grade export record. + * The row contains backend export metadata columns followed by + * one column per assessment criterion score. + * + * @param {Object} record - Prepared grade export record. + * @returns {Object} A flat object suitable for Papa.unparse. + */ +function buildGradeCsvRow(record) { + const criterionScores = record.scores && typeof record.scores === "object" ? record.scores : {}; + return { + projectId: record.projectId, + userId: record.userId, + userExtId: record.userExtId, + userName: record.userName, + displayName: record.displayName, + submissionId: record.submissionId, + submissionExtId: record.submissionExtId, + studySessionId: record.studySessionId, + studyName: record.studyName, + studyStepId: record.studyStepId, + studyStepType: record.studyStepType, + configurationId: record.configurationId, + studyOwner: record.studyOwner, + sessionOwner: record.sessionOwner, + author: record.author, + totalPoints: record.totalPoints, + createdAt: record.createdAt, + ...criterionScores + }; +} + +/** + * Loads the related entities needed to turn raw assessment_result rows into + * export-ready grade records. + * + * @param {Object} server - The server instance with Sequelize models. + * @param {Array} gradeRows - Assessment result rows with attached documents. + * @param {Array} users - The selected document owners for the export. + * @returns {Promise} Lookup maps for related grade-export entities. + */ +async function loadGradeExportContext(server, gradeRows, users) { + const { Op } = server.db.Sequelize; + + const sessionIds = [...new Set(gradeRows.map((row) => row.studySessionId).filter(Boolean))]; + const studySessions = sessionIds.length > 0 + ? await server.db.models.study_session.findAll({ + where: { id: { [Op.in]: sessionIds }, deleted: false }, + raw: true + }) + : []; + const sessionsById = new Map(studySessions.map((session) => [session.id, session])); + + const studyIds = [...new Set(studySessions.map((session) => session.studyId).filter(Boolean))]; + const studies = studyIds.length > 0 + ? await server.db.models.study.findAll({ + where: { id: { [Op.in]: studyIds }, deleted: false }, + raw: true + }) + : []; + const studiesById = new Map(studies.map((study) => [study.id, study])); + + const studyStepIds = [...new Set(gradeRows.map((row) => row.studyStepId).filter(Boolean))]; + const studySteps = studyStepIds.length > 0 + ? await server.db.models.study_step.findAll({ + where: { id: { [Op.in]: studyStepIds }, deleted: false }, + raw: true + }) + : []; + const studyStepsById = new Map(studySteps.map((studyStep) => [studyStep.id, studyStep])); + + const configurationIds = [...new Set( + studySteps + .map((studyStep) => getAssessmentConfigurationId(studyStep.configuration)) + .filter((id) => id !== null) + )]; + const configurations = configurationIds.length > 0 + ? await server.db.models.configuration.findAll({ + where: { id: { [Op.in]: configurationIds }, deleted: false }, + raw: true + }) + : []; + const configurationsById = new Map(configurations.map((configuration) => [configuration.id, configuration])); + + // The export references study/session owners in addition to the selected document owners. + const relatedUserIds = [...new Set([ + ...users.map((user) => user.id), + ...studySessions.map((session) => session.userId), + ...studies.map((study) => study.userId) + ].filter(Boolean))]; + const relatedUsers = relatedUserIds.length > 0 + ? await server.db.models.user.findAll({ where: { id: { [Op.in]: relatedUserIds } }, raw: true }) + : []; + const usersById = new Map(relatedUsers.map((user) => [user.id, user])); + + return { + sessionsById, + studiesById, + studyStepsById, + configurationsById, + usersById + }; +} + +/** + * Resolves which of the given user ids have opted into data sharing. + * + * @param {Object} server - The server instance providing database models. + * @param {Array} candidateUserIds - User ids to check consent for. + * @returns {Promise>} Set of user ids that accepted data sharing. + */ +async function getConsentedUserIds(server, candidateUserIds) { + if (candidateUserIds.length === 0) return new Set(); + const consentedUsers = await server.db.models.user.findAll({ + where: { id: candidateUserIds }, + attributes: ['id', 'acceptDataSharing'], + raw: true, + }); + return new Set(consentedUsers.filter(u => u.acceptDataSharing).map(u => u.id)); +} + +/** + * Orders grade records by session, then step within the session, then creation time. + * @param {Object} a - First grade record to compare. + * @param {Object} b - Second grade record to compare. + * @returns {number} Standard comparator result for Array#sort. + */ +function compareGradeRecords(a, b) { + const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; + const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; + return ( + (a.studySessionId || 0) - (b.studySessionId || 0) || + (a.studyStepId || 0) - (b.studyStepId || 0) || + createdA - createdB + ); +} + +/** + * Appends a stored file (by hash + extension) to the archive if it exists on disk, otherwise warns. + * @param {Object} archive - The archiver instance to append the file to. + * @param {string} hash - The document's storage hash. + * @param {string} extension - File extension including the dot, e.g. ".pdf". + * @param {string} archivePath - Destination path inside the ZIP archive. + * @param {string} typeLabel - Human-readable type label used in the warning log. + * @returns {void} + */ +function appendStoredFileIfExists(archive, hash, extension, archivePath, typeLabel) { + const filePath = path.join(storageDir, `${hash}${extension}`); + if (fs.existsSync(filePath)) { + archive.file(filePath, { name: archivePath }); + } else { + console.warn(`[DocumentExport] ${typeLabel} not found for document ${hash}`); + } +} + +/** + * Resolves whether a user may see other users' full names in exports (admins always can). + * @param {Object} server - The server instance providing database models. + * @param {number} userId - The requesting user's id. + * @returns {Promise} Whether the user has the private-info export right. + */ +async function resolveHasPrivateInfoRight(server, userId) { + const roleIds = await server.db.models["user_role_matching"].getUserRolesById(userId); + const isAdmin = await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); + if (isAdmin) return true; + + const userRightsObj = await server.db.models.user.getUserRights(userId); + if (!userRightsObj) return false; + + const allRights = Object.values(userRightsObj).flat(); + return allRights.includes('frontend.dashboard.studies.view.userPrivateInfo'); +} + +/** + * Parses the raw userIds field from a request body into an array, tolerating a JSON-encoded string. + * @param {*} rawUserIds - The raw value from req.body.userIds. + * @returns {Array} Parsed array of user ids, or an empty array if parsing fails. + */ +function parseUserIds(rawUserIds) { + try { + const parsed = typeof rawUserIds === 'string' ? JSON.parse(rawUserIds) : rawUserIds; + return Array.isArray(parsed) ? parsed : []; + } catch (e) { + console.warn("Could not parse userIds:", rawUserIds); + return []; + } +} + +/** + * Builds flat grade records for the given users/project by resolving each + * assessment_result row's session/study/step/configuration context and score. + * Shared by processGradesExport (grouped by user for JSON/CSV output) and + * processStudyBasedExport (grouped by session for a per-session grades.json). + * + * @param {Object} server - The server instance providing database models and Sequelize operators. + * @param {number} projectId - The project whose grades should be resolved. + * @param {Array} userIds - The selected document owners. + * @param {Array} users - Full user records for the selected users. + * @param {boolean} shouldGenerateAliases - Whether student names should be anonymized. + * @param {boolean} hasPrivateInfoRight - Whether the requester may export real names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @returns {Promise<{records: Array, criteriaReferencesByConfigId: Map}>} + */ +/** + * Collect assessment results and translate them into grade records. + * + * Pass `options.sessionIds` to scope the lookup to a set of study sessions. Without it the lookup + * is scoped by the owner of the assessed document, which only works when that owner is also the + * person the grades are being collected for. That holds for exposé assessments, where the assessed + * document belongs to the study owner, but not for review assessments: there the study belongs to + * the reviewer being assessed while the assessed review document belongs to the reviewed author, + * so an owner-scoped lookup returns nothing and the review grades are silently dropped. + */ +async function buildGradeRecords(server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, options = {}) { + const { Op } = server.db.Sequelize; + const { sessionIds = null } = options; + + const documentWhere = { projectId, deleted: false }; + if (!sessionIds) documentWhere.userId = { [Op.in]: userIds }; + + const gradeRows = await server.db.models.document_data.findAll({ + where: { + key: ASSESSMENT_RESULT_KEY, + deleted: false, + studySessionId: sessionIds ? { [Op.in]: sessionIds } : { [Op.ne]: null } + }, + include: [{ + model: server.db.models.document, + as: "document", + required: true, + where: documentWhere, + include: [{ + model: server.db.models.submission, + as: "submission", + required: false + }] + }], + order: [["studySessionId", "ASC"], ["studyStepId", "ASC"], ["createdAt", "ASC"]] + }); + + const { + sessionsById, + studiesById, + studyStepsById, + configurationsById, + usersById + } = await loadGradeExportContext(server, gradeRows, users); + + const records = []; + const criteriaReferencesByConfigId = new Map(); + for (const row of gradeRows) { + const document = row.document; + const ownerUser = usersById.get(document.userId); + if (!ownerUser) { + console.warn("Skipping grade export row because the document owner could not be resolved.", { + documentId: document.id, + documentUserId: document.userId, + studySessionId: row.studySessionId, + studyStepId: row.studyStepId + }); + continue; + } + const session = sessionsById.get(row.studySessionId); + const reviewerUser = session ? usersById.get(session.userId) : null; + const study = session ? studiesById.get(session.studyId) : null; + const graderUser = study ? usersById.get(study.userId) : null; + const studyStep = studyStepsById.get(row.studyStepId); + const submission = document.submission; + const studyStepConfiguration = studyStep?.configuration; + const configurationId = getAssessmentConfigurationId(studyStepConfiguration); + const studyName = study?.name || `study_${session?.studyId || "unknown"}`; + + const scoreObject = row.value || {}; + const assessmentState = typeof scoreObject === "string" ? parseAssessmentState(scoreObject) : scoreObject; + const flatScores = buildScoresFromState(assessmentState); + const assessmentConfig = resolveAssessmentConfigurationContent( + studyStepConfiguration, + configurationsById + ); + addCriteriaReferenceEntry( + criteriaReferencesByConfigId, + configurationId, + assessmentConfig + ); + const assessmentScore = calculateAssessmentScore(assessmentConfig, flatScores); + const totalPoints = assessmentScore.achieved_points; + + records.push({ + projectId, + userId: ownerUser.id, + userExtId: ownerUser.extId ?? null, + userName: ownerUser.userName ?? "", + displayName: getDisplayName(ownerUser, shouldGenerateAliases, hasPrivateInfoRight, userMapping), + submissionId: submission?.id ?? document.submissionId ?? null, + submissionExtId: submission?.extId ?? null, + studySessionId: row.studySessionId ?? null, + studyStepId: row.studyStepId ?? null, + configurationId, + studyName, + sessionHash: session?.hash ?? null, + studyOwner: getPrivateAwareName(graderUser, hasPrivateInfoRight), + sessionOwner: getPrivateAwareName(reviewerUser, hasPrivateInfoRight), + author: getPrivateAwareName(ownerUser, hasPrivateInfoRight), + scores: flatScores, + totalPoints, + createdAt: row.createdAt ? new Date(row.createdAt).toISOString() : null, + studyStepType: studyStep?.stepType ?? null + }); + } + + return { records, criteriaReferencesByConfigId }; +} + +module.exports = { + replaceAuthorInZip, + buildUserMapping, + sanitizeFolderName, + getPrivateAwareName, + getDisplayName, + calculateSubmissionVersion, + parseAssessmentState, + getAssessmentConfigurationId, + resolveAssessmentConfigurationContent, + addCriteriaReferenceEntry, + buildGradeCsvRow, + loadGradeExportContext, + getConsentedUserIds, + compareGradeRecords, + appendStoredFileIfExists, + resolveHasPrivateInfoRight, + parseUserIds, + loadExportRequestContext, + SUPPORTED_EXPORT_TYPES, + buildGradeRecords, +}; \ No newline at end of file From e0050f5385d4595e9e1e03936c5c3dcdb56ca98b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 13:17:41 +0200 Subject: [PATCH 02/23] updated routes/export.js to the latest version I had --- backend/webserver/routes/export.js | 1096 +++++++++++++++------------- 1 file changed, 578 insertions(+), 518 deletions(-) diff --git a/backend/webserver/routes/export.js b/backend/webserver/routes/export.js index 35dd32376..08d713327 100644 --- a/backend/webserver/routes/export.js +++ b/backend/webserver/routes/export.js @@ -1,13 +1,34 @@ const archiver = require('archiver'); const path = require('path'); const fs = require('fs'); -const { faker } = require('@faker-js/faker'); -const JSZip = require('jszip'); -const { deriveUserSeed } = require('../auth/utils'); const Papa = require('papaparse'); -const { calculateAssessmentScore, buildScoresFromState } = require('assessment-score'); - -const ASSESSMENT_RESULT_KEY = "assessment_result"; +const { dbToDelta, deltaToPlainText, deltaToHtml } = require('editor-delta-conversion'); +const { + replaceAuthorInZip, + buildUserMapping, + sanitizeFolderName, + getPrivateAwareName, + getDisplayName, + calculateSubmissionVersion, + parseAssessmentState, + getAssessmentConfigurationId, + resolveAssessmentConfigurationContent, + addCriteriaReferenceEntry, + buildGradeCsvRow, + loadGradeExportContext, + getConsentedUserIds, + compareGradeRecords, + appendStoredFileIfExists, + resolveHasPrivateInfoRight, + parseUserIds, + loadExportRequestContext, + SUPPORTED_EXPORT_TYPES, + buildGradeRecords, +} = require('../../utils/helper/export.js'); +const storageDir = path.join(__dirname, "..", "..", "..", "files"); + +// Stored files the study export ships per step, by document type: PDF and LaTeX ZIP. +const STUDY_DOCUMENT_EXTENSIONS = { 0: '.pdf', 4: '.zip' }; module.exports = function (server) { @@ -16,76 +37,38 @@ module.exports = function (server) { // Auth checking const currentUserId = req.user?.id; if (!currentUserId) return res.status(401).send("Log in required"); - const currentUser = await server.db.models.user.findByPk(currentUserId); if (!currentUser) return res.status(401).send("User not found"); + const hasPrivateInfoRight = await resolveHasPrivateInfoRight(server, currentUserId); - // check if user has right to see full names - let hasPrivateInfoRight = false; - - const roleIds = await server.db.models["user_role_matching"].getUserRolesById(currentUserId); - const isAdmin = await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); - if (isAdmin) { - // override, admin has all rights - hasPrivateInfoRight = true; - } else { - const userRightsObj = await server.db.models.user.getUserRights(currentUserId); - - if (userRightsObj) { - const allRights = Object.values(userRightsObj).flat(); - hasPrivateInfoRight = allRights.includes('frontend.dashboard.studies.view.userPrivateInfo'); - } - } // Input parsing - const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles } = req.body; - let { userIds = [] } = req.body; + const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles, excludeNonConsentingEdits, excludeNonConsentingAnnotations, includeEmptyStudies, includeDocumentFiles, includeGrades } = req.body; + let { userIds: rawUserIds = [], documentTypes = [0, 1, 2, 4], workflowIds = [] } = req.body; const shouldGenerateAliases = String(generateAliases) === 'true'; const shouldMergeCsvFiles = String(mergeCsvFiles) === "true"; + const shouldExcludeNonConsentingEdits = String(excludeNonConsentingEdits) === 'true'; + const shouldExcludeNonConsentingAnnotations = String(excludeNonConsentingAnnotations) === 'true'; + const shouldIncludeEmptyStudies = String(includeEmptyStudies) === 'true'; + const shouldIncludeDocumentFiles = String(includeDocumentFiles) === 'true'; + const shouldIncludeGrades = String(includeGrades) === 'true'; const normalizedGradeFormat = String(gradeFormat || "json").toLowerCase(); - const supportedExportTypes = new Set(["submissions", "grades"]); - const { Op } = server.db.Sequelize; const parsedProjectId = Number(projectId); + const userIds = parseUserIds(rawUserIds); try { - userIds = typeof userIds === 'string' ? JSON.parse(userIds) : userIds; - if (!Array.isArray(userIds)) userIds = []; - } catch (e) { - console.warn("Could not parse userIds:", userIds); - userIds = []; - } - - try { - if (!Number.isInteger(parsedProjectId)) return res.status(400).send("Missing projectId."); - if (!supportedExportTypes.has(exportType)) { - return res.status(400).send("Unsupported export type."); - } - if (exportType === "grades" && !["json", "csv"].includes(normalizedGradeFormat)) { - return res.status(400).send("Unsupported grade format. Use json or csv."); - } - if (userIds.length === 0) { - console.warn("Export aborted: No valid users selected."); - return res.status(400).send("No valid users selected."); - } - - // check if the project is valid - const projectCheck = await server.db.models.project.findOne({ where: { id: parsedProjectId } }); - if (!projectCheck) { - console.warn(`${parsedProjectId} does not exist.`); - return res.status(403).send("The selected project does not exist."); - } - - const users = await server.db.models.user.findAll({ where: { id: { [Op.in]: userIds } } }); - if (users.length === 0) { - console.warn("Export aborted: No existing users to export."); - return res.status(400).send("No authorized users to export."); + const context = await loadExportRequestContext(server, { parsedProjectId, exportType, normalizedGradeFormat, userIds, workflowIds }); + if (!context.success) { + return res.status(context.status).send(context.message); } + const { users, workflowIds: parsedWorkflowIds } = context; // build user mapping for aliases const { userMapping, mappingCsv } = buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, currentUser.salt); // archiver stream setup - const exportFolderName = `${exportType}_${Date.now()}.zip`; + const timestamp = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14); + const exportFolderName = `${timestamp}_${exportType}.zip`; res.attachment(exportFolderName); const archive = archiver('zip', { zlib: { level: 5 } }); archive.on('error', function(err) { @@ -99,6 +82,8 @@ module.exports = function (server) { archive.append(mappingCsv, { name: 'aliases_mapping.csv' }); } + const baseFolderName = exportFolderName.split('.')[0]; + // process based on type switch (exportType) { case 'submissions': @@ -110,7 +95,7 @@ module.exports = function (server) { shouldGenerateAliases, hasPrivateInfoRight, userMapping, - exportFolderName.split('.')[0], + baseFolderName, archive ); break; @@ -128,6 +113,37 @@ module.exports = function (server) { archive ); break; + case 'documents': + await processDocumentBasedExport( + server, + parsedProjectId, + userIds, + documentTypes, + shouldExcludeNonConsentingEdits, + shouldExcludeNonConsentingAnnotations, + baseFolderName, + archive + ); + break; + case 'studies': + await processStudyBasedExport( + server, + parsedProjectId, + userIds, + users, + shouldGenerateAliases, + hasPrivateInfoRight, + userMapping, + parsedWorkflowIds, + shouldIncludeEmptyStudies, + shouldExcludeNonConsentingEdits, + shouldExcludeNonConsentingAnnotations, + shouldIncludeDocumentFiles, + shouldIncludeGrades, + baseFolderName, + archive + ); + break; default: return res.status(400).send("Unsupported export type."); } @@ -141,93 +157,6 @@ module.exports = function (server) { } }); - // HELPER FUNCTIONS - - /** - * Opens a zip file, replaces the student's real name with a fake name in all .tex files, - * and returns the modified zip as a Buffer. - * @param {string} filePath - Path to the original zip file on disk - * @param {string} realName - The student's real name to search for - * @param {string} fakeName - The generated fake name to insert - * @returns {Promise} - The newly generated zip file buffer - */ - async function replaceAuthorInZip(filePath, realName, fakeName) { - const fileData = fs.readFileSync(filePath); - const zip = await JSZip.loadAsync(fileData); - const getFirstAndLastNameTokens = (name) => { - const parts = String(name || "").trim().split(/\s+/).filter(Boolean); - if (parts.length === 0) return ["", ""]; - if (parts.length === 1) return [parts[0], ""]; - return [parts[0], parts[parts.length - 1]]; - }; - const [realFirstName, realLastName] = getFirstAndLastNameTokens(realName); - const [fakeFirstName, fakeLastName] = getFirstAndLastNameTokens(fakeName); - - const authorRegex = /\\author\s*\{[^}]*\}/g; - - for (const [relativePath, zipEntry] of Object.entries(zip.files)) { - if (!zipEntry.dir && relativePath.toLowerCase().endsWith('.tex')) { - let text = await zipEntry.async("string"); - text = text.replace(authorRegex, `\\author{${fakeName}}`); - if (realFirstName && fakeFirstName) text = text.replace(realFirstName, fakeFirstName); - if (realLastName && fakeLastName) text = text.replace(realLastName, fakeLastName); - - zip.file(relativePath, text); - } - } - - return await zip.generateAsync({ - type: "nodebuffer", - compression: "DEFLATE" - }); - } - - /** - * Constructs a mapping of user IDs to aliases and generates a - * corresponding CSV string. - * @param {Array} users - Array of user objects from the database. - * @param {boolean} shouldGenerateAliases - Whether the export should use fake names. - * @param {boolean} hasPrivateInfoRight - Whether the current user is allowed to see/export full names. - * @param {number|string} fakerSeed - The base integer seed (from the form input). - * @param {string} salt - The hex-encoded salt string from the user's database record. - * @returns {Object} An object containing: - * - userMapping: An object mapping user IDs to their generated fake names. - * - mappingCsv: A CSV-formatted string containing the mapping (conditionally includes real names). - */ - function buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, salt) { - let userMapping = {}; - let csvRows = []; - - if (shouldGenerateAliases) { - if (fakerSeed && !isNaN(parseInt(fakerSeed, 10))) { - const derivedFakerSeed = deriveUserSeed(parseInt(fakerSeed, 10), salt); - faker.seed(derivedFakerSeed); - } - - const sortedUsers = [...users].sort((a, b) => Number(a.id) - Number(b.id)); - sortedUsers.forEach(u => { - const realUsername = u.userName; - const realName = `${u.firstName} ${u.lastName}`; - const fakeName = `${faker.person.firstName()} ${faker.person.lastName()}`; - - userMapping[u.id] = fakeName; - - let rowData = { - "Username": realUsername - }; - if (hasPrivateInfoRight) { - rowData["Real Name"] = realName; - } - - rowData["Generated Alias"] = fakeName; - - csvRows.push(rowData); - }); - } - const mappingCsv = csvRows.length > 0 ? Papa.unparse(csvRows) : ""; - return { userMapping, mappingCsv }; - } - /** * Does the fetching, filtering, and archiving of student submissions for a specific project. * Handles file renaming based on validation rules and manages directory structures @@ -274,7 +203,6 @@ module.exports = function (server) { 1: ".html", 4: ".zip" }; - const storageDir = path.join(__dirname, "..", "..", "..", "files"); for (const submission of submissions) { const student = usersById.get(submission.userId); @@ -325,235 +253,6 @@ module.exports = function (server) { } } - /** - * Normalizes a folder name so it can be used as a ZIP path segment without - * accidentally introducing invalid filename characters or nested paths. - * - * @param {string|number|null|undefined} value - The raw folder name. - * @returns {string} A sanitized folder name with reserved characters replaced. - */ - function sanitizeFolderName(value) { - return String(value || "unknown") - .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_") - .replace(/\s+/g, " ") - .trim(); - } - - /** - * Parses an assessment state payload when it is stored as JSON text. - * - * @param {string} rawAssessmentState - The raw JSON string from document_data. - * @returns {Object} The parsed assessment state or an empty object on failure. - */ - function parseAssessmentState(rawAssessmentState) { - try { - const parsed = JSON.parse(rawAssessmentState); - return parsed && typeof parsed === "object" ? parsed : {}; - } catch (error) { - console.warn("Failed to parse assessment state:", error.message); - return {}; - } - } - - /** - * Resolves the assessment rubric configuration referenced by a study step. - * Study steps are expected to store only a configurationId; rubric content - * is loaded from the configuration table. - * - * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration JSON. - * @param {Map} configurationsById - Loaded configuration records by id. - * @returns {Object|null} Assessment config content (with rubrics) or null. - */ - function resolveAssessmentConfigurationContent(studyStepConfiguration, configurationsById) { - const configurationId = getAssessmentConfigurationId(studyStepConfiguration); - if (configurationId === null) return null; - - const configuration = configurationsById.get(configurationId); - return configuration?.content ?? null; - } - - /** - * Reads the rubric configuration id from a study step configuration payload. - * - * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration object. - * @returns {number|null} The referenced configuration id or null when unavailable. - */ - function getAssessmentConfigurationId(studyStepConfiguration) { - if (!studyStepConfiguration || typeof studyStepConfiguration !== "object") return null; - const rawId = - studyStepConfiguration.settings?.configurationId ?? - studyStepConfiguration.configurationId ?? - null; - const parsedId = Number(rawId); - return Number.isInteger(parsedId) ? parsedId : null; - } - - /** - * Captures the single assessment configuration used by the current grade - * export for inclusion in the shared criteria_reference.json sidecar file. - * - * The first valid configuration becomes the export reference. If another - * different configuration is encountered later, the export aborts because - * grade exports are expected to use exactly one configuration. - * - * @param {{ key: string|null, reference: Object|null }} referenceState - Mutable single-reference state. - * @param {number|null} configurationId - Resolved persisted configuration id. - * @param {Object|null} assessmentConfig - Resolved assessment configuration content. - * @returns {void} - */ - function addCriteriaReferenceEntry(referenceState, configurationId, assessmentConfig) { - if (!assessmentConfig || typeof assessmentConfig !== "object") return; - - const referenceKey = Number.isInteger(configurationId) ? `configuration:${configurationId}` : null; - if (!referenceKey) return; - - if (!referenceState.reference) { - referenceState.key = referenceKey; - referenceState.reference = { - configurationId: Number.isInteger(configurationId) ? configurationId : null, - ...assessmentConfig - }; - return; - } - - if (referenceState.key !== referenceKey) { - throw new Error("Expected exactly one assessment configuration for grade export, found multiple."); - } - } - - /** - * Returns a user's display name based on private info permissions. - * - * @param {Object|null} user - The user record. - * @param {boolean} hasPrivateInfoRight - Whether real names are allowed. - * @returns {string|null} Full name or username depending on permissions. - */ - function getPrivateAwareName(user, hasPrivateInfoRight) { - if (!user) return null; - if (hasPrivateInfoRight) return `${user.firstName} ${user.lastName}`.trim(); - // Usernames are considered anonymous-enough for exports when real names are restricted. - return user.userName ?? null; - } - - /** - * Builds a flat CSV row for a grade export record. - * The row contains backend export metadata columns followed by - * one column per assessment criterion score. - * - * @param {Object} record - Prepared grade export record. - * @returns {Object} A flat object suitable for Papa.unparse. - */ - function buildGradeCsvRow(record) { - const criterionScores = record.scores && typeof record.scores === "object" ? record.scores : {}; - return { - projectId: record.projectId, - userId: record.userId, - userExtId: record.userExtId, - userName: record.userName, - displayName: record.displayName, - submissionId: record.submissionId, - submissionExtId: record.submissionExtId, - studySessionId: record.studySessionId, - studyName: record.studyName, - studyStepId: record.studyStepId, - studyStepType: record.studyStepType, - configurationId: record.configurationId, - studyOwner: record.studyOwner, - sessionOwner: record.sessionOwner, - author: record.author, - totalPoints: record.totalPoints, - createdAt: record.createdAt, - ...criterionScores - }; - } - - /** - * Resolves the display name for a user based on the current export settings. - * This wraps getPrivateAwareName with alias support for anonymized exports. - * - * @param {Object} user - The user record to display. - * @param {boolean} shouldGenerateAliases - Whether aliases should replace real names. - * @param {boolean} hasPrivateInfoRight - Whether the current user may export real names. - * @param {Object} userMapping - Map of user IDs to generated aliases. - * @returns {string} The display name to write into the export. - */ - function getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping) { - if (shouldGenerateAliases) return userMapping[user.id]; - return getPrivateAwareName(user, hasPrivateInfoRight); - } - - /** - * Loads the related entities needed to turn raw assessment_result rows into - * export-ready grade records. - * - * @param {Object} server - The server instance with Sequelize models. - * @param {Array} gradeRows - Assessment result rows with attached documents. - * @param {Array} users - The selected document owners for the export. - * @returns {Promise} Lookup maps for related grade-export entities. - */ - async function loadGradeExportContext(server, gradeRows, users) { - const { Op } = server.db.Sequelize; - - const sessionIds = [...new Set(gradeRows.map((row) => row.studySessionId).filter(Boolean))]; - const studySessions = sessionIds.length > 0 - ? await server.db.models.study_session.findAll({ - where: { id: { [Op.in]: sessionIds }, deleted: false }, - raw: true - }) - : []; - const sessionsById = new Map(studySessions.map((session) => [session.id, session])); - - const studyIds = [...new Set(studySessions.map((session) => session.studyId).filter(Boolean))]; - const studies = studyIds.length > 0 - ? await server.db.models.study.findAll({ - where: { id: { [Op.in]: studyIds }, deleted: false }, - raw: true - }) - : []; - const studiesById = new Map(studies.map((study) => [study.id, study])); - - const studyStepIds = [...new Set(gradeRows.map((row) => row.studyStepId).filter(Boolean))]; - const studySteps = studyStepIds.length > 0 - ? await server.db.models.study_step.findAll({ - where: { id: { [Op.in]: studyStepIds }, deleted: false }, - raw: true - }) - : []; - const studyStepsById = new Map(studySteps.map((studyStep) => [studyStep.id, studyStep])); - - const configurationIds = [...new Set( - studySteps - .map((studyStep) => getAssessmentConfigurationId(studyStep.configuration)) - .filter((id) => id !== null) - )]; - const configurations = configurationIds.length > 0 - ? await server.db.models.configuration.findAll({ - where: { id: { [Op.in]: configurationIds }, deleted: false }, - raw: true - }) - : []; - const configurationsById = new Map(configurations.map((configuration) => [configuration.id, configuration])); - - // The export references study/session owners in addition to the selected document owners. - const relatedUserIds = [...new Set([ - ...users.map((user) => user.id), - ...studySessions.map((session) => session.userId), - ...studies.map((study) => study.userId) - ].filter(Boolean))]; - const relatedUsers = relatedUserIds.length > 0 - ? await server.db.models.user.findAll({ where: { id: { [Op.in]: relatedUserIds } }, raw: true }) - : []; - const usersById = new Map(relatedUsers.map((user) => [user.id, user])); - - return { - sessionsById, - studiesById, - studyStepsById, - configurationsById, - usersById - }; - } - /** * Exports assessment results for the selected users as a ZIP archive. * Each selected user gets one or more hash-named folders containing @@ -582,116 +281,26 @@ module.exports = function (server) { mergeCsvFiles, archive ) { - const { Op } = server.db.Sequelize; - const gradeRows = await server.db.models.document_data.findAll({ - where: { - key: ASSESSMENT_RESULT_KEY, - deleted: false, - studySessionId: { [Op.ne]: null } - }, - include: [{ - model: server.db.models.document, - as: "document", - // required: true turns this include into an inner join. - required: true, - where: { - projectId, - userId: { [Op.in]: userIds }, - deleted: false - }, - include: [{ - model: server.db.models.submission, - as: "submission", - required: false - }] - }], - // Sort by session first, then step within the session, then creation time within the step. - order: [["studySessionId", "ASC"], ["studyStepId", "ASC"], ["createdAt", "ASC"]] - }); - - const { - sessionsById, - studiesById, - studyStepsById, - configurationsById, - usersById - } = await loadGradeExportContext(server, gradeRows, users); + const { records, criteriaReferencesByConfigId } = await buildGradeRecords( + server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping + ); const recordsByUser = new Map(); - // Grade export currently assumes that all exported rows point to one assessment config. - const criteriaReferenceState = { - key: null, - reference: null - }; - for (const row of gradeRows) { - const document = row.document; - const ownerUser = usersById.get(document.userId); - if (!ownerUser) { - console.warn("Skipping grade export row because the document owner could not be resolved.", { - documentId: document.id, - documentUserId: document.userId, - studySessionId: row.studySessionId, - studyStepId: row.studyStepId - }); - continue; - } - const session = sessionsById.get(row.studySessionId); - const reviewerUser = session ? usersById.get(session.userId) : null; - const study = session ? studiesById.get(session.studyId) : null; - const graderUser = study ? usersById.get(study.userId) : null; - const studyStep = studyStepsById.get(row.studyStepId); - const submission = document.submission; - const studyStepConfiguration = studyStep?.configuration; - // configurationId is exported as metadata; assessmentConfig is the rubric content - // needed for score calculation and criteria_reference.json. - const configurationId = getAssessmentConfigurationId(studyStepConfiguration); - const studyName = study?.name || `study_${session?.studyId || "unknown"}`; - - const scoreObject = row.value || {}; - const assessmentState = typeof scoreObject === "string" ? parseAssessmentState(scoreObject) : scoreObject; - const flatScores = buildScoresFromState(assessmentState); - const assessmentConfig = resolveAssessmentConfigurationContent( - studyStepConfiguration, - configurationsById - ); - addCriteriaReferenceEntry( - criteriaReferenceState, - configurationId, - assessmentConfig - ); - const assessmentScore = calculateAssessmentScore(assessmentConfig, flatScores); - const totalPoints = assessmentScore.achieved_points; - - const record = { - projectId, - userId: ownerUser.id, - userExtId: ownerUser.extId ?? null, - userName: ownerUser.userName ?? "", - displayName: getDisplayName(ownerUser, shouldGenerateAliases, hasPrivateInfoRight, userMapping), - submissionId: submission?.id ?? document.submissionId ?? null, - submissionExtId: submission?.extId ?? null, - studySessionId: row.studySessionId ?? null, - studyStepId: row.studyStepId ?? null, - configurationId, - studyName, - sessionHash: session?.hash ?? null, - studyOwner: getPrivateAwareName(graderUser, hasPrivateInfoRight), - sessionOwner: getPrivateAwareName(reviewerUser, hasPrivateInfoRight), - author: getPrivateAwareName(ownerUser, hasPrivateInfoRight), - scores: flatScores, - totalPoints, - createdAt: row.createdAt ? new Date(row.createdAt).toISOString() : null, - studyStepType: studyStep?.stepType ?? null - }; - - if (!recordsByUser.has(ownerUser.id)) recordsByUser.set(ownerUser.id, []); - recordsByUser.get(ownerUser.id).push(record); + for (const record of records) { + if (!recordsByUser.has(record.userId)) recordsByUser.set(record.userId, []); + recordsByUser.get(record.userId).push(record); } - archive.append( - JSON.stringify(criteriaReferenceState.reference || {}, null, 2), - { name: "grades/criteria_reference.json" } - ); + if (criteriaReferencesByConfigId.size > 0) { + for (const [configurationId, reference] of criteriaReferencesByConfigId.entries()) { + archive.append( + JSON.stringify(reference, null, 2), + { name: `grades/criteria_reference_${configurationId}.json` } + ); + } + } else { + archive.append(JSON.stringify({}, null, 2), { name: "grades/criteria_reference.json" }); + } const usedFolderNames = new Set(); const getUniqueHashFolderName = (baseHash, userId, sessionId) => { @@ -722,15 +331,7 @@ module.exports = function (server) { } for (const [groupKey, groupRecords] of mergedGroups.entries()) { - const sortedRecords = [...groupRecords].sort((a, b) => { - const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; - const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; - return ( - (a.studySessionId || 0) - (b.studySessionId || 0) || - (a.studyStepId || 0) - (b.studyStepId || 0) || - createdA - createdB - ); - }); + const sortedRecords = [...groupRecords].sort(compareGradeRecords); const csvRows = sortedRecords.map((record) => buildGradeCsvRow(record)); @@ -742,18 +343,10 @@ module.exports = function (server) { } for (const user of users) { - const records = (recordsByUser.get(user.id) || []).sort((a, b) => { - const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; - const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; - return ( - (a.studySessionId || 0) - (b.studySessionId || 0) || - (a.studyStepId || 0) - (b.studyStepId || 0) || - createdA - createdB - ); - }); + const userRecords = (recordsByUser.get(user.id) || []).sort(compareGradeRecords); const recordsByHash = new Map(); - for (const record of records) { + for (const record of userRecords) { const hashKey = record.sessionHash || null; if (!recordsByHash.has(hashKey)) recordsByHash.set(hashKey, []); recordsByHash.get(hashKey).push(record); @@ -775,22 +368,489 @@ module.exports = function (server) { } /** - * Calculates the version number of a submission by traversing backwards - * through the chain of previous submissions. - * @param {Object} submission - The current submission object to start from. - * @param {Map} submissionMap - A Map containing all related - * submissions for quick lookup by ID. - * @returns {number} - The calculated version number (starting at 1 for the original). + * Exports a single document to the archive based on its type. + * - Type 0 (PDF): exports annotations, comments (with votes), document_data, and the PDF file. + * - Type 1 (HTML) / Type 2 (Modal): exports edits, plain text, HTML, and document_data. + * - Type 4 (ZIP): exports the zip file and document_data. + * @param {Object} server - The server instance providing database models. + * @param {Object} doc - The document record from the database. + * @param {string} docFolder - The target folder path inside the archive. + * @param {Object} archive - The archiver instance to append files to. + * @returns {Promise} */ - function calculateSubmissionVersion(submission, submissionMap) { - let version = 1; - let currentSub = submission; - while (currentSub && currentSub.previousSubmissionId) { - const prevSub = submissionMap.get(currentSub.previousSubmissionId); - if (!prevSub) break; - version++; - currentSub = prevSub; + async function processDocumentForExport(server, doc, docFolder, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, docUserRoles, archive) { + // document_data for all types, at the doc level. + const documentData = await server.db.models.document_data.findAll({ + where: { documentId: doc.id, deleted: false }, + raw: true, + }); + if (documentData.length > 0) { + archive.append(JSON.stringify(documentData, null, 2), { name: `${docFolder}/document_data.json` }); + } + + const docMeta = { + ...doc.toJSON(), + userRoles: docUserRoles, + }; + archive.append(JSON.stringify(docMeta, null, 2), { name: `${docFolder}/meta.json` }); + + switch (doc.type) { + case 0: { // PDF + // Annotations live on study-session copies (parentDocumentId = doc.id), + // not on the root document. Collect all copy IDs and query across them. + const copies = await server.db.models.document.findAll({ + where: { parentDocumentId: doc.id }, + attributes: ['id'], + raw: true, + }); + const allDocIds = [doc.id, ...copies.map(c => c.id)]; + + let [annotations, comments] = await Promise.all([ + server.db.models.annotation.findAll({ where: { documentId: allDocIds }, raw: true }), + server.db.models.comment.findAll({ where: { documentId: allDocIds }, raw: true }), + ]); + + if (shouldExcludeNonConsentingAnnotations) { + const allUserIds = [...new Set([ + ...annotations.map(a => a.userId), + ...comments.map(c => c.userId), + ].filter(Boolean))]; + const consentedUserIds = await getConsentedUserIds(server, allUserIds); + annotations = annotations.filter(a => !a.userId || consentedUserIds.has(a.userId)); + comments = comments.filter(c => !c.userId || consentedUserIds.has(c.userId)); + } + + const commentVotes = await server.db.models.comment_vote.findAll({ + where: { commentId: comments.map(c => c.id), deleted: false }, + raw: true, + }); + const commentsWithVotes = comments.map(c => ({ + ...c, + votes: commentVotes.filter(v => v.commentId === c.id), + })); + + // All annotations and comments go into one file each. + if (annotations.length > 0) { + archive.append(JSON.stringify(annotations, null, 2), { name: `${docFolder}/annotations.json` }); + } + if (commentsWithVotes.length > 0) { + archive.append(JSON.stringify(commentsWithVotes, null, 2), { name: `${docFolder}/comments.json` }); + } + + appendStoredFileIfExists(archive, doc.hash, '.pdf', `${docFolder}/document.pdf`, 'PDF'); + break; + } + + case 1: // HTML + case 2: { // MODAL + // fetch all edits for this document, ordered chronologically + let allEdits = await server.db.models.document_edit.findAll({ + where: { documentId: doc.id, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }); + + // filter by consent unless the option is enabled + if (shouldExcludeNonConsentingEdits) { + const editorUserIds = [...new Set(allEdits.map(e => e.userId).filter(Boolean))]; + const consentedUserIds = await getConsentedUserIds(server, editorUserIds); + allEdits = allEdits.filter(e => !e.userId || consentedUserIds.has(e.userId)); + } + + // group edits by studySessionId (null = template) + const sessionGroups = new Map(); + for (const edit of allEdits) { + const key = edit.studySessionId ?? '__template__'; + if (!sessionGroups.has(key)) sessionGroups.set(key, []); + sessionGroups.get(key).push(edit); + } + + // fetch study sessions to resolve hashes + const sessionIds = [...sessionGroups.keys()].filter(k => k !== '__template__'); + const sessions = sessionIds.length > 0 + ? await server.db.models.study_session.findAll({ + where: { id: sessionIds }, + attributes: ['id', 'hash'], + raw: true, + }) + : []; + const sessionHashMap = new Map(sessions.map(s => [s.id, s.hash])); + + for (const [key, edits] of sessionGroups.entries()) { + const isTemplate = key === '__template__'; + const delta = dbToDelta(edits); + + // skip empty content + const text = deltaToPlainText(delta); + if (!text.trim()) continue; + + const subFolder = isTemplate + ? `${docFolder}/template` + : `${docFolder}/${sessionHashMap.get(key) ?? key}`; + + archive.append(text, { name: `${subFolder}/text.txt` }); + archive.append(deltaToHtml(delta), { name: `${subFolder}/html.html` }); + archive.append(JSON.stringify(edits, null, 2), { name: `${subFolder}/edits.json` }); + } + break; + } + + case 4: { // ZIP + appendStoredFileIfExists(archive, doc.hash, '.zip', `${docFolder}/document.zip`, 'ZIP'); + break; + } + + default: + console.warn(`[DocumentExport] Unhandled document type ${doc.type} for document ${doc.hash}, skipping.`); + } + } + + /** + * Main export function for the "documents" export type. + * Fetches all studies and steps for a project, collects unique documents, + * filters by owner data sharing consent, and exports each document to the archive. + * @param {Object} server - The server instance providing database models. + * @param {number|string} projectId - The ID of the project to export. + * @param {string} baseFolderName - The root folder name inside the ZIP archive. + * @param {Object} archive - The archiver instance to append files to. + * @param {Array} userIds - List of user IDs to filter documents by. + * @param {Array} documentTypes - List of document types to include (0=PDF, 1=HTML, 2=Modal, 4=ZIP). + * @returns {Promise} + */ + async function processDocumentBasedExport(server, projectId, userIds, documentTypes, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, baseFolderName, archive) { + try { + documentTypes = typeof documentTypes === 'string' ? JSON.parse(documentTypes) : documentTypes; + if (!Array.isArray(documentTypes)) documentTypes = [0, 1, 2, 4]; + } catch (e) { + console.warn("Could not parse documentTypes:", documentTypes); + documentTypes = [0, 1, 2, 4]; + } + + const docs = await server.db.models.document.findAll({ + where: { projectId, userId: userIds, deleted: false, parentDocumentId: null }, + }); + + if (docs.length === 0) { + console.warn(`[DocumentExport] No documents found for project ${projectId}`); + return; + } + + const filteredDocs = docs.filter(doc => + documentTypes.includes(doc.type) || documentTypes.includes(String(doc.type)) + ); + + if (filteredDocs.length === 0) { + console.warn(`[DocumentExport] No documents matching selected types found for project ${projectId}`); + return; + } + + const uniqueUserIds = [...new Set(filteredDocs.map(doc => doc.userId).filter(Boolean))]; + + const userRoleRows = await server.db.models.user_role_matching.findAll({ + where: { userId: uniqueUserIds }, + raw: true, + }); + + const rolesMap = {}; + for (const row of userRoleRows) { + if (!rolesMap[row.userId]) rolesMap[row.userId] = []; + rolesMap[row.userId].push(row.userRoleId); + } + + for (const doc of filteredDocs) { + const docFolder = `${baseFolderName}/${doc.hash}`; + const docUserRoles = rolesMap[doc.userId] || []; + await processDocumentForExport(server, doc, docFolder, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, docUserRoles, archive); + } + } + + function sortSteps(items, prevKey) { + const sorted = []; + let current = items.find(item => item[prevKey] === null); + while (current) { + sorted.push(current); + current = items.find(item => item[prevKey] === current.id); + } + return sorted; + } + + async function processStudyBasedExport(server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, workflowIds, shouldIncludeEmptyStudies, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, shouldIncludeDocumentFiles, shouldIncludeGrades, baseFolderName, archive) { + const studyWhere = { userId: userIds, projectId, deleted: false, workflowId: workflowIds }; + + const studies = await server.db.models.study.findAll({ where: studyWhere }); + + if (studies.length === 0) { + console.warn(`[StudyExport] No studies found for selected users in project ${projectId}`); + return; + } + + const writtenCriteriaReferenceIds = new Set(); + + for (const study of studies) { + const studyFolder = `${baseFolderName}/${study.hash}`; + + const allSteps = await server.db.models.study_step.findAll({ + where: { studyId: study.id, deleted: false } + }); + const sortedSteps = sortSteps(allSteps, 'studyStepPrevious'); + + const stepDocumentsById = new Map(); + // A step references one document, but PDF and LaTeX ZIP are separate documents + // of the same submission, each with its own hash. + const submissionDocumentsBySubmissionId = new Map(); + if (shouldIncludeDocumentFiles) { + const stepDocumentIds = [...new Set(sortedSteps.map(step => step.documentId).filter(Boolean))]; + const stepDocuments = stepDocumentIds.length > 0 + ? await server.db.models.document.findAll({ where: { id: stepDocumentIds }, raw: true }) + : []; + for (const doc of stepDocuments) stepDocumentsById.set(doc.id, doc); + + const submissionIds = [...new Set(stepDocuments.map(doc => doc.submissionId).filter(Boolean))]; + const submissionDocuments = submissionIds.length > 0 + ? await server.db.models.document.findAll({ + where: { submissionId: submissionIds, deleted: false }, + raw: true, + }) + : []; + for (const doc of submissionDocuments) { + if (!submissionDocumentsBySubmissionId.has(doc.submissionId)) { + submissionDocumentsBySubmissionId.set(doc.submissionId, []); + } + submissionDocumentsBySubmissionId.get(doc.submissionId).push(doc); + } + } + + const sessions = await server.db.models.study_session.findAll({ + where: { studyId: study.id, deleted: false }, + raw: true, + }); + + const sessionResults = []; + for (const session of sessions) { + const stepResults = []; + let sessionHasContent = false; + + for (let i = 0; i < sortedSteps.length; i++) { + const step = sortedSteps[i]; + const files = []; + + switch (step.stepType) { + case 1: { // Annotator + let annotations = await server.db.models.annotation.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + + let comments = await server.db.models.comment.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + + if (shouldExcludeNonConsentingAnnotations) { + const allUserIds = [...new Set([ + ...annotations.map(a => a.userId), + ...comments.map(c => c.userId) + ].filter(Boolean))]; + const consentedUsers = await server.db.models.user.findAll({ + where: { id: allUserIds }, + attributes: ['id', 'acceptDataSharing'], + raw: true, + }); + const consentedIds = new Set(consentedUsers.filter(u => u.acceptDataSharing).map(u => u.id)); + annotations = annotations.filter(a => !a.userId || consentedIds.has(a.userId)); + comments = comments.filter(c => !c.userId || consentedIds.has(c.userId)); + } + + if (annotations.length > 0) { + files.push({ name: 'annotations.json', content: JSON.stringify(annotations, null, 2) }); + sessionHasContent = true; + } + + if (comments.length > 0) { + const commentVotes = await server.db.models.comment_vote.findAll({ + where: { commentId: comments.map(c => c.id), deleted: false }, + raw: true, + }); + files.push({ + name: 'comments.json', + content: JSON.stringify( + comments.map(c => ({ ...c, votes: commentVotes.filter(v => v.commentId === c.id) })), + null, 2 + ) + }); + sessionHasContent = true; + } + + // Assessments live on the annotator step, so without this the whole + // document_data of an assessment workflow never leaves the database. + const annotatorData = await server.db.models.document_data.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + if (annotatorData.length > 0) { + files.push({ name: 'document_data.json', content: JSON.stringify(annotatorData, null, 2) }); + sessionHasContent = true; + } + break; + } + + case 2: // Editor + case 3: { // Modal + const [templateEdits, sessionEdits] = await Promise.all([ + server.db.models.document_edit.findAll({ + where: { documentId: step.documentId, studySessionId: null, studyStepId: null, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }), + server.db.models.document_edit.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }), + ]); + + let edits = [...templateEdits, ...sessionEdits].sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt)); + + if (shouldExcludeNonConsentingEdits) { + const editorUserIds = [...new Set(edits.map(e => e.userId).filter(Boolean))]; + const editorUsers = await server.db.models.user.findAll({ + where: { id: editorUserIds }, + attributes: ['id', 'acceptDataSharing'], + raw: true, + }); + const consentedIds = new Set(editorUsers.filter(u => u.acceptDataSharing).map(u => u.id)); + edits = edits.filter(e => !e.userId || consentedIds.has(e.userId)); + } + + if (edits.length > 0) { + const delta = dbToDelta(edits); + const text = deltaToPlainText(delta); + if (text.trim()) { + files.push({ name: 'edits.json', content: JSON.stringify(edits, null, 2) }); + files.push({ name: 'text.txt', content: text }); + files.push({ name: 'html.html', content: deltaToHtml(delta) }); + sessionHasContent = true; + } + } + + const documentData = await server.db.models.document_data.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + if (documentData.length > 0) { + files.push({ name: 'document_data.json', content: JSON.stringify(documentData, null, 2) }); + } + break; + } + } + + stepResults.push({ stepIndex: i, files }); + + } + + sessionResults.push({ session, stepResults, hasContent: sessionHasContent }); + } + + const includedSessions = shouldIncludeEmptyStudies + ? sessionResults + : sessionResults.filter(s => s.hasContent); + + if (!shouldIncludeEmptyStudies && includedSessions.length === 0) continue; + + const studyMeta = { + id: study.id, + name: study.name, + userId: study.userId, + workflowId: study.workflowId, + sessions: includedSessions.map(({ session }) => ({ + hash: session.hash, + id: session.id, + userId: session.userId, + numberSteps: session.numberSteps, + steps: sortedSteps.map((step, i) => ({ + id: step.id, + stepNumber: i + 1, + stepType: step.stepType, + configuration: step.configuration, + })) + })) + }; + + archive.append(JSON.stringify(studyMeta, null, 2), { name: `${studyFolder}/meta.json` }); + + let gradeRecordsBySessionId = new Map(); + if (shouldIncludeGrades) { + // Scope by this study's sessions, not by the study owner: review assessments are + // stored on a document owned by the reviewed author, so an owner-scoped lookup + // finds none of them. See buildGradeRecords(). + const { records: ownerGradeRecords, criteriaReferencesByConfigId } = await buildGradeRecords( + server, projectId, [study.userId], users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, + { sessionIds: sessions.map(s => s.id) } + ); + + for (const [configurationId, reference] of criteriaReferencesByConfigId.entries()) { + if (writtenCriteriaReferenceIds.has(configurationId)) continue; + writtenCriteriaReferenceIds.add(configurationId); + archive.append( + JSON.stringify(reference, null, 2), + { name: `${baseFolderName}/criteria_reference_${configurationId}.json` } + ); + } + + for (const record of ownerGradeRecords) { + if (!gradeRecordsBySessionId.has(record.studySessionId)) gradeRecordsBySessionId.set(record.studySessionId, []); + gradeRecordsBySessionId.get(record.studySessionId).push(record); + } + } + + for (const { session, stepResults } of includedSessions) { + const sessionFolder = `${studyFolder}/${session.hash}`; + + if (shouldIncludeGrades) { + const sessionGrades = (gradeRecordsBySessionId.get(session.id) || []) + .map(({ sessionHash, ...rest }) => rest) + .sort(compareGradeRecords); + if (sessionGrades.length > 0) { + archive.append(JSON.stringify(sessionGrades, null, 2), { name: `${sessionFolder}/grades.json` }); + } + } + + for (const { stepIndex, files } of stepResults) { + const stepFolder = `${sessionFolder}/step_${stepIndex + 1}`; + + if (shouldIncludeDocumentFiles) { + const step = sortedSteps[stepIndex]; + const stepDocument = stepDocumentsById.get(step.documentId); + if (stepDocument) { + const submissionSiblings = stepDocument.submissionId + ? (submissionDocumentsBySubmissionId.get(stepDocument.submissionId) || []) + : []; + const candidates = [ + stepDocument, + ...submissionSiblings.filter(doc => doc.id !== stepDocument.id), + ]; + + const appendedExtensions = new Set(); + for (const doc of candidates) { + const extension = STUDY_DOCUMENT_EXTENSIONS[doc.type]; + if (!extension || appendedExtensions.has(extension)) continue; + appendedExtensions.add(extension); + appendStoredFileIfExists( + archive, + doc.hash, + extension, + `${stepFolder}/document${extension}`, + extension.slice(1).toUpperCase(), + ); + } + } + } + + for (const file of files) { + archive.append(file.content, { name: `${stepFolder}/${file.name}` }); + } + } + } } - return version; } -}; +}; \ No newline at end of file From cbf44b70fbfaafc219c82ff1d52c7a1b0cef47ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 13:20:05 +0200 Subject: [PATCH 03/23] adds a different name for each download --- .../projects/export/StepConfirmDownload.vue | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue b/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue index 8afd4df92..30f9b3b56 100644 --- a/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue +++ b/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue @@ -6,7 +6,7 @@
Confirm Selection:
- +
You have selected one or more students who didn't accept data sharing.
@@ -21,14 +21,15 @@
Summary:
- You are about to download submissions for - {{ submissionSelection.length }} student(s). + You are about to download + {{ exportTypeLabel }} + for {{ userSelection.length }} user(s).
    -
  • - {{ row.studentName || row.userName }} ({{ row.fileCount }} files) +
  • + {{ row.name }} ({{ row.suffix }})
@@ -44,13 +45,12 @@ import BasicLoading from "@/basic/Loading.vue"; * * The final confirmation step within the ExportModal. * This component provides a summary of the selected - * submissions intended for download, as well as some + * data intended for download, as well as some * warnings for the user, if they selected generate aliases * or students who didn't accept data sharing. * * @author Mélissa Loew */ - export default { name: "StepConfirmDownload", components: { BasicLoading }, @@ -63,15 +63,40 @@ export default { type: Boolean, default: false }, - submissionSelection: { + userSelection: { type: Array, required: true + }, + exportType: { + type: String, + default: 'submissions' } }, computed: { hasDeclinedSharingSelected() { - return this.submissionSelection.some(row => row.acceptDataSharing === 'No'); - } + return this.userSelection.some(row => row.acceptDataSharing === 'No'); + }, + exportTypeLabel() { + const labels = { + submissions: 'submissions', + grades: 'grades', + documents: 'documents', + studies: 'studies', + }; + return labels[this.exportType] || 'documents'; + }, + userSelectionDisplay() { + const unitByExportType = { + submissions: 'submission(s)', + studies: 'study(ies)', + }; + const unit = unitByExportType[this.exportType] || 'document(s)'; + return this.userSelection.map(row => ({ + userId: row.userId, + name: row.studentName || row.userName, + suffix: this.exportType === 'grades' ? null : `${row.count} ${unit}`, + })); + }, } } \ No newline at end of file From 70959a48f713c97c201acd9c4d20e7ed86036b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 13:21:56 +0200 Subject: [PATCH 04/23] added StepOptionsDocuments.vue for the doc-based export --- .../projects/export/StepOptionsDocuments.vue | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue b/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue new file mode 100644 index 000000000..5389f51c9 --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue @@ -0,0 +1,117 @@ + + + \ No newline at end of file From 3d4627178e69b40dfb2a1646c1b8fb0aca2d94a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 13:23:10 +0200 Subject: [PATCH 05/23] added StepOptionsStudies.vue for the study-based export --- .../projects/export/StepOptionsStudies.vue | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue new file mode 100644 index 000000000..7cd6e641d --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue @@ -0,0 +1,157 @@ + + + \ No newline at end of file From 84929b5eec0a5976d0045869249cb0fd833606cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 13:23:53 +0200 Subject: [PATCH 06/23] updated with config view --- .../projects/export/StepSelectStudents.vue | 220 ++++++++++++++---- 1 file changed, 170 insertions(+), 50 deletions(-) diff --git a/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue b/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue index 8ca0a4962..25bc4c38b 100644 --- a/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue +++ b/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue @@ -1,17 +1,17 @@ + - @@ -87,8 +110,11 @@ import JSZip from 'jszip'; import FileSaver from 'file-saver'; import Quill from "quill"; import {dbToDelta} from "editor-delta-conversion"; -import StepSelectStudents from "@/components/dashboard/projects/export/StepSelectStudents.vue"; +import BasicLoading from "@/basic/Loading.vue"; +import StepSelectUsers from "@/components/dashboard/projects/export/StepSelectUsers.vue"; import StepOptions from "@/components/dashboard/projects/export/StepOptions.vue"; +import StepOptionsDocuments from "@/components/dashboard/projects/export/StepOptionsDocuments.vue"; +import StepOptionsStudies from "@/components/dashboard/projects/export/StepOptionsStudies.vue"; import StepConfirmDownload from "@/components/dashboard/projects/export/StepConfirmDownload.vue"; import getServerURL from "@/assets/serverUrl.js"; @@ -96,11 +122,11 @@ import getServerURL from "@/assets/serverUrl.js"; /** * ProjectModal - modal component for adding and editing projects * - * @author Dennis Zyska, Mélissa Loew, Linyin Huang + * @author Dennis Zyska, Mélissa Loew */ export default { name: "ExportProjectModal", - components: { StepperModal, BasicForm, StepSelectStudents, StepOptions, StepConfirmDownload }, + components: { BasicLoading, StepperModal, BasicForm, StepSelectUsers, StepOptions, StepOptionsDocuments, StepOptionsStudies, StepConfirmDownload }, subscribeTable: [{ table: "document", }, { @@ -119,6 +145,14 @@ export default { table: "tag_set", }, { table: "tag" + }, { + table: "document_data", + }, { + table: "study_step", + }, { + table: "configuration", + }, { + table: "workflow", } ], provide() { @@ -135,11 +169,17 @@ export default { filter: [], wait: false, // Data for Export Submissions - submissionSelection: [], + userSelection: [], generateAliases:false, fakerSeed: 846569412, gradeFormat: "json", - mergeCsvFiles: false + mergeCsvFiles: false, + selectedDocumentTypes: [0, 1, 2, 4], + excludeNonConsentingEdits: false, + excludeNonConsentingAnnotations: false, + selectedWorkflowIds: [], + includeStudyGrades: true, + includeEmptyStudies: false }; }, computed: { @@ -147,10 +187,24 @@ export default { if (["submissions", "grades"].includes(this.dataSelection.exportType)) { return [ !!this.dataSelection.projectId && !!this.dataSelection.exportType, // must select a valid project and export type - this.submissionSelection.length > 0, // must select at least one student + this.userSelection.length > 0, // must select at least one student true, true ]; + } else if (this.dataSelection.exportType === "documents") { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + this.selectedDocumentTypes.length > 0, + true, + ]; + } else if (this.dataSelection.exportType === 'studies') { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + this.selectedWorkflowIds.length > 0, + true, + ]; } return [ !!this.dataSelection.projectId && !!this.dataSelection.exportType, @@ -158,14 +212,14 @@ export default { ]; }, steps() { - if (["submissions", "grades"].includes(this.dataSelection.exportType)) { + if (["submissions", "grades", "documents", "studies"].includes(this.dataSelection.exportType)) { return [ { title: "Settings" }, - { title: "Select Students" }, + { title: "Select Users" }, { title: "Options" }, { title: "Confirm Download" } ]; - } + } return [ {title: "Settings"}, {title: "Confirmation"} @@ -191,6 +245,8 @@ export default { {name: "Export a list of all reviewers", value: "reviewerList"}, {name: "Export submissions", value: "submissions"}, {name: "Export grades", value: "grades"}, + {name: "Export documents", value:"documents"}, + {name: "Export studies", value: "studies"}, {name: "All", value: "all"}, ], required: true, @@ -256,6 +312,16 @@ export default { }, hide() { this.filter = []; + this.userSelection = []; + this.generateAliases = false; + this.fakerSeed = 846569412; + this.selectedDocumentTypes = [0, 1, 2, 4]; + this.excludeNonConsentingEdits = false; + this.excludeNonConsentingAnnotations = false; + this.selectedWorkflowIds = []; + this.includeStudyGrades = true; + this.includeEmptyStudies = false; + this.wait = false; }, downloadData() { if (this.dataSelection.exportType === "reviewerList") { @@ -264,6 +330,10 @@ export default { this.downloadSubmissions(); } else if (this.dataSelection.exportType === "grades") { this.downloadGrades(); + } else if (this.dataSelection.exportType === "documents") { + this.downloadDocuments(); + } else if (this.dataSelection.exportType === 'studies') { + this.downloadStudies(); } else { this.downloadAllData(); } @@ -327,7 +397,7 @@ export default { async downloadSubmissions() { try { // get the selected student's user ids - const selectedUserIds = this.submissionSelection.map(row => row.userId); + const selectedUserIds = this.userSelection.map(row => row.userId); // call helper function to trigger the stream download this.triggerStreamDownload({ @@ -346,7 +416,7 @@ export default { }, async downloadGrades() { try { - const selectedUserIds = this.submissionSelection.map(row => row.userId); + const selectedUserIds = this.userSelection.map(row => row.userId); this.triggerStreamDownload({ projectId: this.dataSelection.projectId, exportType: 'grades', @@ -362,6 +432,44 @@ export default { this.$toast.error("An error occurred starting the stream. Please try again."); } }, + async downloadDocuments() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'documents', + userIds: selectedUserIds, + documentTypes: this.selectedDocumentTypes, + excludeNonConsentingEdits: this.excludeNonConsentingEdits, + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations + }); + + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, + async downloadStudies() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'studies', + userIds: selectedUserIds, + workflowIds: this.selectedWorkflowIds, + includeEmptyStudies: this.includeEmptyStudies, + includeDocumentFiles: this.includeStudyDocumentFiles, + includeGrades: this.includeStudyGrades, + excludeNonConsentingEdits: this.excludeNonConsentingEdits, + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations + }); + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, async downloadAllData() { this.wait = true; From c14978e7d89b280fcb4ee5c67da248ca505ffe08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 13:36:01 +0200 Subject: [PATCH 08/23] file naming error --- backend/package-lock.json | 348 +++++++++++++++++- ...SelectStudents.vue => StepSelectUsers.vue} | 0 .../editor-delta-conversion/package-lock.json | 2 + 3 files changed, 348 insertions(+), 2 deletions(-) rename frontend/src/components/dashboard/projects/export/{StepSelectStudents.vue => StepSelectUsers.vue} (100%) diff --git a/backend/package-lock.json b/backend/package-lock.json index c8ac72d58..324071f0e 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -12,6 +12,7 @@ "@node-saml/passport-saml": "^5.1.0", "archiver": "^7.0.1", "argparse": "^2.0.1", + "assessment-score": "file:../utils/modules/assessment-score", "axios": "^1.13.6", "axios-oauth-client": "^2.2.0", "bcrypt": "^6.0.0", @@ -56,12 +57,21 @@ }, "devDependencies": { "cross-env": "^10.1.0", - "jest": "^30.2.0" + "jest": "^30.2.0", + "nodemon": "^3.1.10" }, "engines": { "node": ">=18.0.0" } }, + "../utils/modules/assessment-score": { + "version": "1.0.0", + "license": "Apache-2.0", + "devDependencies": { + "cross-env": "^7.0.3", + "jest": "^30.3.0" + } + }, "../utils/modules/editor-delta-conversion": { "version": "1.0.0", "license": "Apache-2.0", @@ -105,6 +115,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2099,6 +2110,10 @@ "node": ">=0.8" } }, + "node_modules/assessment-score": { + "resolved": "../utils/modules/assessment-score", + "link": true + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -2134,6 +2149,7 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.15.11", "form-data": "^4.0.5", @@ -2294,6 +2310,7 @@ "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", "license": "Apache-2.0", + "peer": true, "peerDependencies": { "bare-abort-controller": "*" }, @@ -2451,6 +2468,19 @@ "integrity": "sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==", "license": "MIT" }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -2496,6 +2526,19 @@ "balanced-match": "^1.0.0" } }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/browserslist": { "version": "4.28.2", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", @@ -2516,6 +2559,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -2706,6 +2750,31 @@ "node": "*" } }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, "node_modules/ci-info": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", @@ -3677,6 +3746,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -3848,6 +3918,19 @@ "moment": "^2.29.1" } }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/finalhandler": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", @@ -4143,6 +4226,19 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -4289,6 +4385,13 @@ ], "license": "BSD-3-Clause" }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true, + "license": "ISC" + }, "node_modules/immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", @@ -4382,6 +4485,19 @@ "dev": true, "license": "MIT" }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", @@ -4403,6 +4519,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -4422,6 +4548,29 @@ "node": ">=6" } }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -5792,6 +5941,110 @@ "node": ">=6.0.0" } }, + "node_modules/nodemon": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.14.tgz", + "integrity": "sha512-jakjZi93UtB3jHMWsXL68FXSAosbLfY0In5gtKq3niLSkrWznrVBzXFNOEMJUfc9+Ke7SHWoAZsiMkNP3vq6Jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^10.2.1", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/nodemon/node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/nodemon/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/nodemon/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/nodemon/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/nodemon/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/nopt": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.1.tgz", @@ -6204,6 +6457,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -6244,7 +6498,6 @@ "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.19.0.tgz", "integrity": "sha512-J5cF1MUz7LRJ9emOqF/06QjabMHMZy587rSPF0UuA8rCwKeeYl2co8Pp+6k5UU9YrAYHMzWkLxilfZB0hqsWWw==", "license": "MIT", - "peer": true, "peerDependencies": { "pg": "^8" } @@ -6498,6 +6751,13 @@ "node": ">=10" } }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true, + "license": "MIT" + }, "node_modules/pure-rand": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", @@ -6619,6 +6879,32 @@ "node": ">=10" } }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/readdirp/node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -6794,6 +7080,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@types/debug": "^4.1.8", "@types/validator": "^13.7.17", @@ -7213,6 +7500,32 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-update-notifier/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -7806,6 +8119,19 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -7821,6 +8147,16 @@ "integrity": "sha512-OsLcGGbYF3rMjPUf8oKktyvCiUxSbqMMS39m33MAjLTC1DVIH6x3WSt63/M77ihI09+Sdfk1AXvfhCEeUmC7mg==", "license": "MIT" }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "license": "ISC", + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, "node_modules/triple-beam": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", @@ -7914,6 +8250,13 @@ "node": ">=6.0.0" } }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true, + "license": "MIT" + }, "node_modules/undici-types": { "version": "7.19.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", @@ -8135,6 +8478,7 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", "license": "MIT", + "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", diff --git a/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue b/frontend/src/components/dashboard/projects/export/StepSelectUsers.vue similarity index 100% rename from frontend/src/components/dashboard/projects/export/StepSelectStudents.vue rename to frontend/src/components/dashboard/projects/export/StepSelectUsers.vue diff --git a/utils/modules/editor-delta-conversion/package-lock.json b/utils/modules/editor-delta-conversion/package-lock.json index 26dee65da..2c2c24cce 100644 --- a/utils/modules/editor-delta-conversion/package-lock.json +++ b/utils/modules/editor-delta-conversion/package-lock.json @@ -48,6 +48,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -1670,6 +1671,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", From dd0a3c49ca62fc61440314333970b470f69a98d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Thu, 30 Jul 2026 16:42:28 +0200 Subject: [PATCH 09/23] grades.json => scores.json --- backend/utils/helper/export.js | 2 +- backend/webserver/routes/export.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/utils/helper/export.js b/backend/utils/helper/export.js index a41b1ff81..35aa9385c 100644 --- a/backend/utils/helper/export.js +++ b/backend/utils/helper/export.js @@ -465,7 +465,7 @@ function parseUserIds(rawUserIds) { * Builds flat grade records for the given users/project by resolving each * assessment_result row's session/study/step/configuration context and score. * Shared by processGradesExport (grouped by user for JSON/CSV output) and - * processStudyBasedExport (grouped by session for a per-session grades.json). + * processStudyBasedExport (grouped by session for a per-session scores.json). * * @param {Object} server - The server instance providing database models and Sequelize operators. * @param {number} projectId - The project whose grades should be resolved. diff --git a/backend/webserver/routes/export.js b/backend/webserver/routes/export.js index 08d713327..44391b0bf 100644 --- a/backend/webserver/routes/export.js +++ b/backend/webserver/routes/export.js @@ -811,7 +811,7 @@ module.exports = function (server) { .map(({ sessionHash, ...rest }) => rest) .sort(compareGradeRecords); if (sessionGrades.length > 0) { - archive.append(JSON.stringify(sessionGrades, null, 2), { name: `${sessionFolder}/grades.json` }); + archive.append(JSON.stringify(sessionGrades, null, 2), { name: `${sessionFolder}/scores.json` }); } } From 64611c62ac72bb723dd478d2dcab56591c46eee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Fri, 31 Jul 2026 08:48:06 +0200 Subject: [PATCH 10/23] added deltaToHtml --- .../dashboard/projects/ExportModal.vue | 2 +- .../projects/export/StepOptionsStudies.vue | 11 ++++++++- .../modules/editor-delta-conversion/index.js | 24 ++++++++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/dashboard/projects/ExportModal.vue b/frontend/src/components/dashboard/projects/ExportModal.vue index c2e69824a..0edd0dc00 100644 --- a/frontend/src/components/dashboard/projects/ExportModal.vue +++ b/frontend/src/components/dashboard/projects/ExportModal.vue @@ -178,7 +178,7 @@ export default { excludeNonConsentingEdits: false, excludeNonConsentingAnnotations: false, selectedWorkflowIds: [], - includeStudyGrades: true, + includeStudyGrades: false, includeEmptyStudies: false }; }, diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue index 7cd6e641d..67d574f99 100644 --- a/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue +++ b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue @@ -48,7 +48,7 @@ export default { }, includeGrades: { type: Boolean, - default: true + default: false }, excludeNonConsentingEdits: { type: Boolean, @@ -141,6 +141,15 @@ export default { excludeNonConsentingAnnotations(value) { this.optionsData.excludeNonConsentingAnnotations = value; }, + workflows: { + immediate: true, + handler(newWorkflows) { + if (this.optionsData.selectedWorkflowIds.length === 0 && newWorkflows.length > 0) { + this.optionsData.selectedWorkflowIds = newWorkflows.map(wf => wf.id); + this.$emit('update:selectedWorkflowIds', this.optionsData.selectedWorkflowIds); + } + } + }, optionsData: { handler(value) { this.$emit('update:selectedWorkflowIds', value.selectedWorkflowIds); diff --git a/utils/modules/editor-delta-conversion/index.js b/utils/modules/editor-delta-conversion/index.js index c8d722197..763b6e097 100644 --- a/utils/modules/editor-delta-conversion/index.js +++ b/utils/modules/editor-delta-conversion/index.js @@ -2,10 +2,11 @@ * * This module provides methods to convert between Quill Delta objects and database entries. * - * @author Juliane Bechert + * @author Juliane Bechert, Mélissa Loew * */ const Delta = require('quill-delta'); +const { QuillDeltaToHtmlConverter } = require('quill-delta-to-html'); /** * Converts an array of database entries to a Quill Delta object. @@ -134,8 +135,29 @@ function deltaToPlainText(deltaOrOps) { .join(""); } +/** + * Converts a Quill Delta object to an HTML string. + * Each newline in the delta marks the end of a paragraph and is flushed as a

tag. + * Supports bold, italic, underline, and link attributes. + * + * @param {object|array} deltaOrOps - Quill Delta ({ ops: [...] }) or ops array + * @returns {string} A full HTML document string + */ +function deltaToHtml(deltaOrOps) { + if (!deltaOrOps) return ""; + const ops = Array.isArray(deltaOrOps) + ? deltaOrOps + : (deltaOrOps.ops || []); + + const converter = new QuillDeltaToHtmlConverter(ops, {}); + const body = converter.convert(); + + return `\n\n\n${body}\n`; +} + module.exports = { deltaToDb: deltaToDb, dbToDelta: dbToDelta, deltaToPlainText: deltaToPlainText, + deltaToHtml: deltaToHtml, } \ No newline at end of file From 2085c4d985c7aff250851aa2c77e721869e40729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Fri, 31 Jul 2026 09:00:01 +0200 Subject: [PATCH 11/23] added delta to html dependency --- .../editor-delta-conversion/package-lock.json | 12 +++++++++++- utils/modules/editor-delta-conversion/package.json | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/utils/modules/editor-delta-conversion/package-lock.json b/utils/modules/editor-delta-conversion/package-lock.json index 2c2c24cce..46796e133 100644 --- a/utils/modules/editor-delta-conversion/package-lock.json +++ b/utils/modules/editor-delta-conversion/package-lock.json @@ -10,7 +10,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.2", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", @@ -3653,6 +3654,15 @@ "node": ">= 12.0.0" } }, + "node_modules/quill-delta-to-html": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/quill-delta-to-html/-/quill-delta-to-html-0.12.1.tgz", + "integrity": "sha512-QhpeMk9+5ge3HYbL5A0Ewz3pXCsbemqGvIF/kw5D6D4V68AtcUp7yt9xNUkzOk/0IQz43hKy3IkzBzRhLIE+oA==", + "license": "ISC", + "dependencies": { + "lodash.isequal": "^4.5.0" + } + }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", diff --git a/utils/modules/editor-delta-conversion/package.json b/utils/modules/editor-delta-conversion/package.json index d4d5d2ad5..fbf701b85 100644 --- a/utils/modules/editor-delta-conversion/package.json +++ b/utils/modules/editor-delta-conversion/package.json @@ -18,7 +18,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.2", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", From edd21cdbff1f2114decb975a3c9bf7699682ced8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Sat, 1 Aug 2026 22:07:36 +0200 Subject: [PATCH 12/23] added ai score filtering for study export --- backend/utils/helper/export.js | 4 ++- backend/webserver/routes/export.js | 21 ++++++++++----- .../dashboard/projects/ExportModal.vue | 14 +++++++--- .../projects/export/StepOptionsStudies.vue | 26 ++++++++++++++----- 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/backend/utils/helper/export.js b/backend/utils/helper/export.js index 35aa9385c..2ec6f540b 100644 --- a/backend/utils/helper/export.js +++ b/backend/utils/helper/export.js @@ -542,6 +542,7 @@ async function buildGradeRecords(server, projectId, userIds, users, shouldGenera const studyStep = studyStepsById.get(row.studyStepId); const submission = document.submission; const studyStepConfiguration = studyStep?.configuration; + const isAiGraded = Array.isArray(studyStepConfiguration?.services) && studyStepConfiguration.services.some(s => s.type === "nlpRequest"); const configurationId = getAssessmentConfigurationId(studyStepConfiguration); const studyName = study?.name || `study_${session?.studyId || "unknown"}`; @@ -579,7 +580,8 @@ async function buildGradeRecords(server, projectId, userIds, users, shouldGenera scores: flatScores, totalPoints, createdAt: row.createdAt ? new Date(row.createdAt).toISOString() : null, - studyStepType: studyStep?.stepType ?? null + studyStepType: studyStep?.stepType ?? null, + isAiGraded }); } diff --git a/backend/webserver/routes/export.js b/backend/webserver/routes/export.js index 44391b0bf..b8c4b3731 100644 --- a/backend/webserver/routes/export.js +++ b/backend/webserver/routes/export.js @@ -43,7 +43,7 @@ module.exports = function (server) { // Input parsing - const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles, excludeNonConsentingEdits, excludeNonConsentingAnnotations, includeEmptyStudies, includeDocumentFiles, includeGrades } = req.body; + const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles, excludeNonConsentingEdits, excludeNonConsentingAnnotations, includeEmptyStudies, includeDocumentFiles, includeGrades, includeAiScores } = req.body; let { userIds: rawUserIds = [], documentTypes = [0, 1, 2, 4], workflowIds = [] } = req.body; const shouldGenerateAliases = String(generateAliases) === 'true'; const shouldMergeCsvFiles = String(mergeCsvFiles) === "true"; @@ -52,6 +52,7 @@ module.exports = function (server) { const shouldIncludeEmptyStudies = String(includeEmptyStudies) === 'true'; const shouldIncludeDocumentFiles = String(includeDocumentFiles) === 'true'; const shouldIncludeGrades = String(includeGrades) === 'true'; + const shouldIncludeAiScores = includeAiScores === undefined ? true : String(includeAiScores) === 'true'; const normalizedGradeFormat = String(gradeFormat || "json").toLowerCase(); const parsedProjectId = Number(projectId); const userIds = parseUserIds(rawUserIds); @@ -140,6 +141,7 @@ module.exports = function (server) { shouldExcludeNonConsentingAnnotations, shouldIncludeDocumentFiles, shouldIncludeGrades, + shouldIncludeAiScores, baseFolderName, archive ); @@ -574,7 +576,7 @@ module.exports = function (server) { return sorted; } - async function processStudyBasedExport(server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, workflowIds, shouldIncludeEmptyStudies, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, shouldIncludeDocumentFiles, shouldIncludeGrades, baseFolderName, archive) { + async function processStudyBasedExport(server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, workflowIds, shouldIncludeEmptyStudies, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, shouldIncludeDocumentFiles, shouldIncludeGrades, shouldIncludeAiScores, baseFolderName, archive) { const studyWhere = { userId: userIds, projectId, deleted: false, workflowId: workflowIds }; const studies = await server.db.models.study.findAll({ where: studyWhere }); @@ -807,11 +809,16 @@ module.exports = function (server) { const sessionFolder = `${studyFolder}/${session.hash}`; if (shouldIncludeGrades) { - const sessionGrades = (gradeRecordsBySessionId.get(session.id) || []) - .map(({ sessionHash, ...rest }) => rest) - .sort(compareGradeRecords); - if (sessionGrades.length > 0) { - archive.append(JSON.stringify(sessionGrades, null, 2), { name: `${sessionFolder}/scores.json` }); + const allSessionGrades = (gradeRecordsBySessionId.get(session.id) || []).sort(compareGradeRecords); + + const humanGrades = allSessionGrades.filter(r => !r.isAiGraded).map(({ sessionHash, isAiGraded, ...rest }) => rest); + const aiGrades = allSessionGrades.filter(r => r.isAiGraded).map(({ sessionHash, isAiGraded, ...rest }) => rest); + + if (humanGrades.length > 0) { + archive.append(JSON.stringify(humanGrades, null, 2), { name: `${sessionFolder}/scores.json` }); + } + if (shouldIncludeAiScores && aiGrades.length > 0) { + archive.append(JSON.stringify(aiGrades, null, 2), { name: `${sessionFolder}/scores_ai.json` }); } } diff --git a/frontend/src/components/dashboard/projects/ExportModal.vue b/frontend/src/components/dashboard/projects/ExportModal.vue index 0edd0dc00..6112e04eb 100644 --- a/frontend/src/components/dashboard/projects/ExportModal.vue +++ b/frontend/src/components/dashboard/projects/ExportModal.vue @@ -81,6 +81,7 @@ v-model:includeEmptyStudies="includeEmptyStudies" v-model:includeDocumentFiles="includeStudyDocumentFiles" v-model:includeGrades="includeStudyGrades" + v-model:includeAiScores="includeStudyIncludeAiScores" v-model:excludeNonConsentingEdits="excludeNonConsentingEdits" v-model:excludeNonConsentingAnnotations="excludeNonConsentingAnnotations" /> @@ -178,8 +179,10 @@ export default { excludeNonConsentingEdits: false, excludeNonConsentingAnnotations: false, selectedWorkflowIds: [], - includeStudyGrades: false, - includeEmptyStudies: false + includeStudyDocumentFiles: true, + includeStudyGrades: true, + includeStudyIncludeAiScores: true, + includeEmptyStudies: true }; }, computed: { @@ -319,8 +322,10 @@ export default { this.excludeNonConsentingEdits = false; this.excludeNonConsentingAnnotations = false; this.selectedWorkflowIds = []; + includeStudyDocumentFiles = true; this.includeStudyGrades = true; - this.includeEmptyStudies = false; + this.includeStudyIncludeAiScores = true; + this.includeEmptyStudies = true; this.wait = false; }, downloadData() { @@ -462,7 +467,8 @@ export default { includeDocumentFiles: this.includeStudyDocumentFiles, includeGrades: this.includeStudyGrades, excludeNonConsentingEdits: this.excludeNonConsentingEdits, - excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations, + includeAiScores: this.includeStudyIncludeAiScores }); this.$refs.exportStepper.close(); } catch (error) { diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue index 67d574f99..f77a595a8 100644 --- a/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue +++ b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue @@ -40,15 +40,15 @@ export default { }, includeEmptyStudies: { type: Boolean, - default: false + default: true }, includeDocumentFiles: { type: Boolean, - default: false + default: true }, includeGrades: { type: Boolean, - default: false + default: true }, excludeNonConsentingEdits: { type: Boolean, @@ -57,9 +57,13 @@ export default { excludeNonConsentingAnnotations: { type: Boolean, default: false - } + }, + includeAiScores: { + type: Boolean, + default: true + }, }, - emits: ['update:selectedWorkflowIds', 'update:includeEmptyStudies', 'update:includeDocumentFiles', 'update:includeGrades', 'update:excludeNonConsentingEdits', 'update:excludeNonConsentingAnnotations'], + emits: ['update:selectedWorkflowIds', 'update:includeEmptyStudies', 'update:includeDocumentFiles', 'update:includeGrades', 'update:excludeNonConsentingEdits', 'update:excludeNonConsentingAnnotations', 'update:includeAiScores'], data() { return { optionsData: { @@ -68,7 +72,8 @@ export default { includeDocumentFiles: this.includeDocumentFiles, includeGrades: this.includeGrades, excludeNonConsentingEdits: this.excludeNonConsentingEdits, - excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations, + includeAiScores: this.includeAiScores } }; }, @@ -107,6 +112,11 @@ export default { label: "Include grades", type: "switch", }, + ...(this.optionsData.includeGrades ? [{ + key: "includeAiScores", + label: "Include AI-assisted scores", + type: "switch", + }] : []), { key: "excludeNonConsentingEdits", label: "Exclude edits from non-consenting users", @@ -135,6 +145,9 @@ export default { includeGrades(value) { this.optionsData.includeGrades = value; }, + includeAiScores(value) { + this.optionsData.includeAiScores = value; + }, excludeNonConsentingEdits(value) { this.optionsData.excludeNonConsentingEdits = value; }, @@ -158,6 +171,7 @@ export default { this.$emit('update:includeGrades', value.includeGrades); this.$emit('update:excludeNonConsentingEdits', value.excludeNonConsentingEdits); this.$emit('update:excludeNonConsentingAnnotations', value.excludeNonConsentingAnnotations); + this.$emit('update:includeAiScores', value.includeAiScores); }, deep: true } From 6fa0e01cb2a5464402fb0549c665186266654c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Sun, 2 Aug 2026 15:07:19 +0200 Subject: [PATCH 13/23] added tagName along tagId in annotations.json --- backend/utils/helper/export.js | 15 +++++++++++++++ backend/webserver/routes/export.js | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/backend/utils/helper/export.js b/backend/utils/helper/export.js index 2ec6f540b..d0baade80 100644 --- a/backend/utils/helper/export.js +++ b/backend/utils/helper/export.js @@ -588,6 +588,20 @@ async function buildGradeRecords(server, projectId, userIds, users, shouldGenera return { records, criteriaReferencesByConfigId }; } +async function attachTagNames(server, annotations) { + const tagIds = [...new Set(annotations.map(a => a.tagId).filter(Boolean))]; + if (tagIds.length === 0) return annotations; + + const tags = await server.db.models.tag.findAll({ + where: { id: tagIds }, + attributes: ['id', 'name'], + raw: true, + }); + const tagNameById = new Map(tags.map(t => [t.id, t.name])); + + return annotations.map(a => ({ ...a, tagName: tagNameById.get(a.tagId) ?? null })); +} + module.exports = { replaceAuthorInZip, buildUserMapping, @@ -609,4 +623,5 @@ module.exports = { loadExportRequestContext, SUPPORTED_EXPORT_TYPES, buildGradeRecords, + attachTagNames, }; \ No newline at end of file diff --git a/backend/webserver/routes/export.js b/backend/webserver/routes/export.js index b8c4b3731..4b7efc706 100644 --- a/backend/webserver/routes/export.js +++ b/backend/webserver/routes/export.js @@ -24,6 +24,7 @@ const { loadExportRequestContext, SUPPORTED_EXPORT_TYPES, buildGradeRecords, + attachTagNames, } = require('../../utils/helper/export.js'); const storageDir = path.join(__dirname, "..", "..", "..", "files"); @@ -412,6 +413,8 @@ module.exports = function (server) { server.db.models.comment.findAll({ where: { documentId: allDocIds }, raw: true }), ]); + annotations = await attachTagNames(server, annotations); + if (shouldExcludeNonConsentingAnnotations) { const allUserIds = [...new Set([ ...annotations.map(a => a.userId), @@ -642,6 +645,8 @@ module.exports = function (server) { where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, raw: true, }); + + annotations = await attachTagNames(server, annotations); let comments = await server.db.models.comment.findAll({ where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, From 34ac2452dd2b0308d327e971f275a3ef2331ea42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A9lissa=20Loew?= Date: Sun, 2 Aug 2026 17:45:44 +0200 Subject: [PATCH 14/23] added export user behaviour --- backend/utils/helper/export.js | 110 +++++++++++++++++- backend/webserver/routes/export.js | 80 ++++++++++++- .../dashboard/projects/ExportModal.vue | 47 +++++++- .../export/StepOptionsUserBehaviour.vue | 68 +++++++++++ .../projects/export/StepSelectUsers.vue | 3 +- 5 files changed, 299 insertions(+), 9 deletions(-) create mode 100644 frontend/src/components/dashboard/projects/export/StepOptionsUserBehaviour.vue diff --git a/backend/utils/helper/export.js b/backend/utils/helper/export.js index d0baade80..63b6988ed 100644 --- a/backend/utils/helper/export.js +++ b/backend/utils/helper/export.js @@ -5,8 +5,9 @@ const { deriveUserSeed } = require('../../webserver/auth/utils'); const path = require('path'); const storageDir = path.join(__dirname, "..", "..", "..", "files"); const Papa = require('papaparse'); +const { Readable } = require('stream'); -const SUPPORTED_EXPORT_TYPES = new Set(["submissions", "grades", "documents", "studies"]); +const SUPPORTED_EXPORT_TYPES = new Set(["submissions", "grades", "documents", "studies", "userBehaviour"]); const { calculateAssessmentScore, buildScoresFromState } = require('assessment-score'); const ASSESSMENT_RESULT_KEY = "assessment_result"; @@ -602,6 +603,110 @@ async function attachTagNames(server, annotations) { return annotations.map(a => ({ ...a, tagName: tagNameById.get(a.tagId) ?? null })); } +async function resolveIsAdmin(server, userId) { + const roleIds = await server.db.models["user_role_matching"].getUserRolesById(userId); + return await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); +} + +/** + * Builds a Readable that emits a JSON array incrementally, paging through fetchPage(lastId, limit) + * using keyset pagination so the full result set is never held in memory at once. + * @param {(lastId: number, limit: number) => Promise>} fetchPage + */ +function createJsonArrayStream(fetchPage, mapRow = (row) => row, pageSize = 1000) { + let lastId = 0; + let started = false; + let finished = false; + let isFirst = true; + let fetching = false; + + return new Readable({ + read() { + if (fetching || finished) return; + fetching = true; + + (async () => { + try { + if (!started) { + this.push('['); + started = true; + } + + const rows = await fetchPage(lastId, pageSize); + if (rows.length === 0) { + this.push('\n]'); + this.push(null); + finished = true; + return; + } + + let chunk = ''; + for (const row of rows) { + chunk += (isFirst ? '' : ',') + '\n' + JSON.stringify(mapRow(row)); + isFirst = false; + } + lastId = rows[rows.length - 1].id; + + if (rows.length < pageSize) { + chunk += '\n]'; + this.push(chunk); + this.push(null); + finished = true; + } else { + this.push(chunk); + } + } catch (err) { + this.destroy(err); + } finally { + fetching = false; + } + })(); + } + }); +} + +function createCsvRowsStream(fetchPage, mapRow, pageSize = 1000) { + let lastId = 0; + let started = false; + let finished = false; + let fetching = false; + + return new Readable({ + read() { + if (fetching || finished) return; + fetching = true; + + (async () => { + try { + const rows = await fetchPage(lastId, pageSize); + if (rows.length === 0) { + if (!started) this.push(Papa.unparse([mapRow].length ? [] : [])); + this.push(null); + finished = true; + return; + } + + const records = rows.map(mapRow); + const csvChunk = Papa.unparse(records, { header: !started }) + '\n'; + started = true; + lastId = rows[rows.length - 1].id; + + this.push(csvChunk); + + if (rows.length < pageSize) { + this.push(null); + finished = true; + } + } catch (err) { + this.destroy(err); + } finally { + fetching = false; + } + })(); + } + }); +} + module.exports = { replaceAuthorInZip, buildUserMapping, @@ -624,4 +729,7 @@ module.exports = { SUPPORTED_EXPORT_TYPES, buildGradeRecords, attachTagNames, + resolveIsAdmin, + createJsonArrayStream, + createCsvRowsStream }; \ No newline at end of file diff --git a/backend/webserver/routes/export.js b/backend/webserver/routes/export.js index 4b7efc706..1875c442c 100644 --- a/backend/webserver/routes/export.js +++ b/backend/webserver/routes/export.js @@ -25,6 +25,9 @@ const { SUPPORTED_EXPORT_TYPES, buildGradeRecords, attachTagNames, + resolveIsAdmin, + createJsonArrayStream, + createCsvRowsStream } = require('../../utils/helper/export.js'); const storageDir = path.join(__dirname, "..", "..", "..", "files"); @@ -44,7 +47,7 @@ module.exports = function (server) { // Input parsing - const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles, excludeNonConsentingEdits, excludeNonConsentingAnnotations, includeEmptyStudies, includeDocumentFiles, includeGrades, includeAiScores } = req.body; + const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles, excludeNonConsentingEdits, excludeNonConsentingAnnotations, includeEmptyStudies, includeDocumentFiles, includeGrades, includeAiScores, behaviourOutputFormat, behaviourFileFormat } = req.body; let { userIds: rawUserIds = [], documentTypes = [0, 1, 2, 4], workflowIds = [] } = req.body; const shouldGenerateAliases = String(generateAliases) === 'true'; const shouldMergeCsvFiles = String(mergeCsvFiles) === "true"; @@ -54,6 +57,8 @@ module.exports = function (server) { const shouldIncludeDocumentFiles = String(includeDocumentFiles) === 'true'; const shouldIncludeGrades = String(includeGrades) === 'true'; const shouldIncludeAiScores = includeAiScores === undefined ? true : String(includeAiScores) === 'true'; + const normalizedBehaviourOutputFormat = behaviourOutputFormat === 'perUser' ? 'perUser' : 'single'; + const normalizedBehaviourFileFormat = behaviourFileFormat === 'csv' ? 'csv' : 'json'; const normalizedGradeFormat = String(gradeFormat || "json").toLowerCase(); const parsedProjectId = Number(projectId); const userIds = parseUserIds(rawUserIds); @@ -147,6 +152,24 @@ module.exports = function (server) { archive ); break; + case 'userBehaviour': { + const isAdmin = await resolveIsAdmin(server, currentUserId); + if (!isAdmin) { + return res.status(403).send("Admin rights required for this export."); + } + await processUserBehaviourExport( + server, + users, + shouldGenerateAliases, + hasPrivateInfoRight, + userMapping, + normalizedBehaviourOutputFormat, + normalizedBehaviourFileFormat, + baseFolderName, + archive + ); + break; + } default: return res.status(400).send("Unsupported export type."); } @@ -645,7 +668,7 @@ module.exports = function (server) { where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, raw: true, }); - + annotations = await attachTagNames(server, annotations); let comments = await server.db.models.comment.findAll({ @@ -865,4 +888,57 @@ module.exports = function (server) { } } } + /** + * Exports usage statistics for the selected users, respecting each user's acceptStats consent. + * @param {string} behaviourOutputFormat - 'single' for one combined file, 'perUser' for one file per user. + */ + async function processUserBehaviourExport(server, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, behaviourOutputFormat, behaviourFileFormat, baseFolderName, archive) { + const { Op } = server.db.Sequelize; + + const consentedUsers = users.filter(u => u.acceptStats); + if (consentedUsers.length === 0) return; + const usersById = new Map(consentedUsers.map(u => [u.id, u])); + const consentedUserIds = consentedUsers.map(u => u.id); + const extension = behaviourFileFormat === 'csv' ? 'csv' : 'json'; + + const parseStatData = (raw) => { + try { + return JSON.parse(raw); + } catch (e) { + return raw; + } + }; + + const toRecord = (stat) => ({ + action: stat.action, + data: behaviourFileFormat === 'csv' ? stat.data : parseStatData(stat.data), + timestamp: stat.timestamp instanceof Date ? stat.timestamp.toISOString() : stat.timestamp, + user: getDisplayName(usersById.get(stat.userId), shouldGenerateAliases, hasPrivateInfoRight, userMapping), + }); + + const buildStream = (fetchPage) => behaviourFileFormat === 'csv' + ? createCsvRowsStream(fetchPage, toRecord) + : createJsonArrayStream(fetchPage, toRecord); + + if (behaviourOutputFormat === 'perUser') { + for (const user of consentedUsers) { + const folderName = sanitizeFolderName(getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping)); + const fetchPage = (lastId, limit) => server.db.models.statistic.findAll({ + where: { userId: user.id, deleted: false, id: { [Op.gt]: lastId } }, + order: [['id', 'ASC']], + limit, + raw: true, + }); + archive.append(buildStream(fetchPage), { name: `${baseFolderName}/${folderName}/behaviour_data.${extension}` }); + } + } else { + const fetchPage = (lastId, limit) => server.db.models.statistic.findAll({ + where: { userId: { [Op.in]: consentedUserIds }, deleted: false, id: { [Op.gt]: lastId } }, + order: [['id', 'ASC']], + limit, + raw: true, + }); + archive.append(buildStream(fetchPage), { name: `${baseFolderName}/behaviour_data.${extension}` }); + } + } }; \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/ExportModal.vue b/frontend/src/components/dashboard/projects/ExportModal.vue index 6112e04eb..da3d91870 100644 --- a/frontend/src/components/dashboard/projects/ExportModal.vue +++ b/frontend/src/components/dashboard/projects/ExportModal.vue @@ -28,7 +28,7 @@ Total Study Sessions: {{ studySessions.length }}

-
+
+
+ +