diff --git a/backend/package-lock.json b/backend/package-lock.json index ff667874b..1208c9aae 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -77,7 +77,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.2", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", @@ -115,6 +116,7 @@ "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -2160,6 +2162,7 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-1.19.0.tgz", "integrity": "sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw==", "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.6", @@ -2321,6 +2324,7 @@ "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", "license": "Apache-2.0", + "peer": true, "peerDependencies": { "bare-abort-controller": "*" }, @@ -2582,6 +2586,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.44", "caniuse-lite": "^1.0.30001806", @@ -3768,6 +3773,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -6502,6 +6508,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -6542,7 +6549,6 @@ "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.19.0.tgz", "integrity": "sha512-J5cF1MUz7LRJ9emOqF/06QjabMHMZy587rSPF0UuA8rCwKeeYl2co8Pp+6k5UU9YrAYHMzWkLxilfZB0hqsWWw==", "license": "MIT", - "peer": true, "peerDependencies": { "pg": "^8" } @@ -7126,6 +7132,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@types/debug": "^4.1.8", "@types/validator": "^13.7.17", @@ -8540,6 +8547,7 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", "license": "MIT", + "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", diff --git a/backend/utils/helper/export.js b/backend/utils/helper/export.js new file mode 100644 index 000000000..63b6988ed --- /dev/null +++ b/backend/utils/helper/export.js @@ -0,0 +1,735 @@ +const fs = require('fs'); +const { faker } = require('@faker-js/faker'); +const JSZip = require('jszip'); +const { deriveUserSeed } = require('../../webserver/auth/utils'); +const path = require('path'); +const storageDir = path.join(__dirname, "..", "..", "..", "files"); +const Papa = require('papaparse'); +const { Readable } = require('stream'); + +const SUPPORTED_EXPORT_TYPES = new Set(["submissions", "grades", "documents", "studies", "userBehaviour"]); +const { calculateAssessmentScore, buildScoresFromState } = require('assessment-score'); +const ASSESSMENT_RESULT_KEY = "assessment_result"; + + +/** + * Validates an export request and loads the project/users it targets. + * @param {Object} server - The server instance providing database models and Sequelize operators. + * @param {Object} params - Validation inputs. + * @param {number} params.parsedProjectId - The numeric project id. + * @param {string} params.exportType - The requested export type. + * @param {string} params.normalizedGradeFormat - The requested grade format, lowercased. + * @param {Array} params.userIds - Parsed user ids to export. + * @param {*} params.workflowIds - Raw workflowIds value from the request body, parsed here. + * @returns {Promise<{success: boolean, status?: number, message?: string, users?: Array, workflowIds?: Array}>} + */ +async function loadExportRequestContext(server, { parsedProjectId, exportType, normalizedGradeFormat, userIds, workflowIds }) { + if (!Number.isInteger(parsedProjectId)) { + return { success: false, status: 400, message: "Missing projectId." }; + } + if (!SUPPORTED_EXPORT_TYPES.has(exportType)) { + return { success: false, status: 400, message: "Unsupported export type." }; + } + workflowIds = typeof workflowIds === 'string' ? JSON.parse(workflowIds) : workflowIds; + if (!Array.isArray(workflowIds)) workflowIds = []; + if (exportType === "grades" && !["json", "csv"].includes(normalizedGradeFormat)) { + return { success: false, status: 400, message: "Unsupported grade format. Use json or csv." }; + } + if (userIds.length === 0) { + console.warn("Export aborted: No valid users selected."); + return { success: false, status: 400, message: "No valid users selected." }; + } + + const project = await server.db.models.project.findOne({ where: { id: parsedProjectId } }); + if (!project) { + console.warn(`${parsedProjectId} does not exist.`); + return { success: false, status: 403, message: "The selected project does not exist." }; + } + + const { Op } = server.db.Sequelize; + const users = await server.db.models.user.findAll({ where: { id: { [Op.in]: userIds } } }); + if (users.length === 0) { + console.warn("Export aborted: No existing users to export."); + return { success: false, status: 400, message: "No authorized users to export." }; + } + + return { success: true, users, workflowIds }; +} + +/** + * Opens a zip file, replaces the student's real name with a fake name in all .tex files, + * and returns the modified zip as a Buffer. + * @param {string} filePath - Path to the original zip file on disk + * @param {string} realName - The student's real name to search for + * @param {string} fakeName - The generated fake name to insert + * @returns {Promise} - The newly generated zip file buffer + */ +async function replaceAuthorInZip(filePath, realName, fakeName) { + const fileData = fs.readFileSync(filePath); + const zip = await JSZip.loadAsync(fileData); + const getFirstAndLastNameTokens = (name) => { + const parts = String(name || "").trim().split(/\s+/).filter(Boolean); + if (parts.length === 0) return ["", ""]; + if (parts.length === 1) return [parts[0], ""]; + return [parts[0], parts[parts.length - 1]]; + }; + const [realFirstName, realLastName] = getFirstAndLastNameTokens(realName); + const [fakeFirstName, fakeLastName] = getFirstAndLastNameTokens(fakeName); + + const authorRegex = /\\author\s*\{[^}]*\}/g; + + for (const [relativePath, zipEntry] of Object.entries(zip.files)) { + if (!zipEntry.dir && relativePath.toLowerCase().endsWith('.tex')) { + let text = await zipEntry.async("string"); + text = text.replace(authorRegex, `\\author{${fakeName}}`); + if (realFirstName && fakeFirstName) text = text.replace(realFirstName, fakeFirstName); + if (realLastName && fakeLastName) text = text.replace(realLastName, fakeLastName); + + zip.file(relativePath, text); + } + } + + return await zip.generateAsync({ + type: "nodebuffer", + compression: "DEFLATE" + }); +} + +/** + * Constructs a mapping of user IDs to aliases and generates a + * corresponding CSV string. + * @param {Array} users - Array of user objects from the database. + * @param {boolean} shouldGenerateAliases - Whether the export should use fake names. + * @param {boolean} hasPrivateInfoRight - Whether the current user is allowed to see/export full names. + * @param {number|string} fakerSeed - The base integer seed (from the form input). + * @param {string} salt - The hex-encoded salt string from the user's database record. + * @returns {Object} An object containing: + * - userMapping: An object mapping user IDs to their generated fake names. + * - mappingCsv: A CSV-formatted string containing the mapping (conditionally includes real names). + */ +function buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, salt) { + let userMapping = {}; + let csvRows = []; + + if (shouldGenerateAliases) { + if (fakerSeed && !isNaN(parseInt(fakerSeed, 10))) { + const derivedFakerSeed = deriveUserSeed(parseInt(fakerSeed, 10), salt); + faker.seed(derivedFakerSeed); + } + + const sortedUsers = [...users].sort((a, b) => Number(a.id) - Number(b.id)); + sortedUsers.forEach(u => { + const realUsername = u.userName; + const realName = `${u.firstName} ${u.lastName}`; + const fakeName = `${faker.person.firstName()} ${faker.person.lastName()}`; + + userMapping[u.id] = fakeName; + + let rowData = { + "Username": realUsername + }; + if (hasPrivateInfoRight) { + rowData["Real Name"] = realName; + } + + rowData["Generated Alias"] = fakeName; + + csvRows.push(rowData); + }); + } + const mappingCsv = csvRows.length > 0 ? Papa.unparse(csvRows) : ""; + return { userMapping, mappingCsv }; +} + +/** + * Normalizes a folder name so it can be used as a ZIP path segment without + * accidentally introducing invalid filename characters or nested paths. + * + * @param {string|number|null|undefined} value - The raw folder name. + * @returns {string} A sanitized folder name with reserved characters replaced. + */ +function sanitizeFolderName(value) { + return String(value || "unknown") + .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Returns a user's display name based on private info permissions. + * + * @param {Object|null} user - The user record. + * @param {boolean} hasPrivateInfoRight - Whether real names are allowed. + * @returns {string|null} Full name or username depending on permissions. + */ +function getPrivateAwareName(user, hasPrivateInfoRight) { + if (!user) return null; + if (hasPrivateInfoRight) return `${user.firstName} ${user.lastName}`.trim(); + // Usernames are considered anonymous-enough for exports when real names are restricted. + return user.userName ?? null; +} + +/** + * Resolves the display name for a user based on the current export settings. + * This wraps getPrivateAwareName with alias support for anonymized exports. + * + * @param {Object} user - The user record to display. + * @param {boolean} shouldGenerateAliases - Whether aliases should replace real names. + * @param {boolean} hasPrivateInfoRight - Whether the current user may export real names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @returns {string} The display name to write into the export. + */ +function getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping) { + if (shouldGenerateAliases) return userMapping[user.id]; + return getPrivateAwareName(user, hasPrivateInfoRight); +} + +/** + * Calculates the version number of a submission by traversing backwards + * through the chain of previous submissions. + * @param {Object} submission - The current submission object to start from. + * @param {Map} submissionMap - A Map containing all related + * submissions for quick lookup by ID. + * @returns {number} - The calculated version number (starting at 1 for the original). + */ +function calculateSubmissionVersion(submission, submissionMap) { + let version = 1; + let currentSub = submission; + while (currentSub && currentSub.previousSubmissionId) { + const prevSub = submissionMap.get(currentSub.previousSubmissionId); + if (!prevSub) break; + version++; + currentSub = prevSub; + } + return version; +} + +/** + * Parses an assessment state payload when it is stored as JSON text. + * + * @param {string} rawAssessmentState - The raw JSON string from document_data. + * @returns {Object} The parsed assessment state or an empty object on failure. + */ +function parseAssessmentState(rawAssessmentState) { + try { + const parsed = JSON.parse(rawAssessmentState); + return parsed && typeof parsed === "object" ? parsed : {}; + } catch (error) { + console.warn("Failed to parse assessment state:", error.message); + return {}; + } +} + +/** + * Reads the rubric configuration id from a study step configuration payload. + * + * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration object. + * @returns {number|null} The referenced configuration id or null when unavailable. + */ +function getAssessmentConfigurationId(studyStepConfiguration) { + if (!studyStepConfiguration || typeof studyStepConfiguration !== "object") return null; + const rawId = + studyStepConfiguration.settings?.configurationId ?? + studyStepConfiguration.configurationId ?? + null; + const parsedId = Number(rawId); + return Number.isInteger(parsedId) ? parsedId : null; +} + +/** + * Resolves the assessment rubric configuration referenced by a study step. + * Study steps are expected to store only a configurationId; rubric content + * is loaded from the configuration table. + * + * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration JSON. + * @param {Map} configurationsById - Loaded configuration records by id. + * @returns {Object|null} Assessment config content (with rubrics) or null. + */ +function resolveAssessmentConfigurationContent(studyStepConfiguration, configurationsById) { + const configurationId = getAssessmentConfigurationId(studyStepConfiguration); + if (configurationId === null) return null; + + const configuration = configurationsById.get(configurationId); + return configuration?.content ?? null; +} + +/** + * Records the assessment configuration content for a given configuration id, + * so each distinct configuration used across a grade export gets its own + * criteria reference file (a grade export may now span multiple configurations). + * + * @param {Map} referencesByConfigId - Mutable map of configurationId -> reference content. + * @param {number|null} configurationId - Resolved persisted configuration id. + * @param {Object|null} assessmentConfig - Resolved assessment configuration content. + * @returns {void} + */ +function addCriteriaReferenceEntry(referencesByConfigId, configurationId, assessmentConfig) { + if (!assessmentConfig || typeof assessmentConfig !== "object") return; + if (!Number.isInteger(configurationId)) return; + if (referencesByConfigId.has(configurationId)) return; + + referencesByConfigId.set(configurationId, { + configurationId, + ...assessmentConfig + }); +} + +/** + * Builds a flat CSV row for a grade export record. + * The row contains backend export metadata columns followed by + * one column per assessment criterion score. + * + * @param {Object} record - Prepared grade export record. + * @returns {Object} A flat object suitable for Papa.unparse. + */ +function buildGradeCsvRow(record) { + const criterionScores = record.scores && typeof record.scores === "object" ? record.scores : {}; + return { + projectId: record.projectId, + userId: record.userId, + userExtId: record.userExtId, + userName: record.userName, + displayName: record.displayName, + submissionId: record.submissionId, + submissionExtId: record.submissionExtId, + studySessionId: record.studySessionId, + studyName: record.studyName, + studyStepId: record.studyStepId, + studyStepType: record.studyStepType, + configurationId: record.configurationId, + studyOwner: record.studyOwner, + sessionOwner: record.sessionOwner, + author: record.author, + totalPoints: record.totalPoints, + createdAt: record.createdAt, + ...criterionScores + }; +} + +/** + * Loads the related entities needed to turn raw assessment_result rows into + * export-ready grade records. + * + * @param {Object} server - The server instance with Sequelize models. + * @param {Array} gradeRows - Assessment result rows with attached documents. + * @param {Array} users - The selected document owners for the export. + * @returns {Promise} Lookup maps for related grade-export entities. + */ +async function loadGradeExportContext(server, gradeRows, users) { + const { Op } = server.db.Sequelize; + + const sessionIds = [...new Set(gradeRows.map((row) => row.studySessionId).filter(Boolean))]; + const studySessions = sessionIds.length > 0 + ? await server.db.models.study_session.findAll({ + where: { id: { [Op.in]: sessionIds }, deleted: false }, + raw: true + }) + : []; + const sessionsById = new Map(studySessions.map((session) => [session.id, session])); + + const studyIds = [...new Set(studySessions.map((session) => session.studyId).filter(Boolean))]; + const studies = studyIds.length > 0 + ? await server.db.models.study.findAll({ + where: { id: { [Op.in]: studyIds }, deleted: false }, + raw: true + }) + : []; + const studiesById = new Map(studies.map((study) => [study.id, study])); + + const studyStepIds = [...new Set(gradeRows.map((row) => row.studyStepId).filter(Boolean))]; + const studySteps = studyStepIds.length > 0 + ? await server.db.models.study_step.findAll({ + where: { id: { [Op.in]: studyStepIds }, deleted: false }, + raw: true + }) + : []; + const studyStepsById = new Map(studySteps.map((studyStep) => [studyStep.id, studyStep])); + + const configurationIds = [...new Set( + studySteps + .map((studyStep) => getAssessmentConfigurationId(studyStep.configuration)) + .filter((id) => id !== null) + )]; + const configurations = configurationIds.length > 0 + ? await server.db.models.configuration.findAll({ + where: { id: { [Op.in]: configurationIds }, deleted: false }, + raw: true + }) + : []; + const configurationsById = new Map(configurations.map((configuration) => [configuration.id, configuration])); + + // The export references study/session owners in addition to the selected document owners. + const relatedUserIds = [...new Set([ + ...users.map((user) => user.id), + ...studySessions.map((session) => session.userId), + ...studies.map((study) => study.userId) + ].filter(Boolean))]; + const relatedUsers = relatedUserIds.length > 0 + ? await server.db.models.user.findAll({ where: { id: { [Op.in]: relatedUserIds } }, raw: true }) + : []; + const usersById = new Map(relatedUsers.map((user) => [user.id, user])); + + return { + sessionsById, + studiesById, + studyStepsById, + configurationsById, + usersById + }; +} + +/** + * Resolves which of the given user ids have opted into data sharing. + * + * @param {Object} server - The server instance providing database models. + * @param {Array} candidateUserIds - User ids to check consent for. + * @returns {Promise>} Set of user ids that accepted data sharing. + */ +async function getConsentedUserIds(server, candidateUserIds) { + if (candidateUserIds.length === 0) return new Set(); + const consentedUsers = await server.db.models.user.findAll({ + where: { id: candidateUserIds }, + attributes: ['id', 'acceptDataSharing'], + raw: true, + }); + return new Set(consentedUsers.filter(u => u.acceptDataSharing).map(u => u.id)); +} + +/** + * Orders grade records by session, then step within the session, then creation time. + * @param {Object} a - First grade record to compare. + * @param {Object} b - Second grade record to compare. + * @returns {number} Standard comparator result for Array#sort. + */ +function compareGradeRecords(a, b) { + const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; + const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; + return ( + (a.studySessionId || 0) - (b.studySessionId || 0) || + (a.studyStepId || 0) - (b.studyStepId || 0) || + createdA - createdB + ); +} + +/** + * Appends a stored file (by hash + extension) to the archive if it exists on disk, otherwise warns. + * @param {Object} archive - The archiver instance to append the file to. + * @param {string} hash - The document's storage hash. + * @param {string} extension - File extension including the dot, e.g. ".pdf". + * @param {string} archivePath - Destination path inside the ZIP archive. + * @param {string} typeLabel - Human-readable type label used in the warning log. + * @returns {void} + */ +function appendStoredFileIfExists(archive, hash, extension, archivePath, typeLabel) { + const filePath = path.join(storageDir, `${hash}${extension}`); + if (fs.existsSync(filePath)) { + archive.file(filePath, { name: archivePath }); + } else { + console.warn(`[DocumentExport] ${typeLabel} not found for document ${hash}`); + } +} + +/** + * Resolves whether a user may see other users' full names in exports (admins always can). + * @param {Object} server - The server instance providing database models. + * @param {number} userId - The requesting user's id. + * @returns {Promise} Whether the user has the private-info export right. + */ +async function resolveHasPrivateInfoRight(server, userId) { + const roleIds = await server.db.models["user_role_matching"].getUserRolesById(userId); + const isAdmin = await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); + if (isAdmin) return true; + + const userRightsObj = await server.db.models.user.getUserRights(userId); + if (!userRightsObj) return false; + + const allRights = Object.values(userRightsObj).flat(); + return allRights.includes('frontend.dashboard.studies.view.userPrivateInfo'); +} + +/** + * Parses the raw userIds field from a request body into an array, tolerating a JSON-encoded string. + * @param {*} rawUserIds - The raw value from req.body.userIds. + * @returns {Array} Parsed array of user ids, or an empty array if parsing fails. + */ +function parseUserIds(rawUserIds) { + try { + const parsed = typeof rawUserIds === 'string' ? JSON.parse(rawUserIds) : rawUserIds; + return Array.isArray(parsed) ? parsed : []; + } catch (e) { + console.warn("Could not parse userIds:", rawUserIds); + return []; + } +} + +/** + * Builds flat grade records for the given users/project by resolving each + * assessment_result row's session/study/step/configuration context and score. + * Shared by processGradesExport (grouped by user for JSON/CSV output) and + * processStudyBasedExport (grouped by session for a per-session scores.json). + * + * @param {Object} server - The server instance providing database models and Sequelize operators. + * @param {number} projectId - The project whose grades should be resolved. + * @param {Array} userIds - The selected document owners. + * @param {Array} users - Full user records for the selected users. + * @param {boolean} shouldGenerateAliases - Whether student names should be anonymized. + * @param {boolean} hasPrivateInfoRight - Whether the requester may export real names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @returns {Promise<{records: Array, criteriaReferencesByConfigId: Map}>} + */ +/** + * Collect assessment results and translate them into grade records. + * + * Pass `options.sessionIds` to scope the lookup to a set of study sessions. Without it the lookup + * is scoped by the owner of the assessed document, which only works when that owner is also the + * person the grades are being collected for. That holds for exposé assessments, where the assessed + * document belongs to the study owner, but not for review assessments: there the study belongs to + * the reviewer being assessed while the assessed review document belongs to the reviewed author, + * so an owner-scoped lookup returns nothing and the review grades are silently dropped. + */ +async function buildGradeRecords(server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, options = {}) { + const { Op } = server.db.Sequelize; + const { sessionIds = null } = options; + + const documentWhere = { projectId, deleted: false }; + if (!sessionIds) documentWhere.userId = { [Op.in]: userIds }; + + const gradeRows = await server.db.models.document_data.findAll({ + where: { + key: ASSESSMENT_RESULT_KEY, + deleted: false, + studySessionId: sessionIds ? { [Op.in]: sessionIds } : { [Op.ne]: null } + }, + include: [{ + model: server.db.models.document, + as: "document", + required: true, + where: documentWhere, + include: [{ + model: server.db.models.submission, + as: "submission", + required: false + }] + }], + order: [["studySessionId", "ASC"], ["studyStepId", "ASC"], ["createdAt", "ASC"]] + }); + + const { + sessionsById, + studiesById, + studyStepsById, + configurationsById, + usersById + } = await loadGradeExportContext(server, gradeRows, users); + + const records = []; + const criteriaReferencesByConfigId = new Map(); + for (const row of gradeRows) { + const document = row.document; + const ownerUser = usersById.get(document.userId); + if (!ownerUser) { + console.warn("Skipping grade export row because the document owner could not be resolved.", { + documentId: document.id, + documentUserId: document.userId, + studySessionId: row.studySessionId, + studyStepId: row.studyStepId + }); + continue; + } + const session = sessionsById.get(row.studySessionId); + const reviewerUser = session ? usersById.get(session.userId) : null; + const study = session ? studiesById.get(session.studyId) : null; + const graderUser = study ? usersById.get(study.userId) : null; + const studyStep = studyStepsById.get(row.studyStepId); + const submission = document.submission; + const studyStepConfiguration = studyStep?.configuration; + const isAiGraded = Array.isArray(studyStepConfiguration?.services) && studyStepConfiguration.services.some(s => s.type === "nlpRequest"); + const configurationId = getAssessmentConfigurationId(studyStepConfiguration); + const studyName = study?.name || `study_${session?.studyId || "unknown"}`; + + const scoreObject = row.value || {}; + const assessmentState = typeof scoreObject === "string" ? parseAssessmentState(scoreObject) : scoreObject; + const flatScores = buildScoresFromState(assessmentState); + const assessmentConfig = resolveAssessmentConfigurationContent( + studyStepConfiguration, + configurationsById + ); + addCriteriaReferenceEntry( + criteriaReferencesByConfigId, + configurationId, + assessmentConfig + ); + const assessmentScore = calculateAssessmentScore(assessmentConfig, flatScores); + const totalPoints = assessmentScore.achieved_points; + + records.push({ + projectId, + userId: ownerUser.id, + userExtId: ownerUser.extId ?? null, + userName: ownerUser.userName ?? "", + displayName: getDisplayName(ownerUser, shouldGenerateAliases, hasPrivateInfoRight, userMapping), + submissionId: submission?.id ?? document.submissionId ?? null, + submissionExtId: submission?.extId ?? null, + studySessionId: row.studySessionId ?? null, + studyStepId: row.studyStepId ?? null, + configurationId, + studyName, + sessionHash: session?.hash ?? null, + studyOwner: getPrivateAwareName(graderUser, hasPrivateInfoRight), + sessionOwner: getPrivateAwareName(reviewerUser, hasPrivateInfoRight), + author: getPrivateAwareName(ownerUser, hasPrivateInfoRight), + scores: flatScores, + totalPoints, + createdAt: row.createdAt ? new Date(row.createdAt).toISOString() : null, + studyStepType: studyStep?.stepType ?? null, + isAiGraded + }); + } + + return { records, criteriaReferencesByConfigId }; +} + +async function attachTagNames(server, annotations) { + const tagIds = [...new Set(annotations.map(a => a.tagId).filter(Boolean))]; + if (tagIds.length === 0) return annotations; + + const tags = await server.db.models.tag.findAll({ + where: { id: tagIds }, + attributes: ['id', 'name'], + raw: true, + }); + const tagNameById = new Map(tags.map(t => [t.id, t.name])); + + return annotations.map(a => ({ ...a, tagName: tagNameById.get(a.tagId) ?? null })); +} + +async function resolveIsAdmin(server, userId) { + const roleIds = await server.db.models["user_role_matching"].getUserRolesById(userId); + return await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); +} + +/** + * Builds a Readable that emits a JSON array incrementally, paging through fetchPage(lastId, limit) + * using keyset pagination so the full result set is never held in memory at once. + * @param {(lastId: number, limit: number) => Promise>} fetchPage + */ +function createJsonArrayStream(fetchPage, mapRow = (row) => row, pageSize = 1000) { + let lastId = 0; + let started = false; + let finished = false; + let isFirst = true; + let fetching = false; + + return new Readable({ + read() { + if (fetching || finished) return; + fetching = true; + + (async () => { + try { + if (!started) { + this.push('['); + started = true; + } + + const rows = await fetchPage(lastId, pageSize); + if (rows.length === 0) { + this.push('\n]'); + this.push(null); + finished = true; + return; + } + + let chunk = ''; + for (const row of rows) { + chunk += (isFirst ? '' : ',') + '\n' + JSON.stringify(mapRow(row)); + isFirst = false; + } + lastId = rows[rows.length - 1].id; + + if (rows.length < pageSize) { + chunk += '\n]'; + this.push(chunk); + this.push(null); + finished = true; + } else { + this.push(chunk); + } + } catch (err) { + this.destroy(err); + } finally { + fetching = false; + } + })(); + } + }); +} + +function createCsvRowsStream(fetchPage, mapRow, pageSize = 1000) { + let lastId = 0; + let started = false; + let finished = false; + let fetching = false; + + return new Readable({ + read() { + if (fetching || finished) return; + fetching = true; + + (async () => { + try { + const rows = await fetchPage(lastId, pageSize); + if (rows.length === 0) { + if (!started) this.push(Papa.unparse([mapRow].length ? [] : [])); + this.push(null); + finished = true; + return; + } + + const records = rows.map(mapRow); + const csvChunk = Papa.unparse(records, { header: !started }) + '\n'; + started = true; + lastId = rows[rows.length - 1].id; + + this.push(csvChunk); + + if (rows.length < pageSize) { + this.push(null); + finished = true; + } + } catch (err) { + this.destroy(err); + } finally { + fetching = false; + } + })(); + } + }); +} + +module.exports = { + replaceAuthorInZip, + buildUserMapping, + sanitizeFolderName, + getPrivateAwareName, + getDisplayName, + calculateSubmissionVersion, + parseAssessmentState, + getAssessmentConfigurationId, + resolveAssessmentConfigurationContent, + addCriteriaReferenceEntry, + buildGradeCsvRow, + loadGradeExportContext, + getConsentedUserIds, + compareGradeRecords, + appendStoredFileIfExists, + resolveHasPrivateInfoRight, + parseUserIds, + loadExportRequestContext, + SUPPORTED_EXPORT_TYPES, + buildGradeRecords, + attachTagNames, + resolveIsAdmin, + createJsonArrayStream, + createCsvRowsStream +}; \ No newline at end of file diff --git a/backend/webserver/routes/export.js b/backend/webserver/routes/export.js index 35dd32376..4ecf98402 100644 --- a/backend/webserver/routes/export.js +++ b/backend/webserver/routes/export.js @@ -1,13 +1,38 @@ const archiver = require('archiver'); const path = require('path'); const fs = require('fs'); -const { faker } = require('@faker-js/faker'); -const JSZip = require('jszip'); -const { deriveUserSeed } = require('../auth/utils'); const Papa = require('papaparse'); -const { calculateAssessmentScore, buildScoresFromState } = require('assessment-score'); - -const ASSESSMENT_RESULT_KEY = "assessment_result"; +const { dbToDelta, deltaToPlainText, deltaToHtml } = require('editor-delta-conversion'); +const { + replaceAuthorInZip, + buildUserMapping, + sanitizeFolderName, + getPrivateAwareName, + getDisplayName, + calculateSubmissionVersion, + parseAssessmentState, + getAssessmentConfigurationId, + resolveAssessmentConfigurationContent, + addCriteriaReferenceEntry, + buildGradeCsvRow, + loadGradeExportContext, + getConsentedUserIds, + compareGradeRecords, + appendStoredFileIfExists, + resolveHasPrivateInfoRight, + parseUserIds, + loadExportRequestContext, + SUPPORTED_EXPORT_TYPES, + buildGradeRecords, + attachTagNames, + resolveIsAdmin, + createJsonArrayStream, + createCsvRowsStream +} = require('../../utils/helper/export.js'); +const storageDir = path.join(__dirname, "..", "..", "..", "files"); + +// Stored files the study export ships per step, by document type: PDF and LaTeX ZIP. +const STUDY_DOCUMENT_EXTENSIONS = { 0: '.pdf', 4: '.zip' }; module.exports = function (server) { @@ -16,76 +41,41 @@ module.exports = function (server) { // Auth checking const currentUserId = req.user?.id; if (!currentUserId) return res.status(401).send("Log in required"); - const currentUser = await server.db.models.user.findByPk(currentUserId); if (!currentUser) return res.status(401).send("User not found"); + const hasPrivateInfoRight = await resolveHasPrivateInfoRight(server, currentUserId); - // check if user has right to see full names - let hasPrivateInfoRight = false; - - const roleIds = await server.db.models["user_role_matching"].getUserRolesById(currentUserId); - const isAdmin = await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); - if (isAdmin) { - // override, admin has all rights - hasPrivateInfoRight = true; - } else { - const userRightsObj = await server.db.models.user.getUserRights(currentUserId); - - if (userRightsObj) { - const allRights = Object.values(userRightsObj).flat(); - hasPrivateInfoRight = allRights.includes('frontend.dashboard.studies.view.userPrivateInfo'); - } - } // Input parsing - const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles } = req.body; - let { userIds = [] } = req.body; + const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles, excludeNonConsentingEdits, excludeNonConsentingAnnotations, includeEmptyStudies, includeDocumentFiles, includeGrades, includeAiScores, behaviourOutputFormat, behaviourFileFormat } = req.body; + let { userIds: rawUserIds = [], documentTypes = [0, 1, 2, 4], workflowIds = [] } = req.body; const shouldGenerateAliases = String(generateAliases) === 'true'; const shouldMergeCsvFiles = String(mergeCsvFiles) === "true"; + const shouldExcludeNonConsentingEdits = String(excludeNonConsentingEdits) === 'true'; + const shouldExcludeNonConsentingAnnotations = String(excludeNonConsentingAnnotations) === 'true'; + const shouldIncludeEmptyStudies = String(includeEmptyStudies) === 'true'; + const shouldIncludeDocumentFiles = String(includeDocumentFiles) === 'true'; + const shouldIncludeGrades = String(includeGrades) === 'true'; + const shouldIncludeAiScores = includeAiScores === undefined ? true : String(includeAiScores) === 'true'; + const normalizedBehaviourOutputFormat = behaviourOutputFormat === 'perUser' ? 'perUser' : 'single'; + const normalizedBehaviourFileFormat = behaviourFileFormat === 'csv' ? 'csv' : 'json'; const normalizedGradeFormat = String(gradeFormat || "json").toLowerCase(); - const supportedExportTypes = new Set(["submissions", "grades"]); - const { Op } = server.db.Sequelize; const parsedProjectId = Number(projectId); + const userIds = parseUserIds(rawUserIds); try { - userIds = typeof userIds === 'string' ? JSON.parse(userIds) : userIds; - if (!Array.isArray(userIds)) userIds = []; - } catch (e) { - console.warn("Could not parse userIds:", userIds); - userIds = []; - } - - try { - if (!Number.isInteger(parsedProjectId)) return res.status(400).send("Missing projectId."); - if (!supportedExportTypes.has(exportType)) { - return res.status(400).send("Unsupported export type."); - } - if (exportType === "grades" && !["json", "csv"].includes(normalizedGradeFormat)) { - return res.status(400).send("Unsupported grade format. Use json or csv."); - } - if (userIds.length === 0) { - console.warn("Export aborted: No valid users selected."); - return res.status(400).send("No valid users selected."); - } - - // check if the project is valid - const projectCheck = await server.db.models.project.findOne({ where: { id: parsedProjectId } }); - if (!projectCheck) { - console.warn(`${parsedProjectId} does not exist.`); - return res.status(403).send("The selected project does not exist."); - } - - const users = await server.db.models.user.findAll({ where: { id: { [Op.in]: userIds } } }); - if (users.length === 0) { - console.warn("Export aborted: No existing users to export."); - return res.status(400).send("No authorized users to export."); + const context = await loadExportRequestContext(server, { parsedProjectId, exportType, normalizedGradeFormat, userIds, workflowIds }); + if (!context.success) { + return res.status(context.status).send(context.message); } + const { users, workflowIds: parsedWorkflowIds } = context; // build user mapping for aliases const { userMapping, mappingCsv } = buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, currentUser.salt); // archiver stream setup - const exportFolderName = `${exportType}_${Date.now()}.zip`; + const timestamp = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14); + const exportFolderName = `${timestamp}_${exportType}.zip`; res.attachment(exportFolderName); const archive = archiver('zip', { zlib: { level: 5 } }); archive.on('error', function(err) { @@ -99,6 +89,8 @@ module.exports = function (server) { archive.append(mappingCsv, { name: 'aliases_mapping.csv' }); } + const baseFolderName = exportFolderName.split('.')[0]; + // process based on type switch (exportType) { case 'submissions': @@ -110,7 +102,7 @@ module.exports = function (server) { shouldGenerateAliases, hasPrivateInfoRight, userMapping, - exportFolderName.split('.')[0], + baseFolderName, archive ); break; @@ -128,6 +120,56 @@ module.exports = function (server) { archive ); break; + case 'documents': + await processDocumentBasedExport( + server, + parsedProjectId, + userIds, + documentTypes, + shouldExcludeNonConsentingEdits, + shouldExcludeNonConsentingAnnotations, + baseFolderName, + archive + ); + break; + case 'studies': + await processStudyBasedExport( + server, + parsedProjectId, + userIds, + users, + shouldGenerateAliases, + hasPrivateInfoRight, + userMapping, + parsedWorkflowIds, + shouldIncludeEmptyStudies, + shouldExcludeNonConsentingEdits, + shouldExcludeNonConsentingAnnotations, + shouldIncludeDocumentFiles, + shouldIncludeGrades, + shouldIncludeAiScores, + baseFolderName, + archive + ); + break; + case 'userBehaviour': { + const isAdmin = await resolveIsAdmin(server, currentUserId); + if (!isAdmin) { + return res.status(403).send("Admin rights required for this export."); + } + await processUserBehaviourExport( + server, + users, + shouldGenerateAliases, + hasPrivateInfoRight, + userMapping, + normalizedBehaviourOutputFormat, + normalizedBehaviourFileFormat, + baseFolderName, + archive + ); + break; + } default: return res.status(400).send("Unsupported export type."); } @@ -141,93 +183,6 @@ module.exports = function (server) { } }); - // HELPER FUNCTIONS - - /** - * Opens a zip file, replaces the student's real name with a fake name in all .tex files, - * and returns the modified zip as a Buffer. - * @param {string} filePath - Path to the original zip file on disk - * @param {string} realName - The student's real name to search for - * @param {string} fakeName - The generated fake name to insert - * @returns {Promise} - The newly generated zip file buffer - */ - async function replaceAuthorInZip(filePath, realName, fakeName) { - const fileData = fs.readFileSync(filePath); - const zip = await JSZip.loadAsync(fileData); - const getFirstAndLastNameTokens = (name) => { - const parts = String(name || "").trim().split(/\s+/).filter(Boolean); - if (parts.length === 0) return ["", ""]; - if (parts.length === 1) return [parts[0], ""]; - return [parts[0], parts[parts.length - 1]]; - }; - const [realFirstName, realLastName] = getFirstAndLastNameTokens(realName); - const [fakeFirstName, fakeLastName] = getFirstAndLastNameTokens(fakeName); - - const authorRegex = /\\author\s*\{[^}]*\}/g; - - for (const [relativePath, zipEntry] of Object.entries(zip.files)) { - if (!zipEntry.dir && relativePath.toLowerCase().endsWith('.tex')) { - let text = await zipEntry.async("string"); - text = text.replace(authorRegex, `\\author{${fakeName}}`); - if (realFirstName && fakeFirstName) text = text.replace(realFirstName, fakeFirstName); - if (realLastName && fakeLastName) text = text.replace(realLastName, fakeLastName); - - zip.file(relativePath, text); - } - } - - return await zip.generateAsync({ - type: "nodebuffer", - compression: "DEFLATE" - }); - } - - /** - * Constructs a mapping of user IDs to aliases and generates a - * corresponding CSV string. - * @param {Array} users - Array of user objects from the database. - * @param {boolean} shouldGenerateAliases - Whether the export should use fake names. - * @param {boolean} hasPrivateInfoRight - Whether the current user is allowed to see/export full names. - * @param {number|string} fakerSeed - The base integer seed (from the form input). - * @param {string} salt - The hex-encoded salt string from the user's database record. - * @returns {Object} An object containing: - * - userMapping: An object mapping user IDs to their generated fake names. - * - mappingCsv: A CSV-formatted string containing the mapping (conditionally includes real names). - */ - function buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, salt) { - let userMapping = {}; - let csvRows = []; - - if (shouldGenerateAliases) { - if (fakerSeed && !isNaN(parseInt(fakerSeed, 10))) { - const derivedFakerSeed = deriveUserSeed(parseInt(fakerSeed, 10), salt); - faker.seed(derivedFakerSeed); - } - - const sortedUsers = [...users].sort((a, b) => Number(a.id) - Number(b.id)); - sortedUsers.forEach(u => { - const realUsername = u.userName; - const realName = `${u.firstName} ${u.lastName}`; - const fakeName = `${faker.person.firstName()} ${faker.person.lastName()}`; - - userMapping[u.id] = fakeName; - - let rowData = { - "Username": realUsername - }; - if (hasPrivateInfoRight) { - rowData["Real Name"] = realName; - } - - rowData["Generated Alias"] = fakeName; - - csvRows.push(rowData); - }); - } - const mappingCsv = csvRows.length > 0 ? Papa.unparse(csvRows) : ""; - return { userMapping, mappingCsv }; - } - /** * Does the fetching, filtering, and archiving of student submissions for a specific project. * Handles file renaming based on validation rules and manages directory structures @@ -274,7 +229,6 @@ module.exports = function (server) { 1: ".html", 4: ".zip" }; - const storageDir = path.join(__dirname, "..", "..", "..", "files"); for (const submission of submissions) { const student = usersById.get(submission.userId); @@ -325,235 +279,6 @@ module.exports = function (server) { } } - /** - * Normalizes a folder name so it can be used as a ZIP path segment without - * accidentally introducing invalid filename characters or nested paths. - * - * @param {string|number|null|undefined} value - The raw folder name. - * @returns {string} A sanitized folder name with reserved characters replaced. - */ - function sanitizeFolderName(value) { - return String(value || "unknown") - .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_") - .replace(/\s+/g, " ") - .trim(); - } - - /** - * Parses an assessment state payload when it is stored as JSON text. - * - * @param {string} rawAssessmentState - The raw JSON string from document_data. - * @returns {Object} The parsed assessment state or an empty object on failure. - */ - function parseAssessmentState(rawAssessmentState) { - try { - const parsed = JSON.parse(rawAssessmentState); - return parsed && typeof parsed === "object" ? parsed : {}; - } catch (error) { - console.warn("Failed to parse assessment state:", error.message); - return {}; - } - } - - /** - * Resolves the assessment rubric configuration referenced by a study step. - * Study steps are expected to store only a configurationId; rubric content - * is loaded from the configuration table. - * - * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration JSON. - * @param {Map} configurationsById - Loaded configuration records by id. - * @returns {Object|null} Assessment config content (with rubrics) or null. - */ - function resolveAssessmentConfigurationContent(studyStepConfiguration, configurationsById) { - const configurationId = getAssessmentConfigurationId(studyStepConfiguration); - if (configurationId === null) return null; - - const configuration = configurationsById.get(configurationId); - return configuration?.content ?? null; - } - - /** - * Reads the rubric configuration id from a study step configuration payload. - * - * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration object. - * @returns {number|null} The referenced configuration id or null when unavailable. - */ - function getAssessmentConfigurationId(studyStepConfiguration) { - if (!studyStepConfiguration || typeof studyStepConfiguration !== "object") return null; - const rawId = - studyStepConfiguration.settings?.configurationId ?? - studyStepConfiguration.configurationId ?? - null; - const parsedId = Number(rawId); - return Number.isInteger(parsedId) ? parsedId : null; - } - - /** - * Captures the single assessment configuration used by the current grade - * export for inclusion in the shared criteria_reference.json sidecar file. - * - * The first valid configuration becomes the export reference. If another - * different configuration is encountered later, the export aborts because - * grade exports are expected to use exactly one configuration. - * - * @param {{ key: string|null, reference: Object|null }} referenceState - Mutable single-reference state. - * @param {number|null} configurationId - Resolved persisted configuration id. - * @param {Object|null} assessmentConfig - Resolved assessment configuration content. - * @returns {void} - */ - function addCriteriaReferenceEntry(referenceState, configurationId, assessmentConfig) { - if (!assessmentConfig || typeof assessmentConfig !== "object") return; - - const referenceKey = Number.isInteger(configurationId) ? `configuration:${configurationId}` : null; - if (!referenceKey) return; - - if (!referenceState.reference) { - referenceState.key = referenceKey; - referenceState.reference = { - configurationId: Number.isInteger(configurationId) ? configurationId : null, - ...assessmentConfig - }; - return; - } - - if (referenceState.key !== referenceKey) { - throw new Error("Expected exactly one assessment configuration for grade export, found multiple."); - } - } - - /** - * Returns a user's display name based on private info permissions. - * - * @param {Object|null} user - The user record. - * @param {boolean} hasPrivateInfoRight - Whether real names are allowed. - * @returns {string|null} Full name or username depending on permissions. - */ - function getPrivateAwareName(user, hasPrivateInfoRight) { - if (!user) return null; - if (hasPrivateInfoRight) return `${user.firstName} ${user.lastName}`.trim(); - // Usernames are considered anonymous-enough for exports when real names are restricted. - return user.userName ?? null; - } - - /** - * Builds a flat CSV row for a grade export record. - * The row contains backend export metadata columns followed by - * one column per assessment criterion score. - * - * @param {Object} record - Prepared grade export record. - * @returns {Object} A flat object suitable for Papa.unparse. - */ - function buildGradeCsvRow(record) { - const criterionScores = record.scores && typeof record.scores === "object" ? record.scores : {}; - return { - projectId: record.projectId, - userId: record.userId, - userExtId: record.userExtId, - userName: record.userName, - displayName: record.displayName, - submissionId: record.submissionId, - submissionExtId: record.submissionExtId, - studySessionId: record.studySessionId, - studyName: record.studyName, - studyStepId: record.studyStepId, - studyStepType: record.studyStepType, - configurationId: record.configurationId, - studyOwner: record.studyOwner, - sessionOwner: record.sessionOwner, - author: record.author, - totalPoints: record.totalPoints, - createdAt: record.createdAt, - ...criterionScores - }; - } - - /** - * Resolves the display name for a user based on the current export settings. - * This wraps getPrivateAwareName with alias support for anonymized exports. - * - * @param {Object} user - The user record to display. - * @param {boolean} shouldGenerateAliases - Whether aliases should replace real names. - * @param {boolean} hasPrivateInfoRight - Whether the current user may export real names. - * @param {Object} userMapping - Map of user IDs to generated aliases. - * @returns {string} The display name to write into the export. - */ - function getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping) { - if (shouldGenerateAliases) return userMapping[user.id]; - return getPrivateAwareName(user, hasPrivateInfoRight); - } - - /** - * Loads the related entities needed to turn raw assessment_result rows into - * export-ready grade records. - * - * @param {Object} server - The server instance with Sequelize models. - * @param {Array} gradeRows - Assessment result rows with attached documents. - * @param {Array} users - The selected document owners for the export. - * @returns {Promise} Lookup maps for related grade-export entities. - */ - async function loadGradeExportContext(server, gradeRows, users) { - const { Op } = server.db.Sequelize; - - const sessionIds = [...new Set(gradeRows.map((row) => row.studySessionId).filter(Boolean))]; - const studySessions = sessionIds.length > 0 - ? await server.db.models.study_session.findAll({ - where: { id: { [Op.in]: sessionIds }, deleted: false }, - raw: true - }) - : []; - const sessionsById = new Map(studySessions.map((session) => [session.id, session])); - - const studyIds = [...new Set(studySessions.map((session) => session.studyId).filter(Boolean))]; - const studies = studyIds.length > 0 - ? await server.db.models.study.findAll({ - where: { id: { [Op.in]: studyIds }, deleted: false }, - raw: true - }) - : []; - const studiesById = new Map(studies.map((study) => [study.id, study])); - - const studyStepIds = [...new Set(gradeRows.map((row) => row.studyStepId).filter(Boolean))]; - const studySteps = studyStepIds.length > 0 - ? await server.db.models.study_step.findAll({ - where: { id: { [Op.in]: studyStepIds }, deleted: false }, - raw: true - }) - : []; - const studyStepsById = new Map(studySteps.map((studyStep) => [studyStep.id, studyStep])); - - const configurationIds = [...new Set( - studySteps - .map((studyStep) => getAssessmentConfigurationId(studyStep.configuration)) - .filter((id) => id !== null) - )]; - const configurations = configurationIds.length > 0 - ? await server.db.models.configuration.findAll({ - where: { id: { [Op.in]: configurationIds }, deleted: false }, - raw: true - }) - : []; - const configurationsById = new Map(configurations.map((configuration) => [configuration.id, configuration])); - - // The export references study/session owners in addition to the selected document owners. - const relatedUserIds = [...new Set([ - ...users.map((user) => user.id), - ...studySessions.map((session) => session.userId), - ...studies.map((study) => study.userId) - ].filter(Boolean))]; - const relatedUsers = relatedUserIds.length > 0 - ? await server.db.models.user.findAll({ where: { id: { [Op.in]: relatedUserIds } }, raw: true }) - : []; - const usersById = new Map(relatedUsers.map((user) => [user.id, user])); - - return { - sessionsById, - studiesById, - studyStepsById, - configurationsById, - usersById - }; - } - /** * Exports assessment results for the selected users as a ZIP archive. * Each selected user gets one or more hash-named folders containing @@ -582,116 +307,26 @@ module.exports = function (server) { mergeCsvFiles, archive ) { - const { Op } = server.db.Sequelize; - const gradeRows = await server.db.models.document_data.findAll({ - where: { - key: ASSESSMENT_RESULT_KEY, - deleted: false, - studySessionId: { [Op.ne]: null } - }, - include: [{ - model: server.db.models.document, - as: "document", - // required: true turns this include into an inner join. - required: true, - where: { - projectId, - userId: { [Op.in]: userIds }, - deleted: false - }, - include: [{ - model: server.db.models.submission, - as: "submission", - required: false - }] - }], - // Sort by session first, then step within the session, then creation time within the step. - order: [["studySessionId", "ASC"], ["studyStepId", "ASC"], ["createdAt", "ASC"]] - }); - - const { - sessionsById, - studiesById, - studyStepsById, - configurationsById, - usersById - } = await loadGradeExportContext(server, gradeRows, users); + const { records, criteriaReferencesByConfigId } = await buildGradeRecords( + server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping + ); const recordsByUser = new Map(); - // Grade export currently assumes that all exported rows point to one assessment config. - const criteriaReferenceState = { - key: null, - reference: null - }; - for (const row of gradeRows) { - const document = row.document; - const ownerUser = usersById.get(document.userId); - if (!ownerUser) { - console.warn("Skipping grade export row because the document owner could not be resolved.", { - documentId: document.id, - documentUserId: document.userId, - studySessionId: row.studySessionId, - studyStepId: row.studyStepId - }); - continue; - } - const session = sessionsById.get(row.studySessionId); - const reviewerUser = session ? usersById.get(session.userId) : null; - const study = session ? studiesById.get(session.studyId) : null; - const graderUser = study ? usersById.get(study.userId) : null; - const studyStep = studyStepsById.get(row.studyStepId); - const submission = document.submission; - const studyStepConfiguration = studyStep?.configuration; - // configurationId is exported as metadata; assessmentConfig is the rubric content - // needed for score calculation and criteria_reference.json. - const configurationId = getAssessmentConfigurationId(studyStepConfiguration); - const studyName = study?.name || `study_${session?.studyId || "unknown"}`; - - const scoreObject = row.value || {}; - const assessmentState = typeof scoreObject === "string" ? parseAssessmentState(scoreObject) : scoreObject; - const flatScores = buildScoresFromState(assessmentState); - const assessmentConfig = resolveAssessmentConfigurationContent( - studyStepConfiguration, - configurationsById - ); - addCriteriaReferenceEntry( - criteriaReferenceState, - configurationId, - assessmentConfig - ); - const assessmentScore = calculateAssessmentScore(assessmentConfig, flatScores); - const totalPoints = assessmentScore.achieved_points; - - const record = { - projectId, - userId: ownerUser.id, - userExtId: ownerUser.extId ?? null, - userName: ownerUser.userName ?? "", - displayName: getDisplayName(ownerUser, shouldGenerateAliases, hasPrivateInfoRight, userMapping), - submissionId: submission?.id ?? document.submissionId ?? null, - submissionExtId: submission?.extId ?? null, - studySessionId: row.studySessionId ?? null, - studyStepId: row.studyStepId ?? null, - configurationId, - studyName, - sessionHash: session?.hash ?? null, - studyOwner: getPrivateAwareName(graderUser, hasPrivateInfoRight), - sessionOwner: getPrivateAwareName(reviewerUser, hasPrivateInfoRight), - author: getPrivateAwareName(ownerUser, hasPrivateInfoRight), - scores: flatScores, - totalPoints, - createdAt: row.createdAt ? new Date(row.createdAt).toISOString() : null, - studyStepType: studyStep?.stepType ?? null - }; - - if (!recordsByUser.has(ownerUser.id)) recordsByUser.set(ownerUser.id, []); - recordsByUser.get(ownerUser.id).push(record); + for (const record of records) { + if (!recordsByUser.has(record.userId)) recordsByUser.set(record.userId, []); + recordsByUser.get(record.userId).push(record); } - archive.append( - JSON.stringify(criteriaReferenceState.reference || {}, null, 2), - { name: "grades/criteria_reference.json" } - ); + if (criteriaReferencesByConfigId.size > 0) { + for (const [configurationId, reference] of criteriaReferencesByConfigId.entries()) { + archive.append( + JSON.stringify(reference, null, 2), + { name: `grades/criteria_reference_${configurationId}.json` } + ); + } + } else { + archive.append(JSON.stringify({}, null, 2), { name: "grades/criteria_reference.json" }); + } const usedFolderNames = new Set(); const getUniqueHashFolderName = (baseHash, userId, sessionId) => { @@ -722,15 +357,7 @@ module.exports = function (server) { } for (const [groupKey, groupRecords] of mergedGroups.entries()) { - const sortedRecords = [...groupRecords].sort((a, b) => { - const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; - const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; - return ( - (a.studySessionId || 0) - (b.studySessionId || 0) || - (a.studyStepId || 0) - (b.studyStepId || 0) || - createdA - createdB - ); - }); + const sortedRecords = [...groupRecords].sort(compareGradeRecords); const csvRows = sortedRecords.map((record) => buildGradeCsvRow(record)); @@ -742,18 +369,10 @@ module.exports = function (server) { } for (const user of users) { - const records = (recordsByUser.get(user.id) || []).sort((a, b) => { - const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; - const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; - return ( - (a.studySessionId || 0) - (b.studySessionId || 0) || - (a.studyStepId || 0) - (b.studyStepId || 0) || - createdA - createdB - ); - }); + const userRecords = (recordsByUser.get(user.id) || []).sort(compareGradeRecords); const recordsByHash = new Map(); - for (const record of records) { + for (const record of userRecords) { const hashKey = record.sessionHash || null; if (!recordsByHash.has(hashKey)) recordsByHash.set(hashKey, []); recordsByHash.get(hashKey).push(record); @@ -775,22 +394,557 @@ module.exports = function (server) { } /** - * Calculates the version number of a submission by traversing backwards - * through the chain of previous submissions. - * @param {Object} submission - The current submission object to start from. - * @param {Map} submissionMap - A Map containing all related - * submissions for quick lookup by ID. - * @returns {number} - The calculated version number (starting at 1 for the original). + * Exports a single document to the archive based on its type. + * - Type 0 (PDF): exports annotations, comments (with votes), document_data, and the PDF file. + * - Type 1 (HTML) / Type 2 (Modal): exports edits, plain text, HTML, and document_data. + * - Type 4 (ZIP): exports the zip file and document_data. + * @param {Object} server - The server instance providing database models. + * @param {Object} doc - The document record from the database. + * @param {string} docFolder - The target folder path inside the archive. + * @param {Object} archive - The archiver instance to append files to. + * @returns {Promise} */ - function calculateSubmissionVersion(submission, submissionMap) { - let version = 1; - let currentSub = submission; - while (currentSub && currentSub.previousSubmissionId) { - const prevSub = submissionMap.get(currentSub.previousSubmissionId); - if (!prevSub) break; - version++; - currentSub = prevSub; + async function processDocumentForExport(server, doc, docFolder, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, docUserRoles, archive) { + // document_data for all types, at the doc level. + const documentData = await server.db.models.document_data.findAll({ + where: { documentId: doc.id, deleted: false }, + raw: true, + }); + if (documentData.length > 0) { + archive.append(JSON.stringify(documentData, null, 2), { name: `${docFolder}/document_data.json` }); + } + + const docMeta = { + ...doc.toJSON(), + userRoles: docUserRoles, + }; + archive.append(JSON.stringify(docMeta, null, 2), { name: `${docFolder}/meta.json` }); + + switch (doc.type) { + case 0: { // PDF + // Annotations live on study-session copies (parentDocumentId = doc.id), + // not on the root document. Collect all copy IDs and query across them. + const copies = await server.db.models.document.findAll({ + where: { parentDocumentId: doc.id }, + attributes: ['id'], + raw: true, + }); + const allDocIds = [doc.id, ...copies.map(c => c.id)]; + + let [annotations, comments] = await Promise.all([ + server.db.models.annotation.findAll({ where: { documentId: allDocIds }, raw: true }), + server.db.models.comment.findAll({ where: { documentId: allDocIds }, raw: true }), + ]); + + annotations = await attachTagNames(server, annotations); + + if (shouldExcludeNonConsentingAnnotations) { + const allUserIds = [...new Set([ + ...annotations.map(a => a.userId), + ...comments.map(c => c.userId), + ].filter(Boolean))]; + const consentedUserIds = await getConsentedUserIds(server, allUserIds); + annotations = annotations.filter(a => !a.userId || consentedUserIds.has(a.userId)); + comments = comments.filter(c => !c.userId || consentedUserIds.has(c.userId)); + } + + const commentVotes = await server.db.models.comment_vote.findAll({ + where: { commentId: comments.map(c => c.id), deleted: false }, + raw: true, + }); + const commentsWithVotes = comments.map(c => ({ + ...c, + votes: commentVotes.filter(v => v.commentId === c.id), + })); + + // All annotations and comments go into one file each. + if (annotations.length > 0) { + archive.append(JSON.stringify(annotations, null, 2), { name: `${docFolder}/annotations.json` }); + } + if (commentsWithVotes.length > 0) { + archive.append(JSON.stringify(commentsWithVotes, null, 2), { name: `${docFolder}/comments.json` }); + } + + appendStoredFileIfExists(archive, doc.hash, '.pdf', `${docFolder}/document.pdf`, 'PDF'); + break; + } + + case 1: // HTML + case 2: { // MODAL + // fetch all edits for this document, ordered chronologically + let allEdits = await server.db.models.document_edit.findAll({ + where: { documentId: doc.id, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }); + + // filter by consent unless the option is enabled + if (shouldExcludeNonConsentingEdits) { + const editorUserIds = [...new Set(allEdits.map(e => e.userId).filter(Boolean))]; + const consentedUserIds = await getConsentedUserIds(server, editorUserIds); + allEdits = allEdits.filter(e => !e.userId || consentedUserIds.has(e.userId)); + } + + // group edits by studySessionId (null = template) + const sessionGroups = new Map(); + for (const edit of allEdits) { + const key = edit.studySessionId ?? '__template__'; + if (!sessionGroups.has(key)) sessionGroups.set(key, []); + sessionGroups.get(key).push(edit); + } + + // fetch study sessions to resolve hashes + const sessionIds = [...sessionGroups.keys()].filter(k => k !== '__template__'); + const sessions = sessionIds.length > 0 + ? await server.db.models.study_session.findAll({ + where: { id: sessionIds }, + attributes: ['id', 'hash'], + raw: true, + }) + : []; + const sessionHashMap = new Map(sessions.map(s => [s.id, s.hash])); + + for (const [key, edits] of sessionGroups.entries()) { + const isTemplate = key === '__template__'; + const delta = dbToDelta(edits); + + // skip empty content + const text = deltaToPlainText(delta); + if (!text.trim()) continue; + + const subFolder = isTemplate + ? `${docFolder}/template` + : `${docFolder}/${sessionHashMap.get(key) ?? key}`; + + archive.append(text, { name: `${subFolder}/text.txt` }); + archive.append(deltaToHtml(delta), { name: `${subFolder}/html.html` }); + archive.append(JSON.stringify(edits, null, 2), { name: `${subFolder}/edits.json` }); + } + break; + } + + case 4: { // ZIP + appendStoredFileIfExists(archive, doc.hash, '.zip', `${docFolder}/document.zip`, 'ZIP'); + break; + } + + default: + console.warn(`[DocumentExport] Unhandled document type ${doc.type} for document ${doc.hash}, skipping.`); + } + } + + /** + * Main export function for the "documents" export type. + * Fetches all studies and steps for a project, collects unique documents, + * filters by owner data sharing consent, and exports each document to the archive. + * @param {Object} server - The server instance providing database models. + * @param {number|string} projectId - The ID of the project to export. + * @param {string} baseFolderName - The root folder name inside the ZIP archive. + * @param {Object} archive - The archiver instance to append files to. + * @param {Array} userIds - List of user IDs to filter documents by. + * @param {Array} documentTypes - List of document types to include (0=PDF, 1=HTML, 2=Modal, 4=ZIP). + * @returns {Promise} + */ + async function processDocumentBasedExport(server, projectId, userIds, documentTypes, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, baseFolderName, archive) { + try { + documentTypes = typeof documentTypes === 'string' ? JSON.parse(documentTypes) : documentTypes; + if (!Array.isArray(documentTypes)) documentTypes = [0, 1, 2, 4]; + } catch (e) { + console.warn("Could not parse documentTypes:", documentTypes); + documentTypes = [0, 1, 2, 4]; + } + + const docs = await server.db.models.document.findAll({ + where: { projectId, userId: userIds, deleted: false, parentDocumentId: null }, + }); + + if (docs.length === 0) { + console.warn(`[DocumentExport] No documents found for project ${projectId}`); + return; + } + + const filteredDocs = docs.filter(doc => + documentTypes.includes(doc.type) || documentTypes.includes(String(doc.type)) + ); + + if (filteredDocs.length === 0) { + console.warn(`[DocumentExport] No documents matching selected types found for project ${projectId}`); + return; + } + + const uniqueUserIds = [...new Set(filteredDocs.map(doc => doc.userId).filter(Boolean))]; + + const userRoleRows = await server.db.models.user_role_matching.findAll({ + where: { userId: uniqueUserIds }, + raw: true, + }); + + const rolesMap = {}; + for (const row of userRoleRows) { + if (!rolesMap[row.userId]) rolesMap[row.userId] = []; + rolesMap[row.userId].push(row.userRoleId); + } + + for (const doc of filteredDocs) { + const docFolder = `${baseFolderName}/${doc.hash}`; + const docUserRoles = rolesMap[doc.userId] || []; + await processDocumentForExport(server, doc, docFolder, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, docUserRoles, archive); + } + } + + function sortSteps(items, prevKey) { + const sorted = []; + let current = items.find(item => item[prevKey] === null); + while (current) { + sorted.push(current); + current = items.find(item => item[prevKey] === current.id); + } + return sorted; + } + + async function processStudyBasedExport(server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, workflowIds, shouldIncludeEmptyStudies, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, shouldIncludeDocumentFiles, shouldIncludeGrades, shouldIncludeAiScores, baseFolderName, archive) { + const studyWhere = { userId: userIds, projectId, deleted: false, workflowId: workflowIds }; + + const studies = await server.db.models.study.findAll({ where: studyWhere }); + + if (studies.length === 0) { + console.warn(`[StudyExport] No studies found for selected users in project ${projectId}`); + return; + } + + const writtenCriteriaReferenceIds = new Set(); + + for (const study of studies) { + const studyFolder = `${baseFolderName}/${study.hash}`; + + const allSteps = await server.db.models.study_step.findAll({ + where: { studyId: study.id, deleted: false } + }); + const sortedSteps = sortSteps(allSteps, 'studyStepPrevious'); + + const stepDocumentsById = new Map(); + // A step references one document, but PDF and LaTeX ZIP are separate documents + // of the same submission, each with its own hash. + const submissionDocumentsBySubmissionId = new Map(); + if (shouldIncludeDocumentFiles) { + const stepDocumentIds = [...new Set(sortedSteps.map(step => step.documentId).filter(Boolean))]; + const stepDocuments = stepDocumentIds.length > 0 + ? await server.db.models.document.findAll({ where: { id: stepDocumentIds }, raw: true }) + : []; + for (const doc of stepDocuments) stepDocumentsById.set(doc.id, doc); + + const submissionIds = [...new Set(stepDocuments.map(doc => doc.submissionId).filter(Boolean))]; + const submissionDocuments = submissionIds.length > 0 + ? await server.db.models.document.findAll({ + where: { submissionId: submissionIds, deleted: false }, + raw: true, + }) + : []; + for (const doc of submissionDocuments) { + if (!submissionDocumentsBySubmissionId.has(doc.submissionId)) { + submissionDocumentsBySubmissionId.set(doc.submissionId, []); + } + submissionDocumentsBySubmissionId.get(doc.submissionId).push(doc); + } + } + + const sessions = await server.db.models.study_session.findAll({ + where: { studyId: study.id, deleted: false }, + raw: true, + }); + + const sessionResults = []; + for (const session of sessions) { + const stepResults = []; + let sessionHasContent = false; + + for (let i = 0; i < sortedSteps.length; i++) { + const step = sortedSteps[i]; + const files = []; + + switch (step.stepType) { + case 1: { // Annotator + let annotations = await server.db.models.annotation.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + + annotations = await attachTagNames(server, annotations); + + let comments = await server.db.models.comment.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + + if (shouldExcludeNonConsentingAnnotations) { + const allUserIds = [...new Set([ + ...annotations.map(a => a.userId), + ...comments.map(c => c.userId) + ].filter(Boolean))]; + const consentedUsers = await server.db.models.user.findAll({ + where: { id: allUserIds }, + attributes: ['id', 'acceptDataSharing'], + raw: true, + }); + const consentedIds = new Set(consentedUsers.filter(u => u.acceptDataSharing).map(u => u.id)); + annotations = annotations.filter(a => !a.userId || consentedIds.has(a.userId)); + comments = comments.filter(c => !c.userId || consentedIds.has(c.userId)); + } + + if (annotations.length > 0) { + files.push({ name: 'annotations.json', content: JSON.stringify(annotations, null, 2) }); + sessionHasContent = true; + } + + if (comments.length > 0) { + const commentVotes = await server.db.models.comment_vote.findAll({ + where: { commentId: comments.map(c => c.id), deleted: false }, + raw: true, + }); + files.push({ + name: 'comments.json', + content: JSON.stringify( + comments.map(c => ({ ...c, votes: commentVotes.filter(v => v.commentId === c.id) })), + null, 2 + ) + }); + sessionHasContent = true; + } + + // Assessments live on the annotator step, so without this the whole + // document_data of an assessment workflow never leaves the database. + const annotatorData = await server.db.models.document_data.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + if (annotatorData.length > 0) { + files.push({ name: 'document_data.json', content: JSON.stringify(annotatorData, null, 2) }); + sessionHasContent = true; + } + break; + } + + case 2: // Editor + case 3: { // Modal + const [templateEdits, sessionEdits] = await Promise.all([ + server.db.models.document_edit.findAll({ + where: { documentId: step.documentId, studySessionId: null, studyStepId: null, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }), + server.db.models.document_edit.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }), + ]); + + let edits = [...templateEdits, ...sessionEdits].sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt)); + + if (shouldExcludeNonConsentingEdits) { + const editorUserIds = [...new Set(edits.map(e => e.userId).filter(Boolean))]; + const editorUsers = await server.db.models.user.findAll({ + where: { id: editorUserIds }, + attributes: ['id', 'acceptDataSharing'], + raw: true, + }); + const consentedIds = new Set(editorUsers.filter(u => u.acceptDataSharing).map(u => u.id)); + edits = edits.filter(e => !e.userId || consentedIds.has(e.userId)); + } + + if (edits.length > 0) { + const delta = dbToDelta(edits); + const text = deltaToPlainText(delta); + if (text.trim()) { + files.push({ name: 'edits.json', content: JSON.stringify(edits, null, 2) }); + files.push({ name: 'text.txt', content: text }); + files.push({ name: 'html.html', content: deltaToHtml(delta) }); + sessionHasContent = true; + } + } + + const documentData = await server.db.models.document_data.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + if (documentData.length > 0) { + files.push({ name: 'document_data.json', content: JSON.stringify(documentData, null, 2) }); + } + break; + } + } + + stepResults.push({ stepIndex: i, files }); + + } + + sessionResults.push({ session, stepResults, hasContent: sessionHasContent }); + } + + const includedSessions = shouldIncludeEmptyStudies + ? sessionResults + : sessionResults.filter(s => s.hasContent); + + if (!shouldIncludeEmptyStudies && includedSessions.length === 0) continue; + + const studyMeta = { + id: study.id, + name: study.name, + userId: study.userId, + workflowId: study.workflowId, + sessions: includedSessions.map(({ session }) => ({ + hash: session.hash, + id: session.id, + userId: session.userId, + numberSteps: session.numberSteps, + steps: sortedSteps.map((step, i) => ({ + id: step.id, + stepNumber: i + 1, + stepType: step.stepType, + configuration: step.configuration, + })) + })) + }; + + archive.append(JSON.stringify(studyMeta, null, 2), { name: `${studyFolder}/meta.json` }); + + let gradeRecordsBySessionId = new Map(); + if (shouldIncludeGrades) { + // Scope by this study's sessions, not by the study owner: review assessments are + // stored on a document owned by the reviewed author, so an owner-scoped lookup + // finds none of them. See buildGradeRecords(). + const { records: ownerGradeRecords, criteriaReferencesByConfigId } = await buildGradeRecords( + server, projectId, [study.userId], users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, + { sessionIds: sessions.map(s => s.id) } + ); + + for (const [configurationId, reference] of criteriaReferencesByConfigId.entries()) { + if (writtenCriteriaReferenceIds.has(configurationId)) continue; + writtenCriteriaReferenceIds.add(configurationId); + archive.append( + JSON.stringify(reference, null, 2), + { name: `${baseFolderName}/criteria_reference_${configurationId}.json` } + ); + } + + for (const record of ownerGradeRecords) { + if (!gradeRecordsBySessionId.has(record.studySessionId)) gradeRecordsBySessionId.set(record.studySessionId, []); + gradeRecordsBySessionId.get(record.studySessionId).push(record); + } + } + + for (const { session, stepResults } of includedSessions) { + const sessionFolder = `${studyFolder}/${session.hash}`; + + if (shouldIncludeGrades) { + const allSessionGrades = (gradeRecordsBySessionId.get(session.id) || []).sort(compareGradeRecords); + + const humanGrades = allSessionGrades.filter(r => !r.isAiGraded).map(({ sessionHash, isAiGraded, ...rest }) => rest); + const aiGrades = allSessionGrades.filter(r => r.isAiGraded).map(({ sessionHash, isAiGraded, ...rest }) => rest); + + if (humanGrades.length > 0) { + archive.append(JSON.stringify(humanGrades, null, 2), { name: `${sessionFolder}/scores.json` }); + } + if (shouldIncludeAiScores && aiGrades.length > 0) { + archive.append(JSON.stringify(aiGrades, null, 2), { name: `${sessionFolder}/scores_ai.json` }); + } + } + + for (const { stepIndex, files } of stepResults) { + const stepFolder = `${sessionFolder}/step_${stepIndex + 1}`; + + if (shouldIncludeDocumentFiles) { + const step = sortedSteps[stepIndex]; + const stepDocument = stepDocumentsById.get(step.documentId); + if (stepDocument) { + const submissionSiblings = stepDocument.submissionId + ? (submissionDocumentsBySubmissionId.get(stepDocument.submissionId) || []) + : []; + const candidates = [ + stepDocument, + ...submissionSiblings.filter(doc => doc.id !== stepDocument.id), + ]; + + const appendedExtensions = new Set(); + for (const doc of candidates) { + const extension = STUDY_DOCUMENT_EXTENSIONS[doc.type]; + if (!extension || appendedExtensions.has(extension)) continue; + appendedExtensions.add(extension); + appendStoredFileIfExists( + archive, + doc.hash, + extension, + `${stepFolder}/document${extension}`, + extension.slice(1).toUpperCase(), + ); + } + } + } + + for (const file of files) { + archive.append(file.content, { name: `${stepFolder}/${file.name}` }); + } + } + } + } + } + /** + * Exports usage statistics for the selected users, respecting each user's acceptStats consent. + * @param {string} behaviourOutputFormat - 'single' for one combined file, 'perUser' for one file per user. + */ + async function processUserBehaviourExport(server, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, behaviourOutputFormat, behaviourFileFormat, baseFolderName, archive) { + const { Op } = server.db.Sequelize; + + const consentedUsers = users.filter(u => u.acceptStats); + if (consentedUsers.length === 0) return; + const usersById = new Map(consentedUsers.map(u => [u.id, u])); + const consentedUserIds = consentedUsers.map(u => u.id); + const extension = behaviourFileFormat === 'csv' ? 'csv' : 'json'; + + const parseStatData = (raw) => { + try { + return JSON.parse(raw); + } catch (e) { + return raw; + } + }; + + const toRecord = (stat) => { + const user = usersById.get(stat.userId); + return { + action: stat.action, + data: behaviourFileFormat === 'csv' ? stat.data : parseStatData(stat.data), + timestamp: stat.timestamp instanceof Date ? stat.timestamp.toISOString() : stat.timestamp, + user: getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping), + username: user?.userName ?? null, + userId: stat.userId, + session: stat.session, + }; + }; + + const buildStream = (fetchPage) => behaviourFileFormat === 'csv' + ? createCsvRowsStream(fetchPage, toRecord) + : createJsonArrayStream(fetchPage, toRecord); + + if (behaviourOutputFormat === 'perUser') { + for (const user of consentedUsers) { + const folderName = sanitizeFolderName(getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping)); + const fetchPage = (lastId, limit) => server.db.models.statistic.findAll({ + where: { userId: user.id, deleted: false, id: { [Op.gt]: lastId } }, + order: [['id', 'ASC']], + limit, + raw: true, + }); + archive.append(buildStream(fetchPage), { name: `${baseFolderName}/${folderName}/behaviour_data.${extension}` }); + } + } else { + const fetchPage = (lastId, limit) => server.db.models.statistic.findAll({ + where: { userId: { [Op.in]: consentedUserIds }, deleted: false, id: { [Op.gt]: lastId } }, + order: [['id', 'ASC']], + limit, + raw: true, + }); + archive.append(buildStream(fetchPage), { name: `${baseFolderName}/behaviour_data.${extension}` }); } - return version; } -}; +}; \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 17c037c06..87ca2e3ec 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -88,7 +88,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.2", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", @@ -123,6 +124,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -1470,6 +1472,7 @@ "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", "license": "MIT", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -1917,6 +1920,7 @@ "integrity": "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -2286,6 +2290,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2558,6 +2563,7 @@ } ], "license": "MIT", + "peer": true, "peerDependencies": { "@popperjs/core": "^2.11.8" } @@ -2640,6 +2646,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.44", "caniuse-lite": "^1.0.30001806", @@ -3640,6 +3647,7 @@ "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -5812,6 +5820,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -6445,6 +6454,7 @@ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -6983,6 +6993,7 @@ "resolved": "https://registry.npmjs.org/terser/-/terser-5.46.0.tgz", "integrity": "sha512-jTwoImyr/QbOWFFso3YoU3ik0jBBDJ6JTOQiy/J2YxVJdZCc+5u7skhNwiOR3FQIygFqVUPHl7qbbxtjW2K3Qg==", "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", @@ -7317,6 +7328,7 @@ "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", @@ -7456,6 +7468,7 @@ "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.30.tgz", "integrity": "sha512-hTHLc6VNZyzzEH/l7PFGjpcTvUgiaPK5mdLkbjrTeWSRcEfxFrv56g/XckIYlE9ckuobsdwqd5mk2g1sBkMewg==", "license": "MIT", + "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.30", "@vue/compiler-sfc": "3.5.30", @@ -7494,6 +7507,7 @@ "integrity": "sha512-Vxi9pJdbN3ZnVGLODVtZ7y4Y2kzAAE2Cm0CZ3ZDRvydVYxZ6VrnBhLikBsRS+dpwj4Jv4UCv21PTEwF5rQ9WXg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "debug": "^4.4.0", "eslint-scope": "^8.2.0 || ^9.0.0", @@ -7963,6 +7977,7 @@ "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "license": "ISC", + "peer": true, "bin": { "yaml": "bin.mjs" }, diff --git a/frontend/src/components/dashboard/projects/ExportModal.vue b/frontend/src/components/dashboard/projects/ExportModal.vue index b46137148..70c056a26 100644 --- a/frontend/src/components/dashboard/projects/ExportModal.vue +++ b/frontend/src/components/dashboard/projects/ExportModal.vue @@ -18,7 +18,9 @@ :fields="dataSelectionFields" /> + - @@ -87,8 +117,11 @@ import JSZip from 'jszip'; import FileSaver from 'file-saver'; import Quill from "quill"; import {dbToDelta} from "editor-delta-conversion"; -import StepSelectStudents from "@/components/dashboard/projects/export/StepSelectStudents.vue"; +import StepSelectUsers from "@/components/dashboard/projects/export/StepSelectUsers.vue"; import StepOptions from "@/components/dashboard/projects/export/StepOptions.vue"; +import StepOptionsDocuments from "@/components/dashboard/projects/export/StepOptionsDocuments.vue"; +import StepOptionsStudies from "@/components/dashboard/projects/export/StepOptionsStudies.vue"; +import StepOptionsUserBehaviour from "@/components/dashboard/projects/export/StepOptionsUserBehaviour.vue"; import StepConfirmDownload from "@/components/dashboard/projects/export/StepConfirmDownload.vue"; import getServerURL from "@/assets/serverUrl.js"; @@ -96,11 +129,11 @@ import getServerURL from "@/assets/serverUrl.js"; /** * ProjectModal - modal component for adding and editing projects * - * @author Dennis Zyska, Mélissa Loew, Linyin Huang + * @author Dennis Zyska, Mélissa Loew */ export default { name: "ExportProjectModal", - components: { StepperModal, BasicForm, StepSelectStudents, StepOptions, StepConfirmDownload }, + components: { StepperModal, BasicForm, StepSelectUsers, StepOptions, StepOptionsDocuments, StepOptionsStudies, StepOptionsUserBehaviour, StepConfirmDownload }, subscribeTable: [{ table: "document", }, { @@ -119,6 +152,18 @@ export default { table: "tag_set", }, { table: "tag" + }, { + table: "document_data", + }, { + table: "study_step", + }, { + table: "configuration", + }, { + table: "workflow", + }, { + table: "user_role", + }, { + table: "user_role_matching", } ], provide() { @@ -135,11 +180,21 @@ export default { filter: [], wait: false, // Data for Export Submissions - submissionSelection: [], + userSelection: [], generateAliases:false, fakerSeed: 846569412, gradeFormat: "json", - mergeCsvFiles: false + mergeCsvFiles: false, + selectedDocumentTypes: [0, 1, 2, 4], + excludeNonConsentingEdits: false, + excludeNonConsentingAnnotations: false, + selectedWorkflowIds: [], + includeStudyDocumentFiles: true, + includeStudyGrades: true, + includeStudyIncludeAiScores: true, + includeEmptyStudies: true, + behaviourOutputFormat: 'single', + behaviourFileFormat: 'json', }; }, computed: { @@ -147,10 +202,31 @@ export default { if (["submissions", "grades"].includes(this.dataSelection.exportType)) { return [ !!this.dataSelection.projectId && !!this.dataSelection.exportType, // must select a valid project and export type - this.submissionSelection.length > 0, // must select at least one student + this.userSelection.length > 0, // must select at least one student true, true ]; + } else if (this.dataSelection.exportType === "documents") { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + this.selectedDocumentTypes.length > 0, + true, + ]; + } else if (this.dataSelection.exportType === 'studies') { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + this.selectedWorkflowIds.length > 0, + true, + ]; + } else if (this.dataSelection.exportType === 'userBehaviour') { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + true, + true, + ]; } return [ !!this.dataSelection.projectId && !!this.dataSelection.exportType, @@ -158,14 +234,14 @@ export default { ]; }, steps() { - if (["submissions", "grades"].includes(this.dataSelection.exportType)) { + if (["submissions", "grades", "documents", "studies", "userBehaviour"].includes(this.dataSelection.exportType)) { return [ { title: "Settings" }, - { title: "Select Students" }, + { title: "Select Users" }, { title: "Options" }, { title: "Confirm Download" } ]; - } + } return [ {title: "Settings"}, {title: "Confirmation"} @@ -191,6 +267,9 @@ export default { {name: "Export a list of all reviewers", value: "reviewerList"}, {name: "Export submissions", value: "submissions"}, {name: "Export grades", value: "grades"}, + {name: "Export documents", value:"documents"}, + {name: "Export studies", value: "studies"}, + ...(this.$store.getters["auth/isAdmin"] ? [{name: "Export user behaviour", value: "userBehaviour"}] : []), {name: "All", value: "all"}, ], required: true, @@ -249,13 +328,40 @@ export default { return this.$store.getters["table/project/getAll"]; }, }, + watch: { + 'dataSelection.exportType'() { + this.resetOptions(); + }, + 'dataSelection.projectId'() { + this.resetOptions(); + } + }, methods: { + resetOptions() { + this.filter = []; + this.userSelection = []; + this.generateAliases = false; + this.fakerSeed = 846569412; + this.gradeFormat = "json"; + this.mergeCsvFiles = false; + this.selectedDocumentTypes = [0, 1, 2, 4]; + this.excludeNonConsentingEdits = false; + this.excludeNonConsentingAnnotations = false; + this.selectedWorkflowIds = []; + this.includeStudyDocumentFiles = true; + this.includeStudyGrades = true; + this.includeStudyIncludeAiScores = true; + this.includeEmptyStudies = true; + this.behaviourOutputFormat = 'single'; + this.behaviourFileFormat = 'json'; + }, open(projectId) { this.dataSelection.projectId = projectId; this.$refs.exportStepper.open(); }, hide() { - this.filter = []; + this.resetOptions(); + this.wait = false; }, downloadData() { if (this.dataSelection.exportType === "reviewerList") { @@ -264,6 +370,12 @@ export default { this.downloadSubmissions(); } else if (this.dataSelection.exportType === "grades") { this.downloadGrades(); + } else if (this.dataSelection.exportType === "documents") { + this.downloadDocuments(); + } else if (this.dataSelection.exportType === 'studies') { + this.downloadStudies(); + } else if (this.dataSelection.exportType === 'userBehaviour') { + this.downloadUserBehaviour(); } else { this.downloadAllData(); } @@ -327,7 +439,7 @@ export default { async downloadSubmissions() { try { // get the selected student's user ids - const selectedUserIds = this.submissionSelection.map(row => row.userId); + const selectedUserIds = this.userSelection.map(row => row.userId); // call helper function to trigger the stream download this.triggerStreamDownload({ @@ -346,7 +458,7 @@ export default { }, async downloadGrades() { try { - const selectedUserIds = this.submissionSelection.map(row => row.userId); + const selectedUserIds = this.userSelection.map(row => row.userId); this.triggerStreamDownload({ projectId: this.dataSelection.projectId, exportType: 'grades', @@ -362,6 +474,61 @@ export default { this.$toast.error("An error occurred starting the stream. Please try again."); } }, + async downloadDocuments() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'documents', + userIds: selectedUserIds, + documentTypes: this.selectedDocumentTypes, + excludeNonConsentingEdits: this.excludeNonConsentingEdits, + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations + }); + + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, + async downloadStudies() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'studies', + userIds: selectedUserIds, + workflowIds: this.selectedWorkflowIds, + includeEmptyStudies: this.includeEmptyStudies, + includeDocumentFiles: this.includeStudyDocumentFiles, + includeGrades: this.includeStudyGrades, + excludeNonConsentingEdits: this.excludeNonConsentingEdits, + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations, + includeAiScores: this.includeStudyIncludeAiScores + }); + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, + async downloadUserBehaviour() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'userBehaviour', + userIds: selectedUserIds, + behaviourOutputFormat: this.behaviourOutputFormat, + behaviourFileFormat: this.behaviourFileFormat, + }); + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, async downloadAllData() { this.wait = true; diff --git a/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue b/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue index 8afd4df92..672fca32d 100644 --- a/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue +++ b/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue @@ -6,7 +6,7 @@
Confirm Selection:
- +
You have selected one or more students who didn't accept data sharing.
@@ -21,14 +21,15 @@
Summary:
- You are about to download submissions for - {{ submissionSelection.length }} student(s). + You are about to download + {{ exportTypeLabel }} + for {{ userSelection.length }} user(s).
    -
  • - {{ row.studentName || row.userName }} ({{ row.fileCount }} files) +
  • + {{ row.name }} ({{ row.suffix }})
@@ -44,13 +45,12 @@ import BasicLoading from "@/basic/Loading.vue"; * * The final confirmation step within the ExportModal. * This component provides a summary of the selected - * submissions intended for download, as well as some + * data intended for download, as well as some * warnings for the user, if they selected generate aliases * or students who didn't accept data sharing. * * @author Mélissa Loew */ - export default { name: "StepConfirmDownload", components: { BasicLoading }, @@ -63,15 +63,40 @@ export default { type: Boolean, default: false }, - submissionSelection: { + userSelection: { type: Array, required: true + }, + exportType: { + type: String, + default: 'submissions' } }, computed: { hasDeclinedSharingSelected() { - return this.submissionSelection.some(row => row.acceptDataSharing === 'No'); - } + return this.userSelection.some(row => row.acceptDataSharing === 'No'); + }, + exportTypeLabel() { + const labels = { + submissions: 'submissions', + grades: 'grades', + documents: 'documents', + studies: 'studies', + }; + return labels[this.exportType] || 'documents'; + }, + userSelectionDisplay() { + const unitByExportType = { + submissions: 'submission(s)', + studies: 'study(ies)', + }; + const unit = unitByExportType[this.exportType] || 'document(s)'; + return this.userSelection.map(row => ({ + userId: row.userId, + name: row.studentName || row.userName, + suffix: ['grades', 'userBehaviour'].includes(this.exportType) ? null : `${row.count} ${unit}`, + })); + }, } } \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue b/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue new file mode 100644 index 000000000..d18fae9cb --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue @@ -0,0 +1,138 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue new file mode 100644 index 000000000..6183ac734 --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue @@ -0,0 +1,201 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsUserBehaviour.vue b/frontend/src/components/dashboard/projects/export/StepOptionsUserBehaviour.vue new file mode 100644 index 000000000..9a4abd8a9 --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsUserBehaviour.vue @@ -0,0 +1,68 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue b/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue deleted file mode 100644 index 8ca0a4962..000000000 --- a/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue +++ /dev/null @@ -1,147 +0,0 @@ - - - \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepSelectUsers.vue b/frontend/src/components/dashboard/projects/export/StepSelectUsers.vue new file mode 100644 index 000000000..08145b9f3 --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepSelectUsers.vue @@ -0,0 +1,335 @@ + + + \ No newline at end of file diff --git a/utils/modules/editor-delta-conversion/index.js b/utils/modules/editor-delta-conversion/index.js index c8d722197..763b6e097 100644 --- a/utils/modules/editor-delta-conversion/index.js +++ b/utils/modules/editor-delta-conversion/index.js @@ -2,10 +2,11 @@ * * This module provides methods to convert between Quill Delta objects and database entries. * - * @author Juliane Bechert + * @author Juliane Bechert, Mélissa Loew * */ const Delta = require('quill-delta'); +const { QuillDeltaToHtmlConverter } = require('quill-delta-to-html'); /** * Converts an array of database entries to a Quill Delta object. @@ -134,8 +135,29 @@ function deltaToPlainText(deltaOrOps) { .join(""); } +/** + * Converts a Quill Delta object to an HTML string. + * Each newline in the delta marks the end of a paragraph and is flushed as a

tag. + * Supports bold, italic, underline, and link attributes. + * + * @param {object|array} deltaOrOps - Quill Delta ({ ops: [...] }) or ops array + * @returns {string} A full HTML document string + */ +function deltaToHtml(deltaOrOps) { + if (!deltaOrOps) return ""; + const ops = Array.isArray(deltaOrOps) + ? deltaOrOps + : (deltaOrOps.ops || []); + + const converter = new QuillDeltaToHtmlConverter(ops, {}); + const body = converter.convert(); + + return `\n\n\n${body}\n`; +} + module.exports = { deltaToDb: deltaToDb, dbToDelta: dbToDelta, deltaToPlainText: deltaToPlainText, + deltaToHtml: deltaToHtml, } \ No newline at end of file diff --git a/utils/modules/editor-delta-conversion/package-lock.json b/utils/modules/editor-delta-conversion/package-lock.json index 45d704caf..01339b787 100644 --- a/utils/modules/editor-delta-conversion/package-lock.json +++ b/utils/modules/editor-delta-conversion/package-lock.json @@ -10,7 +10,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.2", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", @@ -48,6 +49,7 @@ "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -1670,6 +1672,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.44", "caniuse-lite": "^1.0.30001806", @@ -3654,6 +3657,15 @@ "node": ">= 12.0.0" } }, + "node_modules/quill-delta-to-html": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/quill-delta-to-html/-/quill-delta-to-html-0.12.1.tgz", + "integrity": "sha512-QhpeMk9+5ge3HYbL5A0Ewz3pXCsbemqGvIF/kw5D6D4V68AtcUp7yt9xNUkzOk/0IQz43hKy3IkzBzRhLIE+oA==", + "license": "ISC", + "dependencies": { + "lodash.isequal": "^4.5.0" + } + }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", diff --git a/utils/modules/editor-delta-conversion/package.json b/utils/modules/editor-delta-conversion/package.json index d4d5d2ad5..fbf701b85 100644 --- a/utils/modules/editor-delta-conversion/package.json +++ b/utils/modules/editor-delta-conversion/package.json @@ -18,7 +18,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.2", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3",