-
Notifications
You must be signed in to change notification settings - Fork 10
Tenant Based Micro-Improvement Dashboard Creation #185
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: release-3.1.0
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -245,8 +245,8 @@ class ProjectMetabaseDashboardFunction(config: CombinedDashboardCreatorConfig)(i | |
| */ | ||
| logger.info("-->> Process Program Manager Collection and project solution Dashboard") | ||
| if (targetedSolutionId.nonEmpty && solutionName.nonEmpty && targetedProgramId.nonEmpty && programName.nonEmpty) { | ||
| val programCollectionName = s"$programName [org : $orgId]" | ||
| val programCollectionDescription = s"Program Id: $targetedProgramId\n\nProgram External Id: $programExternalId\n\nCollection For: Program Manager\n\nProgram Description: $programDescription" | ||
| val programCollectionName = s"$programName" | ||
| val programCollectionDescription = s"Program Id: $targetedProgramId\n\nProgram External Id: $programExternalId\n\nCreator Organisation: $orgId\n\nCollection For: Admin\n\nProgram Description: $programDescription" | ||
| val solutionCollectionName = s"$solutionName [Project]" | ||
| val solutionCollectionDescription = s"Solution Id: $targetedSolutionId\n\nSolution External Id: $solutionExternalId\n\nCollection For: Program Manager\n\nSolution Description: $solutionDescription" | ||
|
|
||
|
|
@@ -293,37 +293,49 @@ class ProjectMetabaseDashboardFunction(config: CombinedDashboardCreatorConfig)(i | |
| } | ||
| } | ||
|
|
||
| def createMicroImprovementsCollectionAndDashboard(metaDataTable: String, reportConfig: String, databaseId: Int): Unit = { | ||
| val createDashboardQuery = s"UPDATE $metaDataTable SET status = 'Failed',error_message = 'errorMessage' WHERE id = '$admin';" | ||
| val (mipCollectionName, mipCollectionDescription) = (s"Micro Improvements", s"This collection contains dashboards that track the progress, participation, and effectiveness of Micro Improvement Projects across various programs.\n\nCollection For: Admin") | ||
| val mipCollectionId = Utils.checkAndCreateCollection(mipCollectionName, mipCollectionDescription, metabaseUtil) | ||
| if (mipCollectionId != -1) { | ||
| Utils.createGroupForCollection(metabaseUtil, s"Report_Admin_Micro_Improvement", mipCollectionId) | ||
| val (mipDashboardName, mipDashboardDescription) = (s"Overview - Across States and Programs", s"A consolidated view of project progress and user participation.") | ||
| val (mipDashboardId, projectTabId, userTabId, csvTabId) = Utils.createMicroImprovementsDashboardAndTabs(mipCollectionId, mipDashboardName, mipDashboardDescription, metabaseUtil) | ||
| if (projectTabId != -1 && userTabId != -1 && csvTabId != -1) { | ||
| val stateNameId: Int = getTheColumnId(databaseId, projects, "state_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val districtNameId: Int = getTheColumnId(databaseId, projects, "district_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val programNameId: Int = getTheColumnId(databaseId, projects, "program_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val blockNameId: Int = getTheColumnId(databaseId, projects, "block_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val clusterNameId: Int = getTheColumnId(databaseId, projects, "cluster_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val orgNameId: Int = getTheColumnId(databaseId, projects, "org_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val projectDetailsConfigQuery: String = s"SELECT question_type, config FROM $reportConfig WHERE dashboard_name = 'Admin' AND report_name = 'Project-Details';" | ||
| val projectQuestionCardIdList = ProcessAdminConstructor.ProcessAndUpdateJsonFiles(projectDetailsConfigQuery, mipCollectionId, databaseId, mipDashboardId, projectTabId, stateNameId, districtNameId, programNameId, blockNameId, clusterNameId, orgNameId, projects, solutions, metabaseUtil, postgresUtil) | ||
| val userDetailsConfigQuery: String = s"SELECT question_type, config FROM $reportConfig WHERE dashboard_name = 'Admin' AND report_name = 'User-Details';" | ||
| val userQuestionCardIdList = ProcessAdminConstructor.ProcessAndUpdateJsonFiles(userDetailsConfigQuery, mipCollectionId, databaseId, mipDashboardId, userTabId, stateNameId, districtNameId, programNameId, blockNameId, clusterNameId, orgNameId, projects, solutions, metabaseUtil, postgresUtil) | ||
| val submissionDetailsConfigQuery: String = s"SELECT question_type, config FROM $reportConfig WHERE dashboard_name = 'Admin' AND report_name = 'Submission-Details-CSV';" | ||
| val submissionQuestionCardIdList = ProcessAdminConstructor.ProcessAndUpdateJsonFiles(submissionDetailsConfigQuery, mipCollectionId, databaseId, mipDashboardId, csvTabId, stateNameId, districtNameId, programNameId, blockNameId, clusterNameId, orgNameId, projects, solutions, metabaseUtil, postgresUtil) | ||
| val mainQuestionIdsString = "[" + (projectQuestionCardIdList ++ userQuestionCardIdList ++ submissionQuestionCardIdList).mkString(",") + "]" | ||
| val parametersQuery: String = s"SELECT config FROM $reportConfig WHERE report_name = 'Project-Parameter' AND question_type = 'admin-parameter'" | ||
| UpdateParameters.UpdateAdminParameterFunction(metabaseUtil, parametersQuery, mipDashboardId, postgresUtil) | ||
| val projectMetadataJson = new ObjectMapper().createArrayNode().add(new ObjectMapper().createObjectNode().put("collectionId", mipCollectionId).put("collectionName", mipCollectionName).put("dashboardId", mipDashboardId).put("dashboardName", mipDashboardName).put("questionIds", mainQuestionIdsString)) | ||
| postgresUtil.insertData(s"UPDATE $metaDataTable SET main_metadata = COALESCE(main_metadata::jsonb, '[]'::jsonb) || '$projectMetadataJson' ::jsonb WHERE entity_id = '$admin';") | ||
| } else { | ||
| logger.info(s"Creating tabs failed please check: $projectTabId, $userTabId, $csvTabId") | ||
| def createMicroImprovementsCollectionAndDashboard(metaDataTable: String, reportConfig: String, databaseId: Int): Unit = { | ||
| val createDashboardQuery = s"UPDATE $metaDataTable SET status = 'Failed',error_message = 'errorMessage' WHERE id = '$admin';" | ||
| val (mipCollectionName, mipCollectionDescription) = (s"Micro Improvements [TenantId: $tenantId]", s"This collection contains dashboards that track the progress, participation, and effectiveness of Micro Improvement Projects across various programs.\n\nCollection For: Admin") | ||
| val mipCollectionId = Utils.checkAndCreateCollection(mipCollectionName, mipCollectionDescription, metabaseUtil) | ||
| if (mipCollectionId != -1) { | ||
| Utils.createGroupForCollection(metabaseUtil, s"Report_Admin_Micro_Improvement", mipCollectionId) | ||
| val (mipDashboardName, mipDashboardDescription) = (s"Overview - Across States and Programs", s"A consolidated view of project progress and user participation.") | ||
| val (mipDashboardId, projectTabId, userTabId, csvTabId) = Utils.createMicroImprovementsDashboardAndTabs(mipCollectionId, mipDashboardName, mipDashboardDescription, metabaseUtil) | ||
| if (projectTabId != -1 && userTabId != -1 && csvTabId != -1) { | ||
| val stateNameId: Int = getTheColumnId(databaseId, projects, "state_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val districtNameId: Int = getTheColumnId(databaseId, projects, "district_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val programNameId: Int = getTheColumnId(databaseId, projects, "program_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val blockNameId: Int = getTheColumnId(databaseId, projects, "block_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val clusterNameId: Int = getTheColumnId(databaseId, projects, "cluster_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val orgNameId: Int = getTheColumnId(databaseId, projects, "org_name", metabaseUtil, metabasePostgresUtil, metabaseApiKey, createDashboardQuery) | ||
| val projectDetailsConfigQuery: String = s"SELECT question_type, config FROM $reportConfig WHERE dashboard_name = 'Admin' AND report_name = 'Project-Details';" | ||
| val projectQuestionCardIdList = ProcessAdminConstructor.ProcessAndUpdateJsonFiles(projectDetailsConfigQuery, mipCollectionId, databaseId, mipDashboardId, projectTabId, stateNameId, districtNameId, programNameId, blockNameId, clusterNameId, orgNameId, projects, solutions, tenantId, metabaseUtil, postgresUtil) | ||
| val userDetailsConfigQuery: String = s"SELECT question_type, config FROM $reportConfig WHERE dashboard_name = 'Admin' AND report_name = 'User-Details';" | ||
| val userQuestionCardIdList = ProcessAdminConstructor.ProcessAndUpdateJsonFiles(userDetailsConfigQuery, mipCollectionId, databaseId, mipDashboardId, userTabId, stateNameId, districtNameId, programNameId, blockNameId, clusterNameId, orgNameId, projects, solutions, tenantId, metabaseUtil, postgresUtil) | ||
| val submissionDetailsConfigQuery: String = s"SELECT question_type, config FROM $reportConfig WHERE dashboard_name = 'Admin' AND report_name = 'Submission-Details-CSV';" | ||
| val submissionQuestionCardIdList = ProcessAdminConstructor.ProcessAndUpdateJsonFiles(submissionDetailsConfigQuery, mipCollectionId, databaseId, mipDashboardId, csvTabId, stateNameId, districtNameId, programNameId, blockNameId, clusterNameId, orgNameId, projects, solutions, tenantId, metabaseUtil, postgresUtil) | ||
| val mainQuestionIdsString = "[" + (projectQuestionCardIdList ++ userQuestionCardIdList ++ submissionQuestionCardIdList).mkString(",") + "]" | ||
| val filterQuery: String = s"SELECT config FROM $reportConfig WHERE report_name = 'Project-Filters' AND question_type = 'admin-filter'" | ||
| val filterResults: List[Map[String, Any]] = postgresUtil.fetchData(filterQuery) | ||
| val objectMapper = new ObjectMapper() | ||
| val slugNameToTenantIdFilterMap = mutable.Map[String, Int]() | ||
| for (result <- filterResults) { | ||
| val configString = result.get("config").map(_.toString).getOrElse("") | ||
| val configJson = objectMapper.readTree(configString) | ||
| val slugName = configJson.findValue("name").asText() | ||
| val tenantIdFilter: Int = UpdateAndAddAdminTenantFilter.updateAndAddFilter(metabaseUtil, configJson: JsonNode, s"$tenantId", mipCollectionId, databaseId, projects, solutions) | ||
| slugNameToTenantIdFilterMap(slugName) = tenantIdFilter | ||
| } | ||
| val immutableSlugNameToTenantIdFilterMap: Map[String, Int] = slugNameToTenantIdFilterMap.toMap | ||
| val parametersQuery: String = s"SELECT config FROM $reportConfig WHERE report_name = 'Project-Parameter' AND question_type = 'admin-parameter'" | ||
| UpdateParameters.updateParameterFunction(metabaseUtil, postgresUtil, parametersQuery, immutableSlugNameToTenantIdFilterMap, mipDashboardId) | ||
|
Comment on lines
+318
to
+331
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do not persist invalid filter card IDs into admin parameters.
Proposed fix val slugName = configJson.findValue("name").asText()
val tenantIdFilter: Int = UpdateAndAddAdminTenantFilter.updateAndAddFilter(metabaseUtil, configJson: JsonNode, s"$tenantId", mipCollectionId, databaseId, projects, solutions)
+ if (tenantIdFilter == -1) {
+ throw new IllegalStateException(s"Failed to create admin filter card for '$slugName'")
+ }
slugNameToTenantIdFilterMap(slugName) = tenantIdFilterAs per coding guidelines 🤖 Prompt for AI Agents |
||
| val projectMetadataJson = new ObjectMapper().createArrayNode().add(new ObjectMapper().createObjectNode().put("collectionId", mipCollectionId).put("collectionName", mipCollectionName).put("dashboardId", mipDashboardId).put("dashboardName", mipDashboardName).put("questionIds", mainQuestionIdsString)) | ||
| postgresUtil.insertData(s"UPDATE $metaDataTable SET main_metadata = COALESCE(main_metadata::jsonb, '[]'::jsonb) || '$projectMetadataJson' ::jsonb WHERE entity_id = '$admin';") | ||
| } else { | ||
| logger.error(s"Creating tabs failed please check: $projectTabId, $userTabId, $csvTabId") | ||
| } | ||
| } | ||
| } | ||
|
|
||
| def processStateDashboard(tenantId: String, stateName: String, targetedStateId: String, reportConfig: String, metaDataTable: String, databaseId: Int, projects: String, solutions: String, metabaseUtil: MetabaseUtil, postgresUtil: PostgresUtil, reportFor: String): Int = { | ||
| val collectionName = s"$stateName State [Tenant : $tenantId]" | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| package org.shikshalokam.job.combined.dashboard.creator.functions.project | ||
|
|
||
| import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} | ||
| import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} | ||
| import org.shikshalokam.job.util.MetabaseUtil | ||
|
|
||
| object UpdateAndAddAdminTenantFilter { | ||
| val objectMapper = new ObjectMapper() | ||
|
|
||
| def updateAndAddFilter(metabaseUtil: MetabaseUtil, queryResult: JsonNode, tenantId: String, collectionId: Int, databaseId: Int, projectTable: String, solutionTable: String): Int = { | ||
| println(s"** Started to update and create filter questions for state dashboard") | ||
| val objectMapper = new ObjectMapper() | ||
|
|
||
| def replaceTenantName(json: JsonNode, tenantId: String, projectTable: String, solutionTable: String): JsonNode = { | ||
| def processNode(node: JsonNode): JsonNode = { | ||
| node match { | ||
| case obj: ObjectNode => | ||
| obj.fieldNames().forEachRemaining { fieldName => | ||
| val childNode = obj.get(fieldName) | ||
| if (childNode.isTextual) { | ||
| var updatedText = childNode.asText() | ||
|
|
||
| if (updatedText.contains("TENANTID")) { | ||
| updatedText = updatedText.replace("TENANTID", tenantId) | ||
| } | ||
| if (updatedText.contains("${config.projects}")) { | ||
| updatedText = updatedText.replace("${config.projects}", projectTable) | ||
| } | ||
| if (updatedText.contains("${config.solutions}")) { | ||
| updatedText = updatedText.replace("${config.solutions}", solutionTable) | ||
| } | ||
| obj.put(fieldName, updatedText) | ||
| } else { | ||
| obj.set(fieldName, processNode(childNode)) | ||
| } | ||
| } | ||
| obj | ||
|
|
||
| case array: ArrayNode => | ||
| val newArray = array.deepCopy().asInstanceOf[ArrayNode] | ||
| newArray.removeAll() | ||
| array.elements().forEachRemaining { child => | ||
| newArray.add(processNode(child)) | ||
| } | ||
| newArray | ||
|
|
||
| case _ => node | ||
| } | ||
| } | ||
|
|
||
| val updatedJson = processNode(json.deepCopy()) | ||
| updatedJson | ||
| } | ||
|
|
||
| def updateCollectionIdAndDatabaseId(jsonFile: JsonNode, collectionId: Int, databaseId: Int): JsonNode = { | ||
| try { | ||
| val questionCardNode = jsonFile.get("questionCard").asInstanceOf[ObjectNode] | ||
| if (questionCardNode == null) { | ||
| throw new IllegalArgumentException("'questionCard' node not found.") | ||
| } | ||
| questionCardNode.put("collection_id", collectionId) | ||
| val datasetQueryNode = questionCardNode.get("dataset_query").asInstanceOf[ObjectNode] | ||
| if (datasetQueryNode == null) { | ||
| throw new IllegalArgumentException("'dataset_query' node not found.") | ||
| } | ||
| datasetQueryNode.put("database", databaseId) | ||
| jsonFile | ||
| } catch { | ||
| case ex: Exception => | ||
| throw new RuntimeException(s"Error updating JSON: ${ex.getMessage}", ex) | ||
| } | ||
| } | ||
|
|
||
| def getTheQuestionId(json: JsonNode): Int = { | ||
| try { | ||
| val requestBody = json.get("questionCard") | ||
| val questionCardResponse = metabaseUtil.createQuestionCard(requestBody.toString) | ||
| val responseJson = objectMapper.readTree(questionCardResponse) | ||
| Option(responseJson.get("id")).map(_.asInt()).getOrElse { | ||
| println("Error: 'id' field not found in the response.") | ||
| -1 | ||
| } | ||
| } catch { | ||
| case ex: Exception => | ||
| println(s"Error fetching 'id' from response: ${ex.getMessage}") | ||
| -1 | ||
| } | ||
|
Comment on lines
+74
to
+87
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do not return
🤖 Prompt for AI Agents |
||
| } | ||
|
|
||
|
|
||
| val ReplacedStateNameJson = replaceTenantName(queryResult, tenantId, projectTable, solutionTable) | ||
| val updatedJson = updateCollectionIdAndDatabaseId(ReplacedStateNameJson, collectionId, databaseId) | ||
| val questionId = getTheQuestionId(updatedJson) | ||
| println(s"** Completed to update and create filter questions for state dashboard") | ||
| questionId | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Keep the Program Manager collection tagged as Program Manager.
This branch creates Program Manager collections, but the description now says
Collection For: Admin. The latervalidateCollection(..., "Program Manager", ...)lookup uses that description, so reruns will fail to match the existing collection.Proposed fix
As per coding guidelines
metabase-jobs/**: "Verify: - Idempotency of ETL runs".📝 Committable suggestion
🤖 Prompt for AI Agents