Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions CHANGE-LOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,53 @@
# OpenStudyBuilder (OSB) Commits changelog

## V 2.3

New Features and Enhancements
============

### Fixes and Enhancements

- Minor improvements to activity instance wizard stepper.
- It is now possible to exchange activities in the Study Activities list page, beside on the Detailed Schedule of Activities page.
- In the Schedule of Activities, when a user opens the action menu of a row displaying an activity, that row will also be highlighted.
- In the Schedule of Activities, rows displaying activity placeholder requests will be highlighted in the same way they already are in the Study Activities list page (using orange and yellow colors).
- Activity Placeholders created under detailed SoA is now also visible under Data Specifications, Study activity instances. The library will refer to 'Requested' and while the request is under evaluation it is not possible to select a related activity instance.
- The 'Study Structure' menu has been updated and it is now possible to edit 'Epochs' in the 'Study Visits table' when the table is in 'edit mode'. Furthermore, the 'Study' menu under Manage Study' menu has been updated. The possibility to 'add exiting study as study subpart' has been removed from the front end and it is now possible to create a new study as a subpart.
- Minor updates to API and data model for study data suppliers.

### New Feature

- Define Study, Data Specifications, Study Activity Instances now support defining baseline flags by visits for activity instances in operational SoA.


### End-to-End Automated test enhancements

- Various code improvements to ensure easier maintenance and overall tests stability.
- Studies > Manage Study > Study Data Supplier: Added study data supplier automation test implementation.
- Studies > Define Study > Data Specifications > Study Activity Instances: Defined Gherkins and implemented tests for Baseline Flags.


Solved Bugs
============

### Library

**Code Lists -> CT Catalogues -> Codelist**

- Catalogue Name is cleared when trying to save before filling all mandatory data

**Code Lists > CT Catalogues > Terms**

- Download of SDTM domain abbreviation codelist does not contain the abbreviation


### Studies

**Manage Study > Study Core Attributes**

- The API is very slow when deleting study activities


## V 2.2

New Features and Enhancements
Expand Down
3 changes: 2 additions & 1 deletion clinical-mdr-api/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,5 @@ linting_report.txt
/reports
/consumer_api/reports
traceability.html
Consumer_API_Traceability.html
Consumer_API_Traceability.html
.github
2 changes: 2 additions & 0 deletions clinical-mdr-api/Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ openapi = "python generate_openapi_json.py"
schemathesis = """
schemathesis
run
--experimental=openapi-3.1
--checks=all
--base-url=http://localhost:8000
--request-timeout=200000
Expand All @@ -113,6 +114,7 @@ consumer-openapi = "python generate_openapi.py consumer_api.consumer_api:app con
consumer-api-schemathesis = """
schemathesis
run
--experimental=openapi-3.1
--checks=all
--base-url=http://localhost:8008
--request-timeout=30000
Expand Down
2 changes: 1 addition & 1 deletion clinical-mdr-api/apiVersion
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.0.517
3.0.529
Original file line number Diff line number Diff line change
Expand Up @@ -198,56 +198,80 @@ def _create_new_value_node(self, ar: ActivityItemClassAR) -> ActivityItemClassVa
return new_value

def get_all_for_activity_instance_class(
self, activity_instance_class_uid: str, dataset_uid: str | None = None
self,
activity_instance_class_uid: str,
ig_uid: str | None = None,
dataset_uid: str | None = None,
) -> set[ActivityItemClassRoot]:
"""
Return all Activity Item Class nodes linked to given Activity Instance Class
and its parents.
"""
filter_kwargs: dict[str, str] = {
"has_activity_instance_class__uid": activity_instance_class_uid,
}
if dataset_uid:
filter_kwargs[
"maps_variable_class__has_instance__implemented_by_variable__has_dataset_variable__is_instance_of__uid"
] = dataset_uid

nodes = (
ActivityItemClassRoot.nodes.traverse(
Path("has_latest_value", include_rels_in_return=False)
)
.filter(**filter_kwargs)
.resolve_subgraph()
)

# Then fetch parent UIDs
query = """MATCH (n:ActivityInstanceClassRoot)
WHERE n.uid=$uid
OPTIONAL MATCH (n)-[:PARENT_CLASS]->{1,3}(m:ActivityInstanceClassRoot)
RETURN collect(DISTINCT m.uid)
# Building a Cypher query for performance optimization
# Also, using some or on node labels and relationship types
base_match = """
MATCH (aicv:ActivityItemClassValue)<-[:LATEST]-(aicr:ActivityItemClassRoot)
<-[has_activity_instance_class:HAS_ITEM_CLASS]-(:ActivityInstanceClassRoot{uid:$activity_instance_class_uid})
"""

base_parent_match = """
MATCH (aic:ActivityInstanceClassRoot {uid:$activity_instance_class_uid})-[:PARENT_CLASS]->{1,3}(p_aic:ActivityInstanceClassRoot)
-[has_activity_instance_class:HAS_ITEM_CLASS]->(aicr:ActivityItemClassRoot)-[:LATEST]->(aicv:ActivityItemClassValue)
"""
results, _ = db.cypher_query(query, {"uid": activity_instance_class_uid})
parent_uids: list[str] = []
if results and results[0][0]:
parent_uids += results[0][0]

# Finally, get activity item classes linked to parents
parent_filter_kwargs: dict[str, str | list[str]] = {
"has_activity_instance_class__uid__in": parent_uids,
}
if dataset_uid:
parent_filter_kwargs[
"maps_variable_class__has_instance__implemented_by_variable__has_dataset_variable__is_instance_of__uid"
] = dataset_uid
parent_nodes = (
ActivityItemClassRoot.nodes.traverse(
Path("has_latest_value", include_rels_in_return=False)

match_for_filter = """
MATCH (aicr)-[:MAPS_VARIABLE_CLASS]->(:VariableClass)-[:HAS_INSTANCE]->(:VariableClassInstance)
<-[:IMPLEMENTS_VARIABLE|IMPLEMENTS_VARIABLE_CLASS]-(:DatasetVariableInstance|SponsorModelDatasetVariableInstance)
<-[:HAS_DATASET_VARIABLE]-(di:DatasetInstance|SponsorModelDatasetInstance)
"""

dataset_uid_filter = (
"EXISTS((di)<-[:HAS_INSTANCE]-(:Dataset {uid: $dataset_uid}))"
)
ig_uid_filter = """
(
EXISTS((di)<-[:HAS_DATASET]-(:DataModelIGValue)<-[:HAS_VERSION]-(:DataModelIGRoot {uid: $ig_uid}))
OR
EXISTS((di)<-[:HAS_DATASET]-(:SponsorModelValue)-[:EXTENDS_VERSION]->(:DataModelIGValue)<-[:HAS_VERSION]-(:DataModelIGRoot {uid: $ig_uid}))
)
.exclude(uid__in=[node.uid for node in nodes])
.filter(**parent_filter_kwargs)
.resolve_subgraph()
"""

return_clause = "RETURN DISTINCT aicr, aicv, has_activity_instance_class"

query_elements = [base_match]
filter_clause = ""
if dataset_uid or ig_uid:
query_elements.append(match_for_filter)

filter_elements = []
if dataset_uid:
filter_elements.append(dataset_uid_filter)
if ig_uid:
filter_elements.append(ig_uid_filter)
filter_clause = "WHERE " + " AND ".join(filter_elements)
query_elements.append(filter_clause)

query_elements.append(return_clause)
query_elements.append("UNION")
query_elements.append(base_parent_match)
if filter_clause:
query_elements.append(match_for_filter)
query_elements.append(filter_clause)
query_elements.append(return_clause)

query = " ".join(query_elements)

results, meta = db.cypher_query(
query,
params={
"activity_instance_class_uid": activity_instance_class_uid,
"dataset_uid": dataset_uid,
"ig_uid": ig_uid,
},
)
return set(nodes).union(set(parent_nodes))

return [dict(zip(meta, row)) for row in results]

def _has_data_changed(
self, ar: ActivityItemClassAR, value: ActivityItemClassValue
Expand Down Expand Up @@ -445,13 +469,23 @@ def _maintain_parameters(
pass

def get_referenced_codelist_and_term_uids(
self, activity_item_class_uid: str, dataset_uid: str, use_sponsor_model: bool
self,
activity_item_class_uid: str,
dataset_uid: str,
use_sponsor_model: bool,
ct_catalogue_name: str | None = None,
) -> dict[str, list[str] | None]:

if ct_catalogue_name:
extra_filter_kwargs = {
"maps_variable_class__has_instance__implemented_by_variable__references_codelist__has_codelist__name": ct_catalogue_name,
}
else:
extra_filter_kwargs = {}
uids_for_standard_model = (
ActivityItemClassRoot.nodes.filter(
uid=activity_item_class_uid,
maps_variable_class__has_instance__implemented_by_variable__has_dataset_variable__is_instance_of__uid=dataset_uid,
**extra_filter_kwargs,
)
.traverse(
"maps_variable_class__has_instance__implemented_by_variable__references_codelist",
Expand Down Expand Up @@ -485,7 +519,6 @@ def get_referenced_codelist_and_term_uids(
)
.all()
)

codelist_term_sets: dict[str, set[str] | None] = {}
for cl_uid, term_uid in uids_for_standard_model:
if cl_uid not in codelist_term_sets:
Expand All @@ -499,10 +532,17 @@ def get_referenced_codelist_and_term_uids(
codelist_term_sets[cl_uid].add(term_uid)

if use_sponsor_model:
if ct_catalogue_name:
extra_sponsor_filter_kwargs = {
"maps_variable_class__has_instance__implemented_by_sponsor_variable__references_codelist__has_codelist__name": ct_catalogue_name,
}
else:
extra_sponsor_filter_kwargs = {}
uids_for_sponsor_model = (
ActivityItemClassRoot.nodes.filter(
uid=activity_item_class_uid,
maps_variable_class__has_instance__implemented_by_sponsor_variable__has_variable__is_instance_of__uid=dataset_uid,
**extra_sponsor_filter_kwargs,
)
.traverse(
"maps_variable_class__has_instance__implemented_by_sponsor_variable__references_codelist",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -894,7 +894,8 @@ def specific_alias_clause(self, **kwargs) -> str:
-[:HAS_NAME_ROOT]->(term_name_root:CTTermNameRoot)
-[:LATEST]->(term_name_value:CTTermNameValue)
MATCH (ct_term_context)-[:HAS_SELECTED_CODELIST]->(codelist_root:CTCodelistRoot)
RETURN {uid: term_root.uid, name: term_name_value.name, codelist_uid: codelist_root.uid}
MATCH (ct_codelist_term:CTCodelistTerm)-[:HAS_TERM_ROOT]->(term_root)
RETURN {uid: term_root.uid, name: term_name_value.name, codelist_uid: codelist_root.uid, submission_value: ct_codelist_term.submission_value}
},
unit_definitions: [(activity_item)-[:HAS_UNIT_DEFINITION]->(unit_definition_root:UnitDefinitionRoot)-[:LATEST]->(unit_definition_value:UnitDefinitionValue)-[:HAS_CT_DIMENSION]-(:CTTermRoot)-[:HAS_NAME_ROOT]->(CTTermNamesRoot)-[:LATEST]->(dimension_value:CTTermNameValue) | {uid: unit_definition_root.uid, name: unit_definition_value.name, dimension_name: dimension_value.name}],
is_adam_param_specific: activity_item.is_adam_param_specific,
Expand Down Expand Up @@ -1129,7 +1130,8 @@ def get_activity_instance_overview(
-[:HAS_NAME_ROOT]->(term_name_root:CTTermNameRoot)
-[:LATEST]->(term_name_value:CTTermNameValue)
MATCH (ct_term_context)-[:HAS_SELECTED_CODELIST]->(codelist_root:CTCodelistRoot)
RETURN {uid: term_root.uid, name: term_name_value.name, codelist_uid: codelist_root.uid}
MATCH (ct_codelist_term:CTCodelistTerm)-[:HAS_TERM_ROOT]->(term_root)
RETURN {uid: term_root.uid, name: term_name_value.name, codelist_uid: codelist_root.uid, submission_value: ct_codelist_term.submission_value}
},
unit_definitions: [
(activity_item)-[:HAS_UNIT_DEFINITION]->(unit_definition_root:UnitDefinitionRoot)-[:LATEST]->(unit_definition_value:UnitDefinitionValue)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1344,15 +1344,34 @@ def get_specific_activity_version_groupings(
RETURN last(hvs) AS g_ver
}
CALL {
WITH agrp
MATCH (agrp)<-[:HAS_ACTIVITY]-(aiv:ActivityInstanceValue)<-[hv:HAS_VERSION]-(air:ActivityInstanceRoot)
WHERE NOT EXISTS((aiv)<--(:DeletedActivityInstanceRoot))
WITH aiv, hv, air
ORDER BY hv.start_date
WITH DISTINCT air, collect({aiv: aiv, version: hv.version}) AS aiv_versions
WITH air, last(aiv_versions) AS last_aiv_version
WITH DISTINCT last_aiv_version.aiv AS aiv, air, last_aiv_version.version AS instance_version
WITH {instance_name: aiv.name, instance_uid: air.uid, instance_version: instance_version} AS instance
WITH agrp, av, av_rel, ar
// Calculate the activity version's validity period end date
// This ensures we show the latest instance version active during this activity version's timeframe
OPTIONAL MATCH (ar)-[next_rel:HAS_VERSION]->(:ActivityValue)
WHERE toInteger(split(next_rel.version, '.')[0]) > toInteger(split(av_rel.version, '.')[0])
OR (toInteger(split(next_rel.version, '.')[0]) = toInteger(split(av_rel.version, '.')[0])
AND toInteger(split(next_rel.version, '.')[1]) > toInteger(split(av_rel.version, '.')[1]))
WITH agrp, av_rel, min(next_rel.start_date) as min_next_start_date
WITH agrp, COALESCE(av_rel.end_date, min_next_start_date, datetime()) as version_end_date

// Find unique instance roots that have any version linked to this grouping
MATCH (agrp)<-[:HAS_ACTIVITY]-(:ActivityInstanceValue)<-[:HAS_VERSION]-(air:ActivityInstanceRoot)
WHERE NOT EXISTS((air)<-[:DELETED_CONCEPT]-(:DeletedActivityInstanceRoot))
WITH DISTINCT air, version_end_date

// For each root, find the latest version that was active during the validity period
MATCH (air)-[hv:HAS_VERSION]->(aiv:ActivityInstanceValue)
WHERE hv.start_date <= version_end_date
AND NOT EXISTS((aiv)<--(:DeletedActivityInstanceRoot))
WITH air, aiv, hv
ORDER BY air.uid, hv.start_date DESC,
toInteger(split(hv.version, '.')[0]) DESC,
toInteger(split(hv.version, '.')[1]) DESC

// Group by root and take the first (latest) version
WITH air, collect({aiv: aiv, version: hv.version})[0] AS latest_version
WHERE latest_version IS NOT NULL
WITH {instance_name: latest_version.aiv.name, instance_uid: air.uid, instance_version: latest_version.version} AS instance
RETURN collect(instance) AS activity_instances
}
RETURN
Expand Down Expand Up @@ -1528,7 +1547,9 @@ def get_activity_instances_for_version(

// 1b. Find the minimum start date of subsequent versions (if any)
OPTIONAL MATCH (activity_root)-[next_rel:HAS_VERSION]->(:ActivityValue)
WHERE toFloat(next_rel.version) > toFloat($version) // Use parameter
WHERE toInteger(split(next_rel.version, '.')[0]) > toInteger(split($version, '.')[0])
OR (toInteger(split(next_rel.version, '.')[0]) = toInteger(split($version, '.')[0])
AND toInteger(split(next_rel.version, '.')[1]) > toInteger(split($version, '.')[1]))
WITH activity_value, av_rel, min(next_rel.start_date) as min_next_start_date // Grouping implicitly by activity_value, av_rel

// 1c. Calculate the final version_end_date
Expand Down Expand Up @@ -1607,7 +1628,9 @@ def get_activity_instances_for_version(
WITH ai_root, aihv, ai_val, version_end_date
WHERE aihv.start_date <= version_end_date
WITH ai_root, aihv, ai_val, version_end_date // Pass rows for ordering
ORDER BY ai_root.uid, aihv.start_date DESC, toFloat(aihv.version) DESC
ORDER BY ai_root.uid, aihv.start_date DESC,
toInteger(split(aihv.version, '.')[0]) DESC,
toInteger(split(aihv.version, '.')[1]) DESC

// 6. Collect the ordered versions per root
WITH ai_root, version_end_date, collect({{rel: aihv, val: ai_val}}) as relevant_versions_sorted
Expand All @@ -1628,15 +1651,20 @@ def get_activity_instances_for_version(
OPTIONAL MATCH (ai_root)-[child_aihv:HAS_VERSION]->(child_ai_val:ActivityInstanceValue)
WHERE child_aihv <> display_instance_map.rel AND
(child_aihv.start_date < display_instance_map.rel.start_date
OR (child_aihv.start_date = display_instance_map.rel.start_date AND toFloat(child_aihv.version) < toFloat(display_instance_map.rel.version)))
OR (child_aihv.start_date = display_instance_map.rel.start_date
AND (toInteger(split(child_aihv.version, '.')[0]) < toInteger(split(display_instance_map.rel.version, '.')[0])
OR (toInteger(split(child_aihv.version, '.')[0]) = toInteger(split(display_instance_map.rel.version, '.')[0])
AND toInteger(split(child_aihv.version, '.')[1]) < toInteger(split(display_instance_map.rel.version, '.')[1])))))
// *** NOTE: Add appropriate deletion check here if needed ***

// 11. Get ActivityInstanceClass for children
OPTIONAL MATCH (child_ai_val)-[:ACTIVITY_INSTANCE_CLASS]->(:ActivityInstanceClassRoot)-[:LATEST]->(child_aic_value:ActivityInstanceClassValue)

// 12. Order children (newest first) and collect
WITH ai_root, display_instance_map, library, aic_value, child_aihv, child_ai_val, child_aic_value
ORDER BY child_aihv.start_date DESC, toFloat(child_aihv.version) DESC
ORDER BY child_aihv.start_date DESC,
toInteger(split(child_aihv.version, '.')[0]) DESC,
toInteger(split(child_aihv.version, '.')[1]) DESC
WITH ai_root, display_instance_map, library, aic_value, collect(
CASE WHEN child_aihv IS NULL THEN null ELSE {{
uid: ai_root.uid, version: child_aihv.version, status: child_aihv.status, name: child_ai_val.name,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ def _create(self, item: CTCodelistAttributesAR) -> CTCodelistAttributesAR:
ct_catalogue_node = CTCatalogue.nodes.get_or_none(name=catalogue_name)
if ct_catalogue_node is None:
raise BusinessLogicException(
f"Catalogue with name {catalogue_name} does not exist."
msg=f"Catalogue with name {catalogue_name} does not exist."
)
ct_codelist_root_node.has_codelist.connect(ct_catalogue_node)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ def create_term_codelist_vos_from_cypher_result(term_dict: dict[str, Any]) -> CT
codelist_uid=cl["codelist_uid"],
submission_value=cl["submission_value"],
order=cl["order"],
library_name=cl["library"],
library_name=cl["library_name"],
codelist_name=cl["codelist_name"],
codelist_submission_value=cl["codelist_submission_value"],
codelist_concept_id=cl["codelist_concept_id"],
Expand Down
Loading