Deep Schema Support

tofhir-common/src/main/scala/io/tofhir/common/util/SchemaUtil.scala

@@ -16,6 +16,11 @@ object SchemaUtil {
    * @return
    */
   def convertToStructureDefinitionResource(schemaDefinition: SchemaDefinition, fhirVersion: String): Resource = {
+
+    val deepSchema = isDeepSchema(schemaDefinition)
+    val kindVal        = if (deepSchema) "resource"      else "logical"
+    val derivationVal  = if (deepSchema) "constraint"    else "specialization"
+
     var structureDefinitionResource: Resource =
       ("id" -> schemaDefinition.id) ~
         ("resourceType" -> "StructureDefinition") ~
@@ -28,12 +33,12 @@ object SchemaUtil {
     structureDefinitionResource ~
       ("status" -> "draft") ~
       ("fhirVersion" -> fhirVersion) ~
-      ("kind" -> "logical") ~
+      ("kind" -> kindVal) ~
       ("abstract" -> false) ~
       ("type" -> schemaDefinition.`type`) ~
       ("baseDefinition" -> "http://hl7.org/fhir/StructureDefinition/Element") ~
-      ("derivation" -> "specialization") ~
-      ("differential" -> ("element" -> generateElementArray(schemaDefinition.`type`, schemaDefinition.fieldDefinitions.getOrElse(Seq.empty))))
+      ("derivation"   -> derivationVal) ~
+      ("differential" -> ("element" -> generateElementArray(schemaDefinition.id, schemaDefinition.`type`, schemaDefinition.fieldDefinitions.getOrElse(Seq.empty), deepSchema)))
   }
 
   /**
@@ -44,35 +49,69 @@ object SchemaUtil {
    * @return
    * @throws IllegalArgumentException if a field definition does not have at least one data type
    */
-  private def generateElementArray(`type`: String, fieldDefinitions: Seq[SimpleStructureDefinition]): JArray = {
-    // Check whether all field definitions have at least one data type
-    val integrityCheck = fieldDefinitions.forall(fd => fd.dataTypes.isDefined && fd.dataTypes.get.nonEmpty)
+  private def generateElementArray(id: String, `type`: String, fieldDefinitions: Seq[SimpleStructureDefinition], deepSchema: Boolean): JArray = {
+
+    // Normalize path so that it starts with the schema type
+    def normalizePath(p: String): String = {
+      val raw = Option(p).map(_.trim).getOrElse("")
+      val clean =
+        if (id != null && id.nonEmpty && (raw == id || raw.startsWith(id + ".")))
+          raw.stripPrefix(id + ".").stripPrefix(id)
+        else raw
+      val t = `type`
+      if (clean.isEmpty) t
+      else if (clean == t || clean.startsWith(t + ".")) clean
+      else t + "." + clean
+    }
+
+    // Flatten nested elements into a single list suitable for StructureDefinition.differential.element
+    def flatten(defs: Seq[SimpleStructureDefinition]): Seq[SimpleStructureDefinition] = {
+      defs.flatMap { fd =>
+        val np = normalizePath(fd.path)
+        val self = fd.copy(
+          id       = np,
+          path     = np,
+          elements = None
+        )
+        self +: flatten(fd.elements.getOrElse(Seq.empty))
+      }
+    }
+
+    val flatFds = flatten(fieldDefinitions)
+
+    // Validate types after flatten
+    val integrityCheck = flatFds.forall(fd => fd.dataTypes.isDefined && fd.dataTypes.get.nonEmpty)
     if (!integrityCheck) {
       throw new IllegalArgumentException(s"Missing data type.A field definition must have at least one data type. Element rootPath: ${`type`}")
     }
 
-    val rootElement =
-      ("id" -> `type`) ~
-        ("path" -> `type`) ~
-        ("min" -> 0) ~
-        ("max" -> "*") ~
-        ("type" -> JArray(List("code" -> "Element")))
-
-    val elements = fieldDefinitions.map { fd =>
+    val rootElement: org.json4s.JObject =
+      if (deepSchema) {
+        ("id" -> `type`) ~
+          ("path" -> `type`)
+      } else {
+        ("id" -> `type`) ~
+          ("path" -> `type`) ~
+          ("min" -> 0) ~
+          ("max" -> "*") ~
+          ("type" -> JArray(List(("code" -> "Element"))))
+      }
+    // Children (flat list)
+    val elements = flatFds.map { fd =>
       val max: String = fd.maxCardinality match {
         case Some(v) => v.toString
-        case None => "*"
+        case None    => if (fd.isArray) "*" else "1"
       }
       // create element json
       var elementJson =
         ("id" -> fd.path) ~
           ("path" -> fd.path) ~
           ("min" -> fd.minCardinality) ~
           ("max" -> max) ~
-          ("type" -> fd.dataTypes.get.map { dt =>
+          ("type" -> JArray(fd.dataTypes.get.map { dt =>
             ("code" -> dt.dataType) ~
               ("profile" -> dt.profiles)
-          })
+          }.toList))
       // add the field definition if it exists
       if (fd.definition.nonEmpty) {
         elementJson = elementJson ~ ("definition" -> fd.definition)
@@ -86,4 +125,15 @@ object SchemaUtil {
 
     JArray(rootElement +: elements)
   }
+
+  /**
+   * Helper to check if a schema's field definitions have elements in them
+   * @param schema Schema to be classified
+   * @return
+   */
+  def isDeepSchema(schema: SchemaDefinition): Boolean = {
+    schema.fieldDefinitions
+      .getOrElse(Seq.empty)
+      .exists(fd => fd.elements.exists(_.nonEmpty))
+  }
 }

tofhir-engine/src/main/scala/io/tofhir/engine/mapping/schema/SchemaConverter.scala

@@ -119,7 +119,6 @@ class SchemaConverter(majorFhirVersion: String) {
    * @return SimpleStructureDefinition object that defines a Schema
    */
   def fieldsToSchema(structField: StructField, defaultName: String): SimpleStructureDefinition = {
-    val (dataType, isArray, maxCardinality) = {
       def mapDataTypeToFhir(dataType: DataType): Option[Seq[DataTypeWithProfiles]] = {
         dataType match {
           case DataTypes.ShortType => Some(Seq(DataTypeWithProfiles(dataType = FHIR_DATA_TYPES.INTEGER, profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.INTEGER}")))))
@@ -140,37 +139,119 @@ class SchemaConverter(majorFhirVersion: String) {
         }
       }
 
-      structField.dataType match {
-        case arrayType: ArrayType =>
-          (mapDataTypeToFhir(arrayType.elementType), true, None) // None represents "*" (unbounded)
-        case other =>
-          (mapDataTypeToFhir(other), false, Some(1))
-      }
-    }
+    val path =
+      if (defaultName.isEmpty) structField.name
+      else s"$defaultName.${structField.name}"
+
+    structField.dataType match {
+      case ArrayType(st: StructType, _) =>
+        // Backbone element
+        val children = st.fields.map(cf => fieldsToSchema(cf, path))
+        SimpleStructureDefinition(
+          id = structField.name,
+          path = path,
+          dataTypes = Some(Seq(DataTypeWithProfiles(dataType = "BackboneElement", profiles = None))),
+          isPrimitive = false,
+          isChoiceRoot = false,
+          isArray = true,
+          minCardinality = if (structField.nullable) 0 else 1,
+          maxCardinality = None,
+          boundToValueSet = None,
+          isValueSetBindingRequired = None,
+          referencableProfiles = None,
+          constraintDefinitions = None,
+          sliceDefinition = None,
+          sliceName = None,
+          fixedValue = None,
+          patternValue = None,
+          referringTo = None,
+          short = None,
+          definition = None,
+          comment = None,
+          elements = Some(children)
+        )
 
-    SimpleStructureDefinition(
-      id = structField.name,
-      path = defaultName + "." + structField.name,
-      dataTypes = dataType,
-      isPrimitive = true,
-      isChoiceRoot = false,
-      isArray = isArray,
-      minCardinality = if (structField.nullable) 0 else 1,
-      maxCardinality = maxCardinality,
-      boundToValueSet = None,
-      isValueSetBindingRequired = None,
-      referencableProfiles = None,
-      constraintDefinitions = None,
-      sliceDefinition = None,
-      sliceName = None,
-      fixedValue = None,
-      patternValue = None,
-      referringTo = None,
-      short = None,
-      definition = None,
-      comment = None,
-      elements = None
-    )
+      case st: StructType =>
+        // BackboneElement with children
+        val children = st.fields.map(cf => fieldsToSchema(cf, path))
+        SimpleStructureDefinition(
+          id = structField.name,
+          path = path,
+          dataTypes = Some(Seq(DataTypeWithProfiles(dataType = "BackboneElement", profiles = None))),
+          isPrimitive = false,
+          isChoiceRoot = false,
+          isArray = false,
+          minCardinality = if (structField.nullable) 0 else 1,
+          maxCardinality = Some(1),
+          boundToValueSet = None,
+          isValueSetBindingRequired = None,
+          referencableProfiles = None,
+          constraintDefinitions = None,
+          sliceDefinition = None,
+          sliceName = None,
+          fixedValue = None,
+          patternValue = None,
+          referringTo = None,
+          short = None,
+          definition = None,
+          comment = None,
+          elements = Some(children)
+        )
+
+      case ArrayType(elt, _) =>
+        // Array of primitives
+        val fhirType = mapDataTypeToFhir(elt)
+        SimpleStructureDefinition(
+          id = structField.name,
+          path = path,
+          dataTypes = fhirType,
+          isPrimitive = true,
+          isChoiceRoot = false,
+          isArray = true,
+          minCardinality = if (structField.nullable) 0 else 1,
+          maxCardinality = None, // "*"
+          boundToValueSet = None,
+          isValueSetBindingRequired = None,
+          referencableProfiles = None,
+          constraintDefinitions = None,
+          sliceDefinition = None,
+          sliceName = None,
+          fixedValue = None,
+          patternValue = None,
+          referringTo = None,
+          short = None,
+          definition = None,
+          comment = None,
+          elements = None
+        )
+
+      case other =>
+        // Single primitive
+        val fhirType = mapDataTypeToFhir(other)
+        SimpleStructureDefinition(
+          id = structField.name,
+          path = path,
+          dataTypes = fhirType,
+          isPrimitive = true,
+          isChoiceRoot = false,
+          isArray = false,
+          minCardinality = if (structField.nullable) 0 else 1,
+          maxCardinality = Some(1),
+          boundToValueSet = None,
+          isValueSetBindingRequired = None,
+          referencableProfiles = None,
+          constraintDefinitions = None,
+          sliceDefinition = None,
+          sliceName = None,
+          fixedValue = None,
+          patternValue = None,
+          referringTo = None,
+          short = None,
+          definition = None,
+          comment = None,
+          elements = None
+        )
+    }
   }
 
   /**

tofhir-server/src/main/scala/io/tofhir/server/repository/schema/SchemaFolderRepository.scala

@@ -98,11 +98,14 @@ class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectReposito
     } catch {
       case _: IllegalArgumentException => throw BadRequest("Missing data type.", s"A field definition must have at least one data type. Element rootPath: ${schemaDefinition.`type`}")
     }
-    try {
-      fhirConfigurator.validateGivenInfrastructureResources(baseFhirConfig, api.FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION, Seq(structureDefinitionResource))
-    } catch {
-      case e: Exception =>
-        throw BadRequest("Schema definition is not valid.", s"Schema definition cannot be validated: ${schemaDefinition.url}", Some(e))
+    if (!SchemaUtil.isDeepSchema(schemaDefinition))
+    {
+      try {
+        fhirConfigurator.validateGivenInfrastructureResources(baseFhirConfig, api.FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION, Seq(structureDefinitionResource))
+      } catch {
+        case e: Exception =>
+          throw BadRequest("Schema definition is not valid.", s"Schema definition cannot be validated: ${schemaDefinition.url}", Some(e))
+      }
     }
 
     // Ensure that both the ID and "URL|version" (the canonical URL) of the schema is unique/
@@ -138,11 +141,13 @@ class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectReposito
     } catch {
       case _: IllegalArgumentException => throw BadRequest("Missing data type.", s"A field definition must have at least one data type. Element rootPath: ${schemaDefinition.`type`}")
     }
-    try {
-      fhirConfigurator.validateGivenInfrastructureResources(baseFhirConfig, api.FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION, Seq(structureDefinitionResource))
-    } catch {
-      case e: Exception =>
-        throw BadRequest("Schema definition is not valid.", s"Schema definition cannot be validated: ${schemaDefinition.url}", Some(e))
+    if (!SchemaUtil.isDeepSchema(schemaDefinition)) {
+      try {
+        fhirConfigurator.validateGivenInfrastructureResources(baseFhirConfig, api.FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION, Seq(structureDefinitionResource))
+      } catch {
+        case e: Exception =>
+          throw BadRequest("Schema definition is not valid.", s"Schema definition cannot be validated: ${schemaDefinition.url}", Some(e))
+      }
     }
 
     // Update the file

tofhir-server/src/main/scala/io/tofhir/server/service/SchemaDefinitionService.scala

@@ -165,7 +165,7 @@ class SchemaDefinitionService(schemaRepository: ISchemaRepository, mappingReposi
     // Create unnamed Schema definition by infer the schema from DataFrame
     val unnamedSchema = {
       // Map SQL DataTypes to Fhir DataTypes
-      var fieldDefinitions = effectiveSchema.fields.map(structField => schemaConverter.fieldsToSchema(structField, defaultName))
+      var fieldDefinitions = effectiveSchema.fields.map(structField => schemaConverter.fieldsToSchema(structField, ""))
       // Remove INPUT_VALIDITY_ERROR fieldDefinition that is added by SourceHandler
       fieldDefinitions = fieldDefinitions.filter(fieldDefinition => fieldDefinition.id != SourceHandler.INPUT_VALIDITY_ERROR)
       SchemaDefinition(url = defaultName, version = SchemaDefinition.VERSION_LATEST, `type` = defaultName, name = defaultName, description = Option.empty, rootDefinition = Option.empty, fieldDefinitions = Some(fieldDefinitions))