Release/6.0.0

applications/composer/backend/backend/settings.py

@@ -126,7 +126,7 @@
 
 LANGUAGE_CODE = "en-us"
 
-TIME_ZONE = "CET"
+TIME_ZONE = "Europe/Paris"
 
 USE_I18N = True
 
@@ -412,10 +412,8 @@
 if DEBUG:
     INSTALLED_APPS += [
         "debug_toolbar",
-        'silk',
     ]
     MIDDLEWARE += [
-        'silk.middleware.SilkyMiddleware',
         "debug_toolbar.middleware.DebugToolbarMiddleware",
     ]
     DEBUG_TOOLBAR_CONFIG = {

applications/composer/backend/backend/urls.py

@@ -55,5 +55,4 @@
     urlpatterns += [
         # Debug toolbar
         path("__debug__/", include("debug_toolbar.urls")),
-        path('silk/', include('silk.urls', namespace='silk'))
     ]

applications/composer/backend/composer/admin.py

@@ -21,12 +21,15 @@
 from composer.models import (
     AlertType,
     ConnectivityStatementTriple,
+    ConnectivityStatementText,
+    ConnectivityStatementAnatomicalEntity,
     Phenotype,
     Relationship,
     Sex,
     PopulationSet,
     ConnectivityStatement,
     Provenance,
+    ExpertConsultant,
     ExportBatch,
     Note,
     Profile,
@@ -45,7 +48,7 @@
     Region,
     AnatomicalEntityIntersection,
     AnatomicalEntity,
-    CSState
+    CSState,
 )
 
 
@@ -64,6 +67,11 @@ class ProvenanceInline(admin.StackedInline):
     extra = 1
 
 
+class ExpertConsultantInline(admin.StackedInline):
+    model = ExpertConsultant
+    extra = 1
+
+
 class SynonymInline(admin.StackedInline):
     model = Synonym
     extra = 1
@@ -104,9 +112,50 @@ class StatementAlertInline(admin.StackedInline):
 
 
 class RelationshipAdmin(admin.ModelAdmin):
-    list_display = ("title", "predicate_name", "predicate_uri", "type", "order")
+    list_display = ("title", "predicate_name", "predicate_uri", "type", "order", "has_custom_code")
     ordering = ("order",)
     search_fields = ("title", "predicate_name", "predicate_uri")
+    fieldsets = (
+        (None, {
+            'fields': ('title', 'predicate_name', 'predicate_uri', 'type', 'order')
+        }),
+        ('Custom Ingestion Code', {
+            'classes': ('collapse',),
+            'fields': ('custom_ingestion_code',),
+            'description': (
+                'Add custom Python code to extract data from NeuroDM during ingestion. '
+            ),
+        }),
+    )
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if 'custom_ingestion_code' in form.base_fields:
+            form.base_fields['custom_ingestion_code'].widget = forms.Textarea(attrs={
+                'rows': 15,
+                'cols': 100,
+                'style': 'font-family: monospace; font-size: 12px;'
+            })
+            form.base_fields['custom_ingestion_code'].help_text = mark_safe(
+                "Optional Python code to extract data from NeuroDM for this relationship during ingestion.<br>"
+                "The code has access to:<br>"
+                "• <code>fc</code>: dict with neuron properties (id, label, species, phenotype, etc.)<br>"
+                "• <code>fc[\"_neuron\"]</code>: the NeuroDM neuron object<br><br>"
+                "The code must define a <code>result</code> variable with the output:<br>"
+                "• For TRIPLE relationships: list of dicts [{'name': str, 'uri': str}, ...]<br>"
+                "• For TEXT relationships: list of strings or single string<br>"
+                "• For ANATOMICAL_ENTITY relationships:<br>"
+                "&nbsp;&nbsp;- Simple entities: list of URI strings ['http://purl.obolibrary.org/obo/UBERON_0001234', ...]<br>"
+                "&nbsp;&nbsp;- Region-layer pairs: list of dicts [{'region': 'region_uri', 'layer': 'layer_uri'}, ...]<br>"
+                "&nbsp;&nbsp;- Mixed: list combining both formats<br>"
+                "&nbsp;&nbsp;- Note: Region-layer pairs respect the 'update_anatomical_entities' flag<br><br>"
+                "Errors are logged to the ingestion anomalies file and the relationship will be skipped."
+            )
+        return form
+
+    @admin.display(description="Has Custom Code", boolean=True)
+    def has_custom_code(self, obj):
+        return bool(obj.custom_ingestion_code and obj.custom_ingestion_code.strip())
 
 class TripleAdmin(admin.ModelAdmin):
     list_display = ("name", "uri", "relationship")
@@ -117,7 +166,9 @@ class TripleAdmin(admin.ModelAdmin):
     def get_form(self, request, obj=None, **kwargs):
         form = super().get_form(request, obj, **kwargs)
         if "relationship" in form.base_fields:
-            form.base_fields["relationship"].queryset = Relationship.objects.exclude(type=RelationshipType.TEXT)
+            form.base_fields["relationship"].queryset = Relationship.objects.exclude(
+                type__in=[RelationshipType.TEXT, RelationshipType.ANATOMICAL_MULTI]
+            )
         return form
 
 class ConnectivityStatementInline(nested_admin.NestedStackedInline):
@@ -267,8 +318,52 @@ class DestinationInline(admin.TabularInline):
 class ConnectivityStatementTripleInline(admin.TabularInline):
     model = ConnectivityStatementTriple
     extra = 1
-    autocomplete_fields = ("relationship", "triple")
-    fields = ("relationship", "triple", "free_text")
+    autocomplete_fields = ("relationship",)
+    raw_id_fields = ("triples",)
+    fields = ("relationship", "triples")
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if "relationship" in form.base_fields:
+            # Only show triple relationship types
+            form.base_fields["relationship"].queryset = Relationship.objects.filter(
+                type__in=[RelationshipType.TRIPLE_SINGLE, RelationshipType.TRIPLE_MULTI]
+            )
+        return form
+
+
+class ConnectivityStatementTextInline(admin.TabularInline):
+    model = ConnectivityStatementText
+    extra = 1
+    autocomplete_fields = ("relationship",)
+    fields = ("relationship", "text")
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if "relationship" in form.base_fields:
+            # Only show text relationship types
+            form.base_fields["relationship"].queryset = Relationship.objects.filter(
+                type=RelationshipType.TEXT
+            )
+        return form
+
+
+class ConnectivityStatementAnatomicalEntityInline(admin.TabularInline):
+    model = ConnectivityStatementAnatomicalEntity
+    extra = 1
+    autocomplete_fields = ("relationship",)
+    raw_id_fields = ("anatomical_entities",)
+    fields = ("relationship", "anatomical_entities")
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if "relationship" in form.base_fields:
+            # Only show anatomical entity relationship types
+            form.base_fields["relationship"].queryset = Relationship.objects.filter(
+                type__in=[RelationshipType.ANATOMICAL_MULTI]
+            )
+        return form
+
 
 class ConnectivityStatementAdmin(
     SortableAdminBase, FSMTransitionMixin, admin.ModelAdmin
@@ -279,25 +374,24 @@ class ConnectivityStatementAdmin(
     readonly_fields = (
         "state",
         "curie_id",
-        "has_statement_been_exported",
         "reference_uri",
+        "population_index"
     )
-    exclude = ("journey_path", "statement_prefix", "statement_suffix", "population_index")
+    exclude = ("journey_path", "statement_prefix", "statement_suffix", )
     autocomplete_fields = ("sentence", "origins")
     date_hierarchy = "modified_date"
     list_display = (
         "sentence",
-        "pmid",
-        "pmcid",
         "short_ks",
+        "population_set_name",
+        "population_index",
         "tag_list",
         "state",
-        "has_notes",
         "owner",
     )
-    list_display_links = ("sentence", "pmid", "pmcid", "short_ks", "state")
-    list_filter = ("state", "owner", "tags__tag")
-    list_select_related = ("sentence", "origins", "destinations")
+    list_display_links = ("sentence", "short_ks", "state")
+    list_filter = ("state", "population", "owner", "tags__tag")
+    list_select_related = ("sentence", "population", "owner", "origins", "destinations")
     search_fields = (
         "sentence__title",
         "sentence__text",
@@ -309,8 +403,9 @@ class ConnectivityStatementAdmin(
 
     fieldsets = ()
 
-    inlines = (ProvenanceInline, NoteConnectivityStatementInline,
-               ViaInline, DestinationInline, StatementAlertInline, ConnectivityStatementTripleInline)
+    inlines = (ProvenanceInline, ExpertConsultantInline, NoteConnectivityStatementInline,
+               ViaInline, DestinationInline, StatementAlertInline, ConnectivityStatementTripleInline,
+               ConnectivityStatementTextInline, ConnectivityStatementAnatomicalEntityInline)
 
     def _filter_admin_transitions(self, transitions_generator):
         """
@@ -337,17 +432,9 @@ def delete_queryset(self, request, queryset):
     def short_ks(self, obj):
         return str(obj)
 
-    @admin.display(description="PMID")
-    def pmid(self, obj):
-        return obj.sentence.pmid
-
-    @admin.display(description="PMCID")
-    def pmcid(self, obj):
-        return obj.sentence.pmcid
-
-    @admin.display(description="REFERENCE")
-    def reference(self, obj):
-        return str(obj)
+    @admin.display(description="Population Set")
+    def population_set_name(self, obj):
+        return obj.population.name if obj.population else "-"
 
 
 class ExportBatchAdmin(admin.ModelAdmin):
@@ -391,6 +478,45 @@ class IngestSentenceForm(forms.Form):
     file = forms.FileField(label="CSV file")
 
 
+class IngestStatementsForm(forms.Form):
+    """Form for configuring connectivity statement ingestion parameters"""
+    update_upstream = forms.BooleanField(
+        required=False,
+        initial=False,
+        label="Update upstream statements",
+        help_text="Set this flag to update upstream statements."
+    )
+    update_anatomical_entities = forms.BooleanField(
+        required=False,
+        initial=False,
+        label="Update anatomical entities",
+        help_text="Set this flag to try move anatomical entities to specific layer, region."
+    )
+    disable_overwrite = forms.BooleanField(
+        required=False,
+        initial=False,
+        label="Disable overwrite",
+        help_text="Set this flag to prevent overwriting existing statements."
+    )
+    full_imports = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={'rows': 3, 'placeholder': 'Enter URIs separated by commas or new lines'}),
+        label="Full imports",
+        help_text="List of full imports to include in the ingestion (comma or newline separated)."
+    )
+    label_imports = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={'rows': 3, 'placeholder': 'Enter labels separated by commas or new lines'}),
+        label="Label imports",
+        help_text="List of label imports to include in the ingestion (comma or newline separated)."
+    )
+    population_file = forms.FileField(
+        required=False,
+        label="Population file",
+        help_text="Text file containing population URIs (one per line). Only statements matching these URIs will be processed."
+    )
+
+
 # Custom view for ingesting sentences from a CSV file
 def ingest_sentences_view(request):
     output = None
@@ -423,11 +549,51 @@ def ingest_sentences_view(request):
     return render(request, "admin/ingest_sentences.html", context)
 
 
+# Custom view for downloading ingestion log files
+def download_logs_view(request):
+    """
+    Admin page with links to download ingestion log files.
+    """
+    context = admin.site.each_context(request)
+    context.update({
+        "title": "Download Ingestion Logs",
+        "anomalies_url": reverse('composer-api:ingestion-logs') + '?log_type=anomalies',
+        "ingested_url": reverse('composer-api:ingestion-logs') + '?log_type=ingested',
+    })
+    return render(request, "admin/download_logs.html", context)
+
+
+# Custom view for ingesting connectivity statements
+def ingest_statements_view(request):
+    """
+    Admin page for configuring and triggering connectivity statement ingestion.
+    """
+    context = admin.site.each_context(request)
+    if request.method == "POST":
+        form = IngestStatementsForm(request.POST, request.FILES)
+        if form.is_valid():
+            context.update({
+                "form": form,
+                "title": "Ingest Connectivity Statements",
+            })
+            return render(request, "admin/ingest_statements.html", context)
+    else:
+        form = IngestStatementsForm()
+
+    context.update({
+        "form": form,
+        "title": "Ingest Connectivity Statements",
+    })
+    return render(request, "admin/ingest_statements.html", context)
+
+
 def custom_admin_urls(original_get_urls):
     def get_urls():
         urls = original_get_urls()
         custom_urls = [
             path('ingest-sentences/', admin.site.admin_view(ingest_sentences_view), name='ingest-sentences'),
+            path('ingest-statements/', admin.site.admin_view(ingest_statements_view), name='ingest-statements'),
+            path('download-logs/', admin.site.admin_view(download_logs_view), name='download-logs'),
         ]
         return custom_urls + urls
     return get_urls