Fileglancer support

cell/scrit_f.yaml

@@ -0,0 +1,20 @@
+# data_path: /nrs/cellmap/data/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr/recon-1/em/fibsem-uint8/
+# data_path: /nrs/mengwang/data/jrc_celegans-dlon-2/jrc_celegans-dlon-2.zarr/recon-1/em/fibsem-uint8
+data_path: /nrs/cellmap/data/jrc_mus-salivary-1/jrc_mus-salivary-1.zarr/recon-1/em/fibsem-uint8
+queue: gpu_a100
+charge_group: cellmap
+json_data:
+  input_norm:
+    MinMaxNormalizer:
+      min_value: 0
+      max_value: 250
+      invert: false
+    LambdaNormalizer:
+      expression: x*2-1
+  postprocess: {}
+models:
+  fish:
+    type: script
+    name: cell_16nm
+    script_path: /groups/cellmap/cellmap/zouinkhim/cellmap-flow/cell/get_model.py
+

cellmap_flow/cli/viewer_cli.py

@@ -1,7 +1,7 @@
 """
 Simple CLI for viewing datasets with CellMap Flow without requiring model configs.
 """
-
+import os
 import click
 import logging
 import neuroglancer
@@ -21,6 +21,9 @@
     type=str,
     help="Path to the dataset (zarr or n5)",
 )
+@click.option(
+    "-P", "--project", default=None, help="Project/chargeback group for billing"
+)
 @click.option(
     "--log-level",
     type=click.Choice(
@@ -29,7 +32,7 @@
     default="INFO",
     help="Set the logging level",
 )
-def main(dataset, log_level):
+def main(dataset, project, log_level):
     """
     Start CellMap Flow viewer with a dataset.
 
@@ -39,6 +42,7 @@ def main(dataset, log_level):
     logging.basicConfig(level=getattr(logging, log_level.upper()))
 
     logger.info(f"Starting CellMap Flow viewer with dataset: {dataset}")
+    logger.info(f"Project: {project}")
 
     # Set up neuroglancer server
     neuroglancer.set_server_bind_address("0.0.0.0")
@@ -49,7 +53,10 @@ def main(dataset, log_level):
     # Set dataset path in globals
     g.dataset_path = dataset
     g.viewer = viewer
-
+    if "LSB_PROJECT_NAME" in os.environ:
+        g.charge_group = os.environ["LSB_PROJECT_NAME"]
+    if project:
+        g.charge_group = project
     # Add dataset layer to viewer
     with viewer.txn() as s:
         # Set coordinate space

cellmap_flow/dashboard/app.py

@@ -44,8 +44,21 @@ def create_and_run_app(neuroglancer_url=None, inference_servers=None):
     state.INFERENCE_SERVER = inference_servers
     hostname = socket.gethostname()
     port = 0
-    logger.warning(f"Host name: {hostname}")
-    app.run(host="0.0.0.0", port=port, debug=False, use_reloader=False)
+
+    from werkzeug.serving import make_server
+    server = make_server("0.0.0.0", port, app)
+    actual_port = server.socket.getsockname()[1]
+    url = f"http://{hostname}:{actual_port}"
+    logger.warning(f"Dashboard running at: {url}")
+    print(f"\n * Dashboard URL: {url}\n")
+    try:
+        service_url_path = os.environ.get("SERVICE_URL_PATH")
+        if service_url_path:
+            with open(service_url_path, "w") as f:
+                f.write(url)
+    except Exception as e:
+        logger.warning(f"Failed to write service URL to {service_url_path}: {e}")
+    server.serve_forever()
 
 
 if __name__ == "__main__":

cellmap_flow/models/models_config.py

@@ -740,6 +740,7 @@ def _load_metadata(self):
             self._metadata = {}
         return self._metadata
 
+
     @property
     def command(self) -> str:
         cmd = f"huggingface --repo {self.repo}"

cellmap_flow/server.py

@@ -327,6 +327,7 @@ def run(self, debug=False, port=None, certfile=None, keyfile=None):
 
         address = f"{'https' if ssl_context else 'http'}://{get_public_ip()}:{port}"
         output = f"{IP_PATTERN[0]}{address}{IP_PATTERN[1]}"
+
         logger.error(output)
         print(output, flush=True)
 

cellmap_flow/utils/bsub_utils.py

@@ -26,7 +26,7 @@
 DEFAULT_SECURITY = "http"
 DEFAULT_QUEUE = "gpu_h100"
 DEFAULT_CHARGE_GROUP = "cellmap"
-SERVER_COMMAND = "cellmap_flow_server"
+SERVER_COMMAND = "pixi run cellmap_flow_server"
 
 
 class JobStatus(Enum):

cellmap_flow/utils/ds.py

@@ -190,6 +190,51 @@ def __getattr__(self, attr):
         return at
 
 
+def _clean_zarr_compressor(dataset_path: str):
+    """Read .zarray metadata and strip unknown compressor fields for tensorstore compatibility.
+
+    Tensorstore is strict about compressor fields and rejects unknown ones
+    (e.g. 'checksum' added by newer numcodecs). Returns cleaned metadata dict
+    or None if not applicable.
+    """
+    zarray_path = os.path.join(dataset_path, ".zarray")
+    if not os.path.isfile(zarray_path):
+        return None
+    try:
+        with open(zarray_path) as f:
+            meta = json.load(f)
+    except (json.JSONDecodeError, OSError):
+        return None
+
+    compressor = meta.get("compressor")
+    if not compressor or not isinstance(compressor, dict):
+        return None
+
+    # Known fields per compressor type that tensorstore accepts
+    known_fields = {
+        "zstd": {"id", "level"},
+        "zlib": {"id", "level"},
+        "gzip": {"id", "level"},
+        "bz2": {"id", "level"},
+        "blosc": {"id", "cname", "clevel", "shuffle", "blocksize"},
+    }
+    codec_id = compressor.get("id", "")
+    allowed = known_fields.get(codec_id)
+    if allowed is None:
+        return None
+
+    extra_keys = set(compressor.keys()) - allowed
+    if not extra_keys:
+        return None
+
+    logger.info(
+        f"Stripping unsupported compressor fields {extra_keys} for tensorstore compatibility"
+    )
+    cleaned_compressor = {k: v for k, v in compressor.items() if k in allowed}
+    meta["compressor"] = cleaned_compressor
+    return meta
+
+
 def open_ds_tensorstore(
     dataset_path: str, mode="r", concurrency_limit=None, normalize=True
 ):
@@ -231,6 +276,15 @@ def open_ds_tensorstore(
             "path": os.path.normpath(dataset_path),
         }
 
+    # For local zarr files, clean compressor metadata to remove fields
+    # unsupported by tensorstore (e.g. 'checksum' from newer numcodecs)
+    _assume_metadata = False
+    if filetype == "zarr" and isinstance(kvstore, dict) and kvstore.get("driver") == "file":
+        cleaned_metadata = _clean_zarr_compressor(kvstore["path"])
+        if cleaned_metadata is not None:
+            extra_args["metadata"] = cleaned_metadata
+            _assume_metadata = True
+
     if concurrency_limit:
         spec = {
             "driver": filetype,
@@ -244,10 +298,14 @@ def open_ds_tensorstore(
     else:
         spec = {"driver": filetype, "kvstore": kvstore, **extra_args}
 
+    open_kwargs = {}
+    if _assume_metadata:
+        open_kwargs = {"open": True, "assume_metadata": True}
+
     if mode == "r":
-        dataset_future = ts.open(spec, read=True, write=False)
+        dataset_future = ts.open(spec, read=True, write=False, **open_kwargs)
     else:
-        dataset_future = ts.open(spec, read=False, write=True)
+        dataset_future = ts.open(spec, read=False, write=True, **open_kwargs)
 
     if dataset_path.startswith("gs://"):
         # NOTE: Currently a hack since google store is for some reason stored as mutlichannel

cellmap_flow/utils/neuroglancer_utils.py

@@ -5,6 +5,7 @@
 from cellmap_flow.dashboard.app import create_and_run_app
 from cellmap_flow.utils.scale_pyramid import get_raw_layer
 from cellmap_flow.globals import g
+import os
 
 from cellmap_flow.utils.web_utils import (
     ARGS_KEY,
@@ -60,7 +61,6 @@ def generate_neuroglancer_url(dataset_path,wrap_raw=True):
     # .replace("zouinkhim-lm1", "192.168.1.167")
     print("viewer", viewer_url)
     url = create_and_run_app(neuroglancer_url=viewer_url)
-    show(url)
     return url