From 7beed0878982a320ecf61e3bbf2b81821f45feb4 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 13 Mar 2026 23:24:00 -0700 Subject: [PATCH 01/40] start adding some text --- .gitignore | 1 + docs/Project.toml | 3 +++ docs/make.jl | 10 ++++++++- docs/src/api.md | 5 +++++ docs/src/examples/jwst.md | 38 ++++++++++++++++++++++++++++++++ docs/src/examples/roman.md | 1 + docs/src/index.md | 24 +++++++++++++++++++-- docs/src/intro.md | 44 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 docs/src/api.md create mode 100644 docs/src/examples/jwst.md create mode 100644 docs/src/examples/roman.md create mode 100644 docs/src/intro.md diff --git a/.gitignore b/.gitignore index 0665207..28f78ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ Manifest*.toml +data/ diff --git a/docs/Project.toml b/docs/Project.toml index 2df60e2..8b975ac 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,8 @@ [deps] ASDF = "686f71d1-807d-59a4-a860-28280ea06d7b" +AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" +AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" [compat] diff --git a/docs/make.jl b/docs/make.jl index 80d9484..a8631f6 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -8,9 +8,17 @@ makedocs(; repo = GitHub("JuliaAstro/ASDF.jl"), sitename = "ASDF.jl", format = Documenter.HTML( - prettyurls = true, canonical = "https://juliaastro.org/ASDF/stable/", ), + pages = [ + "Home" => "index.md", + "Introduction" => "intro.md", + "Examples" => [ + "JWST" => "examples/jwst.md", + "Roman" => "examples/roman.md", + ], + "API" => "api.md", + ], ) deploydocs(; diff --git a/docs/src/api.md b/docs/src/api.md new file mode 100644 index 0000000..8149da7 --- /dev/null +++ b/docs/src/api.md @@ -0,0 +1,5 @@ +# API + +```@autodocs +Modules = [ASDF] +``` diff --git a/docs/src/examples/jwst.md b/docs/src/examples/jwst.md new file mode 100644 index 0000000..894a2c7 --- /dev/null +++ b/docs/src/examples/jwst.md @@ -0,0 +1,38 @@ +# JWST + +*Adapted from [ADASS 2024 workshop](https://github.com/asdf-format/asdf-adass2024/blob/main/02_working_with_asdf/Working_with_ASDF.ipynb).* + +## Reading an ASDF file + +```@example jwst +using ASDF + +af = ASDF.load_file("../../data/jwst.asdf") + +af.metadata +``` + +```@example jwst +img_sci = let + img = af.metadata["data"][] + img[img .< 0] .= 1 + img +end +``` + +And plot: + +```@example jwst +using CairoMakie + +fig, ax, hm = heatmap(img_sci; + colorrange = (1, 1e3), + colorscale = log10, + colormap = :cividis, + nan_color = :lime, # NaNs are handled automatically +) + +Colorbar(fig[1, 2], hm) + +fig +``` diff --git a/docs/src/examples/roman.md b/docs/src/examples/roman.md new file mode 100644 index 0000000..f5da645 --- /dev/null +++ b/docs/src/examples/roman.md @@ -0,0 +1 @@ +# Roman diff --git a/docs/src/index.md b/docs/src/index.md index fd82086..79409fc 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,5 +1,25 @@ # ASDF.jl -```@autodocs -Modules = [ASDF] +A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs.io/en/latest/index.html) package, written in Julia. + + +## Quickstart + +```julia +using ASDF +``` + +```julia +# Load +af = ASDF.load_file("") +``` + +```julia +# Modify +af.metadata[""] = +``` + +```julia +# Write +ASDF.write_file("") ``` diff --git a/docs/src/intro.md b/docs/src/intro.md new file mode 100644 index 0000000..7f7709a --- /dev/null +++ b/docs/src/intro.md @@ -0,0 +1,44 @@ +# Introduction + +*Adapted from [ADASS 2024 workshop](https://github.com/asdf-format/asdf-adass2024/blob/main/03_creating_asdf_files/Creating_ASDF_Files.ipynb).* + +## Getting started + +Created via a regular dictionary: + +```@example intro +af_payload = Dict{Any, Any}( # To-do: see if type signature needs to be this general + "meta" => Dict("my" => Dict("nested" => "metadata")), + "data" => [1, 2, 3, 4], +) +``` + +Write and load with `ASDF.write_file` and `ASDF.load_file`, respectively: + +```@example intro +using ASDF + +ASDF.write_file("../data/my_asdf.asdf", af_payload) + +af = ASDF.load_file("../data/my_asdf.asdf") +``` + +This contains a `meta` field, which is a dictionary that merges information about this library with `af_payload`: + +```@example intro +af.metadata +``` + +```@example intro +af.metadata["asdf/library"] +``` + +## Tagged objects + +Supporting custom objects, extensions. + +## Array storage + +```julia +ASDF.NDArrayWrapper(...; inline, compression) +``` From 8683b57ac578360b883a884747e1674778e2d651 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 14 Mar 2026 18:04:34 -0700 Subject: [PATCH 02/40] more text --- docs/src/index.md | 13 ++++++++++- docs/src/intro.md | 59 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index 79409fc..d261013 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -5,6 +5,17 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs ## Quickstart + +### Installation + +```julia-repl +julia> ] + +pkg> add ASDF +``` + +### Usage + ```julia using ASDF ``` @@ -15,7 +26,7 @@ af = ASDF.load_file("") ``` ```julia -# Modify +# Acess and modify af.metadata[""] = ``` diff --git a/docs/src/intro.md b/docs/src/intro.md index 7f7709a..ad854f0 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -2,9 +2,15 @@ *Adapted from [ADASS 2024 workshop](https://github.com/asdf-format/asdf-adass2024/blob/main/03_creating_asdf_files/Creating_ASDF_Files.ipynb).* +The ASDF file format is based on the human-readable [YAML](http://yaml.org/) standard, extended with efficient binary blocks to store array data. Basic arithmetic types (`Bool`, `Int`, `Float`, `Complex`) and `String` types are supported out of the box. Other types (structures) need to be declared to be supported. + +ASDF supports arbitrary array strides, both C (Python) and Fortran (Julia) memory layouts, as well as compression. The YAML metadata can contain arbitrary information corresponding to scalars, arrays, or dictionaries. + +The ASDF file format targets a similar audience as the [HDF5](https://www.hdfgroup.org/solutions/hdf5/) format. + ## Getting started -Created via a regular dictionary: +ASDF files are initially created as a nested dictionary with your specified keys: ```@example intro af_payload = Dict{Any, Any}( # To-do: see if type signature needs to be this general @@ -13,17 +19,52 @@ af_payload = Dict{Any, Any}( # To-do: see if type signature needs to be this gen ) ``` -Write and load with `ASDF.write_file` and `ASDF.load_file`, respectively: +Next, this dictionary can be written to the ASDF file format with `ASDF.write_file`: ```@example intro using ASDF -ASDF.write_file("../data/my_asdf.asdf", af_payload) +ASDF.write_file("../data/my_file.asdf", af_payload) +``` + +which contains the following file contents: -af = ASDF.load_file("../data/my_asdf.asdf") +```yaml +#ASDF 1.0.0 +#ASDF_STANDARD 1.2.0 +# This is an ASDF file +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- +!core/asdf-1.1.0 +meta: + my: + nested: "metadata" +data: + - 1 + - 2 + - 3 + - 4 +asdf/library: !core/software-1.0.0 + version: "2.0.0" + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" +... +#ASDF BLOCK INDEX +%YAML 1.1 +--- +[] +... ``` -This contains a `meta` field, which is a dictionary that merges information about this library with `af_payload`: +This file can be loaded back with `ASDF.load_file`: + +```@example intro +af = ASDF.load_file("../data/my_file.asdf") +``` + +This creates an `ASDF.ASDFFile` object which contains a `meta` field. This is a new dictionary that merges information about this library (stored under the `asdf/library` key) with the original user-defined `af_payload` dictionary: ```@example intro af.metadata @@ -33,6 +74,14 @@ af.metadata af.metadata["asdf/library"] ``` +Since the underlying data is a dictionary, it can be modified in the standard way: + +```@example intro +af.metadata["meta"]["my"]["nested2"] = "metadata2" + +af.metadata +``` + ## Tagged objects Supporting custom objects, extensions. From a6908a81128f584027934e5cc6125fff3d3ee1bc Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 14 Mar 2026 19:07:52 -0700 Subject: [PATCH 03/40] start adding text on NDArrayWrapper and friends --- docs/src/api.md | 4 ++++ docs/src/intro.md | 14 +++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 8149da7..9828355 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -1,5 +1,9 @@ # API +## Public + +## Private ```@autodocs Modules = [ASDF] ``` +### Compression algorithms diff --git a/docs/src/intro.md b/docs/src/intro.md index ad854f0..561a61b 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -82,12 +82,16 @@ af.metadata["meta"]["my"]["nested2"] = "metadata2" af.metadata ``` -## Tagged objects - -Supporting custom objects, extensions. - ## Array storage +By default, array data is written inline as a literal to the ASDF file. This can be stored and later accessed more efficiently by wrapping your data in an `ASDF.NDArrayWrapper`. This allows for your data to be stored as a binary via the `inline = false` keyword. This can be further optimized by specifying a supported compression algorithm to use via the `compression` keyword. In either case, `NDArrayWrapper` data allows for your data to be accessed as a strided view. + ```julia -ASDF.NDArrayWrapper(...; inline, compression) +ASDF.NDArrayWrapper(...; compression = ASDF.C_Bzip2) # The default ``` + +Access view `[]` + +## Tagged objects + +Comming soon. Supporting custom objects, extensions. From 5bbcd1eddab75bc24cf9a7bbe5292b01b2c9d63d Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Thu, 19 Mar 2026 21:25:11 -0700 Subject: [PATCH 04/40] docs: adding some more text --- docs/Project.toml | 1 + docs/src/examples/jwst.md | 15 +++++++--- docs/src/examples/roman.md | 59 ++++++++++++++++++++++++++++++++++++++ docs/src/index.md | 2 -- docs/src/intro.md | 22 ++++++++++---- 5 files changed, 88 insertions(+), 11 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index 8b975ac..a692bb2 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -4,6 +4,7 @@ AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" [compat] Documenter = "1" diff --git a/docs/src/examples/jwst.md b/docs/src/examples/jwst.md index 894a2c7..2877ba6 100644 --- a/docs/src/examples/jwst.md +++ b/docs/src/examples/jwst.md @@ -2,16 +2,23 @@ *Adapted from [ADASS 2024 workshop](https://github.com/asdf-format/asdf-adass2024/blob/main/02_working_with_asdf/Working_with_ASDF.ipynb).* -## Reading an ASDF file +In this example, we show how to use ASDF.jl to load and view some astronomical data taken from the James Webb Space Telescope ([JWST](https://science.nasa.gov/mission/webb/)). + +!!! note "Data availability" + The sample data for this example can be [downloaded here](https://data.science.stsci.edu/redirect/Roman/Roman_Data_Workshop/ADASS2024/jwst.asdf) from the data repository of the Space Telescope Science Institute ([STScI](https://www.stsci.edu/)). Note: it is a moderately large file (~100 MB). + +## Load ```@example jwst using ASDF -af = ASDF.load_file("../../data/jwst.asdf") +af = ASDF.load_file("../../data/jwst.asdf"; strict = false) af.metadata ``` +## Modify + ```@example jwst img_sci = let img = af.metadata["data"][] @@ -20,7 +27,7 @@ img_sci = let end ``` -And plot: +## Plot ```@example jwst using CairoMakie @@ -29,7 +36,7 @@ fig, ax, hm = heatmap(img_sci; colorrange = (1, 1e3), colorscale = log10, colormap = :cividis, - nan_color = :lime, # NaNs are handled automatically + nan_color = :limegreen, # NaNs are handled automatically ) Colorbar(fig[1, 2], hm) diff --git a/docs/src/examples/roman.md b/docs/src/examples/roman.md index f5da645..85568b3 100644 --- a/docs/src/examples/roman.md +++ b/docs/src/examples/roman.md @@ -1 +1,60 @@ # Roman + +*Adapted from [STScI Roman Notebooks](https://github.com/spacetelescope/roman_notebooks/blob/main/notebooks/data_visualization/data_visualization.ipynb).* + +```julia-repl +julia> using ASDF, AWSS3, AWS + +julia> aws_config = AWS.AWSConfig(; creds = nothing, region = "us-east-1") +AWSConfig(nothing, "us-east-1", "json", 3) + +# This is a large file, will take some time +julia> s3_get_file( + aws_config, + "stpubdata", + "roman/nexus/soc_simulations/tutorial_data/r0003201001001001004_0001_wfi01_f106_cal.asdf", + "roman.asdf" +) + +julia> af = ASDF.load_file("roman.asdf"; strict = false); + +julia> af.metadata +Dict{Any, Any} with 3 entries: + "history" => Dict{Any, Any}("extensions"=>Dict{Any, Any}[Dict("software"=>Dict{Any, Any}("name"=>"asdf",… + "asdf_library" => Dict{Any, Any}("name"=>"asdf", "author"=>"The ASDF Developers", "homepage"=>"http://github.… + "roman" => Dict{Any, Any}("dq_border_ref_pix_top"=>NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IO… + +julia> af.metadata["roman"] +Dict{Any, Any} with 16 entries: + "dq_border_ref_pix_top" => NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( Dict{Any, Any}("file_date"=>"2020-01-01T00:00:00.000", "cal_logs"=>["2025-05-28… + "dq_border_ref_pix_right" => NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( af.metadata["roman"]["data"][] + ERROR: AssertionError: all(actual_checksum == header.checksum) + Stacktrace: + [1] read_block(header::ASDF.BlockHeader) + @ ASDF ~/projects/ASDF.jl/src/ASDF.jl:168 + [2] getindex(ndarray::ASDF.NDArray) + @ ASDF ~/projects/ASDF.jl/src/ASDF.jl:394 + [3] top-level scope + @ REPL[10]:1 + ``` diff --git a/docs/src/index.md b/docs/src/index.md index d261013..575d406 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -9,8 +9,6 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs ### Installation ```julia-repl -julia> ] - pkg> add ASDF ``` diff --git a/docs/src/intro.md b/docs/src/intro.md index 561a61b..b67e7d3 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -84,14 +84,26 @@ af.metadata ## Array storage -By default, array data is written inline as a literal to the ASDF file. This can be stored and later accessed more efficiently by wrapping your data in an `ASDF.NDArrayWrapper`. This allows for your data to be stored as a binary via the `inline = false` keyword. This can be further optimized by specifying a supported compression algorithm to use via the `compression` keyword. In either case, `NDArrayWrapper` data allows for your data to be accessed as a strided view. +By default, array data is written inline as a literal to the ASDF file. This can be stored and later accessed more efficiently by wrapping your data in an `ASDF.NDArrayWrapper`. This allows for your data to be stored as a binary via the `inline = false` keyword, which can be further optimized by specifying a supported [compression algorithm](@ref "Compression algorithms") to use via the `compression` keyword: -```julia -ASDF.NDArrayWrapper(...; compression = ASDF.C_Bzip2) # The default +```@example intro +af_payload = Dict{Any, Any}( + "meta" => Dict("my" => Dict("nested" => "metadata")), + # Default + "data" => ASDF.NDArrayWrapper([1, 2, 3, 4]; inline = false, compression = ASDF.C_Bzip2), +) + +ASDF.write_file("../data/my_file_compressed.asdf", af_payload) ``` -Access view `[]` +Saving your data as an `NDArrayWrapper` allows for it to be lazily accessed as a strided view. To access the underlying data, use the `[]` (dereference) syntax: + +```@example intro +af = ASDF.load_file("../data/my_file_compressed.asdf") + +af.metadata["data"][] +``` ## Tagged objects -Comming soon. Supporting custom objects, extensions. +Coming soon. Supporting custom objects, extensions. From 015aad4020b91b4de0f0634380ce4a65aaceee36 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 02:23:11 -0700 Subject: [PATCH 05/40] docs: autodownload data --- docs/Project.toml | 3 ++ docs/src/examples/jwst.md | 24 ++++++--- docs/src/examples/roman.md | 106 +++++++++++++++++-------------------- 3 files changed, 69 insertions(+), 64 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index a692bb2..720a4e2 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,5 +6,8 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +[sources] +ASDF = {url = "https://github.com/JuliaAstro/ASDF.jl", rev = "strict"} + [compat] Documenter = "1" diff --git a/docs/src/examples/jwst.md b/docs/src/examples/jwst.md index 2877ba6..bc18dcf 100644 --- a/docs/src/examples/jwst.md +++ b/docs/src/examples/jwst.md @@ -9,28 +9,36 @@ In this example, we show how to use ASDF.jl to load and view some astronomical d ## Load +```@example jwst + +fpath = joinpath("..", "..", "data", "jwst.asdf") + +if !isfile(fpath) + @info "Downloading JWST data" + using Downloads: download + + download("https://data.science.stsci.edu/redirect/Roman/Roman_Data_Workshop/ADASS2024/jwst.asdf", fpath) +end +``` + ```@example jwst using ASDF -af = ASDF.load_file("../../data/jwst.asdf"; strict = false) +af = ASDF.load_file(fpath; extensions = true) af.metadata ``` -## Modify +## Plot ```@example jwst +using CairoMakie + img_sci = let img = af.metadata["data"][] img[img .< 0] .= 1 img end -``` - -## Plot - -```@example jwst -using CairoMakie fig, ax, hm = heatmap(img_sci; colorrange = (1, 1e3), diff --git a/docs/src/examples/roman.md b/docs/src/examples/roman.md index 85568b3..da1db6a 100644 --- a/docs/src/examples/roman.md +++ b/docs/src/examples/roman.md @@ -1,60 +1,54 @@ # Roman -*Adapted from [STScI Roman Notebooks](https://github.com/spacetelescope/roman_notebooks/blob/main/notebooks/data_visualization/data_visualization.ipynb).* - -```julia-repl -julia> using ASDF, AWSS3, AWS - -julia> aws_config = AWS.AWSConfig(; creds = nothing, region = "us-east-1") -AWSConfig(nothing, "us-east-1", "json", 3) - -# This is a large file, will take some time -julia> s3_get_file( - aws_config, - "stpubdata", - "roman/nexus/soc_simulations/tutorial_data/r0003201001001001004_0001_wfi01_f106_cal.asdf", - "roman.asdf" -) - -julia> af = ASDF.load_file("roman.asdf"; strict = false); - -julia> af.metadata -Dict{Any, Any} with 3 entries: - "history" => Dict{Any, Any}("extensions"=>Dict{Any, Any}[Dict("software"=>Dict{Any, Any}("name"=>"asdf",… - "asdf_library" => Dict{Any, Any}("name"=>"asdf", "author"=>"The ASDF Developers", "homepage"=>"http://github.… - "roman" => Dict{Any, Any}("dq_border_ref_pix_top"=>NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IO… - -julia> af.metadata["roman"] -Dict{Any, Any} with 16 entries: - "dq_border_ref_pix_top" => NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( Dict{Any, Any}("file_date"=>"2020-01-01T00:00:00.000", "cal_logs"=>["2025-05-28… - "dq_border_ref_pix_right" => NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( NDArray(LazyBlockHeaders(BlockHeader[BlockHeader(IOStream( s3_get_file( + aws_config, + "stpubdata", + "roman/nexus/soc_simulations/tutorial_data/r0003201001001001004_0001_wfi01_f106_cal.asdf", + fpath + ) +end +``` + +```@example roman +using ASDF + +af = ASDF.load_file(fpath; extensions = true, validate_checksum = false) + +af.metadata["roman"] +``` + +## Plot + +```@example roman +using CairoMakie + +img = af.metadata["roman"]["data"][] + +let + fig, ax, hm = heatmap(img[begin:1000, begin:1000]; colorscale = asinh, colorrange = (0.5, 4)) + Colorbar(fig[1, 2], hm) + fig +end ``` -!!! todo - Fix checksum issue: - - ```julia-repl - julia> af.metadata["roman"]["data"][] - ERROR: AssertionError: all(actual_checksum == header.checksum) - Stacktrace: - [1] read_block(header::ASDF.BlockHeader) - @ ASDF ~/projects/ASDF.jl/src/ASDF.jl:168 - [2] getindex(ndarray::ASDF.NDArray) - @ ASDF ~/projects/ASDF.jl/src/ASDF.jl:394 - [3] top-level scope - @ REPL[10]:1 - ``` +!!! note + Some ASDF files produced by the Python implementation of ASDF may save a checksum in its header block computed from the original decompressed file. This will cause ASDF.jl to fail because in constrast, it computes the checksum based on the compressed (i.e., "used data"), as per the [current specification for ASDF](https://www.asdf-format.org/projects/asdf-standard/en/1.0.1/file_layout.html#block-header). To handle this potenial failure mode, we pass `validate_checksum = false` to avoid running the default checksum. From dd18ff8fd9e386d218b9dc6452d0587ff4428a63 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 02:31:04 -0700 Subject: [PATCH 06/40] docs: update README [skip ci] --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 84707e6..9145e61 100644 --- a/README.md +++ b/README.md @@ -10,4 +10,25 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs.io/en/latest/index.html) package, written in Julia. +## Quickstart + +```julia +using ASDF +``` + +```julia +# Load +af = ASDF.load_file("") +``` + +```julia +# Acess and modify +af.metadata[""] = +``` + +```julia +# Write +ASDF.write_file("") +``` + See [`ASDF.jl v1`](https://github.com/JuliaAstro/ASDF.jl/releases/tag/v1.1.3) for the older one which is based on the Python [`asdf`](https://github.com/spacetelescope/asdf) Python package. From f2081367c505cc7ddef037d0c1ed1ae22a8eb340 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 02:31:17 -0700 Subject: [PATCH 07/40] docs: cleanup --- docs/src/examples/jwst.md | 1 - docs/src/examples/roman.md | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/src/examples/jwst.md b/docs/src/examples/jwst.md index bc18dcf..740a9bc 100644 --- a/docs/src/examples/jwst.md +++ b/docs/src/examples/jwst.md @@ -14,7 +14,6 @@ In this example, we show how to use ASDF.jl to load and view some astronomical d fpath = joinpath("..", "..", "data", "jwst.asdf") if !isfile(fpath) - @info "Downloading JWST data" using Downloads: download download("https://data.science.stsci.edu/redirect/Roman/Roman_Data_Workshop/ADASS2024/jwst.asdf", fpath) diff --git a/docs/src/examples/roman.md b/docs/src/examples/roman.md index da1db6a..1b5214b 100644 --- a/docs/src/examples/roman.md +++ b/docs/src/examples/roman.md @@ -13,13 +13,12 @@ In this example, we show how to use ASDF.jl to load and view some simulated astr fpath = joinpath("..", "..", "data", "roman.asdf") if !isfile(fpath) - @info "Downloading Roman data" using AWSS3, AWS aws_config = AWS.AWSConfig(; creds = nothing, region = "us-east-1") AWSConfig(nothing, "us-east-1", "json", 3) # This is a large file, will take some time to download - julia> s3_get_file( + s3_get_file( aws_config, "stpubdata", "roman/nexus/soc_simulations/tutorial_data/r0003201001001001004_0001_wfi01_f106_cal.asdf", From 1069a519fa7022c1dc84fcf8ae95048d76ec9283 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 03:17:44 -0700 Subject: [PATCH 08/40] ci(docs): temporarily disable package dev so that feature branch can be used --- .github/workflows/docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index cee328b..1e5fc2c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,6 +22,8 @@ jobs: - uses: julia-actions/cache@v3 - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 + with: + devpackage: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key From 3022cbfb0e7c8fff52781203c24c7384d5c0010f Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 03:19:34 -0700 Subject: [PATCH 09/40] ci(docs): typo --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1e5fc2c..8045b5e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,7 +23,7 @@ jobs: - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 with: - devpackage: false + install-package: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key From 5a7be47e434d46c7780f2e60f73d68dc63a92601 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 03:23:51 -0700 Subject: [PATCH 10/40] ci(docs): towards temp feature branch activation --- .github/workflows/docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8045b5e..6299b80 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,6 +20,8 @@ jobs: with: version: '1.12' - uses: julia-actions/cache@v3 + - name: Instantiate docs environment + run: julia --project=docs -e 'using Pkg; Pkg.instantiate()' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 with: From 1ac9a071a7ae9df0ae50be2760cac12b2e2664ff Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 03:52:59 -0700 Subject: [PATCH 11/40] docs: update READEME [skip ci] --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9145e61..72f58b3 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ af.metadata[""] = ```julia # Write -ASDF.write_file("") +doc = Dict() +ASDF.write_file("", doc) ``` See [`ASDF.jl v1`](https://github.com/JuliaAstro/ASDF.jl/releases/tag/v1.1.3) for the older one which is based on the Python [`asdf`](https://github.com/spacetelescope/asdf) Python package. From 73a47458ec4aeb8d7f5e00579e5d9c4838ba89e7 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 03:53:24 -0700 Subject: [PATCH 12/40] docs: update data downloads --- docs/src/examples/jwst.md | 4 +++- docs/src/examples/roman.md | 4 +++- docs/src/index.md | 3 ++- docs/src/intro.md | 13 +++++++++---- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/src/examples/jwst.md b/docs/src/examples/jwst.md index 740a9bc..ee6b798 100644 --- a/docs/src/examples/jwst.md +++ b/docs/src/examples/jwst.md @@ -11,7 +11,9 @@ In this example, we show how to use ASDF.jl to load and view some astronomical d ```@example jwst -fpath = joinpath("..", "..", "data", "jwst.asdf") +data_dir = joinpath("..", "..", "data") +mkpath(data_dir) +fpath = joinpath(data_dir, "jwst.asdf") if !isfile(fpath) using Downloads: download diff --git a/docs/src/examples/roman.md b/docs/src/examples/roman.md index 1b5214b..8e20027 100644 --- a/docs/src/examples/roman.md +++ b/docs/src/examples/roman.md @@ -10,7 +10,9 @@ In this example, we show how to use ASDF.jl to load and view some simulated astr ## Load ```@example roman -fpath = joinpath("..", "..", "data", "roman.asdf") +data_dir = joinpath("..", "..", "data") +mkpath(data_dir) +fpath = joinpath(data_dir, "roman.asdf") if !isfile(fpath) using AWSS3, AWS diff --git a/docs/src/index.md b/docs/src/index.md index 575d406..fae5cc6 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -30,5 +30,6 @@ af.metadata[""] = ```julia # Write -ASDF.write_file("") +doc = Dict() +ASDF.write_file("", doc) ``` diff --git a/docs/src/intro.md b/docs/src/intro.md index b67e7d3..62cb9bd 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -24,7 +24,11 @@ Next, this dictionary can be written to the ASDF file format with `ASDF.write_fi ```@example intro using ASDF -ASDF.write_file("../data/my_file.asdf", af_payload) +data_dir = joinpath("..", "data") +mkpath(data_dir) +fpath = joinpath(data_dir, "my_file.asdf") + +ASDF.write_file(fpath, af_payload) ``` which contains the following file contents: @@ -61,7 +65,7 @@ asdf/library: !core/software-1.0.0 This file can be loaded back with `ASDF.load_file`: ```@example intro -af = ASDF.load_file("../data/my_file.asdf") +af = ASDF.load_file(fpath) ``` This creates an `ASDF.ASDFFile` object which contains a `meta` field. This is a new dictionary that merges information about this library (stored under the `asdf/library` key) with the original user-defined `af_payload` dictionary: @@ -93,13 +97,14 @@ af_payload = Dict{Any, Any}( "data" => ASDF.NDArrayWrapper([1, 2, 3, 4]; inline = false, compression = ASDF.C_Bzip2), ) -ASDF.write_file("../data/my_file_compressed.asdf", af_payload) +fpath = joinpath(data_dir, "my_file_compressed.asdf") +ASDF.write_file(fpath, af_payload) ``` Saving your data as an `NDArrayWrapper` allows for it to be lazily accessed as a strided view. To access the underlying data, use the `[]` (dereference) syntax: ```@example intro -af = ASDF.load_file("../data/my_file_compressed.asdf") +af = ASDF.load_file(fpath) af.metadata["data"][] ``` From c99408abc3f9de86f1e50446b3816f5f353c00b4 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 04:09:47 -0700 Subject: [PATCH 13/40] ci: for real this time --- .github/workflows/docs.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6299b80..992cbde 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,7 +21,13 @@ jobs: version: '1.12' - uses: julia-actions/cache@v3 - name: Instantiate docs environment - run: julia --project=docs -e 'using Pkg; Pkg.instantiate()' + run: | + julia --project=docs -e ' + using Pkg + Pkg.rm("ASDF") + Pkg.add(url="https://github.com/JuliaAstro/ASDF.jl", rev="strict") + Pkg.instantiate() + ' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 with: From ab80df9e243c8a845ef5f2a3fb892f7bd513cb69 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 04:13:14 -0700 Subject: [PATCH 14/40] bruh --- .github/workflows/docs.yml | 2 +- docs/Project.toml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 992cbde..3581c7c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,9 +24,9 @@ jobs: run: | julia --project=docs -e ' using Pkg - Pkg.rm("ASDF") Pkg.add(url="https://github.com/JuliaAstro/ASDF.jl", rev="strict") Pkg.instantiate() + Pkg.resolve() ' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 diff --git a/docs/Project.toml b/docs/Project.toml index 720a4e2..a692bb2 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,8 +6,5 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -[sources] -ASDF = {url = "https://github.com/JuliaAstro/ASDF.jl", rev = "strict"} - [compat] Documenter = "1" From d378b76f4fc34df9e5f8252b9198a704129cfbc1 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 04:19:28 -0700 Subject: [PATCH 15/40] ci: temporarily try with [workspace] --- .github/workflows/docs.yml | 10 ---------- Project.toml | 2 +- docs/Project.toml | 3 +++ 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3581c7c..cee328b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,18 +20,8 @@ jobs: with: version: '1.12' - uses: julia-actions/cache@v3 - - name: Instantiate docs environment - run: | - julia --project=docs -e ' - using Pkg - Pkg.add(url="https://github.com/JuliaAstro/ASDF.jl", rev="strict") - Pkg.instantiate() - Pkg.resolve() - ' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 - with: - install-package: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key diff --git a/Project.toml b/Project.toml index 4c609d5..9883356 100644 --- a/Project.toml +++ b/Project.toml @@ -4,7 +4,7 @@ version = "2.0.0" authors = ["Erik Schnetter "] [workspace] -projects = ["test", "docs"] +projects = ["test"] [deps] ChunkCodecLibBlosc = "c6a955be-ab7f-4fbb-b38f-caf93db6b928" diff --git a/docs/Project.toml b/docs/Project.toml index a692bb2..720a4e2 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,5 +6,8 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +[sources] +ASDF = {url = "https://github.com/JuliaAstro/ASDF.jl", rev = "strict"} + [compat] Documenter = "1" From c37b71405c9657eb73cb4c59de942ba38a0407dd Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 04:24:38 -0700 Subject: [PATCH 16/40] Update docs/Project.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- docs/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 720a4e2..cf74f60 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -7,7 +7,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" [sources] -ASDF = {url = "https://github.com/JuliaAstro/ASDF.jl", rev = "strict"} +ASDF = {path = ".."} [compat] Documenter = "1" From 1401c8ace8f2c889637c41bca089777631a7a646 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 21 Mar 2026 04:32:17 -0700 Subject: [PATCH 17/40] ci(docs): finish putting [workspace] back --- .github/workflows/docs.yml | 9 +++++++++ Project.toml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index cee328b..42e604f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,8 +20,17 @@ jobs: with: version: '1.12' - uses: julia-actions/cache@v3 + - name: Instantiate feature branch (remove after merge) + run: | + julia --project=docs -e ' + using Pkg + Pkg.add(url="https://github.com/JuliaAstro/ASDF.jl", rev="strict") + Pkg.instantiate() + ' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 + with: + install-package: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key diff --git a/Project.toml b/Project.toml index 9883356..4c609d5 100644 --- a/Project.toml +++ b/Project.toml @@ -4,7 +4,7 @@ version = "2.0.0" authors = ["Erik Schnetter "] [workspace] -projects = ["test"] +projects = ["test", "docs"] [deps] ChunkCodecLibBlosc = "c6a955be-ab7f-4fbb-b38f-caf93db6b928" From 96e1b56ac0f5e11ead1594c77e1cda29d78234da Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 10:17:29 -0700 Subject: [PATCH 18/40] try explicit PackageSpec to other branch, drop [sources] --- .github/workflows/docs.yml | 2 +- docs/Project.toml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 42e604f..0f694c3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ jobs: run: | julia --project=docs -e ' using Pkg - Pkg.add(url="https://github.com/JuliaAstro/ASDF.jl", rev="strict") + Pkg.add(PackageSpec(url="https://github.com/JuliaAstro/ASDF.jl", rev="strict")); Pkg.instantiate() ' - name: Build and deploy diff --git a/docs/Project.toml b/docs/Project.toml index cf74f60..a692bb2 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,8 +6,5 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -[sources] -ASDF = {path = ".."} - [compat] Documenter = "1" From 344009b66357c229496e493cf42f42f958432568 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 10:27:57 -0700 Subject: [PATCH 19/40] try explicit checkout of strict branch --- .github/workflows/docs.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0f694c3..9e28bdf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,6 +16,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + - uses: actions/checkout@v6 # Check out 'strict' branch alongside + with: + ref: strict + path: asdf-strict + - uses: julia-actions/setup-julia@v2 with: version: '1.12' @@ -24,7 +29,7 @@ jobs: run: | julia --project=docs -e ' using Pkg - Pkg.add(PackageSpec(url="https://github.com/JuliaAstro/ASDF.jl", rev="strict")); + Pkg.add(path = "asdf-strict"); Pkg.instantiate() ' - name: Build and deploy From 76485ae7c141d71c15c3eebf4972eb8e0640e7da Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 10:31:24 -0700 Subject: [PATCH 20/40] typo: add --> develop --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9e28bdf..77b3cea 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,7 +29,7 @@ jobs: run: | julia --project=docs -e ' using Pkg - Pkg.add(path = "asdf-strict"); + Pkg.develop(path = "asdf-strict"); Pkg.instantiate() ' - name: Build and deploy From 5356d54cb8d304f6006b383a904b1692765542e8 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 10:41:54 -0700 Subject: [PATCH 21/40] try explicit clone --- .github/workflows/docs.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 77b3cea..1a3d338 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,20 +16,17 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: actions/checkout@v6 # Check out 'strict' branch alongside - with: - ref: strict - path: asdf-strict - - uses: julia-actions/setup-julia@v2 with: version: '1.12' - uses: julia-actions/cache@v3 + - name: Clone strict branch (remove after merge) + run: git clone --branch strict https://github.com/JuliaAstro/ASDF.jl $RUNNER_TEMP/asdf-strict - name: Instantiate feature branch (remove after merge) run: | julia --project=docs -e ' using Pkg - Pkg.develop(path = "asdf-strict"); + Pkg.develop(path=ENV["RUNNER_TEMP"] * "/asdf-strict") Pkg.instantiate() ' - name: Build and deploy From b95a08f247159783469549f4672adc94d37b883e Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 10:48:56 -0700 Subject: [PATCH 22/40] try wrapping path in PackageSpec --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1a3d338..6b3c5bd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: run: | julia --project=docs -e ' using Pkg - Pkg.develop(path=ENV["RUNNER_TEMP"] * "/asdf-strict") + Pkg.develop(PackageSpec(path=ENV["RUNNER_TEMP"] * "/asdf-strict")) Pkg.instantiate() ' - name: Build and deploy From 2da91fbe70d87aa21b2a395ee097d6430dca7af6 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 10:55:38 -0700 Subject: [PATCH 23/40] try just copying the src over from the other branch --- .github/workflows/docs.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6b3c5bd..8ba85b9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,15 +20,12 @@ jobs: with: version: '1.12' - uses: julia-actions/cache@v3 - - name: Clone strict branch (remove after merge) - run: git clone --branch strict https://github.com/JuliaAstro/ASDF.jl $RUNNER_TEMP/asdf-strict - - name: Instantiate feature branch (remove after merge) + - name: Copy strict branch src over usage branch's (remove after merge) run: | - julia --project=docs -e ' - using Pkg - Pkg.develop(PackageSpec(path=ENV["RUNNER_TEMP"] * "/asdf-strict")) - Pkg.instantiate() - ' + git clone --branch strict https://github.com/JuliaAstro/ASDF.jl $RUNNER_TEMP/asdf-strict + cp -r $RUNNER_TEMP/asdf-strict/src/* src/ + - name: Instantiate docs environment + run: julia --project=docs -e 'using Pkg; Pkg.instantiate()' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 with: From 67614fbdfce45d59e2edfd35e8cb5b0408a6b4a3 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 15:50:41 -0700 Subject: [PATCH 24/40] try just omitting docs from [workspace] --- .github/workflows/docs.yml | 8 -------- Project.toml | 3 ++- docs/Project.toml | 4 ++++ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8ba85b9..cee328b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,16 +20,8 @@ jobs: with: version: '1.12' - uses: julia-actions/cache@v3 - - name: Copy strict branch src over usage branch's (remove after merge) - run: | - git clone --branch strict https://github.com/JuliaAstro/ASDF.jl $RUNNER_TEMP/asdf-strict - cp -r $RUNNER_TEMP/asdf-strict/src/* src/ - - name: Instantiate docs environment - run: julia --project=docs -e 'using Pkg; Pkg.instantiate()' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 - with: - install-package: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key diff --git a/Project.toml b/Project.toml index 4c609d5..1870d7b 100644 --- a/Project.toml +++ b/Project.toml @@ -4,7 +4,8 @@ version = "2.0.0" authors = ["Erik Schnetter "] [workspace] -projects = ["test", "docs"] +# TODO: add "docs" after `strict` branch is merged +projects = ["test"] [deps] ChunkCodecLibBlosc = "c6a955be-ab7f-4fbb-b38f-caf93db6b928" diff --git a/docs/Project.toml b/docs/Project.toml index a692bb2..6b604c5 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,5 +6,9 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +# TODO: remove after `strict` branch is merged +[sources] +ASDF = {url = "https://github.com/JuliaAstro/ASDF.jl", rev = "strict"} + [compat] Documenter = "1" From 984b0f5c6c50d76c825fdd5e28ed079fb7545e3c Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 15:54:17 -0700 Subject: [PATCH 25/40] nvm This reverts commit 039df6d3dfe49415f0fe1fd7dd8d59cbe373d411. --- .github/workflows/docs.yml | 8 ++++++++ Project.toml | 3 +-- docs/Project.toml | 4 ---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index cee328b..8ba85b9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,8 +20,16 @@ jobs: with: version: '1.12' - uses: julia-actions/cache@v3 + - name: Copy strict branch src over usage branch's (remove after merge) + run: | + git clone --branch strict https://github.com/JuliaAstro/ASDF.jl $RUNNER_TEMP/asdf-strict + cp -r $RUNNER_TEMP/asdf-strict/src/* src/ + - name: Instantiate docs environment + run: julia --project=docs -e 'using Pkg; Pkg.instantiate()' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 + with: + install-package: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key diff --git a/Project.toml b/Project.toml index 1870d7b..4c609d5 100644 --- a/Project.toml +++ b/Project.toml @@ -4,8 +4,7 @@ version = "2.0.0" authors = ["Erik Schnetter "] [workspace] -# TODO: add "docs" after `strict` branch is merged -projects = ["test"] +projects = ["test", "docs"] [deps] ChunkCodecLibBlosc = "c6a955be-ab7f-4fbb-b38f-caf93db6b928" diff --git a/docs/Project.toml b/docs/Project.toml index 6b604c5..a692bb2 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,9 +6,5 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -# TODO: remove after `strict` branch is merged -[sources] -ASDF = {url = "https://github.com/JuliaAstro/ASDF.jl", rev = "strict"} - [compat] Documenter = "1" From b7b3ccde43a6010b30a1f7727ec5b4237a71563f Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 16:56:19 -0700 Subject: [PATCH 26/40] ci: cleanup --- .github/workflows/docs.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8ba85b9..cee328b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,16 +20,8 @@ jobs: with: version: '1.12' - uses: julia-actions/cache@v3 - - name: Copy strict branch src over usage branch's (remove after merge) - run: | - git clone --branch strict https://github.com/JuliaAstro/ASDF.jl $RUNNER_TEMP/asdf-strict - cp -r $RUNNER_TEMP/asdf-strict/src/* src/ - - name: Instantiate docs environment - run: julia --project=docs -e 'using Pkg; Pkg.instantiate()' - name: Build and deploy uses: julia-actions/julia-docdeploy@v1 - with: - install-package: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key From 2f983b2b8ca5f365212cc83101b2a69fb4900225 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sun, 22 Mar 2026 23:33:51 -0700 Subject: [PATCH 27/40] docs: added compression scheme docstring --- docs/src/api.md | 1 - docs/src/intro.md | 2 +- src/ASDF.jl | 15 +++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 9828355..78cdb17 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -6,4 +6,3 @@ ```@autodocs Modules = [ASDF] ``` -### Compression algorithms diff --git a/docs/src/intro.md b/docs/src/intro.md index 62cb9bd..581fc1c 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -88,7 +88,7 @@ af.metadata ## Array storage -By default, array data is written inline as a literal to the ASDF file. This can be stored and later accessed more efficiently by wrapping your data in an `ASDF.NDArrayWrapper`. This allows for your data to be stored as a binary via the `inline = false` keyword, which can be further optimized by specifying a supported [compression algorithm](@ref "Compression algorithms") to use via the `compression` keyword: +By default, array data is written inline as a literal to the ASDF file. This can be stored and later accessed more efficiently by wrapping your data in an `ASDF.NDArrayWrapper`. This allows for your data to be stored as a binary via the `inline = false` keyword, which can be further optimized by specifying a supported [compression algorithm](@ref ASDF.Compression) to use via the `compression` keyword: ```@example intro af_payload = Dict{Any, Any}( diff --git a/src/ASDF.jl b/src/ASDF.jl index 7743a07..ad72950 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -20,6 +20,21 @@ const software_version = string(PkgVersion.@Version) ################################################################################ +""" +Identifies the compression algorithm used for a data block. Available variants: + +| Scheme | 4-byte key | Backend | Description | +| :--------------- | :------------- | :--------------------- | :------------------------------------------------------------ | +| `C_None` | `\\0\\0\\0\\0` | -- | Fast I/O speed, no CPU overhead | +| `C_Blosc` | `blsc` | ChunkCodecLibBlosc.jl | Multi-threaded, shuffle-aware, best with typed numeric arrays | +| `C_Blosc2` | `bls2` | See [Issue #49](https://github.com/JuliaIO/ChunkCodecs.jl/issues/49) | Like Blosc but supports more than 2 GB of data | +| `C_Bzip2` | `bzp2` | ChunkCodecLibBzip2.jl | Good ratio, moderate speed (default) | +| `C_Lz4` (:block) | `lz4\\0` | ChunkCodecLibLz4.jl | Fastest decompression, Python-compatible | +| `C_Lz4` (:frame) | `lz4\\0` | ChunkCodecLibLz4.jl | LZ4 frame format for non-Python consumers | +| `C_Xz` | `xz\\0\\0` | CodecXz.jl | Highest compression ratio, slowest | +| `C_Zlib` | `zlib` | ChunkCodecLibZlib.jl | Broad compatibility | +| `C_Zstd` | `zstd` | ChunkCodecLibZstd.jl | Best ratio/speed trade-off | +""" @enum Compression C_None C_Blosc C_Blosc2 C_Bzip2 C_Lz4 C_Xz C_Zlib C_Zstd const compression_keys = Dict{Compression,Vector{UInt8}}( From 62f64748588dbf55a18863099d3588fcee2c781e Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Mon, 23 Mar 2026 00:46:08 -0700 Subject: [PATCH 28/40] docs: more docstrings --- src/ASDF.jl | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index ad72950..f67e9b7 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -261,7 +261,12 @@ end ################################################################################ """ - @enum ASDF.Byteorder Byteorder_little Byteorder_big + ASDF.Byteorder + +Represents the byte order of array data stored in a block. Available variants: + +- `Byteorder_little` : Little-endian +- `Byteorder_big`: Big-endian """ @enum Byteorder Byteorder_little Byteorder_big const byteorder_string_dict = Dict{Byteorder,String}(Byteorder_little => "little", Byteorder_big => "big") @@ -269,21 +274,54 @@ const string_byteorder_dict = Dict{String,Byteorder}(val => key for (key, val) i """ ASDF.Byteorder(str::AbstractString)::Byteorder + +Convenience conversion for [ASDF.Byteorder](@ref). Inverse of [`Base.string(byteorder::Byteorder)`](@ref). + +# Examples + +```jldoctest +julia> ASDF.Byteorder("little") +Byteorder_little::Byteorder = 0 +``` """ Byteorder(str::AbstractString) = string_byteorder_dict[str] """ string(byteorder::Byteorder)::AbstractString + +Convenience conversion for [ASDF.Byteorder](@ref). Inverse of [`ASDF.Byteorder(str::AbstractString)`](@ref). + +# Examples + +```jldoctest +julia> string(ASDF.Byteorder_little) +"little" +``` """ Base.string(byteorder::Byteorder) = byteorder_string_dict[byteorder] Base.show(io::IO, byteorder::Byteorder) = show(io, string(byteorder)) +""" + host_byteorder + +Native byte order of the running machine, detected at load time. Defined by [`ASDF.Byteorder`](@ref). +""" const host_byteorder = reinterpret(UInt8, UInt16[1])[1] == 1 ? Byteorder_little : Byteorder_big ################################################################################ """ -Careful, there is also `Base.DataType`, which is a different type. +Maps ASDF datatype strings to Julia types. Note this is unrelated to `Base.DataType`. Defined mapping: + +| ASDF string | Julia type | +| :---------------------- | :---------------------- | +| `bool8` | `Bool` | +| `int8` ... `int128` | `Int8` ... `Int128` | +| `uint8` ... `uint128` | `UInt8` ... `UInt128` | +| `float16` ... `float64` | `Float16` ... `Float64` | +| `complex32` | `Complex{Float16}` | +| `complex64` | `Complex{Float32}` | +| `complex128` | `Complex{Float64}` | """ @enum Datatype begin Datatype_bool8 @@ -354,6 +392,19 @@ Datatype(type::Type) = type_datatype_dict[type] ################################################################################ +""" + NDArray + +A dense N-dimensional array, either backed lazily by a file block (`source`) or by an in-memory array (`data`). Exactly one of `source` and `data` is non-nothing. The array can be stored inline or in an ASDF binary block. + +- `source` -- zero-based index into the file's block list (mutually exclusive with `data`). +- `data` -- in-memory array (mutually exclusive with `source`). +- `shape` -- dimensions in Python/C order (outermost first). The Julia array has dimensions `reverse(shape)`. +- `strides` -- byte strides in Python/C order. +- `offset` -- byte offset into the block. + +The Julia array shape is `reverse(shape)` (column-major). Strides are always stored in row-major order to match the ASDF/Python convention. +""" struct NDArray lazy_block_headers::LazyBlockHeaders @@ -444,6 +495,17 @@ function make_construct_yaml_ndarray(block_headers::LazyBlockHeaders) return construct_yaml_ndarray end +""" + Base.getindex(ndarray::NDArray) + +Returns the fully materialized array. See [`ASDF.NDArray`](@ref) for definitions. When block-backed (`source` !== `nothing`), this reads and decompresses the block, applies offset and strides via a [`StridedViews.StridedView`](https://github.com/QuantumKitHub/StridedViews.jl), reinterprets bytes to `Type(datatype)`, and byte-swaps if `byteorder != host_byteorder`. The returned array satisfies: + +```julia +size(result) == Tuple(reverse(ndarray.shape)) +eltype(result) == Type(ndarray.datatype) +sizeof(eltype) .* strides(result) == Tuple(reverse(ndarray.strides)) +``` +""" function Base.getindex(ndarray::NDArray) if ndarray.data !== nothing data = ndarray.data @@ -480,6 +542,11 @@ end ################################################################################ +""" + NDArrayChunk + +A positioned tile within an [ASDF.ChunkedNDArray](@ref). `start` gives the zero-based origin of this chunk in the coordinate space of the parent array (Python/C order). +""" struct NDArrayChunk start::Vector{Int64} ndarray::NDArray @@ -505,6 +572,11 @@ function make_construct_yaml_ndarray_chunk(block_headers::LazyBlockHeaders) return construct_yaml_ndarray_chunk end +""" + ChunkedNDArray + +An array composed of arbitrarily-positioned tiles. Gaps between chunks are left uninitialised. Overlapping chunks are written in order. +""" struct ChunkedNDArray shape::Vector{Int64} datatype::Datatype @@ -543,6 +615,11 @@ function make_construct_yaml_chunked_ndarray(block_headers::LazyBlockHeaders) return construct_yaml_chunked_ndarray end +""" + Base.getindex(chunked_ndarray::ChunkedNDArray) + +Allocates a dense array of shape `reverse(shape)` and fills it by calling `chunk.ndarray[]` for each chunk, placing the result at the correct offset. +""" function Base.getindex(chunked_ndarray::ChunkedNDArray) shape = chunked_ndarray.shape datatype = Type(chunked_ndarray.datatype) From 3c10fc72232a3bacc9e5f3f18eda999c45c0c172 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Mon, 23 Mar 2026 01:11:18 -0700 Subject: [PATCH 29/40] docs: add ASDF.jl to doctests --- docs/make.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/make.jl b/docs/make.jl index a8631f6..c9dd024 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,6 +2,8 @@ using ASDF using Documenter using Documenter.Remotes: GitHub +DocMeta.setdocmeta!(ASDF, :DocTestSetup, setup; recursive = true) + makedocs(; modules = [ASDF], authors = "Erik Schnetter", From 9d8ecea98e2b3260ea229ba53863f56f874d1e95 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Mon, 23 Mar 2026 01:17:32 -0700 Subject: [PATCH 30/40] typo --- docs/make.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index c9dd024..62599be 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,7 +2,7 @@ using ASDF using Documenter using Documenter.Remotes: GitHub -DocMeta.setdocmeta!(ASDF, :DocTestSetup, setup; recursive = true) +DocMeta.setdocmeta!(ASDF, :DocTestSetup, :(using ASDF); recursive = true) makedocs(; modules = [ASDF], From ad82da6ef8620397fb4c6fd26f9516099f92cd62 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Mon, 23 Mar 2026 01:53:13 -0700 Subject: [PATCH 31/40] docs: more docstrings --- src/ASDF.jl | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index f67e9b7..c02c375 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -275,7 +275,7 @@ const string_byteorder_dict = Dict{String,Byteorder}(val => key for (key, val) i """ ASDF.Byteorder(str::AbstractString)::Byteorder -Convenience conversion for [ASDF.Byteorder](@ref). Inverse of [`Base.string(byteorder::Byteorder)`](@ref). +Convenience conversion for [`ASDF.Byteorder`](@ref). Inverse of [`Base.string(byteorder::Byteorder)`](@ref). # Examples @@ -289,7 +289,7 @@ Byteorder(str::AbstractString) = string_byteorder_dict[str] """ string(byteorder::Byteorder)::AbstractString -Convenience conversion for [ASDF.Byteorder](@ref). Inverse of [`ASDF.Byteorder(str::AbstractString)`](@ref). +Convenience conversion for [`ASDF.Byteorder`](@ref). Inverse of [`ASDF.Byteorder(str::AbstractString)`](@ref). # Examples @@ -545,7 +545,7 @@ end """ NDArrayChunk -A positioned tile within an [ASDF.ChunkedNDArray](@ref). `start` gives the zero-based origin of this chunk in the coordinate space of the parent array (Python/C order). +A positioned tile within an [`ASDF.ChunkedNDArray`](@ref). `start` gives the zero-based origin of this chunk in the coordinate space of the parent array (Python/C order). """ struct NDArrayChunk start::Vector{Int64} @@ -634,6 +634,13 @@ end ################################################################################ +""" + ASDFFile + +An open ASDF file. `metadata` is the parsed YAML tree. Any [`ASDF.NDArray`](@ref) or [`ASDF.ChunkedNDArray`](@ref) nodes appear as values and are lazily backed by lazy_block_headers. + +`YAML.write(file::ASDFFile)` returns a human-readable summary string. +""" struct ASDFFile filename::AbstractString metadata::Dict{Any,Any} @@ -646,6 +653,26 @@ end ################################################################################ +""" + load_file(filename::AbstractString; extensions = false, validate_checksum = true) + +Reads an ASDF file from disk. + +| Parameter | Description | +| :------------------ | :--------------------------------------------------------------------------------------------------------------- | +| `filename` | Path to the `.asdf` file | +| `extensions` | When `true`, unknown YAML tags are parsed leniently (as maps, sequences, or scalars) instead of raising an error | +| `validate_checksum` | When `true`, each block's MD5 checksum is verified against the stored value | + +Block data is located lazily. Block headers are scanned after the YAML is parsed, and array data (`ndarray`) is read only when [`Base.getindex(ndarray::NDArray)`](@ref) is called, i.e., `ndarray[]`. + +Supported YAML tags: + +- `tag:stsci.edu:asdf/core/ndarray-1.0.0` +- `tag:stsci.edu:asdf/core/ndarray-1.1.0` +- `tag:stsci.edu:asdf/core/ndarray-chunk-1.0.0` +- `tag:stsci.edu:asdf/core/chunked-ndarray-1.0.0` +""" function load_file(filename::AbstractString; extensions = false, validate_checksum = true) asdf_constructors = copy(YAML.default_yaml_constructors) asdf_constructors["tag:stsci.edu:asdf/core/asdf-1.1.0"] = asdf_constructors["tag:yaml.org,2002:map"] @@ -687,6 +714,11 @@ end ################################################################################ ################################################################################ +""" + ASDFLibrary + +Software provenance metadata, serialised as a `!core/software-1.0.0 YAML` tag. [`ASDF.write_file`] inserts an entry automatically under the key `"asdf/library"` if one is not already present, using the package's own name, author, homepage, and version. +""" struct ASDFLibrary name::AbstractString author::AbstractString @@ -699,6 +731,20 @@ function YAML._print(io::IO, val::ASDFLibrary, level::Int=0, ignore_level::Bool= YAML._print(io, library, level, ignore_level) end +""" + NDArrayWrapper + +A write-side wrapper around a Julia array that carries compression and layout options. Used as the value type when building a document dict for [`ASDF.write_file`](@ref). + +Parameter | Default | Description | +| :---------- | :-------- | :------------------------------------------------------------------------ | +`compression` | `C_Bzip2` | Applied compression scheme | +`inline` | `false` | Embed data in YAML instead of a binary block | +`lz4_layout` | `:block` | `:block` for Python-compatible chunked LZ4, `:frame` for LZ4 frame format | + +!!! note + If the compressed output is larger than the raw input, the block is stored uncompressed regardless of the chosen compression. +""" struct NDArrayWrapper array::AbstractArray compression::Compression @@ -789,6 +835,18 @@ function encode_Lz4_block(input::AbstractVector{UInt8}; chunk_size::Int = 1024 * return out end +""" + write_file(filename::AbstractString, document::Dict{Any,Any}) + +Writes an ASDF file to disk. `document` is a plain `Dict` whose values may include [`NDArrayWrapper`](@ref) instances. These are serialised as binary blocks with appropriate compression. + +Layout of the output file: + +1. ASDF/YAML header (`#ASDF 1.0.0, #ASDF_STANDARD 1.2.0, %YAML 1.1`) +1. YAML tree (`!core/asdf-1.1.0`) +1. Binary blocks — one per [`NDArrayWrapper`](@ref) that has `inline == false` +1. Block index (`#ASDF BLOCK INDEX`) +""" function write_file(filename::AbstractString, document::Dict{Any,Any}) # Set up block descriptors global blocks From ddb5b934cd985dfc2988883a6cfea519074a041a Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Mon, 23 Mar 2026 02:41:40 -0700 Subject: [PATCH 32/40] docs: more docstrings --- src/ASDF.jl | 108 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 14 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index c02c375..1b4f8fa 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -48,16 +48,53 @@ const compression_keys = Dict{Compression,Vector{UInt8}}( C_Zstd => Vector{UInt8}("zstd"), ) @assert all(length(val)==4 for val in values(compression_keys)) + const compression_enums = Dict{Vector{UInt8},Compression}( value => key for (key, value) in compression_keys ) ################################################################################ +""" + BlockHeader + +Parsed representation of a single ASDF binary block header. + +Every binary block in an ASDF file begins with a fixed-layout header that describes the block's compression, size, and integrity checksum. `BlockHeader` captures all decoded fields from that header, together with the `IO` handle and file position needed to subsequently read the block's data payload. + +# Fields + +| Field | Description | +| :------------------ | :---------- | +| `io` | The open file handle from which the block's data can be read. | +| `position` | Absolute byte offset of the block magic token within `io`. | +| `token` | 4-byte magic token. Always equal to [`block_magic_token`](@ref) (`\\323BLK`). | +| `header_size` | Size of the extended header region in bytes (excludes the 6-byte prefix). | +| `flags` | Block flags. Bit 0 (`0x1`) indicates a streamed block (not currently supported). | +| `compression` | 4-byte compression key (e.g. `"zstd"`, `"bzp2"`). | +| `allocated_size` | Number of bytes allocated in the file for this block's data (≥ `used_size`). | +| `used_size` | Number of bytes of (compressed) data actually written. | +| `data_size` | Number of bytes of the *uncompressed* data. | +| `checksum` | 16-byte MD5 digest of the compressed data, or all zeros if omitted. | +| `validate_checksum` | When `true`, [`ASDF.read_block`](@ref) verifies the MD5 digest before returning data. | + +# File layout + +The block occupies the following byte range within `io`, starting at `position`: +| | | | +| :-- | :-- | :-- | +| `[position]` | 4 bytes | magic token | +| `[position + 4]` | 2 bytes | header_size (big-endian UInt16) | +| `[position + 6]` | header_size bytes | extended header fields | +| `[position + 6 + header_size]` | allocated_size bytes | data payload | + +The extended header (always 48 bytes in the current implementation) contains, in order: `flags` (4 B), `compression` (4 B), `allocated_size` (8 B), `used_size` (8 B), `data_size` (8 B), and `checksum` (16 B). + +All multi-byte integers in the header are stored in **big-endian** byte order. +""" struct BlockHeader io::IO position::Int64 - token::AbstractVector{UInt8} # length 4 header_size::UInt16 flags::UInt32 @@ -74,6 +111,11 @@ mutable struct LazyBlockHeaders LazyBlockHeaders() = new(BlockHeader[]) end +""" + block_magic_token + +The 4-byte sentinel `UInt8[0xd3, 0x42, 0x4c, 0x4b]`. +""" const block_magic_token = UInt8[0xd3, 0x42, 0x4c, 0x4b] # "\323BLK" find_first_block(io::IO) = find_next_block(io, Int64(0)) @@ -127,6 +169,11 @@ native2big_U16(val::Integer) = native2big_U16(UInt16(val)) native2big_U32(val::Integer) = native2big_U32(UInt32(val)) native2big_U64(val::Integer) = native2big_U64(UInt64(val)) +""" + read_block_header(io::IO, position::Int64; validate_checksum::Bool) + +Constructs an [`ASDF.BlockHeader`](@ref) by parsing the file at a given offset. +""" function read_block_header(io::IO, position::Int64; validate_checksum::Bool) # Read block header max_header_size = 6 + 48 @@ -159,6 +206,11 @@ function read_block_header(io::IO, position::Int64; validate_checksum::Bool) return BlockHeader(io, position, token, header_size, flags, compression, allocated_size, used_size, data_size, checksum, validate_checksum) end +""" + find_all_blocks(io::IO, pos::Int64=Int64(0); validate_checksum::Bool) + +Scans an `IO` stream and returns all [`ASDF.BlockHeader`](@ref) values found. +""" function find_all_blocks(io::IO, pos::Int64=Int64(0); validate_checksum::Bool) headers = BlockHeader[] pos = find_next_block(io, pos) @@ -171,6 +223,11 @@ function find_all_blocks(io::IO, pos::Int64=Int64(0); validate_checksum::Bool) return headers end +""" + read_block(header::BlockHeader) + +Uses an [`ASDF.BlockHeader`](@ref) to read, verify, and decompress the data payload. +""" function read_block(header::BlockHeader) block_data_start = header.position + 6 + header.header_size seek(header.io, block_data_start) @@ -311,7 +368,7 @@ const host_byteorder = reinterpret(UInt8, UInt16[1])[1] == 1 ? Byteorder_little ################################################################################ """ -Maps ASDF datatype strings to Julia types. Note this is unrelated to `Base.DataType`. Defined mapping: +Maps ASDF datatype strings to Julia types. Note this is unrelated to `Base.DataType`. Defined mappings: | ASDF string | Julia type | | :---------------------- | :---------------------- | @@ -395,19 +452,25 @@ Datatype(type::Type) = type_datatype_dict[type] """ NDArray -A dense N-dimensional array, either backed lazily by a file block (`source`) or by an in-memory array (`data`). Exactly one of `source` and `data` is non-nothing. The array can be stored inline or in an ASDF binary block. +A lazily-materialized N-dimensional array stored in an ASDF file, either as a binary block or inline within the ASDF file. -- `source` -- zero-based index into the file's block list (mutually exclusive with `data`). -- `data` -- in-memory array (mutually exclusive with `source`). -- `shape` -- dimensions in Python/C order (outermost first). The Julia array has dimensions `reverse(shape)`. -- `strides` -- byte strides in Python/C order. -- `offset` -- byte offset into the block. +`NDArray` is the in-memory representation of an `!core/ndarray-1.0.0` YAML node. It holds the array's shape, type, and layout metadata, but defers reading and decompressing block data until the array is explicitly materialized by calling [`Base.getindex(ndarray::NDArray)`](@ref). -The Julia array shape is `reverse(shape)` (column-major). Strides are always stored in row-major order to match the ASDF/Python convention. +# Fields + +| Field | Description | +| :-------------------- | :---------- | +| `lazy_block_headers` | Reference to the file's block header list. Used to resolve `source` indices at materialization time. | +| `source` | Zero-based index of the backing binary block, or `nothing` for inline arrays. | +| `data` | In-memory array for inline data, or `nothing` for block-backed arrays. | +| `shape` | Array dimensions in **Python/C (row-major) order** — outermost dimension first. The equivalent Julia shape is `reverse(shape)`. | +| `datatype` | Element type, as an [`ASDF.Datatype`](@ref) enum value. Convert to a Julia type with `Type(datatype)`. | +| `byteorder` | Byte order of the stored data (`Byteorder_little` or `Byteorder_big`). | +| `offset` | Byte offset from the start of the block to the first array element. Non-negative. | +| `strides` | Byte strides in **Python/C (row-major) order**. Must all be positive and `length(strides) == length(shape)`. | """ struct NDArray lazy_block_headers::LazyBlockHeaders - source::Union{Nothing,Int64,AbstractString} data::Union{Nothing,AbstractArray} shape::Vector{Int64} @@ -545,7 +608,14 @@ end """ NDArrayChunk -A positioned tile within an [`ASDF.ChunkedNDArray`](@ref). `start` gives the zero-based origin of this chunk in the coordinate space of the parent array (Python/C order). +A positioned tile within an [`ASDF.ChunkedNDArray`](@ref), pairing an [`ASDF.NDArray`](@ref) with a zero-based origin that locates the tile in the parent array's coordinate space. + +# Fields + +| Field | Description | +| :-------- | :---------- | +| `start` | Zero-based origin of the tile in **Python/C (row-major) order**, outermost dimension first. All elements must be non-negative. `length(start)` must equal `length(ndarray.strides)`. | +| `ndarray` | The tile data, including its own shape, datatype, byte order, and backing block reference. | """ struct NDArrayChunk start::Vector{Int64} @@ -575,7 +645,17 @@ end """ ChunkedNDArray -An array composed of arbitrarily-positioned tiles. Gaps between chunks are left uninitialised. Overlapping chunks are written in order. +A logical N-dimensional array assembled from a collection of arbitrarily-positioned [`ASDF.NDArrayChunk`](@ref) tiles, each backed by its own binary block or inline data. + +`ChunkedNDArray` is the in-memory representation of a `!core/chunked-ndarray-1.X.Y` YAML node. It defines the shape and element type of the full logical array, but defers all data access to the individual chunks. The full array is only allocated and populated when [`Base.getindex(ndarray::NDArray)`](@ref) is called. + +# Fields + +| Field | Description | +| :--------- | :---------- | +| `shape` | Dimensions of the full logical array in **Python/C (row-major) order**, outermost dimension first. All elements must be non-negative. The equivalent Julia shape is `reverse(shape)`. | +| `datatype` | Element type shared by all chunks. Convert to a Julia type with `Type(datatype)`. | +| `chunks` | Ordered collection of tiles that together populate the logical array. Tiles are written in iteration order when materializing. Later tiles overwrite earlier ones in any overlapping region. | """ struct ChunkedNDArray shape::Vector{Int64} @@ -717,7 +797,7 @@ end """ ASDFLibrary -Software provenance metadata, serialised as a `!core/software-1.0.0 YAML` tag. [`ASDF.write_file`] inserts an entry automatically under the key `"asdf/library"` if one is not already present, using the package's own name, author, homepage, and version. +Software provenance metadata, serialized as a `!core/software-1.0.0 YAML` tag. [`ASDF.write_file`] inserts an entry automatically under the key `"asdf/library"` if one is not already present, using the package's own name, author, homepage, and version. """ struct ASDFLibrary name::AbstractString @@ -838,7 +918,7 @@ end """ write_file(filename::AbstractString, document::Dict{Any,Any}) -Writes an ASDF file to disk. `document` is a plain `Dict` whose values may include [`NDArrayWrapper`](@ref) instances. These are serialised as binary blocks with appropriate compression. +Writes an ASDF file to disk. `document` is a plain `Dict` whose values may include [`NDArrayWrapper`](@ref) instances. These are serialized as binary blocks with appropriate compression. Layout of the output file: From 898558e4f34ee6f65d9690b34ba9b67801bb0ad3 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Mon, 23 Mar 2026 03:07:23 -0700 Subject: [PATCH 33/40] docs: more docstrings --- src/ASDF.jl | 57 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index 1b4f8fa..50e64d8 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -106,6 +106,27 @@ struct BlockHeader validate_checksum::Bool end +""" + LazyBlockHeaders + +A mutable container holding the complete list of [`ASDF.BlockHeader`](@ref) values scanned from an ASDF file, shared by reference with every [`ASDF.NDArray`](@ref) +and [`ASDF.ChunkedNDArray`](@ref) constructed during parsing. + +# Reference sharing + +Every [`ASDF.NDArray`](@ref) created during parsing of a given file holds a reference to the same `LazyBlockHeaders` instance. When `ndarray[]` is called, it indexes into `lazy_block_headers.block_headers` using the array's zero-based `source` field: + +```julia +header = ndarray.lazy_block_headers.block_headers[ndarray.source + 1] +data = ASDF.read_block(header) +``` + +Because `block_headers` is populated after all `NDArray` objects are constructed, no array needs to be updated individually when block scanning completes. The shared mutable reference propagates the result automatically. + +# Mutability + +`LazyBlockHeaders` is a `mutable struct` solely to allow `block_headers` to be assigned after construction. The field is written exactly once per file load, immediately after `YAML.load` within [`ASDF.load_file`](@ref) returns. It is never modified again during normal use. Treat it as effectively immutable after [`load_file`](@ref) returns. +""" mutable struct LazyBlockHeaders block_headers::Vector{BlockHeader} LazyBlockHeaders() = new(BlockHeader[]) @@ -717,9 +738,15 @@ end """ ASDFFile -An open ASDF file. `metadata` is the parsed YAML tree. Any [`ASDF.NDArray`](@ref) or [`ASDF.ChunkedNDArray`](@ref) nodes appear as values and are lazily backed by lazy_block_headers. +The in-memory representation of a loaded ASDF file, combining the parsed YAML metadata tree with the binary block infrastructure needed to lazily materialize any array data it references. + +# Fields -`YAML.write(file::ASDFFile)` returns a human-readable summary string. +| Field | Description | +| :-------------------- | :---------- | +| `filename` | Path of the file on disk, as passed to [`ASDF.load_file`](@ref). Used for display and diagnostics only. The file is not kept open after loading. | +| `metadata` | The fully parsed YAML tree. Keys are strings matching the top-level YAML keys. Values may be any Julia type produced by the YAML constructors, including [`ASDF.NDArray`](@ref), [`ASDF.ChunkedNDArray`](@ref), nested `Dict`s, `Vector`s, and scalars. | +| `lazy_block_headers` | All binary block headers found in the file, scanned once at load time. Shared by reference with every [`ASDF.NDArray`](@ref) in `metadata`, allowing them to locate and read their backing blocks on demand. | """ struct ASDFFile filename::AbstractString @@ -746,12 +773,8 @@ Reads an ASDF file from disk. Block data is located lazily. Block headers are scanned after the YAML is parsed, and array data (`ndarray`) is read only when [`Base.getindex(ndarray::NDArray)`](@ref) is called, i.e., `ndarray[]`. -Supported YAML tags: - -- `tag:stsci.edu:asdf/core/ndarray-1.0.0` -- `tag:stsci.edu:asdf/core/ndarray-1.1.0` -- `tag:stsci.edu:asdf/core/ndarray-chunk-1.0.0` -- `tag:stsci.edu:asdf/core/chunked-ndarray-1.0.0` +!!! note "File handle lifetime" + The file handle opened by `load_file` is retained for the lifetime of the returned [`ASDF.ASDFFile`](@ref) so that block data can be read on demand. Do not move, truncate, or overwrite the source file while any [`ASDF.NDArray`](@ref) from it may still be accessed. """ function load_file(filename::AbstractString; extensions = false, validate_checksum = true) asdf_constructors = copy(YAML.default_yaml_constructors) @@ -836,6 +859,24 @@ function NDArrayWrapper(array::AbstractArray; compression::Compression=C_Bzip2, end Base.getindex(val::NDArrayWrapper) = val.array +""" + Blocks + +Module-level accumulator that collects [`ASDF.NDArrayWrapper`](@ref) values and their corresponding file positions during a single call to [`ASDF.write_file`](@ref). + +`Blocks` acts as a two-phase write buffer. In the first phase, as the YAML tree is serialized, each non-inline [`ASDF.NDArrayWrapper`](@ref) appends itself to `arrays` and reserves a sequential source index. In the second phase, [`ASDF.write_file`](@ref) iterates over `arrays`, compresses and writes each block to disk, and records the resulting file offsets in `positions`. The finalized `positions` vector is then written as the ASDF block index at the end of the file. + +!!! warning "Not thread-safe" + A single instance of `Blocks` is held in the module-level constant `ASDF.blocks`. Because this global state is mutated by [`ASDF.write_file`](@ref), concurrent calls to `write_file` from multiple threads will corrupt each other's block lists. Do not call `write_file` concurrently. + +# Fields + +| Field | Description | +| :---------- | :---------- | +| `arrays` | Ordered list of arrays to be written as binary blocks, accumulated during YAML serialisation. The position of each wrapper in this vector is its zero-based block source index. | +| `positions` | Absolute byte offsets of each written block within the output file, populated during the block-writing phase of [`ASDF.write_file`](@ref). `positions[i]` corresponds to `arrays[i]`. | + +""" struct Blocks arrays::Vector{NDArrayWrapper} positions::Vector{Int64} From 02c5599800fb3a13d61146a8e977301da86d2dcb Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Mon, 23 Mar 2026 03:16:44 -0700 Subject: [PATCH 34/40] docs: add FileIO.jl todo for public API --- docs/src/api.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/api.md b/docs/src/api.md index 78cdb17..c7aaa23 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -2,6 +2,9 @@ ## Public +!!! todo + `load` / `save` from FileIO.jl. [#26](https://github.com/JuliaAstro/ASDF.jl/pull/26) + ## Private ```@autodocs Modules = [ASDF] From a3ed4cdff8e1d8969851fc197f734ab3f4db78ec Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 16:24:58 -0700 Subject: [PATCH 35/40] feature: add convenience setindex! and save method --- src/ASDF.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ASDF.jl b/src/ASDF.jl index ef686bf..d033029 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -813,6 +813,7 @@ function YAML.write(file::ASDFFile) end Base.getindex(af::ASDFFile, key) = af.metadata[key] +Base.setindex!(af::ASDFFile, value, key) = (af.metadata[key] = value) struct ASDFTreeNode key::Any @@ -1344,6 +1345,8 @@ function fileio_save(f::File{format"ASDF"}, data) return write_file(f.filename, data) end +fileio_save(f::File{format"ASDF"}, af::ASDFFile) = save(f, af.metadata) + @doc (@doc fileio_save) save end From e3a5ad7842819256f6abc603c146c3a62312fcd9 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 16:25:38 -0700 Subject: [PATCH 36/40] build: switch to master branch of FileIO.jl now that PR is merged --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a6286f4..3ca4184 100644 --- a/Project.toml +++ b/Project.toml @@ -23,7 +23,7 @@ YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" # TODO: Remove after https://github.com/JuliaIO/FileIO.jl/pull/427 [sources] -FileIO = {url = "https://github.com/icweaver/FileIO.jl", rev = "asdf"} +FileIO = {url = "https://github.com/JuliaIO/FileIO.jl", rev = "master"} [compat] AbstractTrees = "0.4.5" From aad118548c5fc6e661a8a759fa0809bff7369dfa Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 16:26:12 -0700 Subject: [PATCH 37/40] docs: update index and intro --- docs/src/index.md | 31 +++--- docs/src/intro.md | 261 +++++++++++++++++++++++++++++++++------------- 2 files changed, 204 insertions(+), 88 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index fae5cc6..dad8695 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -9,27 +9,26 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs ### Installation ```julia-repl -pkg> add ASDF +pkg> add ASDF, OrderedCollections ``` ### Usage -```julia -using ASDF -``` +```@repl +using ASDF, OrderedCollections -```julia -# Load -af = ASDF.load_file("") -``` +doc = OrderedDict( + "field_1" => [5, 6, 7, 8], + "field_2" => ["up", "down", "left", "right"], + "field_3" => OrderedDict( + "field_3a" => ["apple", "orange", "pear"], + "field_3b" => [1.0, 2.0, 3.0], + ) +); -```julia -# Acess and modify -af.metadata[""] = -``` +save("example.asdf", doc) + +af = load("example.asdf") -```julia -# Write -doc = Dict() -ASDF.write_file("", doc) +ASDF.info(af; max_rows = 3) ``` diff --git a/docs/src/intro.md b/docs/src/intro.md index 581fc1c..8d799b7 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -10,105 +10,222 @@ The ASDF file format targets a similar audience as the [HDF5](https://www.hdfgro ## Getting started -ASDF files are initially created as a nested dictionary with your specified keys: +ASDF files are initially created as a dictionary with arbitrarily nested data: -```@example intro -af_payload = Dict{Any, Any}( # To-do: see if type signature needs to be this general - "meta" => Dict("my" => Dict("nested" => "metadata")), - "data" => [1, 2, 3, 4], -) +```jldoctest intro +julia> using OrderedCollections + +julia> af_payload = OrderedDict("field_1" => [5, 6, 7, 8], "field_2" => ["up", "down", "left", "right"], "field_3" => OrderedDict("field_3a" => ["apple", "orange", "pear"], "field_3b" => [1.0, 2.0, 3.0])); ``` -Next, this dictionary can be written to the ASDF file format with `ASDF.write_file`: +!!! note + We use an `OrderedDict` from [OrderedCollections.jl](https://github.com/JuliaCollections/OrderedCollections.jl) to preserve the order of data on write and load, and to maximize compatibility with YAML.jl. For a potential alternative using [Dictionaries.jl](https://github.com/andyferris/Dictionaries.jl), see this [experimental branch](https://github.com/JuliaAstro/ASDF.jl/tree/dictionaries). -```@example intro -using ASDF +ASDF.jl is registered with [FileIO.jl](https://juliaio.github.io/FileIO.jl/stable/), so this data can be written to the ASDF file format with the generic [`save`](@ref) function: -data_dir = joinpath("..", "data") -mkpath(data_dir) -fpath = joinpath(data_dir, "my_file.asdf") +```jldoctest intro +julia> using ASDF -ASDF.write_file(fpath, af_payload) +julia> save("intro.asdf", af_payload) ``` -which contains the following file contents: - -```yaml -#ASDF 1.0.0 -#ASDF_STANDARD 1.2.0 -# This is an ASDF file -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- -!core/asdf-1.1.0 -meta: - my: - nested: "metadata" -data: - - 1 - - 2 - - 3 - - 4 -asdf/library: !core/software-1.0.0 - version: "2.0.0" - name: "ASDF.jl" - author: "Erik Schnetter " - homepage: "https://github.com/JuliaAstro/ASDF.jl" -... -#ASDF BLOCK INDEX -%YAML 1.1 ---- -[] -... +The saved file contains the following human-readable contents: + +!!! details "View file" + ```jldoctest intro + julia> read("intro.asdf", String) |> print + #ASDF 1.0.0 + #ASDF_STANDARD 1.2.0 + # This is an ASDF file + %YAML 1.1 + %TAG ! tag:stsci.edu:asdf/ + --- + !core/asdf-1.1.0 + field_1: + - 5 + - 6 + - 7 + - 8 + field_2: + - "up" + - "down" + - "left" + - "right" + field_3: + field_3a: + - "apple" + - "orange" + - "pear" + field_3b: + - 1.0 + - 2.0 + - 3.0 + asdf/library: !core/software-1.0.0 + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" + version: "2.0.0" + ... + #ASDF BLOCK INDEX + %YAML 1.1 + --- + [] + ... + ``` + +which can be loaded back with FileIO.jl's generic [`load`](@ref) function: + +```jldoctest intro +julia> af = load("intro.asdf") +intro.asdf +├─ field_1::Vector{Int64} | shape = (4,) +├─ field_2::Vector{String} | shape = (4,) +├─ field_3::String +│ ├─ field_3a::Vector{String} | shape = (3,) +│ └─ field_3b::Vector{Float64} | shape = (3,) +└─ asdf/library::String + ├─ author::String | Erik Schnetter + ├─ homepage::String | https://github.com/JuliaAstro/ASDF.jl + ├─ name::String | ASDF.jl + └─ version::String | 2.0.0 ``` -This file can be loaded back with `ASDF.load_file`: +This is stored as an [`ASDF.ASDFFile`](@ref). To change the number of rows shown, pass this object to [`ASDF.info`](@ref): -```@example intro -af = ASDF.load_file(fpath) +```jldoctest intro +julia> ASDF.info(af; max_rows = 3) +intro.asdf +├─ field_1::Vector{Int64} | shape = (4,) +├─ field_2::Vector{String} | shape = (4,) + ⋮ (8) more rows ``` -This creates an `ASDF.ASDFFile` object which contains a `meta` field. This is a new dictionary that merges information about this library (stored under the `asdf/library` key) with the original user-defined `af_payload` dictionary: +It contains a `metadata` field, which is a new dictionary that merges information about this library (stored under the `asdf/library` key) with the original user-defined `af_payload` dictionary. For convenience, `af.metadata[]` can be accessed directly as `af[key]`. Since the underlying data is a dictionary, it can be modified in the standard way: -```@example intro -af.metadata -``` +```jldoctest intro +julia> af["field_1"] = [50, 60, 70, 80]; -```@example intro -af.metadata["asdf/library"] +julia> af["field_1"] +4-element Vector{Int64}: + 50 + 60 + 70 + 80 ``` -Since the underlying data is a dictionary, it can be modified in the standard way: +The convenience syntax can also be used to save the modified `ASDF.ASDFFile` object directly: -```@example intro -af.metadata["meta"]["my"]["nested2"] = "metadata2" - -af.metadata +```jldoctest intro +julia> save("intro_modified.asdf", af) ``` -## Array storage +!!! details "View file" + ```jldoctest intro + julia> read("intro_modified.asdf", String) |> print + #ASDF 1.0.0 + #ASDF_STANDARD 1.2.0 + # This is an ASDF file + %YAML 1.1 + %TAG ! tag:stsci.edu:asdf/ + --- + !core/asdf-1.1.0 + field_1: + - 50 + - 60 + - 70 + - 80 + field_2: + - "up" + - "down" + - "left" + - "right" + field_3: + field_3a: + - "apple" + - "orange" + - "pear" + field_3b: + - 1.0 + - 2.0 + - 3.0 + asdf/library: !core/software-1.0.0 + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" + version: "2.0.0" + ... + #ASDF BLOCK INDEX + %YAML 1.1 + --- + [] + ... + ``` -By default, array data is written inline as a literal to the ASDF file. This can be stored and later accessed more efficiently by wrapping your data in an `ASDF.NDArrayWrapper`. This allows for your data to be stored as a binary via the `inline = false` keyword, which can be further optimized by specifying a supported [compression algorithm](@ref ASDF.Compression) to use via the `compression` keyword: +## Array storage -```@example intro -af_payload = Dict{Any, Any}( - "meta" => Dict("my" => Dict("nested" => "metadata")), - # Default - "data" => ASDF.NDArrayWrapper([1, 2, 3, 4]; inline = false, compression = ASDF.C_Bzip2), -) +By default, array data is written inline as a literal to the ASDF file. This can be stored and later accessed more efficiently by wrapping your data in an [`ASDF.NDArrayWrapper`](@ref). This allows for your data to be stored as a binary via the `inline = false` keyword (default), which can be further optimized by specifying a supported [compression algorithm](@ref ASDF.Compression) to use via the `compression` keyword: -fpath = joinpath(data_dir, "my_file_compressed.asdf") -ASDF.write_file(fpath, af_payload) -``` +```jldoctest intro +julia> af_payload = OrderedDict("meta" => OrderedDict("my" => OrderedDict("nested" => "metadata")), "data" => ASDF.NDArrayWrapper([1, 2, 3, 4]; compression = ASDF.C_Bzip2)); -Saving your data as an `NDArrayWrapper` allows for it to be lazily accessed as a strided view. To access the underlying data, use the `[]` (dereference) syntax: +julia> save("intro_compressed.asdf", af_payload) -```@example intro -af = ASDF.load_file(fpath) +julia> af = load("intro_compressed.asdf") +intro_compressed.asdf +├─ meta::String +│ └─ my::String +│ └─ nested::String | metadata +├─ data::ASDF.NDArray | shape = [4] +└─ asdf/library::String + ├─ author::String | Erik Schnetter + ├─ homepage::String | https://github.com/JuliaAstro/ASDF.jl + ├─ name::String | ASDF.jl + └─ version::String | 2.0.0 +``` -af.metadata["data"][] +!!! details "View file" + ```julia-repl + julia> read("intro_compressed.asdf", String) |> print + #ASDF 1.0.0 + #ASDF_STANDARD 1.2.0 + # This is an ASDF file + %YAML 1.1 + %TAG ! tag:stsci.edu:asdf/ + --- + !core/asdf-1.1.0 + meta: + my: + nested: "metadata" + data: !core/ndarray-1.0.0 + source: 0 + shape: + - 4 + datatype: "int64" + byteorder: "little" + asdf/library: !core/software-1.0.0 + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" + version: "2.0.0" + ... + �BLK0 f�0xj�sq���r#ASDF BLOCK INDEX + %YAML 1.1 + --- + [463,] + ... + ``` + +Using `NDArrayWrapper` allows for the wrapped data to be lazily accessed as a strided view. To access the underlying data, use the `[]` (dereference) syntax: + +```jldoctest intro +julia> af["data"][] +4-element reshape(reinterpret(Int64, ::StridedViews.StridedView{UInt8, 2, Memory{UInt8}, typeof(identity)}), 4) with eltype Int64: + 1 + 2 + 3 + 4 ``` ## Tagged objects -Coming soon. Supporting custom objects, extensions. +Come back soon to see how custom Julia objects can be handled in ASDF.jl. From bcc13c0e673e327ffc63c251ed1c4c97891b4106 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 16:58:24 -0700 Subject: [PATCH 38/40] fix: forward kwargs to load command + update interpolation in docstrings to work with Documenter --- src/ASDF.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index d033029..93375ec 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -841,7 +841,7 @@ Display up to `max_rows` lines of `af` tree. `Base.show` calls this function int ```jldoctest julia> using OrderedCollections: OrderedDict -julia> doc = OrderedDict("field_\$(i)" => rand(10) for i in 1:25); +julia> doc = OrderedDict(string("field_", i) => rand(10) for i in 1:25); julia> save("long.asdf", doc) @@ -996,7 +996,7 @@ Load an asdf file at filepath `f`. ```jldoctest julia> using OrderedCollections: OrderedDict -julia> doc = OrderedDict("field_\$(i)" => rand(10) for i in 1:5); # Create some sample data +julia> doc = OrderedDict(string("field_", i) => rand(10) for i in 1:5); # Create some sample data julia> save("myfile.asdf", doc) @@ -1014,8 +1014,8 @@ myfile.asdf └─ version::String | 2.0.0 ``` """ -function fileio_load(f::File{format"ASDF"}) - return load_file(f.filename) +function fileio_load(f::File{format"ASDF"}; kwargs...) + return load_file(f.filename; kwargs...) end @doc (@doc fileio_load) load @@ -1336,7 +1336,7 @@ Save `data` to an asdf file at filepath `f`. ```jldoctest julia> using OrderedCollections: OrderedDict -julia> data = OrderedDict("field_\$(i)" => rand(10) for i in 1:5); # Create some sample data +julia> data = OrderedDict(string("field_", i) => rand(10) for i in 1:5); # Create some sample data julia> save("myfile.asdf", data) ``` From 7c8453133bb2c98e3ffd93486348995bf496c3e6 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 16:58:55 -0700 Subject: [PATCH 39/40] docs: updated jwst, roman, and api docs --- docs/src/api.md | 7 +++++-- docs/src/examples/jwst.md | 18 +++++++++--------- docs/src/examples/roman.md | 28 ++++++++++++++-------------- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index c7aaa23..93b75ca 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -2,10 +2,13 @@ ## Public -!!! todo - `load` / `save` from FileIO.jl. [#26](https://github.com/JuliaAstro/ASDF.jl/pull/26) +```@autodocs +Modules = [ASDF] +Private = false +``` ## Private ```@autodocs Modules = [ASDF] +Public = false ``` diff --git a/docs/src/examples/jwst.md b/docs/src/examples/jwst.md index ee6b798..7c6d884 100644 --- a/docs/src/examples/jwst.md +++ b/docs/src/examples/jwst.md @@ -9,11 +9,13 @@ In this example, we show how to use ASDF.jl to load and view some astronomical d ## Load -```@example jwst +```@repl jwst -data_dir = joinpath("..", "..", "data") -mkpath(data_dir) -fpath = joinpath(data_dir, "jwst.asdf") +fpath = let + data_dir = joinpath("..", "..", "data") + mkpath(data_dir) + joinpath(data_dir, "jwst.asdf") +end; if !isfile(fpath) using Downloads: download @@ -22,12 +24,10 @@ if !isfile(fpath) end ``` -```@example jwst +```@repl jwst using ASDF -af = ASDF.load_file(fpath; extensions = true) - -af.metadata +af = load(fpath; extensions = true) ``` ## Plot @@ -36,7 +36,7 @@ af.metadata using CairoMakie img_sci = let - img = af.metadata["data"][] + img = af["data"][] img[img .< 0] .= 1 img end diff --git a/docs/src/examples/roman.md b/docs/src/examples/roman.md index 8e20027..3382cd5 100644 --- a/docs/src/examples/roman.md +++ b/docs/src/examples/roman.md @@ -9,10 +9,12 @@ In this example, we show how to use ASDF.jl to load and view some simulated astr ## Load -```@example roman -data_dir = joinpath("..", "..", "data") -mkpath(data_dir) -fpath = joinpath(data_dir, "roman.asdf") +```@repl roman +fpath = let + data_dir = joinpath("..", "..", "data") + mkpath(data_dir) + joinpath(data_dir, "roman.asdf") +end; if !isfile(fpath) using AWSS3, AWS @@ -29,12 +31,10 @@ if !isfile(fpath) end ``` -```@example roman +```@repl roman using ASDF -af = ASDF.load_file(fpath; extensions = true, validate_checksum = false) - -af.metadata["roman"] +af = load(fpath; extensions = true, validate_checksum = false) ``` ## Plot @@ -42,13 +42,13 @@ af.metadata["roman"] ```@example roman using CairoMakie -img = af.metadata["roman"]["data"][] +img = af["roman"]["data"][] -let - fig, ax, hm = heatmap(img[begin:1000, begin:1000]; colorscale = asinh, colorrange = (0.5, 4)) - Colorbar(fig[1, 2], hm) - fig -end +fig, ax, hm = heatmap(img[begin:1000, begin:1000]; colorscale = asinh, colorrange = (0.5, 4)) + +Colorbar(fig[1, 2], hm) + +fig ``` !!! note From f14267e6c3c989a3d15dd48ab8c54503483508a3 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 17:19:19 -0700 Subject: [PATCH 40/40] build(docs): add OrderedCollections.jl to deps --- docs/Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Project.toml b/docs/Project.toml index a692bb2..4044da6 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -5,6 +5,7 @@ AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" [compat] Documenter = "1"