diff --git a/.github/scripts/smoke_install.sh b/.github/scripts/smoke_install.sh index 70ecde3..47ed9a6 100755 --- a/.github/scripts/smoke_install.sh +++ b/.github/scripts/smoke_install.sh @@ -6,5 +6,5 @@ # here; the ceremony lives in the reusable workflow. set -e -pip install ./PyAutoConf ./PyAutoFit +pip install ./PyAutoNerves ./PyAutoFit pip install numba diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index e21998b..2eb2a1e 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -11,5 +11,5 @@ jobs: smoke: uses: PyAutoLabs/PyAutoHeart/.github/workflows/smoke-tests.yml@main with: - chain: "PyAutoConf PyAutoFit" + chain: "PyAutoNerves PyAutoFit" secrets: inherit diff --git a/config/general.yaml b/config/general.yaml index 7832c39..3fe639b 100644 --- a/config/general.yaml +++ b/config/general.yaml @@ -33,7 +33,7 @@ test: version: # The compatibility FLOOR: the oldest library release whose API this # workspace's scripts require. Preferred over workspace_version - # (autoconf/workspace.py). Bump DELIBERATELY — only when a script + # (autonerves/workspace.py). Bump DELIBERATELY — only when a script # starts needing new API — never per release. Must always name an # INSTALLABLE (non-yanked) release. minimum_library_version: 2026.7.9.1 diff --git a/markdown/chapter_1_introduction/tutorial_1_models.md b/markdown/chapter_1_introduction/tutorial_1_models.md index 4f5b485..e828bfc 100644 --- a/markdown/chapter_1_introduction/tutorial_1_models.md +++ b/markdown/chapter_1_introduction/tutorial_1_models.md @@ -80,7 +80,7 @@ import matplotlib.pyplot as plt import autofit as af ``` - .../PyAutoConf/autoconf/workspace.py:206: UserWarning: Cannot verify the workspace at HowToFit/scripts/chapter_1_introduction is compatible with the installed library version (2026.7.6.649): no `version.minimum_library_version` or `version.workspace_version` key in config/general.yaml and no version.txt at the workspace root. + .../PyAutoNerves/autonerves/workspace.py:206: UserWarning: Cannot verify the workspace at HowToFit/scripts/chapter_1_introduction is compatible with the installed library version (2026.7.6.649): no `version.minimum_library_version` or `version.workspace_version` key in config/general.yaml and no version.txt at the workspace root. If you cloned the workspace from `main` rather than a release tag, set `version.workspace_version_check: False` in config/general.yaml to silence this warning. The `main` branch updates more frequently than library releases, so version mismatches are expected and not actionable for `main`-branch users. diff --git a/markdown/chapter_1_introduction/tutorial_2_fitting_data.md b/markdown/chapter_1_introduction/tutorial_2_fitting_data.md index 71b2a15..00cd7b5 100644 --- a/markdown/chapter_1_introduction/tutorial_2_fitting_data.md +++ b/markdown/chapter_1_introduction/tutorial_2_fitting_data.md @@ -58,7 +58,7 @@ This tutorial is split into the following sections: ```python -from autoconf import setup_notebook; setup_notebook() +from autofit import setup_notebook; setup_notebook() from os import path import matplotlib.pyplot as plt diff --git a/markdown/chapter_1_introduction/tutorial_3_non_linear_search.md b/markdown/chapter_1_introduction/tutorial_3_non_linear_search.md index 3d158ff..9683d71 100644 --- a/markdown/chapter_1_introduction/tutorial_3_non_linear_search.md +++ b/markdown/chapter_1_introduction/tutorial_3_non_linear_search.md @@ -156,10 +156,10 @@ from os import path import autofit as af -from autoconf import setup_notebook; setup_notebook() +from autofit import setup_notebook; setup_notebook() ``` - .../PyAutoConf/autoconf/workspace.py:206: UserWarning: Cannot verify the workspace at HowToFit/scripts/chapter_1_introduction is compatible with the installed library version (2026.7.6.649): no `version.minimum_library_version` or `version.workspace_version` key in config/general.yaml and no version.txt at the workspace root. + .../PyAutoNerves/autonerves/workspace.py:206: UserWarning: Cannot verify the workspace at HowToFit/scripts/chapter_1_introduction is compatible with the installed library version (2026.7.6.649): no `version.minimum_library_version` or `version.workspace_version` key in config/general.yaml and no version.txt at the workspace root. If you cloned the workspace from `main` rather than a release tag, set `version.workspace_version_check: False` in config/general.yaml to silence this warning. The `main` branch updates more frequently than library releases, so version mismatches are expected and not actionable for `main`-branch users. diff --git a/markdown/chapter_1_introduction/tutorial_4_why_modeling_is_hard.md b/markdown/chapter_1_introduction/tutorial_4_why_modeling_is_hard.md index 9e6e740..38ec396 100644 --- a/markdown/chapter_1_introduction/tutorial_4_why_modeling_is_hard.md +++ b/markdown/chapter_1_introduction/tutorial_4_why_modeling_is_hard.md @@ -47,7 +47,7 @@ __Contents__ ```python -from autoconf import setup_notebook; setup_notebook() +from autofit import setup_notebook; setup_notebook() from os import path import numpy as np diff --git a/markdown/chapter_1_introduction/tutorial_5_results_and_samples.md b/markdown/chapter_1_introduction/tutorial_5_results_and_samples.md index bc38c57..61aba63 100644 --- a/markdown/chapter_1_introduction/tutorial_5_results_and_samples.md +++ b/markdown/chapter_1_introduction/tutorial_5_results_and_samples.md @@ -39,7 +39,7 @@ This tutorial is split into the following sections: ```python -from autoconf import setup_notebook; setup_notebook() +from autofit import setup_notebook; setup_notebook() import autofit as af import autofit.plot as aplt diff --git a/notebooks/chapter_1_introduction/tutorial_2_fitting_data.ipynb b/notebooks/chapter_1_introduction/tutorial_2_fitting_data.ipynb index 068a177..58d759c 100644 --- a/notebooks/chapter_1_introduction/tutorial_2_fitting_data.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_2_fitting_data.ipynb @@ -100,7 +100,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "from os import path\n", "import matplotlib.pyplot as plt\n", diff --git a/notebooks/chapter_1_introduction/tutorial_3_non_linear_search.ipynb b/notebooks/chapter_1_introduction/tutorial_3_non_linear_search.ipynb index a870c9e..6152757 100644 --- a/notebooks/chapter_1_introduction/tutorial_3_non_linear_search.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_3_non_linear_search.ipynb @@ -198,7 +198,7 @@ "\n", "import autofit as af\n", "\n", - "# from autoconf import setup_notebook; setup_notebook()" + "# from autofit import setup_notebook; setup_notebook()" ], "outputs": [], "execution_count": null diff --git a/notebooks/chapter_1_introduction/tutorial_4_why_modeling_is_hard.ipynb b/notebooks/chapter_1_introduction/tutorial_4_why_modeling_is_hard.ipynb index 15134ac..8a2dc26 100644 --- a/notebooks/chapter_1_introduction/tutorial_4_why_modeling_is_hard.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_4_why_modeling_is_hard.ipynb @@ -89,7 +89,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "from os import path\n", "import numpy as np\n", diff --git a/notebooks/chapter_1_introduction/tutorial_5_results_and_samples.ipynb b/notebooks/chapter_1_introduction/tutorial_5_results_and_samples.ipynb index 299bd2c..a602028 100644 --- a/notebooks/chapter_1_introduction/tutorial_5_results_and_samples.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_5_results_and_samples.ipynb @@ -81,7 +81,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import autofit as af\n", "import autofit.plot as aplt\n", diff --git a/notebooks/chapter_3_graphical_models/tutorial_1_individual_models.ipynb b/notebooks/chapter_3_graphical_models/tutorial_1_individual_models.ipynb index 6ee9fdc..169158a 100644 --- a/notebooks/chapter_3_graphical_models/tutorial_1_individual_models.ipynb +++ b/notebooks/chapter_3_graphical_models/tutorial_1_individual_models.ipynb @@ -98,7 +98,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "from os import path\n", "import numpy as np\n", diff --git a/notebooks/chapter_3_graphical_models/tutorial_2_graphical_model.ipynb b/notebooks/chapter_3_graphical_models/tutorial_2_graphical_model.ipynb index c6c495e..1fa63c1 100644 --- a/notebooks/chapter_3_graphical_models/tutorial_2_graphical_model.ipynb +++ b/notebooks/chapter_3_graphical_models/tutorial_2_graphical_model.ipynb @@ -82,7 +82,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from os import path\n", diff --git a/notebooks/chapter_3_graphical_models/tutorial_3_graphical_benefits.ipynb b/notebooks/chapter_3_graphical_models/tutorial_3_graphical_benefits.ipynb index f6828f7..ae32fae 100644 --- a/notebooks/chapter_3_graphical_models/tutorial_3_graphical_benefits.ipynb +++ b/notebooks/chapter_3_graphical_models/tutorial_3_graphical_benefits.ipynb @@ -85,7 +85,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from os import path\n", diff --git a/notebooks/chapter_3_graphical_models/tutorial_4_hierachical_models.ipynb b/notebooks/chapter_3_graphical_models/tutorial_4_hierachical_models.ipynb index 74c10e3..f471626 100644 --- a/notebooks/chapter_3_graphical_models/tutorial_4_hierachical_models.ipynb +++ b/notebooks/chapter_3_graphical_models/tutorial_4_hierachical_models.ipynb @@ -82,7 +82,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from os import path\n", diff --git a/notebooks/chapter_3_graphical_models/tutorial_5_expectation_propagation.ipynb b/notebooks/chapter_3_graphical_models/tutorial_5_expectation_propagation.ipynb index c166f9e..d13b94d 100644 --- a/notebooks/chapter_3_graphical_models/tutorial_5_expectation_propagation.ipynb +++ b/notebooks/chapter_3_graphical_models/tutorial_5_expectation_propagation.ipynb @@ -82,7 +82,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from os import path\n", diff --git a/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.ipynb b/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.ipynb index ce1ed0d..08cd26d 100644 --- a/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.ipynb +++ b/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.ipynb @@ -67,7 +67,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from os import path\n", diff --git a/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.ipynb b/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.ipynb index 9bb2980..dada37d 100644 --- a/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.ipynb +++ b/notebooks/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.ipynb @@ -72,7 +72,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from os import path\n", diff --git a/notebooks/simulators/simulators.ipynb b/notebooks/simulators/simulators.ipynb index d4ea99f..eca37e0 100644 --- a/notebooks/simulators/simulators.ipynb +++ b/notebooks/simulators/simulators.ipynb @@ -73,7 +73,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import util\n", "from os import path\n", diff --git a/notebooks/simulators/simulators_sample.ipynb b/notebooks/simulators/simulators_sample.ipynb index f710adb..970af56 100644 --- a/notebooks/simulators/simulators_sample.ipynb +++ b/notebooks/simulators/simulators_sample.ipynb @@ -60,7 +60,7 @@ "metadata": {}, "source": [ "\n", - "# from autoconf import setup_notebook; setup_notebook()\n", + "# from autofit import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from os import path\n", diff --git a/notebooks/simulators/util.ipynb b/notebooks/simulators/util.ipynb index 7273e96..45fe4b5 100644 --- a/notebooks/simulators/util.ipynb +++ b/notebooks/simulators/util.ipynb @@ -41,7 +41,7 @@ "cell_type": "code", "metadata": {}, "source": [ - "from autoconf.dictable import to_dict\n", + "from autofit import to_dict\n", "\n", "import autofit as af\n", "\n", diff --git a/scripts/chapter_1_introduction/tutorial_2_fitting_data.py b/scripts/chapter_1_introduction/tutorial_2_fitting_data.py index 4f6ae1e..4be9b87 100644 --- a/scripts/chapter_1_introduction/tutorial_2_fitting_data.py +++ b/scripts/chapter_1_introduction/tutorial_2_fitting_data.py @@ -1,653 +1,653 @@ -""" -Tutorial 2: Fitting Data -======================== - -We've learned that a model consists of equations, numerical processes, and assumptions that describe a physical system. -Using **PyAutoFit**, we defined simple 1D models like the Gaussian, composed them into models using `Model` and -`Collection` objects, and generated model data by varying their parameters. - -To apply our model to real-world situations, we must fit it to data. Fitting involves assessing how well the model -matches observed data. A good fit indicates that the model's parameter values accurately describe the physical system. -Conversely, a poor fit suggests that adjustments are needed to better reflect reality. - -Model-fitting is a cyclical process: define the model, fit it to data, and refine the model based on insights gained. -Iteratively improving the model's complexity enhances its ability to accurately represent the system under study. -This iterative process lies at the core of model-fitting in scientific analysis. - -__Astronomy Example__ - -In Astronomy, this process has been crucial for understanding the distributions of stars within galaxies. By -fitting high-quality images of galaxies with increasingly sophisticated models, astronomers have determined that -stars within galaxies are organized into structures such as disks, bars, and bulges. This approach has also revealed -that stars appear differently in red and blue images due to variations in their age and composition. - -__Overview__ - -In this tutorial, we will explore how to fit the `model_data` generated by a model to actual data. Specifically, we will: - -- Load data representing a 1D Gaussian signal, which serves as our target dataset for fitting. - -- Compute quantities such as residuals by subtracting the model data from the observed data. - -- Quantitatively assess the goodness-of-fit using a critical measure in model-fitting known as the `log_likelihood`. - -All these steps will utilize the **PyAutoFit** API for model composition, introduced in the previous tutorial. - -__Contents__ - -This tutorial is split into the following sections: - -- **Data**: Load and plot the 1D Gaussian dataset we will fit. -- **Model Data**: Generate model data of the `Gaussian` model using a forward model. -- **Residuals**: Compute and visualize residuals between the model data and observed data. -- **Normalized Residuals**: Compute and visualize normalized residuals, which account for the noise properties of the data. -- **Chi Squared**: Compute and visualize the chi-squared map, a measure of the overall goodness-of-fit. -- **Noise Normalization**: Compute the noise normalization term which describes the noise properties of the data. -- **Likelihood**: Compute the log likelihood, a key measure of the goodness-of-fit of the model to the data. -- **Recap**: Summarize the standard metrics for quantifying model fit quality. -- **Fitting Models**: Fit the `Gaussian` model to the 1D data and compute the log likelihood, by guessing parameters. -- **Guess 1**: A first parameter guess with an explanation of the resulting log likelihood. -- **Guess 2**: An improved parameter guess with a better log likelihood. -- **Guess 3**: The optimal parameter guess providing the best fit to the data. -- **Extensibility**: Use the `Collection` object for fitting models with multiple components. -- **Wrap Up**: Summarize the key concepts of this tutorial. -""" - -# from autoconf import setup_notebook; setup_notebook() - -from os import path -import matplotlib.pyplot as plt -import numpy as np - -import autofit as af - -""" -__Data__ - -Our dataset consists of noisy 1D data containing a signal, where the underlying signal can be modeled using -equations such as a 1D Gaussian, a 1D Exponential, or a combination of multiple 1D profiles. - -We load this dataset from .json files, where: - -- `data` is a 1D NumPy array containing values representing the observed signal. - -- `noise_map` is a 1D NumPy array containing values representing the estimated root mean squared (RMS) noise level at - each data point. - -These datasets are generated using scripts located in `HowToFit/scripts/simulators`. Feel free to explore -these scripts for more details! -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1") - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists(dataset_path): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - -noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") -) - -""" -Next, we visualize the 1D signal using `matplotlib`. - -The signal is observed over uniformly spaced `xvalues`, computed using the `arange` function and `data.shape[0]` method. - -We will reuse these `xvalues` shortly when generating model data from the model. -""" -xvalues = np.arange(data.shape[0]) -plt.plot(xvalues, data, color="k") -plt.title("1D Dataset Containing a Gaussian.") -plt.xlabel("x values of profile") -plt.ylabel("Signal Value") -plt.show() - -""" -The earlier plot depicted only the signal without indicating the estimated noise at each data point. - -To visualize both the signal and its `noise_map`, we can use `matplotlib`'s `errorbar` function. -""" -plt.errorbar( - xvalues, - data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.title("1D Gaussian dataset with errors from the noise-map.") -plt.xlabel("x values of profile") -plt.ylabel("Signal Value") -plt.show() - - -""" -__Model Data__ - -To fit our `Gaussian` model to this data, we start by generating `model_data` from the 1D `Gaussian` model, -following the same steps as outlined in the previous tutorial. - -We begin by again defining the `Gaussian` class, following the **PyAutoFit** format for model components. -""" - - -class Gaussian: - def __init__( - self, - centre: float = 30.0, # <- **PyAutoFit** recognises these constructor arguments - normalization: float = 1.0, # <- are the Gaussian`s model parameters. - sigma: float = 5.0, - ): - """ - Represents a 1D Gaussian profile. - - This is a model-component of example models in the **HowToFit** lectures and is used to perform model-fitting - of example datasets. - - Parameters - ---------- - centre - The x coordinate of the profile centre. - normalization - Overall normalization of the profile. - sigma - The sigma value controlling the size of the Gaussian. - """ - self.centre = centre - self.normalization = normalization - self.sigma = sigma - - def model_data_from(self, xvalues: np.ndarray) -> np.ndarray: - """ - Returns a 1D Gaussian on an input list of Cartesian x coordinates. - - The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. - - The output is referred to as the `model_data` to signify that it is a representation of the data from the - model. - - Parameters - ---------- - xvalues - The x coordinates in the original reference frame of the data. - - Returns - ------- - np.array - The Gaussian values at the input x coordinates. - """ - transformed_xvalues = np.subtract(xvalues, self.centre) - return np.multiply( - np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), - np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), - ) - - -""" -To create `model_data` for the `Gaussian`, we use the model by providing it with `xvalues` corresponding to the -observed data, as demonstrated in the previous tutorial. - -The following code essentially utilizes a forward model to generate the model data based on a specified set of -parameters. -""" -model = af.Model(Gaussian) - -gaussian = model.instance_from_vector(vector=[60.0, 20.0, 15.0]) - -model_data = gaussian.model_data_from(xvalues=xvalues) - -plt.plot(xvalues, model_data, color="r") -plt.title("1D Gaussian model.") -plt.xlabel("x values of profile") -plt.ylabel("Profile Normalization") -plt.show() -plt.clf() - -""" -For comparison purposes, it is more informative to plot both the `data` and `model_data` on the same plot. -""" -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.plot(xvalues, model_data, color="r") -plt.title("Model-data fit to 1D Gaussian data.") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.close() - -""" -Changing the values of `centre`, `normalization`, and `sigma` alters the appearance of the `Gaussian`. - -You can modify the parameters passed into `instance_from_vector()` above. After recomputing the `model_data`, plot -it again to observe how these changes affect the Gaussian's appearance. - -__Residuals__ - -While it's informative to compare the `data` and `model_data` above, gaining insights from the residuals can be even -more useful. - -Residuals are calculated as `data - model_data` in 1D: -""" -residual_map = data - model_data -plt.plot(xvalues, residual_map, color="k") -plt.title("Residuals of model-data fit to 1D Gaussian data.") -plt.xlabel("x values of profile") -plt.ylabel("Residuals") -plt.show() -plt.clf() - -""" -Are these residuals indicative of a good fit to the data? Without considering the noise in the data, it's difficult -to ascertain. - -We can plot the residuals with error bars based on the noise map. The plot below reveals that the model is a poor fit, -as many residuals deviate significantly from zero even after accounting for the noise in each data point. - -A blue line through zero is included on the plot, to make it clear where residuals are not constent with zero -above the noise level. -""" -residual_map = data - model_data -plt.plot(range(data.shape[0]), np.zeros(data.shape[0]), "--", color="b") -plt.errorbar( - x=xvalues, - y=residual_map, - yerr=noise_map, - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - linestyle="", -) -plt.title("Residuals of model-data fit to 1D Gaussian data.") -plt.xlabel("x values of profile") -plt.ylabel("Residuals") -plt.show() -plt.clf() - -""" -__Normalized Residuals__ - -Another method to quantify and visualize the quality of the fit is using the normalized residual map, also known as -standardized residuals. - -The normalized residual map is computed as the residual map divided by the noise map: - -\[ \text{normalized\_residual} = \frac{\text{residual\_map}}{\text{noise\_map}} = \frac{\text{data} - \text{model\_data}}{\text{noise\_map}} \] - -If you're familiar with the concept of standard deviations (sigma) in statistics, the normalized residual map represents -how many standard deviations the residual is from zero. For instance, a normalized residual of 2.0 (corresponding -to a 95% confidence interval) means that the probability of the model underestimating the data by that amount is only 5%. - -Both the residual map with error bars and the normalized residual map convey the same information. However, -the normalized residual map is particularly useful for visualization in multidimensional problems, as plotting -error bars in 2D or higher dimensions is not straightforward. -""" -normalized_residual_map = residual_map / noise_map -plt.plot(xvalues, normalized_residual_map, color="k") -plt.title("Normalized residuals of model-data fit to 1D Gaussian data.") -plt.xlabel("x values of profile") -plt.ylabel("Normalized Residuals") -plt.show() -plt.clf() - -""" -__Chi Squared__ - -Next, we define the `chi_squared_map`, which is obtained by squaring the `normalized_residual_map` and serves as a -measure of goodness of fit. - -The chi-squared map is calculated as: - -\[ \chi^2 = \left(\frac{\text{data} - \text{model\_data}}{\text{noise\_map}}\right)^2 \] - -The purpose of squaring the normalized residual map is to ensure all values are positive. For instance, both a -normalized residual of -0.2 and 0.2 would square to 0.04, indicating the same level of fit in terms of `chi_squared`. - -As seen from the normalized residual map, it's evident that the model does not provide a good fit to the data. -""" -chi_squared_map = (normalized_residual_map) ** 2 -plt.plot(xvalues, chi_squared_map, color="k") -plt.title("Chi-Squared Map of model-data fit to 1D Gaussian data.") -plt.xlabel("x values of profile") -plt.ylabel("Chi-Squareds") -plt.show() -plt.clf() - -""" -Now, we consolidate all the information in our `chi_squared_map` into a single measure of goodness-of-fit -called `chi_squared`. - -It is defined as the sum of all values in the `chi_squared_map` and is computed as: - -\[ \chi^2 = \sum \left(\frac{\text{data} - \text{model\_data}}{\text{noise\_map}}\right)^2 \] - -This summing process highlights why ensuring all values in the chi-squared map are positive is crucial. If we -didn't square the values (making them positive), positive and negative residuals would cancel each other out, -leading to an inaccurate assessment of the model's fit to the data. -""" -chi_squared = np.sum(chi_squared_map) -print("Chi-squared = ", chi_squared) - -""" - -The lower the `chi_squared`, the fewer residuals exist between the model's fit and the data, indicating a better -overall fit! - -__Noise Normalization__ - -Next, we introduce another quantity that contributes to our final assessment of the goodness-of-fit: -the `noise_normalization`. - -The `noise_normalization` is computed as the logarithm of the sum of squared noise values in our data: - -\[ -\text{{noise\_normalization}} = \sum \log(2 \pi \text{{noise\_map}}^2) -\] - -This quantity is fixed because the noise-map remains constant throughout the fitting process. Despite this, -including the `noise_normalization` is considered good practice due to its statistical significance. - -Understanding the exact meaning of `noise_normalization` isn't critical for our primary goal of successfully -fitting a model to a dataset. Essentially, it provides a measure of how well the noise properties of our data align -with a Gaussian distribution. -""" -noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) - -""" -__Likelihood__ - -From the `chi_squared` and `noise_normalization`, we can define a final goodness-of-fit measure known as -the `log_likelihood`. - -This measure is calculated by taking the sum of the `chi_squared` and `noise_normalization`, and then multiplying the -result by -0.5: - -\[ \text{log\_likelihood} = -0.5 \times \left( \chi^2 + \text{noise\_normalization} \right) \] - -Why multiply by -0.5? The exact rationale behind this factor isn't critical for our current understanding. -""" -log_likelihood = -0.5 * (chi_squared + noise_normalization) -print("Log Likelihood = ", log_likelihood) - -""" -Above, we mentioned that a lower `chi_squared` indicates a better fit of the model to the data. - -When calculating the `log_likelihood`, we multiply the `chi_squared` by -0.5. Therefore, a higher log likelihood -corresponds to a better model fit. This is what we aim for when fitting models to data, we want to maximize the -log likelihood! - -__Recap__ - -If you're familiar with model-fitting, you've likely encountered terms like 'residuals', 'chi-squared', -and 'log_likelihood' before. - -These metrics are standard ways to quantify the quality of a model fit. They are applicable not only to 1D data but -also to more complex data structures like 2D images, 3D data cubes, or any other multidimensional datasets. - -If these terms are new to you, it's important to understand their meanings as they form the basis of all -model-fitting operations in **PyAutoFit** (and in statistical inference more broadly). - -Let's recap what we've learned so far: - -- We can define models, such as a 1D `Gaussian`, using Python classes that follow a specific format. - -- Models can be organized using `Collection` and `Model` objects, with parameters mapped to instances of their - respective model classes (e.g., `Gaussian`). - -- Using these model instances, we can generate model data, compare it to observed data, and quantify the - goodness-of-fit using the log likelihood. - -__Fitting Models__ - -Now, armed with this knowledge, we are ready to fit our model to our data! - -But how do we find the best-fit model, which maximizes the log likelihood? - -The simplest approach is to guess parameters. Starting with initial parameter values that yield a good -fit (i.e., a higher log likelihood), we iteratively adjust these values to refine our model until we achieve an -optimal fit. - -For a 1D `Gaussian`, this iterative process works effectively. Below, we fit three different `Gaussian` models and -identify the best-fit model—the one that matches the original dataset most closely. - -To streamline this process, I've developed functions that compute the `log_likelihood` of a model fit and visualize -the data alongside the model predictions, complete with error bars. -""" - - -def log_likelihood_from( - data: np.ndarray, noise_map: np.ndarray, model_data: np.ndarray -) -> float: - """ - Compute the log likelihood of a model fit to data given the noise map. - - Parameters - ---------- - data - The observed data. - noise_map - The root mean square noise (or uncertainty) associated with each data point. - model_data - The model's predicted data for the given data x points. - - Returns - ------- - float - The log likelihood of the model fit to the data. - """ - # Calculate residuals and normalized residuals - residual_map = data - model_data - normalized_residual_map = residual_map / noise_map - - # Compute chi-squared and noise normalization - chi_squared_map = normalized_residual_map**2 - chi_squared = np.sum(chi_squared_map) - noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) - - # Compute log likelihood - log_likelihood = -0.5 * (chi_squared + noise_normalization) - - return log_likelihood - - -def plot_model_fit( - xvalues: np.ndarray, - data: np.ndarray, - noise_map: np.ndarray, - model_data: np.ndarray, - color: str = "k", -): - """ - Plot the observed data, model predictions, and error bars. - - Parameters - ---------- - xvalues - The x-axis values where the data is observed and model is predicted. - data - The observed data points. - noise_map - The root mean squared noise (or uncertainty) associated with each data point. - model_data - The model's predicted data for the given data x points. - color - The color for plotting (default is "k" for black). - """ - plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color=color, - ecolor="k", - elinewidth=1, - capsize=2, - ) - plt.plot(xvalues, model_data, color="r") - plt.title("Fit of model-data to data") - plt.xlabel("x values of profile") - plt.ylabel("Profile Value") - plt.show() - plt.clf() # Clear figure to prevent overlapping plots - - -""" -__Guess 1__ - -The first guess correctly pinpoints that the Gaussian's peak is at 50.0, but the width and normalization are off. - -The `log_likelihood` is computed and printed, however because we don't have a value to compare it to yet, its hard -to assess if it is a large or small value. -""" - -gaussian = model.instance_from_vector(vector=[50.0, 10.0, 5.0]) -model_data = gaussian.model_data_from(xvalues=xvalues) -plot_model_fit( - xvalues=xvalues, - data=data, - noise_map=noise_map, - model_data=model_data, - color="r", -) - -log_likelihood = log_likelihood_from( - data=data, noise_map=noise_map, model_data=model_data -) -print(f"Log Likelihood: {log_likelihood}") - -""" -__Guess 2__ - -The second guess refines the width and normalization, but the size of the Gaussian is still off. - -The `log_likelihood` is computed and printed, and increases a lot compared to the previous guess, indicating that -the fit is better. -""" - -gaussian = model.instance_from_vector(vector=[50.0, 25.0, 5.0]) -model_data = gaussian.model_data_from(xvalues=xvalues) -plot_model_fit( - xvalues=xvalues, - data=data, - noise_map=noise_map, - model_data=model_data, - color="r", -) - -log_likelihood = log_likelihood_from( - data=data, noise_map=noise_map, model_data=model_data -) -print(f"Log Likelihood: {log_likelihood}") - -""" -__Guess 3__ - -The third guess provides a good fit to the data, with the Gaussian's peak, width, and normalization all accurately -representing the observed signal. - -The `log_likelihood` is computed and printed, and is the highest value yet, indicating that this model provides the -best fit to the data. -""" - -gaussian = model.instance_from_vector(vector=[50.0, 25.0, 10.0]) -model_data = gaussian.model_data_from(xvalues=xvalues) -plot_model_fit( - xvalues=xvalues, - data=data, - noise_map=noise_map, - model_data=model_data, - color="r", -) - -log_likelihood = log_likelihood_from( - data=data, noise_map=noise_map, model_data=model_data -) -print(f"Log Likelihood: {log_likelihood}") - -""" -__Extensibility__ - -Fitting models composed of multiple components is straightforward with PyAutoFit. Using the `Collection` object, -we can define complex models consisting of several components. Once defined, we generate `model_data` -from this collection and fit it to the observed data to compute the log likelihood. -""" -model = af.Collection(gaussian_0=Gaussian, gaussian_1=Gaussian) - -instance = model.instance_from_vector(vector=[40.0, 0.2, 0.3, 60.0, 0.5, 1.0]) - -model_data_0 = instance.gaussian_0.model_data_from(xvalues=xvalues) -model_data_1 = instance.gaussian_1.model_data_from(xvalues=xvalues) - -model_data = model_data_0 + model_data_1 - -""" -We plot the data and model data below, showing that we get a bad fit (a low log likelihood) for this model. - -We could attempt to improve the model-fit and find a higher log likelihood solution by varying the parameters of -the two Gaussians. However, with 6 parameters, this would be a challenging and cumbersome task to perform by eye. -""" -plot_model_fit( - xvalues=xvalues, - data=data, - noise_map=noise_map, - model_data=model_data, - color="r", -) - -log_likelihood = log_likelihood_from( - data=data, noise_map=noise_map, model_data=model_data -) -print(f"Log Likelihood: {log_likelihood}") - - -""" -When our model consisted of only 3 parameters, it was manageable to visually guess their values and achieve a good -fit to the data. However, as we expanded our model to include six parameters, this approach quickly became -inefficient. Attempting to manually optimize models with even more parameters would effectively become impossible, -and a more systematic approach is required. - -In the next tutorial, we will introduce an automated approach for fitting models to data. This method will enable -us to systematically determine the optimal values of model parameters that best describe the observed data, without -relying on manual guesswork. - -__Wrap Up__ - -To conclude, take a moment to reflect on the model you ultimately aim to fit using **PyAutoFit**. What does your -data look like? Is it one-dimensional, like a spectrum or a time series? Or is it two-dimensional, such as an image -or a map? Visualize the nature of your data and consider whether you can define a mathematical model that -accurately generates similar data. - -Can you imagine what a residual map would look like if you were to compare your model's predictions against this -data? A residual map shows the differences between observed data and the model's predictions, often revealing -patterns or areas where the model fits well or poorly. - -Furthermore, can you foresee how you would calculate a log likelihood from this residual map? The log likelihood q -uantifies how well your model fits the data, incorporating both the residual values and the noise characteristics of -your observations. - -If you find it challenging to visualize these aspects right now, that's perfectly fine. The first step is to -grasp the fundamentals of fitting a model to data using **PyAutoFit**, which will provide you with the tools -and understanding needed to address these questions effectively in the future. -""" +""" +Tutorial 2: Fitting Data +======================== + +We've learned that a model consists of equations, numerical processes, and assumptions that describe a physical system. +Using **PyAutoFit**, we defined simple 1D models like the Gaussian, composed them into models using `Model` and +`Collection` objects, and generated model data by varying their parameters. + +To apply our model to real-world situations, we must fit it to data. Fitting involves assessing how well the model +matches observed data. A good fit indicates that the model's parameter values accurately describe the physical system. +Conversely, a poor fit suggests that adjustments are needed to better reflect reality. + +Model-fitting is a cyclical process: define the model, fit it to data, and refine the model based on insights gained. +Iteratively improving the model's complexity enhances its ability to accurately represent the system under study. +This iterative process lies at the core of model-fitting in scientific analysis. + +__Astronomy Example__ + +In Astronomy, this process has been crucial for understanding the distributions of stars within galaxies. By +fitting high-quality images of galaxies with increasingly sophisticated models, astronomers have determined that +stars within galaxies are organized into structures such as disks, bars, and bulges. This approach has also revealed +that stars appear differently in red and blue images due to variations in their age and composition. + +__Overview__ + +In this tutorial, we will explore how to fit the `model_data` generated by a model to actual data. Specifically, we will: + +- Load data representing a 1D Gaussian signal, which serves as our target dataset for fitting. + +- Compute quantities such as residuals by subtracting the model data from the observed data. + +- Quantitatively assess the goodness-of-fit using a critical measure in model-fitting known as the `log_likelihood`. + +All these steps will utilize the **PyAutoFit** API for model composition, introduced in the previous tutorial. + +__Contents__ + +This tutorial is split into the following sections: + +- **Data**: Load and plot the 1D Gaussian dataset we will fit. +- **Model Data**: Generate model data of the `Gaussian` model using a forward model. +- **Residuals**: Compute and visualize residuals between the model data and observed data. +- **Normalized Residuals**: Compute and visualize normalized residuals, which account for the noise properties of the data. +- **Chi Squared**: Compute and visualize the chi-squared map, a measure of the overall goodness-of-fit. +- **Noise Normalization**: Compute the noise normalization term which describes the noise properties of the data. +- **Likelihood**: Compute the log likelihood, a key measure of the goodness-of-fit of the model to the data. +- **Recap**: Summarize the standard metrics for quantifying model fit quality. +- **Fitting Models**: Fit the `Gaussian` model to the 1D data and compute the log likelihood, by guessing parameters. +- **Guess 1**: A first parameter guess with an explanation of the resulting log likelihood. +- **Guess 2**: An improved parameter guess with a better log likelihood. +- **Guess 3**: The optimal parameter guess providing the best fit to the data. +- **Extensibility**: Use the `Collection` object for fitting models with multiple components. +- **Wrap Up**: Summarize the key concepts of this tutorial. +""" + +# from autofit import setup_notebook; setup_notebook() + +from os import path +import matplotlib.pyplot as plt +import numpy as np + +import autofit as af + +""" +__Data__ + +Our dataset consists of noisy 1D data containing a signal, where the underlying signal can be modeled using +equations such as a 1D Gaussian, a 1D Exponential, or a combination of multiple 1D profiles. + +We load this dataset from .json files, where: + +- `data` is a 1D NumPy array containing values representing the observed signal. + +- `noise_map` is a 1D NumPy array containing values representing the estimated root mean squared (RMS) noise level at + each data point. + +These datasets are generated using scripts located in `HowToFit/scripts/simulators`. Feel free to explore +these scripts for more details! +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1") + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists(dataset_path): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + +noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") +) + +""" +Next, we visualize the 1D signal using `matplotlib`. + +The signal is observed over uniformly spaced `xvalues`, computed using the `arange` function and `data.shape[0]` method. + +We will reuse these `xvalues` shortly when generating model data from the model. +""" +xvalues = np.arange(data.shape[0]) +plt.plot(xvalues, data, color="k") +plt.title("1D Dataset Containing a Gaussian.") +plt.xlabel("x values of profile") +plt.ylabel("Signal Value") +plt.show() + +""" +The earlier plot depicted only the signal without indicating the estimated noise at each data point. + +To visualize both the signal and its `noise_map`, we can use `matplotlib`'s `errorbar` function. +""" +plt.errorbar( + xvalues, + data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.title("1D Gaussian dataset with errors from the noise-map.") +plt.xlabel("x values of profile") +plt.ylabel("Signal Value") +plt.show() + + +""" +__Model Data__ + +To fit our `Gaussian` model to this data, we start by generating `model_data` from the 1D `Gaussian` model, +following the same steps as outlined in the previous tutorial. + +We begin by again defining the `Gaussian` class, following the **PyAutoFit** format for model components. +""" + + +class Gaussian: + def __init__( + self, + centre: float = 30.0, # <- **PyAutoFit** recognises these constructor arguments + normalization: float = 1.0, # <- are the Gaussian`s model parameters. + sigma: float = 5.0, + ): + """ + Represents a 1D Gaussian profile. + + This is a model-component of example models in the **HowToFit** lectures and is used to perform model-fitting + of example datasets. + + Parameters + ---------- + centre + The x coordinate of the profile centre. + normalization + Overall normalization of the profile. + sigma + The sigma value controlling the size of the Gaussian. + """ + self.centre = centre + self.normalization = normalization + self.sigma = sigma + + def model_data_from(self, xvalues: np.ndarray) -> np.ndarray: + """ + Returns a 1D Gaussian on an input list of Cartesian x coordinates. + + The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. + + The output is referred to as the `model_data` to signify that it is a representation of the data from the + model. + + Parameters + ---------- + xvalues + The x coordinates in the original reference frame of the data. + + Returns + ------- + np.array + The Gaussian values at the input x coordinates. + """ + transformed_xvalues = np.subtract(xvalues, self.centre) + return np.multiply( + np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), + np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), + ) + + +""" +To create `model_data` for the `Gaussian`, we use the model by providing it with `xvalues` corresponding to the +observed data, as demonstrated in the previous tutorial. + +The following code essentially utilizes a forward model to generate the model data based on a specified set of +parameters. +""" +model = af.Model(Gaussian) + +gaussian = model.instance_from_vector(vector=[60.0, 20.0, 15.0]) + +model_data = gaussian.model_data_from(xvalues=xvalues) + +plt.plot(xvalues, model_data, color="r") +plt.title("1D Gaussian model.") +plt.xlabel("x values of profile") +plt.ylabel("Profile Normalization") +plt.show() +plt.clf() + +""" +For comparison purposes, it is more informative to plot both the `data` and `model_data` on the same plot. +""" +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.plot(xvalues, model_data, color="r") +plt.title("Model-data fit to 1D Gaussian data.") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.close() + +""" +Changing the values of `centre`, `normalization`, and `sigma` alters the appearance of the `Gaussian`. + +You can modify the parameters passed into `instance_from_vector()` above. After recomputing the `model_data`, plot +it again to observe how these changes affect the Gaussian's appearance. + +__Residuals__ + +While it's informative to compare the `data` and `model_data` above, gaining insights from the residuals can be even +more useful. + +Residuals are calculated as `data - model_data` in 1D: +""" +residual_map = data - model_data +plt.plot(xvalues, residual_map, color="k") +plt.title("Residuals of model-data fit to 1D Gaussian data.") +plt.xlabel("x values of profile") +plt.ylabel("Residuals") +plt.show() +plt.clf() + +""" +Are these residuals indicative of a good fit to the data? Without considering the noise in the data, it's difficult +to ascertain. + +We can plot the residuals with error bars based on the noise map. The plot below reveals that the model is a poor fit, +as many residuals deviate significantly from zero even after accounting for the noise in each data point. + +A blue line through zero is included on the plot, to make it clear where residuals are not constent with zero +above the noise level. +""" +residual_map = data - model_data +plt.plot(range(data.shape[0]), np.zeros(data.shape[0]), "--", color="b") +plt.errorbar( + x=xvalues, + y=residual_map, + yerr=noise_map, + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + linestyle="", +) +plt.title("Residuals of model-data fit to 1D Gaussian data.") +plt.xlabel("x values of profile") +plt.ylabel("Residuals") +plt.show() +plt.clf() + +""" +__Normalized Residuals__ + +Another method to quantify and visualize the quality of the fit is using the normalized residual map, also known as +standardized residuals. + +The normalized residual map is computed as the residual map divided by the noise map: + +\[ \text{normalized\_residual} = \frac{\text{residual\_map}}{\text{noise\_map}} = \frac{\text{data} - \text{model\_data}}{\text{noise\_map}} \] + +If you're familiar with the concept of standard deviations (sigma) in statistics, the normalized residual map represents +how many standard deviations the residual is from zero. For instance, a normalized residual of 2.0 (corresponding +to a 95% confidence interval) means that the probability of the model underestimating the data by that amount is only 5%. + +Both the residual map with error bars and the normalized residual map convey the same information. However, +the normalized residual map is particularly useful for visualization in multidimensional problems, as plotting +error bars in 2D or higher dimensions is not straightforward. +""" +normalized_residual_map = residual_map / noise_map +plt.plot(xvalues, normalized_residual_map, color="k") +plt.title("Normalized residuals of model-data fit to 1D Gaussian data.") +plt.xlabel("x values of profile") +plt.ylabel("Normalized Residuals") +plt.show() +plt.clf() + +""" +__Chi Squared__ + +Next, we define the `chi_squared_map`, which is obtained by squaring the `normalized_residual_map` and serves as a +measure of goodness of fit. + +The chi-squared map is calculated as: + +\[ \chi^2 = \left(\frac{\text{data} - \text{model\_data}}{\text{noise\_map}}\right)^2 \] + +The purpose of squaring the normalized residual map is to ensure all values are positive. For instance, both a +normalized residual of -0.2 and 0.2 would square to 0.04, indicating the same level of fit in terms of `chi_squared`. + +As seen from the normalized residual map, it's evident that the model does not provide a good fit to the data. +""" +chi_squared_map = (normalized_residual_map) ** 2 +plt.plot(xvalues, chi_squared_map, color="k") +plt.title("Chi-Squared Map of model-data fit to 1D Gaussian data.") +plt.xlabel("x values of profile") +plt.ylabel("Chi-Squareds") +plt.show() +plt.clf() + +""" +Now, we consolidate all the information in our `chi_squared_map` into a single measure of goodness-of-fit +called `chi_squared`. + +It is defined as the sum of all values in the `chi_squared_map` and is computed as: + +\[ \chi^2 = \sum \left(\frac{\text{data} - \text{model\_data}}{\text{noise\_map}}\right)^2 \] + +This summing process highlights why ensuring all values in the chi-squared map are positive is crucial. If we +didn't square the values (making them positive), positive and negative residuals would cancel each other out, +leading to an inaccurate assessment of the model's fit to the data. +""" +chi_squared = np.sum(chi_squared_map) +print("Chi-squared = ", chi_squared) + +""" + +The lower the `chi_squared`, the fewer residuals exist between the model's fit and the data, indicating a better +overall fit! + +__Noise Normalization__ + +Next, we introduce another quantity that contributes to our final assessment of the goodness-of-fit: +the `noise_normalization`. + +The `noise_normalization` is computed as the logarithm of the sum of squared noise values in our data: + +\[ +\text{{noise\_normalization}} = \sum \log(2 \pi \text{{noise\_map}}^2) +\] + +This quantity is fixed because the noise-map remains constant throughout the fitting process. Despite this, +including the `noise_normalization` is considered good practice due to its statistical significance. + +Understanding the exact meaning of `noise_normalization` isn't critical for our primary goal of successfully +fitting a model to a dataset. Essentially, it provides a measure of how well the noise properties of our data align +with a Gaussian distribution. +""" +noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) + +""" +__Likelihood__ + +From the `chi_squared` and `noise_normalization`, we can define a final goodness-of-fit measure known as +the `log_likelihood`. + +This measure is calculated by taking the sum of the `chi_squared` and `noise_normalization`, and then multiplying the +result by -0.5: + +\[ \text{log\_likelihood} = -0.5 \times \left( \chi^2 + \text{noise\_normalization} \right) \] + +Why multiply by -0.5? The exact rationale behind this factor isn't critical for our current understanding. +""" +log_likelihood = -0.5 * (chi_squared + noise_normalization) +print("Log Likelihood = ", log_likelihood) + +""" +Above, we mentioned that a lower `chi_squared` indicates a better fit of the model to the data. + +When calculating the `log_likelihood`, we multiply the `chi_squared` by -0.5. Therefore, a higher log likelihood +corresponds to a better model fit. This is what we aim for when fitting models to data, we want to maximize the +log likelihood! + +__Recap__ + +If you're familiar with model-fitting, you've likely encountered terms like 'residuals', 'chi-squared', +and 'log_likelihood' before. + +These metrics are standard ways to quantify the quality of a model fit. They are applicable not only to 1D data but +also to more complex data structures like 2D images, 3D data cubes, or any other multidimensional datasets. + +If these terms are new to you, it's important to understand their meanings as they form the basis of all +model-fitting operations in **PyAutoFit** (and in statistical inference more broadly). + +Let's recap what we've learned so far: + +- We can define models, such as a 1D `Gaussian`, using Python classes that follow a specific format. + +- Models can be organized using `Collection` and `Model` objects, with parameters mapped to instances of their + respective model classes (e.g., `Gaussian`). + +- Using these model instances, we can generate model data, compare it to observed data, and quantify the + goodness-of-fit using the log likelihood. + +__Fitting Models__ + +Now, armed with this knowledge, we are ready to fit our model to our data! + +But how do we find the best-fit model, which maximizes the log likelihood? + +The simplest approach is to guess parameters. Starting with initial parameter values that yield a good +fit (i.e., a higher log likelihood), we iteratively adjust these values to refine our model until we achieve an +optimal fit. + +For a 1D `Gaussian`, this iterative process works effectively. Below, we fit three different `Gaussian` models and +identify the best-fit model—the one that matches the original dataset most closely. + +To streamline this process, I've developed functions that compute the `log_likelihood` of a model fit and visualize +the data alongside the model predictions, complete with error bars. +""" + + +def log_likelihood_from( + data: np.ndarray, noise_map: np.ndarray, model_data: np.ndarray +) -> float: + """ + Compute the log likelihood of a model fit to data given the noise map. + + Parameters + ---------- + data + The observed data. + noise_map + The root mean square noise (or uncertainty) associated with each data point. + model_data + The model's predicted data for the given data x points. + + Returns + ------- + float + The log likelihood of the model fit to the data. + """ + # Calculate residuals and normalized residuals + residual_map = data - model_data + normalized_residual_map = residual_map / noise_map + + # Compute chi-squared and noise normalization + chi_squared_map = normalized_residual_map**2 + chi_squared = np.sum(chi_squared_map) + noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) + + # Compute log likelihood + log_likelihood = -0.5 * (chi_squared + noise_normalization) + + return log_likelihood + + +def plot_model_fit( + xvalues: np.ndarray, + data: np.ndarray, + noise_map: np.ndarray, + model_data: np.ndarray, + color: str = "k", +): + """ + Plot the observed data, model predictions, and error bars. + + Parameters + ---------- + xvalues + The x-axis values where the data is observed and model is predicted. + data + The observed data points. + noise_map + The root mean squared noise (or uncertainty) associated with each data point. + model_data + The model's predicted data for the given data x points. + color + The color for plotting (default is "k" for black). + """ + plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color=color, + ecolor="k", + elinewidth=1, + capsize=2, + ) + plt.plot(xvalues, model_data, color="r") + plt.title("Fit of model-data to data") + plt.xlabel("x values of profile") + plt.ylabel("Profile Value") + plt.show() + plt.clf() # Clear figure to prevent overlapping plots + + +""" +__Guess 1__ + +The first guess correctly pinpoints that the Gaussian's peak is at 50.0, but the width and normalization are off. + +The `log_likelihood` is computed and printed, however because we don't have a value to compare it to yet, its hard +to assess if it is a large or small value. +""" + +gaussian = model.instance_from_vector(vector=[50.0, 10.0, 5.0]) +model_data = gaussian.model_data_from(xvalues=xvalues) +plot_model_fit( + xvalues=xvalues, + data=data, + noise_map=noise_map, + model_data=model_data, + color="r", +) + +log_likelihood = log_likelihood_from( + data=data, noise_map=noise_map, model_data=model_data +) +print(f"Log Likelihood: {log_likelihood}") + +""" +__Guess 2__ + +The second guess refines the width and normalization, but the size of the Gaussian is still off. + +The `log_likelihood` is computed and printed, and increases a lot compared to the previous guess, indicating that +the fit is better. +""" + +gaussian = model.instance_from_vector(vector=[50.0, 25.0, 5.0]) +model_data = gaussian.model_data_from(xvalues=xvalues) +plot_model_fit( + xvalues=xvalues, + data=data, + noise_map=noise_map, + model_data=model_data, + color="r", +) + +log_likelihood = log_likelihood_from( + data=data, noise_map=noise_map, model_data=model_data +) +print(f"Log Likelihood: {log_likelihood}") + +""" +__Guess 3__ + +The third guess provides a good fit to the data, with the Gaussian's peak, width, and normalization all accurately +representing the observed signal. + +The `log_likelihood` is computed and printed, and is the highest value yet, indicating that this model provides the +best fit to the data. +""" + +gaussian = model.instance_from_vector(vector=[50.0, 25.0, 10.0]) +model_data = gaussian.model_data_from(xvalues=xvalues) +plot_model_fit( + xvalues=xvalues, + data=data, + noise_map=noise_map, + model_data=model_data, + color="r", +) + +log_likelihood = log_likelihood_from( + data=data, noise_map=noise_map, model_data=model_data +) +print(f"Log Likelihood: {log_likelihood}") + +""" +__Extensibility__ + +Fitting models composed of multiple components is straightforward with PyAutoFit. Using the `Collection` object, +we can define complex models consisting of several components. Once defined, we generate `model_data` +from this collection and fit it to the observed data to compute the log likelihood. +""" +model = af.Collection(gaussian_0=Gaussian, gaussian_1=Gaussian) + +instance = model.instance_from_vector(vector=[40.0, 0.2, 0.3, 60.0, 0.5, 1.0]) + +model_data_0 = instance.gaussian_0.model_data_from(xvalues=xvalues) +model_data_1 = instance.gaussian_1.model_data_from(xvalues=xvalues) + +model_data = model_data_0 + model_data_1 + +""" +We plot the data and model data below, showing that we get a bad fit (a low log likelihood) for this model. + +We could attempt to improve the model-fit and find a higher log likelihood solution by varying the parameters of +the two Gaussians. However, with 6 parameters, this would be a challenging and cumbersome task to perform by eye. +""" +plot_model_fit( + xvalues=xvalues, + data=data, + noise_map=noise_map, + model_data=model_data, + color="r", +) + +log_likelihood = log_likelihood_from( + data=data, noise_map=noise_map, model_data=model_data +) +print(f"Log Likelihood: {log_likelihood}") + + +""" +When our model consisted of only 3 parameters, it was manageable to visually guess their values and achieve a good +fit to the data. However, as we expanded our model to include six parameters, this approach quickly became +inefficient. Attempting to manually optimize models with even more parameters would effectively become impossible, +and a more systematic approach is required. + +In the next tutorial, we will introduce an automated approach for fitting models to data. This method will enable +us to systematically determine the optimal values of model parameters that best describe the observed data, without +relying on manual guesswork. + +__Wrap Up__ + +To conclude, take a moment to reflect on the model you ultimately aim to fit using **PyAutoFit**. What does your +data look like? Is it one-dimensional, like a spectrum or a time series? Or is it two-dimensional, such as an image +or a map? Visualize the nature of your data and consider whether you can define a mathematical model that +accurately generates similar data. + +Can you imagine what a residual map would look like if you were to compare your model's predictions against this +data? A residual map shows the differences between observed data and the model's predictions, often revealing +patterns or areas where the model fits well or poorly. + +Furthermore, can you foresee how you would calculate a log likelihood from this residual map? The log likelihood q +uantifies how well your model fits the data, incorporating both the residual values and the noise characteristics of +your observations. + +If you find it challenging to visualize these aspects right now, that's perfectly fine. The first step is to +grasp the fundamentals of fitting a model to data using **PyAutoFit**, which will provide you with the tools +and understanding needed to address these questions effectively in the future. +""" diff --git a/scripts/chapter_1_introduction/tutorial_3_non_linear_search.py b/scripts/chapter_1_introduction/tutorial_3_non_linear_search.py index ab52d1a..316dd78 100644 --- a/scripts/chapter_1_introduction/tutorial_3_non_linear_search.py +++ b/scripts/chapter_1_introduction/tutorial_3_non_linear_search.py @@ -1,816 +1,816 @@ -""" -Tutorial 3: Non Linear Search -============================= - -In the previous tutorials, we laid the groundwork by defining a model and manually fitting it to data using fitting -functions. We quantified the goodness of fit using the log likelihood and demonstrated that for models with only a few -free parameters, we could achieve satisfactory fits by manually guessing parameter values. However, as the complexity -of our models increased, this approach quickly became impractical. - -In this tutorial, we will delve into a more systematic approach for fitting models to data. This technique is designed -to handle models with a larger number of parameters—ranging from tens to hundreds. By adopting this approach, we aim -to achieve more efficient and reliable model fits, ensuring that our models accurately capture the underlying -structure of the data. - -This approach not only improves the accuracy of our fits but also allows us to explore more complex models that better -represent the systems we are studying. - -__Overview__ - -In this tutorial, we will use a non-linear search to fit a 1D Gaussian profile to noisy data. Specifically, we will: - -- Introduce concept like a "parameter space", "likelihood surface" and "priors", and relate them to how a non-linear - search works. - -- Introduce the `Analysis` class, which defines the `log_likelihood_function` that quantifies the goodness of fit of a - model instance to the data. - -- Fit a 1D Gaussian model to 1D data with different non-linear searches, including a maximum likelihood estimator (MLE), - Markok Chain Monte Carlo (MCMC) and nested sampling. - -All these steps utilize **PyAutoFit**'s API for model-fitting. - -__Contents__ - -This tutorial is split into the following sections: - -- **Parameter Space**: Introduce the concept of a "parameter space" and how it relates to model-fitting. -- **Non-Linear Search**: Introduce the concept of a "non-linear search" and how it fits models to data. -- **Search Types**: Introduce the maximum likelihood estimator (MLE), Markov Chain Monte Carlo (MCMC) and nested sampling search algorithms used in this tutorial. -- **Deeper Background**: Provide links to resources that more thoroughly describe the statistical principles that underpin non-linear searches. -- **Data**: Load and plot the 1D Gaussian dataset we'll fit. -- **Model**: Introduce the 1D `Gaussian` model we'll fit to the data. -- **Priors**: Introduce priors and how they are used to define the parameter space and guide the non-linear search. -- **Analysis**: Introduce the `Analysis` class, which contains the `log_likelihood_function` used to fit the model to the data. -- **Searches**: An overview of the searches used in this tutorial. -- **Maximum Likelihood Estimation (MLE)**: Perform a model-fit using the MLE search. -- **Markov Chain Monte Carlo (MCMC)**: Perform a model-fit using the MCMC search. -- **Nested Sampling**: Perform a model-fit using the nested sampling search. -- **What is The Best Search To Use?**: Compare the strengths and weaknesses of each search method. -- **Wrap Up**: A summary of the concepts introduced in this tutorial. - -__Parameter Space__ - -In mathematics, a function is defined by its parameters, which relate inputs to outputs. - -For example, consider a simple function: - -\[ f(x) = x^2 \] - -Here, \( x \) is the parameter input into the function \( f \), and \( f(x) \) returns \( x^2 \). This -mapping between \( x \) and \( f(x) \) defines the "parameter space" of the function, which in this case is a parabola. - -Functions can have multiple parameters, such as \( x \), \( y \), and \( z \): - -\[ f(x, y, z) = x + y^2 - z^3 \] - -Here, the mapping between \( x \), \( y \), \( z \), and \( f(x, y, z) \) defines a parameter space with three -dimensions. - -This concept of a parameter space relates closely to how we define and use instances of models in model-fitting. -For instance, in our previous tutorial, we used instances of a 1D Gaussian profile with -parameters \( (x, I, \sigma) \) to fit data and compute a log likelihood. - -This process can be thought of as complete analogous to a function \( f(x, y, z) \), where the output value is the -log likelihood. This key function, which maps parameter values to a log likelihood, is called the "likelihood function" -in statistical inference, albeit we will refer to it hereafter as the `log_likelihood_function` to be explicit -that it is the log of the likelihood function. - -By expressing the likelihood in this manner, we can consider our model as having a parameter space -— a -multidimensional surface that spans all possible values of the model parameters \( x, I, \sigma \). - -This surface is often referred to as the "likelihood surface", and our objective during model-fitting is to find -its peak. - -This parameter space is "non-linear", meaning the relationship between the input parameters and the log likelihood -does not behave linearly. This non-linearity implies that we cannot predict the log likelihood from a set of model -parameters without actually performing a fit to the data by performing the forward model calculation. - -__Non-Linear Search__ - -Now that we understand our problem in terms of a non-linear parameter space with a likelihood surface, we can -introduce the method used to fit the model to the data—the "non-linear search". - -Previously, our approach involved manually guessing models until finding one with a good fit and high log likelihood. -Surprisingly, this random guessing forms the basis of how model-fitting using a non-linear search actually works! - -A non-linear search involves systematically guessing many models while tracking their log likelihoods. As the -algorithm progresses, it tends to favor models with parameter combinations that have previously yielded higher -log likelihoods. This iterative refinement helps to efficiently explore the vast parameter space. - -There are two key differences between guessing random models and using a non-linear search: - -- **Computational Efficiency**: The non-linear search can evaluate the log likelihood of a model parameter - combinations in milliseconds and therefore many thousands of models in minutes. This computational speed enables - it to thoroughly sample potential solutions, which would be impractical for a human. - -- **Effective Sampling**: The search algorithm maintains a robust memory of previously guessed models and their log - likelihoods. This allows it to sample potential solutions more thoroughly and converge on the highest - likelihood solutions more efficiently, which is again impractical for a human. - -Think of the non-linear search as systematically exploring parameter space to pinpoint regions with the highest log -likelihood values. Its primary goal is to identify and converge on the parameter values that best describe the data. - -__Search Types__ - -There are different types of non-linear searches, each of which explores parameter space in a unique way. -In this example, we will use three types of searches, which broadly represent the various approaches to non-linear -searches used in statistical inference. - -These are: - -- **Maximum Likelihood Estimation (MLE)**: This method aims to find the model that maximizes the likelihood function. - It does so by testing nearby models and adjusting parameters in the direction that increases the likelihood. - -- **Markov Chain Monte Carlo (MCMC)**: This approach uses a group of "walkers" that explore parameter space randomly. - The likelihood at each walker's position influences the probability of the walker moving to a new position. - -- **Nested Sampling**: This technique samples points from the parameter space iteratively. Lower likelihood points - are replaced by higher likelihood ones, gradually concentrating the samples in regions of high likelihood. - -We will provide more details on each of these searches below. - -__Deeper Background__ - -**The descriptions of how searches work in this example are simplfied and phoenomenological and do not give a full -description of how they work at a deep statistical level. The goal is to provide you with an intuition for how to use -them and when different searches are appropriate for different problems. Later tutorials will provide a more formal -description of how these searches work.** - -If you're interested in learning more about these principles, you can explore resources such as: - -- [Markov Chain Monte Carlo (MCMC)](https://en.wikipedia.org/wiki/Markov_chain_Monte_Carlo) -- [Introduction to MCMC Sampling](https://twiecki.io/blog/2015/11/10/mcmc-sampling/) -- [Nested Sampling](https://www.imperial.ac.uk/media/imperial-college/research-centres-and-groups/astrophysics/public/icic/data-analysis-workshop/2016/NestedSampling_JRP.pdf) -- [A Zero-Math Introduction to MCMC Methods](https://towardsdatascience.com/a-zero-math-introduction-to-markov-chain-monte-carlo-methods-dcba889e0c50) -""" - -import numpy as np -import matplotlib.pyplot as plt -from os import path - -import autofit as af - -# from autoconf import setup_notebook; setup_notebook() - -""" -__Data__ - -Load and plot the dataset from the `HowToFit/dataset` folder. -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1") - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists(dataset_path): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) -noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") -) - -xvalues = np.arange(data.shape[0]) - -plt.errorbar( - xvalues, - data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.title("1D Gaussian dataset.") -plt.xlabel("x values of profile") -plt.ylabel("Profile Normalization") -plt.show() -plt.clf() - -""" -__Model__ - -Create the `Gaussian` class from which we will compose model components using the standard format. -""" - - -class Gaussian: - def __init__( - self, - centre: float = 30.0, # <- **PyAutoFit** recognises these constructor arguments - normalization: float = 1.0, # <- are the Gaussian`s model parameters. - sigma: float = 5.0, - ): - """ - Represents a 1D Gaussian profile. - - This is a model-component of example models in the **HowToFit** lectures and is used to perform model-fitting - of example datasets. - - Parameters - ---------- - centre - The x coordinate of the profile centre. - normalization - Overall normalization of the profile. - sigma - The sigma value controlling the size of the Gaussian. - """ - self.centre = centre - self.normalization = normalization - self.sigma = sigma - - def model_data_from(self, xvalues: np.ndarray) -> np.ndarray: - """ - Returns a 1D Gaussian on an input list of Cartesian x coordinates. - - The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. - - The output is referred to as the `model_data` to signify that it is a representation of the data from the - model. - - Parameters - ---------- - xvalues - The x coordinates in the original reference frame of the data. - - Returns - ------- - np.array - The Gaussian values at the input x coordinates. - """ - transformed_xvalues = np.subtract(xvalues, self.centre) - return np.multiply( - np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), - np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), - ) - - -""" -We now compose our model, a single 1D Gaussian, which we will fit to the data via the non-linear search. -""" -model = af.Model(Gaussian) - -print(model.info) - -""" -__Priors__ - -When we examine the `.info` of our model, we notice that each parameter (like `centre`, `normalization`, -and `sigma` in our Gaussian model) is associated with priors, such as `UniformPrior`. These priors define the -range of permissible values that each parameter can assume during the model fitting process. - -The priors displayed above use default values defined in the `config/priors` directory. These default values have -been chosen to be broad, and contain all plausible solutions contained in the simulated 1D Gaussian datasets. - -For instance, consider the `centre` parameter of our Gaussian. In theory, it could take on any value from -negative to positive infinity. However, upon inspecting our dataset, we observe that valid values for `centre` -fall strictly between 0.0 and 100.0. By using a `UniformPrior` with `lower_limit=0.0` and `upper_limit=100.0`, -we restrict our parameter space to include only physically plausible values. - -Priors serve two primary purposes: - -**Defining Valid Parameter Space:** Priors specify the range of parameter values that constitute valid solutions. -This ensures that our model explores only those solutions that are consistent with our observed data and physical -constraints. - -**Incorporating Prior Knowledge:** Priors also encapsulate our prior beliefs or expectations about the model -parameters. For instance, if we have previously fitted a similar model to another dataset and obtained certain -parameter values, we can incorporate this knowledge into our priors for a new dataset. This approach guides the -model fitting process towards parameter values that are more probable based on our prior understanding. - -While we are using `UniformPriors` in this tutorial due to their simplicity, **PyAutoFit** offers various other -priors like `TruncatedGaussianPrior` and `LogUniformPrior`. These priors are useful for encoding different forms of prior -information, such as normally distributed values around a mean (`TruncatedGaussianPrior`) or parameters spanning multiple -orders of magnitude (`LogUniformPrior`). -""" -model.centre = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) -model.normalization = af.UniformPrior(lower_limit=0.0, upper_limit=10.0) -model.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=10.0) - -""" -__Analysis__ - -In **PyAutoFit**, the `Analysis` class plays a crucial role in interfacing between the data being fitted and the -model under consideration. Its primary responsibilities include: - -**Receiving Data:** The `Analysis` class is initialized with the data (`data`) and noise map (`noise_map`) that - the model aims to fit. - -**Defining the Log Likelihood Function:** The `Analysis` class defines the `log_likelihood_function`, which - computes the log likelihood of a model instance given the data. It evaluates how well the model, for a given set of - parameters, fits the observed data. - -**Interface with Non-linear Search:** The `log_likelihood_function` is repeatedly called by the non-linear search - algorithm to assess the goodness of fit of different parameter combinations. The search algorithm call this function - many times and maps out regions of parameter space that yield high likelihood solutions. - -Below is a suitable `Analysis` class for fitting a 1D gaussian to the data loaded above. -""" - - -class Analysis(af.Analysis): - def __init__(self, data: np.ndarray, noise_map: np.ndarray): - """ - The `Analysis` class acts as an interface between the data and model in **PyAutoFit**. - - Its `log_likelihood_function` defines how the model is fitted to the data and it is called many times by - the non-linear search fitting algorithm. - - In this example the `Analysis` `__init__` constructor only contains the `data` and `noise-map`, but it can be - easily extended to include other quantities. - - Parameters - ---------- - data - A 1D numpy array containing the data (e.g. a noisy 1D signal) fitted in the workspace examples. - noise_map - A 1D numpy array containing the noise values of the data, used for computing the goodness of fit - metric, the log likelihood. - """ - super().__init__() - - self.data = data - self.noise_map = noise_map - - def log_likelihood_function(self, instance) -> float: - """ - Returns the log likelihood of a fit of a 1D Gaussian to the dataset. - - The `instance` that comes into this method is an instance of the `Gaussian` model above. The parameter values - are chosen by the non-linear search, based on where it thinks the high likelihood regions of parameter - space are. - - The lines of Python code are commented out below to prevent excessive print statements when we run the - non-linear search, but feel free to uncomment them and run the search to see the parameters of every instance - that it fits. - - print("Gaussian Instance:") - print("Centre = ", instance.centre) - print("Normalization = ", instance.normalization) - print("Sigma = ", instance.sigma) - - The data is fitted using an `instance` of the `Gaussian` class where its `model_data_from` - is called in order to create a model data representation of the Gaussian that is fitted to the data. - """ - xvalues = np.arange(self.data.shape[0]) - - model_data = instance.model_data_from(xvalues=xvalues) - residual_map = self.data - model_data - chi_squared_map = (residual_map / self.noise_map) ** 2.0 - chi_squared = sum(chi_squared_map) - noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) - log_likelihood = -0.5 * (chi_squared + noise_normalization) - - return log_likelihood - - -""" -We create an instance of the `Analysis` class by simply passing it the `data` and `noise_map`: -""" -analysis = Analysis(data=data, noise_map=noise_map) - -""" -__Searches__ - -To perform a non-linear search, we create an instance of a `NonLinearSearch` object. **PyAutoFit** offers many options -for this. A detailed description of each search method and guidance on when to use them can be found in -the [search cookbook](https://pyautofit.readthedocs.io/en/latest/cookbooks/search.html). - -In this tutorial, we’ll focus on three searches that represent different approaches to model fitting: - -1. **Maximum Likelihood Estimation (MLE)** using the `LBFGS` non-linear search. -2. **Markov Chain Monte Carlo (MCMC)** using the `Emcee` non-linear search. -3. **Nested Sampling** using the `Dynesty` non-linear search. - -In this example, non-linear search results are stored in memory rather and not written to hard disk because the fits -are fast and can therefore be easily regenerated. The next tutorial will perform fits which write results to the -hard-disk. - -__Maximum Likelihood Estimation (MLE)__ - -Maximum likelihood estimation (MLE) is the most straightforward type of non-linear search. Here’s a simplified -overview of how it works: - -1. Starts at a point in parameter space with a set of initial values for the model parameters. -2. Calculates the likelihood of the model at this starting point. -3. Evaluates the likelihood at nearby points to estimate the gradient, determining the direction in which to move "up" in parameter space. -4. Moves to a new point where, based on the gradient, the likelihood is higher. - -This process repeats until the search finds a point where the likelihood can no longer be improved, indicating that -the maximum likelihood has been reached. - -The `LBFGS` search is an example of an MLE algorithm that follows this iterative procedure. Let’s see how it -performs on our 1D Gaussian model. - -In the example below, we don’t specify a starting point for the MLE, so it begins at the center of the prior -range for each parameter. -""" -search = af.LBFGS() - -""" -To begin the model-fit via the non-linear search, we pass it our model and analysis and begin the fit. - -The fit will take a minute or so to run. -""" -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -model = af.Model(Gaussian) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -""" -Upon completion the non-linear search returns a `Result` object, which contains information about the model-fit. - -The `info` attribute shows the result in a readable format. - -[Above, we discussed that the `info_whitespace_length` parameter in the config files could b changed to make -the `model.info` attribute display optimally on your computer. This attribute also controls the whitespace of the -`result.info` attribute.] -""" -print(result.info) - -""" -The result has a "maximum log likelihood instance", which refers to the specific set of model parameters (e.g., -for a `Gaussian`) that yielded the highest log likelihood among all models tested by the non-linear search. -""" -print("Maximum Likelihood Model:\n") -max_log_likelihood_instance = result.samples.max_log_likelihood() -print("Centre = ", max_log_likelihood_instance.centre) -print("Normalization = ", max_log_likelihood_instance.normalization) -print("Sigma = ", max_log_likelihood_instance.sigma) - -""" -We can use this to plot the maximum log likelihood fit over the data and determine the quality of fit was inferred: -""" -model_data = result.max_log_likelihood_instance.model_data_from( - xvalues=np.arange(data.shape[0]) -) -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.plot(xvalues, model_data, color="r") -plt.title("Dynesty model fit to 1D Gaussian dataset.") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.close() - -""" -The fit quality was poor, and the MLE failed to identify the correct model. - -This happened because the starting point of the search was a poor match to the data, placing it far from the true -solution in parameter space. As a result, after moving "up" the likelihood gradient several times, the search -settled into a "local maximum," where it couldn't find a better solution. - -To achieve a better fit with MLE, the search needs to begin in a region of parameter space where the log likelihood -is higher. This process is known as "initialization," and it involves providing the search with an -appropriate "starting point" in parameter space. -""" -initializer = af.InitializerParamStartPoints( - { - model.centre: 55.0, - model.normalization: 20.0, - model.sigma: 8.0, - } -) - -search = af.LBFGS(initializer=initializer) - -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -model = af.Model(Gaussian) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -""" -By printing `result.info` and looking at the maximum log likelihood model, we can confirm the search provided a -good model fit with a much higher likelihood than the incorrect model above. -""" -print(result.info) - -model_data = result.max_log_likelihood_instance.model_data_from( - xvalues=np.arange(data.shape[0]) -) -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.plot(xvalues, model_data, color="r") -plt.title("Dynesty model fit to 1D Gaussian dataset.") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.close() - -""" -MLE is a great starting point for model-fitting because it’s fast, conceptually simple, and often yields -accurate results. It is especially effective if you can provide a good initialization, allowing it to find the -best-fit solution quickly. - -However, MLE has its limitations. As seen above, it can get "stuck" in a local maximum, particularly if the -starting point is poorly chosen. In complex model-fitting problems, providing a suitable starting point can be -challenging. While MLE performed well in the example with just three parameters, it struggles with models that have -many parameters, as the complexity of the likelihood surface makes simply moving "up" the gradient less effective. - -The MLE also does not provide any information on the errors on the parameters, which is a significant limitation. -The next two types of searches "map out" the likelihood surface, such that they not only infer the maximum likelihood -solution but also quantify the errors on the parameters. - -__Markov Chain Monte Carlo (MCMC)__ - -Markov Chain Monte Carlo (MCMC) is a more powerful method for model-fitting, though it is also more computationally -intensive and conceptually complex. Here’s a simplified overview: - -1. Place a set of "walkers" in parameter space, each with random parameter values. -2. Calculate the likelihood of each walker's position. -3. Move the walkers to new positions, guided by the likelihood of their current positions. Walkers in high-likelihood -regions encourage those in lower regions to move closer to them. - -This process repeats, with the walkers converging on the highest-likelihood regions of parameter space. - -Unlike MLE, MCMC thoroughly explores parameter space. While MLE moves a single point up the likelihood gradient, -MCMC uses many walkers to explore high-likelihood regions, making it more effective at finding the global maximum, -though slower. - -In the example below, we use the `Emcee` MCMC search to fit the 1D Gaussian model. The search starts with walkers -initialized in a "ball" around the center of the model’s priors, similar to the MLE search that failed earlier. -""" -search = af.Emcee( - nwalkers=10, # The number of walkers we'll use to sample parameter space. - nsteps=200, # The number of steps each walker takes, after which 10 * 200 = 2000 steps the non-linear search ends. -) - -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -model = af.Model(Gaussian) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -print(result.info) - -model_data = result.max_log_likelihood_instance.model_data_from( - xvalues=np.arange(data.shape[0]) -) -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.plot(xvalues, model_data, color="r") -plt.title("Dynesty model fit to 1D Gaussian dataset.") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.close() - -""" -The MCMC search succeeded, finding the same high-likelihood model that the MLE search with a good starting point -identified, even without a good initialization. Its use of multiple walkers exploring parameter space allowed it to -avoid the local maxima that had trapped the MLE search. - -A major advantage of MCMC is that it provides estimates of parameter uncertainties by "mapping out" the likelihood -surface, unlike MLE, which only finds the maximum likelihood solution. These error estimates are accessible in -the `result.info` string and through the `result.samples` object, with further details in tutorial 5. - -While a good starting point wasn't necessary for this simple model, it becomes essential for efficiently mapping the -likelihood surface in more complex models with many parameters. The code below shows an MCMC fit using a good starting -point, with two key differences from the MLE initialization: - -1. Instead of single starting values, we provide bounds for each parameter. MCMC initializes each walker in a -small "ball" in parameter space, requiring a defined range for each parameter from which values are randomly drawn. - -2. We do not specify a starting point for the sigma parameter, allowing its initial values to be drawn from its -priors. This illustrates that with MCMC, it’s not necessary to know a good starting point for every parameter. -""" -initializer = af.InitializerParamBounds( - { - model.centre: (54.0, 56.0), - model.normalization: (19.0, 21.0), - } -) - -search = af.Emcee( - nwalkers=10, # The number of walkers we'll use to sample parameter space. - nsteps=200, # The number of steps each walker takes, after which 10 * 200 = 2000 steps the non-linear search ends. - initializer=initializer, -) - -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -model = af.Model(Gaussian) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -print(result.info) - -""" -MCMC is a powerful tool for model-fitting, providing accurate parameter estimates and uncertainties. For simple models -without a starting point, MCMC can still find the correct solution, and if a good starting point is provided, it can -efficiently scale to more complex models with more parameters. - -The main limitation of MCMC is that one has to supply the number of steps the walkers take (`nsteps`). If this value -is too low, the walkers may not explore the likelihood surface sufficiently. It can be challenging to know the right -number of steps, especially if models of different complexity are being fitted or if datasets of varying quality are -used. One often ends up having to perform "trial and error" to verify a sufficient number of steps are used. - -MCMC can perform badly in parameter spaces with certain types of complexity, for example when there are -are local maxima "peaks" the walkers can become stuck walking around them. - -__Nested Sampling__ - -**Nested Sampling** is an advanced method for model-fitting that excels in handling complex models with intricate -parameter spaces. Here’s a simplified overview of its process: - -1. Start with a set of "live points" in parameter space, each initialized with random parameter values drawn from their respective priors. - -2. Compute the log likelihood for each live point. - -3. Draw a new point based on the likelihood of the current live points, favoring regions of higher likelihood. - -4. If the new point has a higher likelihood than any existing live point, it becomes a live point, and the lowest likelihood live point is discarded. - -This iterative process continues, gradually focusing the live points around higher likelihood regions of parameter -space until they converge on the highest likelihood solution. - -Like MCMC, Nested Sampling effectively maps out parameter space, providing accurate estimates of parameters and -their uncertainties. -""" -search = af.DynestyStatic( - sample="rwalk", # This makes dynesty run faster, dont worry about what it means for now! -) - -""" -To begin the model-fit via the non-linear search, we pass it our model and analysis and begin the fit. - -The fit will take a minute or so to run. -""" -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -model = af.Model(Gaussian) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -print(result.info) - -model_data = result.max_log_likelihood_instance.model_data_from( - xvalues=np.arange(data.shape[0]) -) -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.plot(xvalues, model_data, color="r") -plt.title("Dynesty model fit to 1D Gaussian dataset.") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.close() - -""" -The **Nested Sampling** search was successful, identifying the same high-likelihood model as the MLE and MCMC searches. -One of the main benefits of Nested Sampling is its ability to provide accurate parameter estimates and uncertainties, -similar to MCMC. Additionally, it features a built-in stopping criterion, which eliminates the need for users to -specify the number of steps the search should take. - -This method also excels in handling complex parameter spaces, particularly those with multiple peaks. This is because -the live points will identify each peak and converge around them, but then begin to be discard from a peak if higher -likelihood points are found elsewhere in parameter space. In MCMC, the walkers can get stuck indefinitely around a -peak, causing the method to stall. - -Another significant advantage is that Nested Sampling estimates an important statistical quantity -known as "evidence." This value quantifies how well the model fits the data while considering the model's complexity, -making it essential for Bayesian model comparison, which will be covered in later tutorials. - -Nested sampling cannot use a starting point, as it always samples parameter space from scratch by drawing live points -from the priors. This is both good and bad, depending on if you have access to a good starting point or not. If you do -not, your MCMC / MLE fit will likely struggle with initialization compared to Nested Sampling. Conversely, if you do -possess a robust starting point, it can significantly enhance the performance of MCMC, allowing it to begin closer to -the highest likelihood regions of parameter space. This proximity can lead to faster convergence and more reliable results. - -However, Nested Sampling does have limitations; it often scales poorly with increased model complexity. For example, -once a model has around 50 or more parameters, Nested Sampling can become very slow, whereas MCMC remains efficient -even in such complex parameter spaces. - -__What is The Best Search To Use?__ - -The choice of the best search method depends on several factors specific to the problem at hand. Here are key -considerations that influence which search may be optimal: - -Firstly, consider the speed of the fit regardless of the search method. If the fitting process runs efficiently, -nested sampling could be advantageous for low-dimensional parameter spaces due to its ability to handle complex -parameter spaces and its built-in stopping criterion. However, in high-dimensional scenarios, MCMC may be more -suitable, as it scales better with the number of parameters. - -Secondly, evaluate whether you have access to a robust starting point for your model fit. A strong initialization can -make MCMC more appealing, allowing the algorithm to bypass the initial sampling stage and leading to quicker convergence. - -Additionally, think about the importance of error estimation in your analysis. If error estimation is not a priority, -MLE might suffice, but this approach heavily relies on having a solid starting point and may struggle with more complex models. - -Ultimately, every model-fitting problem is unique, making it impossible to provide a one-size-fits-all answer regarding -the best search method. This variability is why **PyAutoFit** offers a diverse array of search options, all -standardized with a consistent interface. This standardization allows users to experiment with different searches on the -same model-fitting problem and determine which yields the best results. - -Finally, it’s important to note that MLE, MCMC, and nested sampling represent only three categories of non-linear -searches, each containing various algorithms. Each algorithm has its strengths and weaknesses, so experimenting with -them can reveal the most effective approach for your specific model-fitting challenge. For further guidance, a detailed -description of each search method can be found in the [search cookbook](https://pyautofit.readthedocs.io/en/latest/cookbooks/search.html). - -__Wrap Up__ - -This tutorial has laid the foundation with several fundamental concepts in model fitting and statistical inference: - -1. **Parameter Space**: This refers to the range of possible values that each parameter in a model can take. It -defines the dimensions over which the likelihood of different parameter values is evaluated. - -2. **Likelihood Surface**: This surface represents how the likelihood of the model varies across the parameter space. -It helps in identifying the best-fit parameters that maximize the likelihood of the model given the data. - -3. **Non-linear Search**: This is an optimization technique used to explore the parameter space and find the -combination of parameter values that best describe the data. It iteratively adjusts the parameters to maximize the -likelihood. Many different search algorithms exist, each with their own strengths and weaknesses, and this tutorial -used the MLE, MCMC, and nested sampling searches. - -4. **Priors**: Priors are probabilities assigned to different values of parameters before considering the data. -They encapsulate our prior knowledge or assumptions about the parameter values. Priors can constrain the parameter -space, making the search more efficient and realistic. - -5. **Model Fitting**: The process of adjusting model parameters to minimize the difference between model predictions -and observed data, quantified by the likelihood function. - -Understanding these concepts is crucial as they form the backbone of model fitting and parameter estimation in -scientific research and data analysis. In the next tutorials, these concepts will be further expanded upon to -deepen your understanding and provide more advanced techniques for model fitting and analysis. -""" +""" +Tutorial 3: Non Linear Search +============================= + +In the previous tutorials, we laid the groundwork by defining a model and manually fitting it to data using fitting +functions. We quantified the goodness of fit using the log likelihood and demonstrated that for models with only a few +free parameters, we could achieve satisfactory fits by manually guessing parameter values. However, as the complexity +of our models increased, this approach quickly became impractical. + +In this tutorial, we will delve into a more systematic approach for fitting models to data. This technique is designed +to handle models with a larger number of parameters—ranging from tens to hundreds. By adopting this approach, we aim +to achieve more efficient and reliable model fits, ensuring that our models accurately capture the underlying +structure of the data. + +This approach not only improves the accuracy of our fits but also allows us to explore more complex models that better +represent the systems we are studying. + +__Overview__ + +In this tutorial, we will use a non-linear search to fit a 1D Gaussian profile to noisy data. Specifically, we will: + +- Introduce concept like a "parameter space", "likelihood surface" and "priors", and relate them to how a non-linear + search works. + +- Introduce the `Analysis` class, which defines the `log_likelihood_function` that quantifies the goodness of fit of a + model instance to the data. + +- Fit a 1D Gaussian model to 1D data with different non-linear searches, including a maximum likelihood estimator (MLE), + Markok Chain Monte Carlo (MCMC) and nested sampling. + +All these steps utilize **PyAutoFit**'s API for model-fitting. + +__Contents__ + +This tutorial is split into the following sections: + +- **Parameter Space**: Introduce the concept of a "parameter space" and how it relates to model-fitting. +- **Non-Linear Search**: Introduce the concept of a "non-linear search" and how it fits models to data. +- **Search Types**: Introduce the maximum likelihood estimator (MLE), Markov Chain Monte Carlo (MCMC) and nested sampling search algorithms used in this tutorial. +- **Deeper Background**: Provide links to resources that more thoroughly describe the statistical principles that underpin non-linear searches. +- **Data**: Load and plot the 1D Gaussian dataset we'll fit. +- **Model**: Introduce the 1D `Gaussian` model we'll fit to the data. +- **Priors**: Introduce priors and how they are used to define the parameter space and guide the non-linear search. +- **Analysis**: Introduce the `Analysis` class, which contains the `log_likelihood_function` used to fit the model to the data. +- **Searches**: An overview of the searches used in this tutorial. +- **Maximum Likelihood Estimation (MLE)**: Perform a model-fit using the MLE search. +- **Markov Chain Monte Carlo (MCMC)**: Perform a model-fit using the MCMC search. +- **Nested Sampling**: Perform a model-fit using the nested sampling search. +- **What is The Best Search To Use?**: Compare the strengths and weaknesses of each search method. +- **Wrap Up**: A summary of the concepts introduced in this tutorial. + +__Parameter Space__ + +In mathematics, a function is defined by its parameters, which relate inputs to outputs. + +For example, consider a simple function: + +\[ f(x) = x^2 \] + +Here, \( x \) is the parameter input into the function \( f \), and \( f(x) \) returns \( x^2 \). This +mapping between \( x \) and \( f(x) \) defines the "parameter space" of the function, which in this case is a parabola. + +Functions can have multiple parameters, such as \( x \), \( y \), and \( z \): + +\[ f(x, y, z) = x + y^2 - z^3 \] + +Here, the mapping between \( x \), \( y \), \( z \), and \( f(x, y, z) \) defines a parameter space with three +dimensions. + +This concept of a parameter space relates closely to how we define and use instances of models in model-fitting. +For instance, in our previous tutorial, we used instances of a 1D Gaussian profile with +parameters \( (x, I, \sigma) \) to fit data and compute a log likelihood. + +This process can be thought of as complete analogous to a function \( f(x, y, z) \), where the output value is the +log likelihood. This key function, which maps parameter values to a log likelihood, is called the "likelihood function" +in statistical inference, albeit we will refer to it hereafter as the `log_likelihood_function` to be explicit +that it is the log of the likelihood function. + +By expressing the likelihood in this manner, we can consider our model as having a parameter space -— a +multidimensional surface that spans all possible values of the model parameters \( x, I, \sigma \). + +This surface is often referred to as the "likelihood surface", and our objective during model-fitting is to find +its peak. + +This parameter space is "non-linear", meaning the relationship between the input parameters and the log likelihood +does not behave linearly. This non-linearity implies that we cannot predict the log likelihood from a set of model +parameters without actually performing a fit to the data by performing the forward model calculation. + +__Non-Linear Search__ + +Now that we understand our problem in terms of a non-linear parameter space with a likelihood surface, we can +introduce the method used to fit the model to the data—the "non-linear search". + +Previously, our approach involved manually guessing models until finding one with a good fit and high log likelihood. +Surprisingly, this random guessing forms the basis of how model-fitting using a non-linear search actually works! + +A non-linear search involves systematically guessing many models while tracking their log likelihoods. As the +algorithm progresses, it tends to favor models with parameter combinations that have previously yielded higher +log likelihoods. This iterative refinement helps to efficiently explore the vast parameter space. + +There are two key differences between guessing random models and using a non-linear search: + +- **Computational Efficiency**: The non-linear search can evaluate the log likelihood of a model parameter + combinations in milliseconds and therefore many thousands of models in minutes. This computational speed enables + it to thoroughly sample potential solutions, which would be impractical for a human. + +- **Effective Sampling**: The search algorithm maintains a robust memory of previously guessed models and their log + likelihoods. This allows it to sample potential solutions more thoroughly and converge on the highest + likelihood solutions more efficiently, which is again impractical for a human. + +Think of the non-linear search as systematically exploring parameter space to pinpoint regions with the highest log +likelihood values. Its primary goal is to identify and converge on the parameter values that best describe the data. + +__Search Types__ + +There are different types of non-linear searches, each of which explores parameter space in a unique way. +In this example, we will use three types of searches, which broadly represent the various approaches to non-linear +searches used in statistical inference. + +These are: + +- **Maximum Likelihood Estimation (MLE)**: This method aims to find the model that maximizes the likelihood function. + It does so by testing nearby models and adjusting parameters in the direction that increases the likelihood. + +- **Markov Chain Monte Carlo (MCMC)**: This approach uses a group of "walkers" that explore parameter space randomly. + The likelihood at each walker's position influences the probability of the walker moving to a new position. + +- **Nested Sampling**: This technique samples points from the parameter space iteratively. Lower likelihood points + are replaced by higher likelihood ones, gradually concentrating the samples in regions of high likelihood. + +We will provide more details on each of these searches below. + +__Deeper Background__ + +**The descriptions of how searches work in this example are simplfied and phoenomenological and do not give a full +description of how they work at a deep statistical level. The goal is to provide you with an intuition for how to use +them and when different searches are appropriate for different problems. Later tutorials will provide a more formal +description of how these searches work.** + +If you're interested in learning more about these principles, you can explore resources such as: + +- [Markov Chain Monte Carlo (MCMC)](https://en.wikipedia.org/wiki/Markov_chain_Monte_Carlo) +- [Introduction to MCMC Sampling](https://twiecki.io/blog/2015/11/10/mcmc-sampling/) +- [Nested Sampling](https://www.imperial.ac.uk/media/imperial-college/research-centres-and-groups/astrophysics/public/icic/data-analysis-workshop/2016/NestedSampling_JRP.pdf) +- [A Zero-Math Introduction to MCMC Methods](https://towardsdatascience.com/a-zero-math-introduction-to-markov-chain-monte-carlo-methods-dcba889e0c50) +""" + +import numpy as np +import matplotlib.pyplot as plt +from os import path + +import autofit as af + +# from autofit import setup_notebook; setup_notebook() + +""" +__Data__ + +Load and plot the dataset from the `HowToFit/dataset` folder. +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1") + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists(dataset_path): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) +noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") +) + +xvalues = np.arange(data.shape[0]) + +plt.errorbar( + xvalues, + data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.title("1D Gaussian dataset.") +plt.xlabel("x values of profile") +plt.ylabel("Profile Normalization") +plt.show() +plt.clf() + +""" +__Model__ + +Create the `Gaussian` class from which we will compose model components using the standard format. +""" + + +class Gaussian: + def __init__( + self, + centre: float = 30.0, # <- **PyAutoFit** recognises these constructor arguments + normalization: float = 1.0, # <- are the Gaussian`s model parameters. + sigma: float = 5.0, + ): + """ + Represents a 1D Gaussian profile. + + This is a model-component of example models in the **HowToFit** lectures and is used to perform model-fitting + of example datasets. + + Parameters + ---------- + centre + The x coordinate of the profile centre. + normalization + Overall normalization of the profile. + sigma + The sigma value controlling the size of the Gaussian. + """ + self.centre = centre + self.normalization = normalization + self.sigma = sigma + + def model_data_from(self, xvalues: np.ndarray) -> np.ndarray: + """ + Returns a 1D Gaussian on an input list of Cartesian x coordinates. + + The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. + + The output is referred to as the `model_data` to signify that it is a representation of the data from the + model. + + Parameters + ---------- + xvalues + The x coordinates in the original reference frame of the data. + + Returns + ------- + np.array + The Gaussian values at the input x coordinates. + """ + transformed_xvalues = np.subtract(xvalues, self.centre) + return np.multiply( + np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), + np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), + ) + + +""" +We now compose our model, a single 1D Gaussian, which we will fit to the data via the non-linear search. +""" +model = af.Model(Gaussian) + +print(model.info) + +""" +__Priors__ + +When we examine the `.info` of our model, we notice that each parameter (like `centre`, `normalization`, +and `sigma` in our Gaussian model) is associated with priors, such as `UniformPrior`. These priors define the +range of permissible values that each parameter can assume during the model fitting process. + +The priors displayed above use default values defined in the `config/priors` directory. These default values have +been chosen to be broad, and contain all plausible solutions contained in the simulated 1D Gaussian datasets. + +For instance, consider the `centre` parameter of our Gaussian. In theory, it could take on any value from +negative to positive infinity. However, upon inspecting our dataset, we observe that valid values for `centre` +fall strictly between 0.0 and 100.0. By using a `UniformPrior` with `lower_limit=0.0` and `upper_limit=100.0`, +we restrict our parameter space to include only physically plausible values. + +Priors serve two primary purposes: + +**Defining Valid Parameter Space:** Priors specify the range of parameter values that constitute valid solutions. +This ensures that our model explores only those solutions that are consistent with our observed data and physical +constraints. + +**Incorporating Prior Knowledge:** Priors also encapsulate our prior beliefs or expectations about the model +parameters. For instance, if we have previously fitted a similar model to another dataset and obtained certain +parameter values, we can incorporate this knowledge into our priors for a new dataset. This approach guides the +model fitting process towards parameter values that are more probable based on our prior understanding. + +While we are using `UniformPriors` in this tutorial due to their simplicity, **PyAutoFit** offers various other +priors like `TruncatedGaussianPrior` and `LogUniformPrior`. These priors are useful for encoding different forms of prior +information, such as normally distributed values around a mean (`TruncatedGaussianPrior`) or parameters spanning multiple +orders of magnitude (`LogUniformPrior`). +""" +model.centre = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) +model.normalization = af.UniformPrior(lower_limit=0.0, upper_limit=10.0) +model.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=10.0) + +""" +__Analysis__ + +In **PyAutoFit**, the `Analysis` class plays a crucial role in interfacing between the data being fitted and the +model under consideration. Its primary responsibilities include: + +**Receiving Data:** The `Analysis` class is initialized with the data (`data`) and noise map (`noise_map`) that + the model aims to fit. + +**Defining the Log Likelihood Function:** The `Analysis` class defines the `log_likelihood_function`, which + computes the log likelihood of a model instance given the data. It evaluates how well the model, for a given set of + parameters, fits the observed data. + +**Interface with Non-linear Search:** The `log_likelihood_function` is repeatedly called by the non-linear search + algorithm to assess the goodness of fit of different parameter combinations. The search algorithm call this function + many times and maps out regions of parameter space that yield high likelihood solutions. + +Below is a suitable `Analysis` class for fitting a 1D gaussian to the data loaded above. +""" + + +class Analysis(af.Analysis): + def __init__(self, data: np.ndarray, noise_map: np.ndarray): + """ + The `Analysis` class acts as an interface between the data and model in **PyAutoFit**. + + Its `log_likelihood_function` defines how the model is fitted to the data and it is called many times by + the non-linear search fitting algorithm. + + In this example the `Analysis` `__init__` constructor only contains the `data` and `noise-map`, but it can be + easily extended to include other quantities. + + Parameters + ---------- + data + A 1D numpy array containing the data (e.g. a noisy 1D signal) fitted in the workspace examples. + noise_map + A 1D numpy array containing the noise values of the data, used for computing the goodness of fit + metric, the log likelihood. + """ + super().__init__() + + self.data = data + self.noise_map = noise_map + + def log_likelihood_function(self, instance) -> float: + """ + Returns the log likelihood of a fit of a 1D Gaussian to the dataset. + + The `instance` that comes into this method is an instance of the `Gaussian` model above. The parameter values + are chosen by the non-linear search, based on where it thinks the high likelihood regions of parameter + space are. + + The lines of Python code are commented out below to prevent excessive print statements when we run the + non-linear search, but feel free to uncomment them and run the search to see the parameters of every instance + that it fits. + + print("Gaussian Instance:") + print("Centre = ", instance.centre) + print("Normalization = ", instance.normalization) + print("Sigma = ", instance.sigma) + + The data is fitted using an `instance` of the `Gaussian` class where its `model_data_from` + is called in order to create a model data representation of the Gaussian that is fitted to the data. + """ + xvalues = np.arange(self.data.shape[0]) + + model_data = instance.model_data_from(xvalues=xvalues) + residual_map = self.data - model_data + chi_squared_map = (residual_map / self.noise_map) ** 2.0 + chi_squared = sum(chi_squared_map) + noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) + log_likelihood = -0.5 * (chi_squared + noise_normalization) + + return log_likelihood + + +""" +We create an instance of the `Analysis` class by simply passing it the `data` and `noise_map`: +""" +analysis = Analysis(data=data, noise_map=noise_map) + +""" +__Searches__ + +To perform a non-linear search, we create an instance of a `NonLinearSearch` object. **PyAutoFit** offers many options +for this. A detailed description of each search method and guidance on when to use them can be found in +the [search cookbook](https://pyautofit.readthedocs.io/en/latest/cookbooks/search.html). + +In this tutorial, we’ll focus on three searches that represent different approaches to model fitting: + +1. **Maximum Likelihood Estimation (MLE)** using the `LBFGS` non-linear search. +2. **Markov Chain Monte Carlo (MCMC)** using the `Emcee` non-linear search. +3. **Nested Sampling** using the `Dynesty` non-linear search. + +In this example, non-linear search results are stored in memory rather and not written to hard disk because the fits +are fast and can therefore be easily regenerated. The next tutorial will perform fits which write results to the +hard-disk. + +__Maximum Likelihood Estimation (MLE)__ + +Maximum likelihood estimation (MLE) is the most straightforward type of non-linear search. Here’s a simplified +overview of how it works: + +1. Starts at a point in parameter space with a set of initial values for the model parameters. +2. Calculates the likelihood of the model at this starting point. +3. Evaluates the likelihood at nearby points to estimate the gradient, determining the direction in which to move "up" in parameter space. +4. Moves to a new point where, based on the gradient, the likelihood is higher. + +This process repeats until the search finds a point where the likelihood can no longer be improved, indicating that +the maximum likelihood has been reached. + +The `LBFGS` search is an example of an MLE algorithm that follows this iterative procedure. Let’s see how it +performs on our 1D Gaussian model. + +In the example below, we don’t specify a starting point for the MLE, so it begins at the center of the prior +range for each parameter. +""" +search = af.LBFGS() + +""" +To begin the model-fit via the non-linear search, we pass it our model and analysis and begin the fit. + +The fit will take a minute or so to run. +""" +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +model = af.Model(Gaussian) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +Upon completion the non-linear search returns a `Result` object, which contains information about the model-fit. + +The `info` attribute shows the result in a readable format. + +[Above, we discussed that the `info_whitespace_length` parameter in the config files could b changed to make +the `model.info` attribute display optimally on your computer. This attribute also controls the whitespace of the +`result.info` attribute.] +""" +print(result.info) + +""" +The result has a "maximum log likelihood instance", which refers to the specific set of model parameters (e.g., +for a `Gaussian`) that yielded the highest log likelihood among all models tested by the non-linear search. +""" +print("Maximum Likelihood Model:\n") +max_log_likelihood_instance = result.samples.max_log_likelihood() +print("Centre = ", max_log_likelihood_instance.centre) +print("Normalization = ", max_log_likelihood_instance.normalization) +print("Sigma = ", max_log_likelihood_instance.sigma) + +""" +We can use this to plot the maximum log likelihood fit over the data and determine the quality of fit was inferred: +""" +model_data = result.max_log_likelihood_instance.model_data_from( + xvalues=np.arange(data.shape[0]) +) +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.plot(xvalues, model_data, color="r") +plt.title("Dynesty model fit to 1D Gaussian dataset.") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.close() + +""" +The fit quality was poor, and the MLE failed to identify the correct model. + +This happened because the starting point of the search was a poor match to the data, placing it far from the true +solution in parameter space. As a result, after moving "up" the likelihood gradient several times, the search +settled into a "local maximum," where it couldn't find a better solution. + +To achieve a better fit with MLE, the search needs to begin in a region of parameter space where the log likelihood +is higher. This process is known as "initialization," and it involves providing the search with an +appropriate "starting point" in parameter space. +""" +initializer = af.InitializerParamStartPoints( + { + model.centre: 55.0, + model.normalization: 20.0, + model.sigma: 8.0, + } +) + +search = af.LBFGS(initializer=initializer) + +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +model = af.Model(Gaussian) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +By printing `result.info` and looking at the maximum log likelihood model, we can confirm the search provided a +good model fit with a much higher likelihood than the incorrect model above. +""" +print(result.info) + +model_data = result.max_log_likelihood_instance.model_data_from( + xvalues=np.arange(data.shape[0]) +) +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.plot(xvalues, model_data, color="r") +plt.title("Dynesty model fit to 1D Gaussian dataset.") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.close() + +""" +MLE is a great starting point for model-fitting because it’s fast, conceptually simple, and often yields +accurate results. It is especially effective if you can provide a good initialization, allowing it to find the +best-fit solution quickly. + +However, MLE has its limitations. As seen above, it can get "stuck" in a local maximum, particularly if the +starting point is poorly chosen. In complex model-fitting problems, providing a suitable starting point can be +challenging. While MLE performed well in the example with just three parameters, it struggles with models that have +many parameters, as the complexity of the likelihood surface makes simply moving "up" the gradient less effective. + +The MLE also does not provide any information on the errors on the parameters, which is a significant limitation. +The next two types of searches "map out" the likelihood surface, such that they not only infer the maximum likelihood +solution but also quantify the errors on the parameters. + +__Markov Chain Monte Carlo (MCMC)__ + +Markov Chain Monte Carlo (MCMC) is a more powerful method for model-fitting, though it is also more computationally +intensive and conceptually complex. Here’s a simplified overview: + +1. Place a set of "walkers" in parameter space, each with random parameter values. +2. Calculate the likelihood of each walker's position. +3. Move the walkers to new positions, guided by the likelihood of their current positions. Walkers in high-likelihood +regions encourage those in lower regions to move closer to them. + +This process repeats, with the walkers converging on the highest-likelihood regions of parameter space. + +Unlike MLE, MCMC thoroughly explores parameter space. While MLE moves a single point up the likelihood gradient, +MCMC uses many walkers to explore high-likelihood regions, making it more effective at finding the global maximum, +though slower. + +In the example below, we use the `Emcee` MCMC search to fit the 1D Gaussian model. The search starts with walkers +initialized in a "ball" around the center of the model’s priors, similar to the MLE search that failed earlier. +""" +search = af.Emcee( + nwalkers=10, # The number of walkers we'll use to sample parameter space. + nsteps=200, # The number of steps each walker takes, after which 10 * 200 = 2000 steps the non-linear search ends. +) + +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +model = af.Model(Gaussian) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +print(result.info) + +model_data = result.max_log_likelihood_instance.model_data_from( + xvalues=np.arange(data.shape[0]) +) +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.plot(xvalues, model_data, color="r") +plt.title("Dynesty model fit to 1D Gaussian dataset.") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.close() + +""" +The MCMC search succeeded, finding the same high-likelihood model that the MLE search with a good starting point +identified, even without a good initialization. Its use of multiple walkers exploring parameter space allowed it to +avoid the local maxima that had trapped the MLE search. + +A major advantage of MCMC is that it provides estimates of parameter uncertainties by "mapping out" the likelihood +surface, unlike MLE, which only finds the maximum likelihood solution. These error estimates are accessible in +the `result.info` string and through the `result.samples` object, with further details in tutorial 5. + +While a good starting point wasn't necessary for this simple model, it becomes essential for efficiently mapping the +likelihood surface in more complex models with many parameters. The code below shows an MCMC fit using a good starting +point, with two key differences from the MLE initialization: + +1. Instead of single starting values, we provide bounds for each parameter. MCMC initializes each walker in a +small "ball" in parameter space, requiring a defined range for each parameter from which values are randomly drawn. + +2. We do not specify a starting point for the sigma parameter, allowing its initial values to be drawn from its +priors. This illustrates that with MCMC, it’s not necessary to know a good starting point for every parameter. +""" +initializer = af.InitializerParamBounds( + { + model.centre: (54.0, 56.0), + model.normalization: (19.0, 21.0), + } +) + +search = af.Emcee( + nwalkers=10, # The number of walkers we'll use to sample parameter space. + nsteps=200, # The number of steps each walker takes, after which 10 * 200 = 2000 steps the non-linear search ends. + initializer=initializer, +) + +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +model = af.Model(Gaussian) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +print(result.info) + +""" +MCMC is a powerful tool for model-fitting, providing accurate parameter estimates and uncertainties. For simple models +without a starting point, MCMC can still find the correct solution, and if a good starting point is provided, it can +efficiently scale to more complex models with more parameters. + +The main limitation of MCMC is that one has to supply the number of steps the walkers take (`nsteps`). If this value +is too low, the walkers may not explore the likelihood surface sufficiently. It can be challenging to know the right +number of steps, especially if models of different complexity are being fitted or if datasets of varying quality are +used. One often ends up having to perform "trial and error" to verify a sufficient number of steps are used. + +MCMC can perform badly in parameter spaces with certain types of complexity, for example when there are +are local maxima "peaks" the walkers can become stuck walking around them. + +__Nested Sampling__ + +**Nested Sampling** is an advanced method for model-fitting that excels in handling complex models with intricate +parameter spaces. Here’s a simplified overview of its process: + +1. Start with a set of "live points" in parameter space, each initialized with random parameter values drawn from their respective priors. + +2. Compute the log likelihood for each live point. + +3. Draw a new point based on the likelihood of the current live points, favoring regions of higher likelihood. + +4. If the new point has a higher likelihood than any existing live point, it becomes a live point, and the lowest likelihood live point is discarded. + +This iterative process continues, gradually focusing the live points around higher likelihood regions of parameter +space until they converge on the highest likelihood solution. + +Like MCMC, Nested Sampling effectively maps out parameter space, providing accurate estimates of parameters and +their uncertainties. +""" +search = af.DynestyStatic( + sample="rwalk", # This makes dynesty run faster, dont worry about what it means for now! +) + +""" +To begin the model-fit via the non-linear search, we pass it our model and analysis and begin the fit. + +The fit will take a minute or so to run. +""" +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +model = af.Model(Gaussian) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +print(result.info) + +model_data = result.max_log_likelihood_instance.model_data_from( + xvalues=np.arange(data.shape[0]) +) +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.plot(xvalues, model_data, color="r") +plt.title("Dynesty model fit to 1D Gaussian dataset.") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.close() + +""" +The **Nested Sampling** search was successful, identifying the same high-likelihood model as the MLE and MCMC searches. +One of the main benefits of Nested Sampling is its ability to provide accurate parameter estimates and uncertainties, +similar to MCMC. Additionally, it features a built-in stopping criterion, which eliminates the need for users to +specify the number of steps the search should take. + +This method also excels in handling complex parameter spaces, particularly those with multiple peaks. This is because +the live points will identify each peak and converge around them, but then begin to be discard from a peak if higher +likelihood points are found elsewhere in parameter space. In MCMC, the walkers can get stuck indefinitely around a +peak, causing the method to stall. + +Another significant advantage is that Nested Sampling estimates an important statistical quantity +known as "evidence." This value quantifies how well the model fits the data while considering the model's complexity, +making it essential for Bayesian model comparison, which will be covered in later tutorials. + +Nested sampling cannot use a starting point, as it always samples parameter space from scratch by drawing live points +from the priors. This is both good and bad, depending on if you have access to a good starting point or not. If you do +not, your MCMC / MLE fit will likely struggle with initialization compared to Nested Sampling. Conversely, if you do +possess a robust starting point, it can significantly enhance the performance of MCMC, allowing it to begin closer to +the highest likelihood regions of parameter space. This proximity can lead to faster convergence and more reliable results. + +However, Nested Sampling does have limitations; it often scales poorly with increased model complexity. For example, +once a model has around 50 or more parameters, Nested Sampling can become very slow, whereas MCMC remains efficient +even in such complex parameter spaces. + +__What is The Best Search To Use?__ + +The choice of the best search method depends on several factors specific to the problem at hand. Here are key +considerations that influence which search may be optimal: + +Firstly, consider the speed of the fit regardless of the search method. If the fitting process runs efficiently, +nested sampling could be advantageous for low-dimensional parameter spaces due to its ability to handle complex +parameter spaces and its built-in stopping criterion. However, in high-dimensional scenarios, MCMC may be more +suitable, as it scales better with the number of parameters. + +Secondly, evaluate whether you have access to a robust starting point for your model fit. A strong initialization can +make MCMC more appealing, allowing the algorithm to bypass the initial sampling stage and leading to quicker convergence. + +Additionally, think about the importance of error estimation in your analysis. If error estimation is not a priority, +MLE might suffice, but this approach heavily relies on having a solid starting point and may struggle with more complex models. + +Ultimately, every model-fitting problem is unique, making it impossible to provide a one-size-fits-all answer regarding +the best search method. This variability is why **PyAutoFit** offers a diverse array of search options, all +standardized with a consistent interface. This standardization allows users to experiment with different searches on the +same model-fitting problem and determine which yields the best results. + +Finally, it’s important to note that MLE, MCMC, and nested sampling represent only three categories of non-linear +searches, each containing various algorithms. Each algorithm has its strengths and weaknesses, so experimenting with +them can reveal the most effective approach for your specific model-fitting challenge. For further guidance, a detailed +description of each search method can be found in the [search cookbook](https://pyautofit.readthedocs.io/en/latest/cookbooks/search.html). + +__Wrap Up__ + +This tutorial has laid the foundation with several fundamental concepts in model fitting and statistical inference: + +1. **Parameter Space**: This refers to the range of possible values that each parameter in a model can take. It +defines the dimensions over which the likelihood of different parameter values is evaluated. + +2. **Likelihood Surface**: This surface represents how the likelihood of the model varies across the parameter space. +It helps in identifying the best-fit parameters that maximize the likelihood of the model given the data. + +3. **Non-linear Search**: This is an optimization technique used to explore the parameter space and find the +combination of parameter values that best describe the data. It iteratively adjusts the parameters to maximize the +likelihood. Many different search algorithms exist, each with their own strengths and weaknesses, and this tutorial +used the MLE, MCMC, and nested sampling searches. + +4. **Priors**: Priors are probabilities assigned to different values of parameters before considering the data. +They encapsulate our prior knowledge or assumptions about the parameter values. Priors can constrain the parameter +space, making the search more efficient and realistic. + +5. **Model Fitting**: The process of adjusting model parameters to minimize the difference between model predictions +and observed data, quantified by the likelihood function. + +Understanding these concepts is crucial as they form the backbone of model fitting and parameter estimation in +scientific research and data analysis. In the next tutorials, these concepts will be further expanded upon to +deepen your understanding and provide more advanced techniques for model fitting and analysis. +""" diff --git a/scripts/chapter_1_introduction/tutorial_4_why_modeling_is_hard.py b/scripts/chapter_1_introduction/tutorial_4_why_modeling_is_hard.py index 603c9c3..88d2b84 100644 --- a/scripts/chapter_1_introduction/tutorial_4_why_modeling_is_hard.py +++ b/scripts/chapter_1_introduction/tutorial_4_why_modeling_is_hard.py @@ -1,934 +1,934 @@ -""" -Tutorial 4: Why Modeling Is Hard -================================ - -We have successfully fitted a simple 1D Gaussian profile to a dataset using a non-linear search. While achieving an -accurate model fit has been straightforward, the reality is that model fitting is a challenging problem where many things can go wrong. - -This tutorial will illustrate why modeling is challenging, highlight common problems that occur when fitting complex -models, and show how a good scientific approach can help us overcome these challenges. - -We will build on concepts introduced in previous tutorials, such as the non-linear parameter space, likelihood surface, -and the role of priors. - -__Overview__ - -In this tutorial, we will fit complex models with up to 15 free parameters and consider the following: - -- Why more complex models are more difficult to fit and may lead the non-linear search to infer an incorrect solution. - -- Strategies for ensuring the non-linear search estimates the correct solution. - -- What drives the run-times of a model fit and how to carefully balance run-times with model complexity. - -__Contents__ - -- **Data**: Load and plot the 1D Gaussian dataset we'll fit, which is more complex than the previous tutorial. -- **Model**: The `Gaussian` model component that we will fit to the data. -- **Analysis**: The log likelihood function used to fit the model to the data. -- **Alternative Syntax**: An alternative loop-based approach for creating a summed profile from multiple model components. -- **Collection**: The `Collection` model used to compose the model-fit. -- **Search**: Set up the nested sampling search (Dynesty) for the model-fit. -- **Model Fit**: Perform the model-fit and examine the results. -- **Result**: Determine if the model-fit was successful and what can be done to ensure a good model-fit. -- **Why Modeling is Hard**: Introduce the concept of randomness and local maxima and why they make model-fitting challenging. -- **Prior Tuning**: Adjust the priors of the model to help the non-linear search find the global maxima solution. -- **Reducing Complexity**: Simplify the model to reduce the dimensionality of the parameter space. -- **Search More Thoroughly**: Adjust the non-linear search settings to search parameter space more thoroughly. -- **Summary**: Summarize the three strategies for ensuring successful model-fitting. -- **Run Times**: Discuss how the likelihood function and complexity of a model impacts the run-time of a model-fit. -- **Model Mismatch**: Introduce the concept of model mismatches and how it makes inferring the correct model challenging. -- **Astronomy Example**: How the concepts of this tutorial are applied to real astronomical problems. -- **Wrap Up**: A summary of the key takeaways of this tutorial. -""" - -# from autoconf import setup_notebook; setup_notebook() - -from os import path -import numpy as np -import matplotlib.pyplot as plt - -import autofit as af - -""" -__Data__ - -Load the dataset we fit. - -This is a new `dataset` where the underlying signal is a sum of five `Gaussian` profiles. -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x5") - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists(dataset_path): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) -noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") -) - -""" -Plotting the data reveals that the signal is more complex than a simple 1D Gaussian, as the wings to the left and -right are more extended than what a single Gaussian profile can account for. -""" -xvalues = np.arange(data.shape[0]) -plt.errorbar( - xvalues, - data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.title("1D Gaussian dataset with errors from the noise-map.") -plt.xlabel("x values of profile") -plt.ylabel("Signal Value") -plt.show() -plt.clf() -plt.close() - -""" -__Model__ - -Create the `Gaussian` class from which we will compose model components using the standard format. -""" - - -class Gaussian: - def __init__( - self, - centre: float = 30.0, # <- **PyAutoFit** recognises these constructor arguments - normalization: float = 1.0, # <- are the Gaussian`s model parameters. - sigma: float = 5.0, - ): - """ - Represents a 1D Gaussian profile. - - This is a model-component of example models in the **HowToFit** lectures and is used to perform model-fitting - of example datasets. - - Parameters - ---------- - centre - The x coordinate of the profile centre. - normalization - Overall normalization of the profile. - sigma - The sigma value controlling the size of the Gaussian. - """ - self.centre = centre - self.normalization = normalization - self.sigma = sigma - - def model_data_from(self, xvalues: np.ndarray) -> np.ndarray: - """ - Returns a 1D Gaussian on an input list of Cartesian x coordinates. - - The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. - - The output is referred to as the `model_data` to signify that it is a representation of the data from the - model. - - Parameters - ---------- - xvalues - The x coordinates in the original reference frame of the data. - - Returns - ------- - np.array - The Gaussian values at the input x coordinates. - """ - transformed_xvalues = np.subtract(xvalues, self.centre) - return np.multiply( - np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), - np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), - ) - - -""" -__Analysis__ - -To define the Analysis class for this model-fit, we need to ensure that the `log_likelihood_function` can handle an -instance containing multiple 1D profiles. Below is an expanded explanation and the corresponding class definition: - -The log_likelihood_function will now assume that the instance it receives consists of multiple Gaussian profiles. -For each Gaussian in the instance, it will compute the model_data and then sum these to create the overall `model_data` -that is compared to the observed data. -""" - - -class Analysis(af.Analysis): - def __init__(self, data: np.ndarray, noise_map: np.ndarray): - """ - The `Analysis` class acts as an interface between the data and model in **PyAutoFit**. - - Its `log_likelihood_function` defines how the model is fitted to the data and it is called many times by - the non-linear search fitting algorithm. - - In this example, the `log_likelihood_function` receives an instance containing multiple instances of - the `Gaussian` class and sums the `model_data` of each to create the overall model fit to the data. - - In this example the `Analysis` `__init__` constructor only contains the `data` and `noise-map`, but it can be - easily extended to include other quantities. - - Parameters - ---------- - data - A 1D numpy array containing the data (e.g. a noisy 1D signal) fitted in the workspace examples. - noise_map - A 1D numpy array containing the noise values of the data, used for computing the goodness of fit - metric, the log likelihood. - """ - super().__init__() - - self.data = data - self.noise_map = noise_map - - def log_likelihood_function(self, instance) -> float: - """ - Returns the log likelihood of a fit of a 1D Gaussian to the dataset. - - In the previous tutorial, the instance was a single `Gaussian` profile, however this function now assumes - the instance contains multiple `Gaussian` profiles. - - The `model_data` is therefore the summed `model_data` of all individual Gaussians in the model. - - The docstring below describes this in more detail. - - Parameters - ---------- - instance - A list of 1D profiles with parameters set via the non-linear search. - - Returns - ------- - float - The log likelihood value indicating how well this model fit the `MaskedDataset`. - """ - - """ - In the previous tutorial the instance was a single `Gaussian` profile, meaning we could create the model data - using the line: - - model_data = instance.gaussian.model_data_from(xvalues=self.data.xvalues) - - In this tutorial our instance is comprised of three 1D Gaussians, because we will use a `Collection` to - compose the model: - - model = Collection(gaussian_0=Gaussian, gaussian_1=Gaussian, gaussian_2=Gaussian). - - By using a Collection, this means the instance parameter input into the fit function is a - dictionary where individual profiles (and their parameters) can be accessed as followed: - - print(instance.gaussian_0) - print(instance.gaussian_1) - print(instance.gaussian_2) - - print(instance.gaussian_0.centre) - print(instance.gaussian_1.centre) - print(instance.gaussian_2.centre) - - The `model_data` is therefore the summed `model_data` of all individual Gaussians in the model. - - The function `model_data_from_instance` performs this summation. - """ - model_data = self.model_data_from_instance(instance=instance) - - residual_map = self.data - model_data - chi_squared_map = (residual_map / self.noise_map) ** 2.0 - chi_squared = sum(chi_squared_map) - noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) - log_likelihood = -0.5 * (chi_squared + noise_normalization) - - return log_likelihood - - def model_data_from_instance(self, instance): - """ - To create the summed profile of all individual profiles, we use a list comprehension to iterate over - all profiles in the instance. - - The `instance` has the properties of a Python `iterator` and therefore can be looped over using the standard - Python for syntax (e.g. `for profile in instance`). - - __Alternative Syntax__ - - For those not familiar with Python list comprehensions, the code below shows how to use the instance to - create the summed profile using a for loop and numpy array: - - model_data = np.zeros(shape=self.data.xvalues.shape[0]) - - for profile in instance: - model_data += profile.model_data_from(xvalues=self.data.xvalues) - - return model_data - """ - xvalues = np.arange(self.data.shape[0]) - - return sum([profile.model_data_from(xvalues=xvalues) for profile in instance]) - - -""" -__Collection__ - -In the previous tutorial, we fitted a single `Gaussian` profile to the dataset by turning it into a model -component using the `Model` class. - -In this tutorial, we will fit a model composed of five `Gaussian` profiles. To do this, we need to combine -five `Gaussian` model components into a single model. - -This can be achieved using a `Collection` object, which was introduced in tutorial 1. The `Collection` object allows -us to group together multiple model components—in this case, five `Gaussian` profiles—into one model that can be -passed to the non-linear search. -""" -model = af.Collection( - gaussian_0=Gaussian, - gaussian_1=Gaussian, - gaussian_2=Gaussian, - gaussian_3=Gaussian, - gaussian_4=Gaussian, -) - -""" -The `model.info` confirms the model is composed of 5 `Gaussian` profiles. -""" -print(model.info) - -""" -__Search__ - -We again use the nested sampling algorithm Dynesty to fit the model to the data. -""" -search = af.DynestyStatic( - sample="rwalk", # This makes dynesty run faster, don't worry about what it means for now! -) - -""" -__Model Fit__ - -Perform the fit using our five `Gaussian` model, which has 15 free parameters. - -This means the non-linear parameter space has a dimensionality of N=15, making it significantly more complex -than the simpler model we fitted in the previous tutorial. - -Consequently, the non-linear search takes slightly longer to run but still completes in under a minute. -""" -analysis = Analysis(data=data, noise_map=noise_map) - -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -""" -__Result__ - -The `info` attribute shows the result in a readable format, which contains information on the full collection -of all 5 model components. -""" -print(result.info) - -""" -From the result info, it is hard to assess if the model fit was good or not. - -A good way to evaluate the fit is through a visual inspection of the model data plotted over the actual data. - -If the model data (red line) consistently aligns with the data points (black error bars), the fit is good. -However, if the model misses certain features of the data, such as peaks or regions of high intensity, -the fit was not successful. -""" -instance = result.max_log_likelihood_instance - -model_data_0 = instance.gaussian_0.model_data_from(xvalues=np.arange(data.shape[0])) -model_data_1 = instance.gaussian_1.model_data_from(xvalues=np.arange(data.shape[0])) -model_data_2 = instance.gaussian_2.model_data_from(xvalues=np.arange(data.shape[0])) -model_data_3 = instance.gaussian_3.model_data_from(xvalues=np.arange(data.shape[0])) -model_data_4 = instance.gaussian_4.model_data_from(xvalues=np.arange(data.shape[0])) - -model_data_list = [model_data_0, model_data_1, model_data_2, model_data_3, model_data_4] - -model_data = sum(model_data_list) - -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, -) -plt.plot(range(data.shape[0]), model_data, color="r") -for model_data_1d_individual in model_data_list: - plt.plot(range(data.shape[0]), model_data_1d_individual, "--") -plt.title(f"Fit (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.close() - -""" -It's challenging to determine from the plot whether the data and model data perfectly overlap across the entire dataset. - -To clarify this, the residual map introduced in tutorial 2 is useful. It provides a clear representation of where -the differences between the model and data exceed the noise level. - -Regions where the black error bars do not align with the zero line in the residual map indicate areas where the model -did not fit the data well and is inconsistent with the data above the noise level. Furthermore, regions where -larger values of residuals are next to one another indicate that the model failed to accurate fit that -region of the data. -""" -residual_map = data - model_data -plt.plot(range(data.shape[0]), np.zeros(data.shape[0]), "--", color="b") -plt.errorbar( - x=xvalues, - y=residual_map, - yerr=noise_map, - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - linestyle="", -) -plt.title(f"Residuals (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Residuals") -plt.show() -plt.clf() -plt.close() - -""" -The normalized residual map, as discussed in tutorial 2, provides an alternative visualization of the fit quality. - -Normalized residuals indicate the standard deviation (σ) level at which the residuals could have been drawn from the -noise. For instance, a normalized residual of 2.0 suggests that a residual value is 2.0σ away from the noise, -implying there is a 5% chance such a residual would occur due to noise. - -Values of normalized residuals above 3.0 are particularly improbable (occurring only 0.3% of the time), which is -generally considered a threshold where issues with the model-fit are likely the cause of the residual as opposed -to it being a noise fluctuation. -""" -residual_map = data - model_data -normalized_residual_map = residual_map / noise_map -plt.plot(xvalues, normalized_residual_map, color="k") -plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Normalized Residuals ($\sigma$)") -plt.show() -plt.clf() -plt.close() - -""" -So, did you achieve a good fit? Maybe a bad one? Or just an okay one? - -The truth is, I don't know, and I can't tell you for sure. Modeling is inherently random. It's not uncommon to -fit the same model to the same dataset using the same non-linear search and get a different result each time. - -When I ran the model fit above, that's exactly what happened. It produced a range of fits: some bad, some okay, and -some good, as shown in the images below: - - - - - -
- -Distinguishing between the good and okay fit is difficult, however the normalized residuals make this easier. They show -that for the okay fit there are residuals above 3.0 sigma, indicating that the model did not perfectly fit the data. - - - - - -
- -You should quickly rerun the code above a couple of times to see this variability for yourself. - -__Why Modeling is Hard__ - -This variability is at the heart of why modeling is challenging. The process of model-fitting is stochastic, -meaning it's hard as the scientist to determine if a better fit is possible or not. - -Why does modeling produce different results each time, and why might it sometimes infer solutions that are not good fits? - -In the previous tutorial, the non-linear search consistently found models that visually matched the data well, -minimizing residuals and yielding high log likelihood values. These optimal solutions are called 'global maxima', -they are where the model parameters correspond to the highest likelihood regions across the entire parameter space. -This ideal scenario is illustrated in the `good_fit.png` image above. - -However, non-linear searches do not always find these global maxima. Instead, they might settle on 'local maxima' -solutions, which have high log likelihood values relative to nearby models in parameter space but are significantly -lower than the true global maxima found elsewhere. - -This is what happened for the okay and bad fits above. The non-linear search converged on solutions that were locally -peaks on the likelihood surface but were not the global maximum solution. This is why the residuals were higher and -the normalized residuals above 3.0 sigma. - -Why does a non-linear search end up at local maxima? As discussed, the search iterates through many models, -focusing more on regions where previous guesses yielded higher likelihoods. It gradually converges around -solutions with higher likelihoods compared to surrounding models. If the search isn't exhaustive enough, it might -converge on a local maxima that appears good compared to nearby models but isn't the global maximum. - -Modeling is challenging because the parameter spaces of complex models are typically filled with local maxima, -making it hard for a non-linear search to locate the global maximum. - -Fortunately, there are strategies to help non-linear searches find the global maxima, and we'll now explore three of -them. - -__Prior Tuning__ - -First, let's assist our non-linear search by tuning our priors. Priors provide guidance to the search on where to -explore in the parameter space. By setting more accurate priors ('tuning' them), we can help the search find the -global solution instead of settling for a local maximum. - -For instance, from the data itself, it's evident that all `Gaussian` profiles are centered around pixel 50. In our -previous fit, the `centre` parameter of each `Gaussian` had a `UniformPrior` spanning from 0.0 to 100.0, which is -much broader than necessary given the data's range. - -Additionally, the peak value of the data's `normalization` parameter was around 17.5. This indicates that -the `normalization` values of our `Gaussians` do not exceed 500.0, allowing us to refine our prior accordingly. - -The following code snippet adjusts these priors for the `centre` and `normalization` parameters of -each `Gaussian` using **PyAutoFit**'s API for model and prior customization: -""" -gaussian_0 = af.Model(Gaussian) - -gaussian_0.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) -gaussian_0.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) - -gaussian_1 = af.Model(Gaussian) - -gaussian_1.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) -gaussian_1.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) - -gaussian_2 = af.Model(Gaussian) - -gaussian_2.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) -gaussian_2.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) - -gaussian_3 = af.Model(Gaussian) - -gaussian_3.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) -gaussian_3.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) - -gaussian_4 = af.Model(Gaussian) - -gaussian_4.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) -gaussian_4.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) - -model = af.Collection( - gaussian_0=gaussian_0, - gaussian_1=gaussian_1, - gaussian_2=gaussian_2, - gaussian_3=gaussian_3, - gaussian_4=gaussian_4, -) - -""" -The `info` attribute shows the model is now using the priors specified above. -""" -print(model.info) - -""" -We now repeat the model-fit using these updated priors. - -First, you should note that the run time of the fit is significantly faster than the previous fit. This is because -the prior is telling the non-linear search where to look, meaning it converges on solutions more quickly -and spends less time searching regions of parameter space that do not contain solutions. - -Second, the model-fit consistently produces a good model-fit more often, because our tuned priors are centred -on the global maxima solution ensuring the non-linear search is less likely to converge on a local maxima. -""" -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -""" -Lets print the result info and plot the fit to the dataset to confirm the priors have provided a better model-fit. -""" -print(result.info) - -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - linestyle="", -) -plt.plot(range(data.shape[0]), model_data, color="r") -for model_data_1d_individual in model_data_list: - plt.plot(range(data.shape[0]), model_data_1d_individual, "--") -plt.title(f"Fit (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.clf() -plt.close() - -residual_map = data - model_data -normalized_residual_map = residual_map / noise_map -plt.plot(xvalues, normalized_residual_map, color="k") -plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Normalized Residuals ($\sigma$)") -plt.show() -plt.clf() -plt.close() - -""" -Lets consider the advantages and disadvantages of prior tuning: - -**Advantages:** - -- Higher likelihood of finding the global maximum log likelihood solutions in parameter space. - -- Faster search times, as the non-linear search explores less of the parameter space. - -**Disadvantages:** - -- Incorrectly specified priors could lead the non-linear search to an incorrect solution. - -- It is not always clear how the priors should be tuned, especially for complex models. - -- Priors tuning must be applied to each dataset fitted. For large datasets, this process would be very time-consuming. - -__Reducing Complexity__ - -The non-linear search may fail because the model is too complex, making its parameter space too difficult to -sample accurately consistent. To address this, we may be able to simplify the model while ensuring it remains -realistic enough for our scientific study. By making certain assumptions, we can reduce the number of model -parameters, thereby lowering the dimensionality of the parameter space and improving the search's performance. - -For example, we may know that the `Gaussian`'s in our model are aligned at the same `centre`. We can therefore -compose a model that assumes that the `centre` of each `Gaussian` is the same, reducing the dimensionality of the -model from N=15 to N=11. - -The code below shows how we can customize the model components to ensure the `centre` of each `Gaussian` is the same: -""" -gaussian_0 = af.Model(Gaussian) -gaussian_1 = af.Model(Gaussian) -gaussian_2 = af.Model(Gaussian) -gaussian_3 = af.Model(Gaussian) -gaussian_4 = af.Model(Gaussian) - -gaussian_1.centre = gaussian_0.centre -gaussian_2.centre = gaussian_0.centre -gaussian_3.centre = gaussian_0.centre -gaussian_4.centre = gaussian_0.centre - -model = af.Collection( - gaussian_0=gaussian_0, - gaussian_1=gaussian_1, - gaussian_2=gaussian_2, - gaussian_3=gaussian_3, - gaussian_4=gaussian_4, -) - -""" -The `info` attribute shows the model is now using the same `centre` for all `Gaussian`'s and has 11 free parameters. -""" -print(model.info) - -""" -We now repeat the model-fit using this updated model where the `centre` of each `Gaussian` is the same. - -You should again note that the run time of the fit is significantly faster than the previous fits -and that it consistently produces a good model-fit more often. This is because the model is less complex, -non-linear parameter space is less difficult to sample and the search is less likely to converge on a local maxima. -""" -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -""" -Lets print the result info and plot the fit to the dataset to confirm the reduced model complexity has -provided a better model-fit. -""" -print(result.info) - -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - linestyle="", -) -plt.plot(range(data.shape[0]), model_data, color="r") -for model_data_1d_individual in model_data_list: - plt.plot(range(data.shape[0]), model_data_1d_individual, "--") -plt.title(f"Fit (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.clf() -plt.close() - -residual_map = data - model_data -normalized_residual_map = residual_map / noise_map -plt.plot(xvalues, normalized_residual_map, color="k") -plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Normalized Residuals ($\sigma$)") -plt.show() -plt.clf() -plt.close() - -""" -Let’s consider the advantages and disadvantages of simplifying the model: - -Advantages: - -- By reducing the complexity of the parameter space, we increase the chances of finding the global maximum log -likelihood, and the search requires less time to do so. - -- Unlike with tuned priors, this approach is not specific to a single dataset, allowing us to use it on many datasets. - -Disadvantages: - -- Our model is less realistic, which may negatively impact the accuracy of our fit and the scientific results we -derive from it. - -__Search More Thoroughly__ - -In approaches 1 and 2, we assisted our non-linear search to find the highest log likelihood regions of parameter -space. In approach 3, we're simply going to tell the search to look more thoroughly through parameter space. - -Every non-linear search has settings that control how thoroughly it explores parameter space. For Dynesty, the -primary setting is the number of live points `nlive`. The more thoroughly the search examines the space, the more -likely it is to find the global maximum model. However, this also means the search will take longer! - -Below, we configure a more thorough Dynesty search with `nlive=500`. It is currently unclear what changing -this setting actually does and what the number of live points actually means. These will be covered in chapter 2 -of the **HowToFit** lectures, where we will also expand on how a non-linear search actually works and the different -types of methods that can be used to search parameter space. -""" -model = af.Collection( - gaussian_0=Gaussian, - gaussian_1=Gaussian, - gaussian_2=Gaussian, - gaussian_3=Gaussian, - gaussian_4=Gaussian, -) - -""" -The `model.info` confirms the model is the same model fitted initially, composed of 5 `Gaussian` profiles. -""" -print(model.info) - -""" -__Search__ - -We again use the nested sampling algorithm Dynesty to fit the model to the data, but now increase the number of live -points to 300 meaning it will search parameter space more thoroughly. -""" -search = af.DynestyStatic( - nlive=300, - sample="rwalk", # This makes dynesty run faster, don't worry about what it means for now! -) - -""" -__Model Fit__ - -Perform the fit using our five `Gaussian` model, which has 15 free parameters. -""" -analysis = Analysis(data=data, noise_map=noise_map) - -print( - """ - The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - - -""" -Lets print the result info and plot the fit to the dataset to confirm the more thorough search has provided a better -model-fit. -""" -print(result.info) - -plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - linestyle="", -) -plt.plot(range(data.shape[0]), model_data, color="r") -for model_data_1d_individual in model_data_list: - plt.plot(range(data.shape[0]), model_data_1d_individual, "--") -plt.title(f"Fit (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.clf() -plt.close() - -residual_map = data - model_data -normalized_residual_map = residual_map / noise_map -plt.plot(xvalues, normalized_residual_map, color="k") -plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") -plt.xlabel("x values of profile") -plt.ylabel("Normalized Residuals ($\sigma$)") -plt.show() -plt.clf() -plt.close() - -""" -If you repeat the fit multiple times, you will find that the model-fit is more likely to produce a good fit than -previously. - -However, the run-time of the search is noticeably longer, taking a few minutes to complete, owining -to the increased number of live points and fact it is searching parameter space more thoroughly. - -Let's list the advantages and disadvantages of simply adjusting the non-linear search: - -**Advantages:** - -- It’s easy to set up; just change the settings of the non-linear search. - -- It generalizes to any dataset. - -- We can retain a more complex model. - -**Disadvantage:** - -- It can be very expensive in terms of run time, producing run-times that are five, tens or even longer than the - original run-time. - -__Summary__ - -We have covered three strategies for ensuring the non-linear search estimates the correct solution: - -1) Prior Tuning: By setting our priors more accurately, we can help the search find the global solution instead of - getting stuck at a local maxima. - -2) Reducing Complexity: By making certain assumptions, we can reduce the number of model parameters, thereby lowering - the dimensionality of the parameter space and improving the search's performance. - -3) Search More Thoroughly: By increasing the number of live points, we can make the search explore parameter space - more thoroughly, increasing the likelihood of finding the global maximum model. - -Each of these strategies has its advantages and disadvantages, and your ability to fit models successfully will -likely rely on a combination of these strategies. Which one works best depends on the specific model, dataset and -scientific question you are trying to answer. Therefore, when you begin your own model-fitting, it is a good idea to -try each of these strategies and assess which works best for your model-fit. - -__Run Times__ - -One challenging aspect of model-fitting which was not properly covered in this tutorial is the run-time of a model-fit. -This example fits simple 1D datasets, which are computationally inexpensive to fit. That is, the `log_likelihood_function` -is evaluated in a fraction of a second, meaning the non-linear search fitted the model in mere minutes. - -Many model-fitting tasks are not as fast. For example, when fitting a model to a 2D image, the `log_likelihood_function` -may take of order seconds, or longer, because it comprises a number of expensive calculations (e.g. a Fourier transform, -2D convolution, etc.). Depending on the model complexity, this means that the non-linear search may take hours, days -or even weeks to fit the model. - -Run times are also dictated by the complexity of the model and the nature of the log likelihood function. For models -with many more dimensions than the simple 1D model used in this tutorial (e.g. hundreds or thousands of free parameters), -non-linear search may take tens or hundreds of more iterations to converge on a solution. This is because the parameter -space is significantly more complex and difficult to sample accurately. More iterations mean longer run times, -which in combination with a slow likelihood function can make model-fitting infeasible. - -Whether or not run times will pose a challenge to your model-fitting task depends on the complexity of the model and -nature of the log likelihood function. If your problem is computationally expensive, **PyAutoFit** provides many -tools to help, which will be the topic of tutorials in chapter 2 of the **HowToFit** lectures. - -__Model Mismatch__ - -In this example, interpreting how well the model fitted the data, and whether it found the global maxima, was -relatively straightforward. This is because the same model was used to simulate the data and fit it, meaning the -global maxima fit corresponded to one where the normalized residuals were minimized and consistent with the noise -(e.g. they went to sigma values below 3.0 or so). - -In many scientific studies, the data that one is fitting may have come from an instrument or simulation where the -exact physical processes that generate the data are not perfectly known. This then means that the model is -not a perfect representation of the data, and it may not ever be possible to fit the data perfectly. In this case, -we might infer a fit with significant residuals, but it may still correspond to the global maxima solution, -at least for that particular model. - -This makes it even more difficult to be certain if the non-linear search is sampling parameter space correctly, -and therefore requires even more care and attention to the strategies we have discussed above. - -Whether or not this is the case for your model-fitting task is something you will have to determine yourself. -**PyAutoFit** provides many tools to help assess the quality of a model-fit, which will be the topic of tutorials -in chapter 2 of the **HowToFit** lectures. - -__Astronomy Example__ - -At the end of chapter 1, we will fit a complex model to a real astronomical dataset in order to quantify -the distribution of stars in 2D images of galaxies. - -This example will illustrate many of the challenges discussed in this tutorial, including: - -- Fits using more complex models consisting of 15-20 parameters often infer local maxima, unless we assist the - non-linear search with tuned priors, reduced complexity or a more thorough search. - -- Fitting 2D imaging data requires a 2D convolution, which is somewhat computationally expensive and means run times - become something we must balance with model complexity. - -- The model is not a perfect representation of the data. For example, the model assumes the galaxy is elliptically - symmetric, whereas the real galaxy may not be. In certain examples, this means that the global maxima solution - actually leaves significant residuals, above 3.0 $\sigma$, in the data. - -__Wrap Up__ - -Now is a good time to assess how straightforward or difficult you think your model-fitting task will be. - -Are the models you will be fitting made up of tens of parameters? or thousands? Are there ways you can simplify -the model parameter or tune priors to make the model-fitting task more feasible? Will run times be an issue, or is -your likelihood function computationally cheap? And how confident are you that the model you are fitting is a good -representation of the data? - -These are all questions you should be asking yourself before beginning your model-fitting task, but they will -become easier to answer as you gain experience with model-fitting and **PyAutoFit**. -""" +""" +Tutorial 4: Why Modeling Is Hard +================================ + +We have successfully fitted a simple 1D Gaussian profile to a dataset using a non-linear search. While achieving an +accurate model fit has been straightforward, the reality is that model fitting is a challenging problem where many things can go wrong. + +This tutorial will illustrate why modeling is challenging, highlight common problems that occur when fitting complex +models, and show how a good scientific approach can help us overcome these challenges. + +We will build on concepts introduced in previous tutorials, such as the non-linear parameter space, likelihood surface, +and the role of priors. + +__Overview__ + +In this tutorial, we will fit complex models with up to 15 free parameters and consider the following: + +- Why more complex models are more difficult to fit and may lead the non-linear search to infer an incorrect solution. + +- Strategies for ensuring the non-linear search estimates the correct solution. + +- What drives the run-times of a model fit and how to carefully balance run-times with model complexity. + +__Contents__ + +- **Data**: Load and plot the 1D Gaussian dataset we'll fit, which is more complex than the previous tutorial. +- **Model**: The `Gaussian` model component that we will fit to the data. +- **Analysis**: The log likelihood function used to fit the model to the data. +- **Alternative Syntax**: An alternative loop-based approach for creating a summed profile from multiple model components. +- **Collection**: The `Collection` model used to compose the model-fit. +- **Search**: Set up the nested sampling search (Dynesty) for the model-fit. +- **Model Fit**: Perform the model-fit and examine the results. +- **Result**: Determine if the model-fit was successful and what can be done to ensure a good model-fit. +- **Why Modeling is Hard**: Introduce the concept of randomness and local maxima and why they make model-fitting challenging. +- **Prior Tuning**: Adjust the priors of the model to help the non-linear search find the global maxima solution. +- **Reducing Complexity**: Simplify the model to reduce the dimensionality of the parameter space. +- **Search More Thoroughly**: Adjust the non-linear search settings to search parameter space more thoroughly. +- **Summary**: Summarize the three strategies for ensuring successful model-fitting. +- **Run Times**: Discuss how the likelihood function and complexity of a model impacts the run-time of a model-fit. +- **Model Mismatch**: Introduce the concept of model mismatches and how it makes inferring the correct model challenging. +- **Astronomy Example**: How the concepts of this tutorial are applied to real astronomical problems. +- **Wrap Up**: A summary of the key takeaways of this tutorial. +""" + +# from autofit import setup_notebook; setup_notebook() + +from os import path +import numpy as np +import matplotlib.pyplot as plt + +import autofit as af + +""" +__Data__ + +Load the dataset we fit. + +This is a new `dataset` where the underlying signal is a sum of five `Gaussian` profiles. +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x5") + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists(dataset_path): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) +noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") +) + +""" +Plotting the data reveals that the signal is more complex than a simple 1D Gaussian, as the wings to the left and +right are more extended than what a single Gaussian profile can account for. +""" +xvalues = np.arange(data.shape[0]) +plt.errorbar( + xvalues, + data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.title("1D Gaussian dataset with errors from the noise-map.") +plt.xlabel("x values of profile") +plt.ylabel("Signal Value") +plt.show() +plt.clf() +plt.close() + +""" +__Model__ + +Create the `Gaussian` class from which we will compose model components using the standard format. +""" + + +class Gaussian: + def __init__( + self, + centre: float = 30.0, # <- **PyAutoFit** recognises these constructor arguments + normalization: float = 1.0, # <- are the Gaussian`s model parameters. + sigma: float = 5.0, + ): + """ + Represents a 1D Gaussian profile. + + This is a model-component of example models in the **HowToFit** lectures and is used to perform model-fitting + of example datasets. + + Parameters + ---------- + centre + The x coordinate of the profile centre. + normalization + Overall normalization of the profile. + sigma + The sigma value controlling the size of the Gaussian. + """ + self.centre = centre + self.normalization = normalization + self.sigma = sigma + + def model_data_from(self, xvalues: np.ndarray) -> np.ndarray: + """ + Returns a 1D Gaussian on an input list of Cartesian x coordinates. + + The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. + + The output is referred to as the `model_data` to signify that it is a representation of the data from the + model. + + Parameters + ---------- + xvalues + The x coordinates in the original reference frame of the data. + + Returns + ------- + np.array + The Gaussian values at the input x coordinates. + """ + transformed_xvalues = np.subtract(xvalues, self.centre) + return np.multiply( + np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), + np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), + ) + + +""" +__Analysis__ + +To define the Analysis class for this model-fit, we need to ensure that the `log_likelihood_function` can handle an +instance containing multiple 1D profiles. Below is an expanded explanation and the corresponding class definition: + +The log_likelihood_function will now assume that the instance it receives consists of multiple Gaussian profiles. +For each Gaussian in the instance, it will compute the model_data and then sum these to create the overall `model_data` +that is compared to the observed data. +""" + + +class Analysis(af.Analysis): + def __init__(self, data: np.ndarray, noise_map: np.ndarray): + """ + The `Analysis` class acts as an interface between the data and model in **PyAutoFit**. + + Its `log_likelihood_function` defines how the model is fitted to the data and it is called many times by + the non-linear search fitting algorithm. + + In this example, the `log_likelihood_function` receives an instance containing multiple instances of + the `Gaussian` class and sums the `model_data` of each to create the overall model fit to the data. + + In this example the `Analysis` `__init__` constructor only contains the `data` and `noise-map`, but it can be + easily extended to include other quantities. + + Parameters + ---------- + data + A 1D numpy array containing the data (e.g. a noisy 1D signal) fitted in the workspace examples. + noise_map + A 1D numpy array containing the noise values of the data, used for computing the goodness of fit + metric, the log likelihood. + """ + super().__init__() + + self.data = data + self.noise_map = noise_map + + def log_likelihood_function(self, instance) -> float: + """ + Returns the log likelihood of a fit of a 1D Gaussian to the dataset. + + In the previous tutorial, the instance was a single `Gaussian` profile, however this function now assumes + the instance contains multiple `Gaussian` profiles. + + The `model_data` is therefore the summed `model_data` of all individual Gaussians in the model. + + The docstring below describes this in more detail. + + Parameters + ---------- + instance + A list of 1D profiles with parameters set via the non-linear search. + + Returns + ------- + float + The log likelihood value indicating how well this model fit the `MaskedDataset`. + """ + + """ + In the previous tutorial the instance was a single `Gaussian` profile, meaning we could create the model data + using the line: + + model_data = instance.gaussian.model_data_from(xvalues=self.data.xvalues) + + In this tutorial our instance is comprised of three 1D Gaussians, because we will use a `Collection` to + compose the model: + + model = Collection(gaussian_0=Gaussian, gaussian_1=Gaussian, gaussian_2=Gaussian). + + By using a Collection, this means the instance parameter input into the fit function is a + dictionary where individual profiles (and their parameters) can be accessed as followed: + + print(instance.gaussian_0) + print(instance.gaussian_1) + print(instance.gaussian_2) + + print(instance.gaussian_0.centre) + print(instance.gaussian_1.centre) + print(instance.gaussian_2.centre) + + The `model_data` is therefore the summed `model_data` of all individual Gaussians in the model. + + The function `model_data_from_instance` performs this summation. + """ + model_data = self.model_data_from_instance(instance=instance) + + residual_map = self.data - model_data + chi_squared_map = (residual_map / self.noise_map) ** 2.0 + chi_squared = sum(chi_squared_map) + noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) + log_likelihood = -0.5 * (chi_squared + noise_normalization) + + return log_likelihood + + def model_data_from_instance(self, instance): + """ + To create the summed profile of all individual profiles, we use a list comprehension to iterate over + all profiles in the instance. + + The `instance` has the properties of a Python `iterator` and therefore can be looped over using the standard + Python for syntax (e.g. `for profile in instance`). + + __Alternative Syntax__ + + For those not familiar with Python list comprehensions, the code below shows how to use the instance to + create the summed profile using a for loop and numpy array: + + model_data = np.zeros(shape=self.data.xvalues.shape[0]) + + for profile in instance: + model_data += profile.model_data_from(xvalues=self.data.xvalues) + + return model_data + """ + xvalues = np.arange(self.data.shape[0]) + + return sum([profile.model_data_from(xvalues=xvalues) for profile in instance]) + + +""" +__Collection__ + +In the previous tutorial, we fitted a single `Gaussian` profile to the dataset by turning it into a model +component using the `Model` class. + +In this tutorial, we will fit a model composed of five `Gaussian` profiles. To do this, we need to combine +five `Gaussian` model components into a single model. + +This can be achieved using a `Collection` object, which was introduced in tutorial 1. The `Collection` object allows +us to group together multiple model components—in this case, five `Gaussian` profiles—into one model that can be +passed to the non-linear search. +""" +model = af.Collection( + gaussian_0=Gaussian, + gaussian_1=Gaussian, + gaussian_2=Gaussian, + gaussian_3=Gaussian, + gaussian_4=Gaussian, +) + +""" +The `model.info` confirms the model is composed of 5 `Gaussian` profiles. +""" +print(model.info) + +""" +__Search__ + +We again use the nested sampling algorithm Dynesty to fit the model to the data. +""" +search = af.DynestyStatic( + sample="rwalk", # This makes dynesty run faster, don't worry about what it means for now! +) + +""" +__Model Fit__ + +Perform the fit using our five `Gaussian` model, which has 15 free parameters. + +This means the non-linear parameter space has a dimensionality of N=15, making it significantly more complex +than the simpler model we fitted in the previous tutorial. + +Consequently, the non-linear search takes slightly longer to run but still completes in under a minute. +""" +analysis = Analysis(data=data, noise_map=noise_map) + +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +__Result__ + +The `info` attribute shows the result in a readable format, which contains information on the full collection +of all 5 model components. +""" +print(result.info) + +""" +From the result info, it is hard to assess if the model fit was good or not. + +A good way to evaluate the fit is through a visual inspection of the model data plotted over the actual data. + +If the model data (red line) consistently aligns with the data points (black error bars), the fit is good. +However, if the model misses certain features of the data, such as peaks or regions of high intensity, +the fit was not successful. +""" +instance = result.max_log_likelihood_instance + +model_data_0 = instance.gaussian_0.model_data_from(xvalues=np.arange(data.shape[0])) +model_data_1 = instance.gaussian_1.model_data_from(xvalues=np.arange(data.shape[0])) +model_data_2 = instance.gaussian_2.model_data_from(xvalues=np.arange(data.shape[0])) +model_data_3 = instance.gaussian_3.model_data_from(xvalues=np.arange(data.shape[0])) +model_data_4 = instance.gaussian_4.model_data_from(xvalues=np.arange(data.shape[0])) + +model_data_list = [model_data_0, model_data_1, model_data_2, model_data_3, model_data_4] + +model_data = sum(model_data_list) + +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, +) +plt.plot(range(data.shape[0]), model_data, color="r") +for model_data_1d_individual in model_data_list: + plt.plot(range(data.shape[0]), model_data_1d_individual, "--") +plt.title(f"Fit (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.close() + +""" +It's challenging to determine from the plot whether the data and model data perfectly overlap across the entire dataset. + +To clarify this, the residual map introduced in tutorial 2 is useful. It provides a clear representation of where +the differences between the model and data exceed the noise level. + +Regions where the black error bars do not align with the zero line in the residual map indicate areas where the model +did not fit the data well and is inconsistent with the data above the noise level. Furthermore, regions where +larger values of residuals are next to one another indicate that the model failed to accurate fit that +region of the data. +""" +residual_map = data - model_data +plt.plot(range(data.shape[0]), np.zeros(data.shape[0]), "--", color="b") +plt.errorbar( + x=xvalues, + y=residual_map, + yerr=noise_map, + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + linestyle="", +) +plt.title(f"Residuals (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Residuals") +plt.show() +plt.clf() +plt.close() + +""" +The normalized residual map, as discussed in tutorial 2, provides an alternative visualization of the fit quality. + +Normalized residuals indicate the standard deviation (σ) level at which the residuals could have been drawn from the +noise. For instance, a normalized residual of 2.0 suggests that a residual value is 2.0σ away from the noise, +implying there is a 5% chance such a residual would occur due to noise. + +Values of normalized residuals above 3.0 are particularly improbable (occurring only 0.3% of the time), which is +generally considered a threshold where issues with the model-fit are likely the cause of the residual as opposed +to it being a noise fluctuation. +""" +residual_map = data - model_data +normalized_residual_map = residual_map / noise_map +plt.plot(xvalues, normalized_residual_map, color="k") +plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Normalized Residuals ($\sigma$)") +plt.show() +plt.clf() +plt.close() + +""" +So, did you achieve a good fit? Maybe a bad one? Or just an okay one? + +The truth is, I don't know, and I can't tell you for sure. Modeling is inherently random. It's not uncommon to +fit the same model to the same dataset using the same non-linear search and get a different result each time. + +When I ran the model fit above, that's exactly what happened. It produced a range of fits: some bad, some okay, and +some good, as shown in the images below: + + + + + +
+ +Distinguishing between the good and okay fit is difficult, however the normalized residuals make this easier. They show +that for the okay fit there are residuals above 3.0 sigma, indicating that the model did not perfectly fit the data. + + + + + +
+ +You should quickly rerun the code above a couple of times to see this variability for yourself. + +__Why Modeling is Hard__ + +This variability is at the heart of why modeling is challenging. The process of model-fitting is stochastic, +meaning it's hard as the scientist to determine if a better fit is possible or not. + +Why does modeling produce different results each time, and why might it sometimes infer solutions that are not good fits? + +In the previous tutorial, the non-linear search consistently found models that visually matched the data well, +minimizing residuals and yielding high log likelihood values. These optimal solutions are called 'global maxima', +they are where the model parameters correspond to the highest likelihood regions across the entire parameter space. +This ideal scenario is illustrated in the `good_fit.png` image above. + +However, non-linear searches do not always find these global maxima. Instead, they might settle on 'local maxima' +solutions, which have high log likelihood values relative to nearby models in parameter space but are significantly +lower than the true global maxima found elsewhere. + +This is what happened for the okay and bad fits above. The non-linear search converged on solutions that were locally +peaks on the likelihood surface but were not the global maximum solution. This is why the residuals were higher and +the normalized residuals above 3.0 sigma. + +Why does a non-linear search end up at local maxima? As discussed, the search iterates through many models, +focusing more on regions where previous guesses yielded higher likelihoods. It gradually converges around +solutions with higher likelihoods compared to surrounding models. If the search isn't exhaustive enough, it might +converge on a local maxima that appears good compared to nearby models but isn't the global maximum. + +Modeling is challenging because the parameter spaces of complex models are typically filled with local maxima, +making it hard for a non-linear search to locate the global maximum. + +Fortunately, there are strategies to help non-linear searches find the global maxima, and we'll now explore three of +them. + +__Prior Tuning__ + +First, let's assist our non-linear search by tuning our priors. Priors provide guidance to the search on where to +explore in the parameter space. By setting more accurate priors ('tuning' them), we can help the search find the +global solution instead of settling for a local maximum. + +For instance, from the data itself, it's evident that all `Gaussian` profiles are centered around pixel 50. In our +previous fit, the `centre` parameter of each `Gaussian` had a `UniformPrior` spanning from 0.0 to 100.0, which is +much broader than necessary given the data's range. + +Additionally, the peak value of the data's `normalization` parameter was around 17.5. This indicates that +the `normalization` values of our `Gaussians` do not exceed 500.0, allowing us to refine our prior accordingly. + +The following code snippet adjusts these priors for the `centre` and `normalization` parameters of +each `Gaussian` using **PyAutoFit**'s API for model and prior customization: +""" +gaussian_0 = af.Model(Gaussian) + +gaussian_0.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) +gaussian_0.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) + +gaussian_1 = af.Model(Gaussian) + +gaussian_1.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) +gaussian_1.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) + +gaussian_2 = af.Model(Gaussian) + +gaussian_2.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) +gaussian_2.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) + +gaussian_3 = af.Model(Gaussian) + +gaussian_3.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) +gaussian_3.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) + +gaussian_4 = af.Model(Gaussian) + +gaussian_4.centre = af.UniformPrior(lower_limit=45.0, upper_limit=55.0) +gaussian_4.normalization = af.LogUniformPrior(lower_limit=0.1, upper_limit=500.0) + +model = af.Collection( + gaussian_0=gaussian_0, + gaussian_1=gaussian_1, + gaussian_2=gaussian_2, + gaussian_3=gaussian_3, + gaussian_4=gaussian_4, +) + +""" +The `info` attribute shows the model is now using the priors specified above. +""" +print(model.info) + +""" +We now repeat the model-fit using these updated priors. + +First, you should note that the run time of the fit is significantly faster than the previous fit. This is because +the prior is telling the non-linear search where to look, meaning it converges on solutions more quickly +and spends less time searching regions of parameter space that do not contain solutions. + +Second, the model-fit consistently produces a good model-fit more often, because our tuned priors are centred +on the global maxima solution ensuring the non-linear search is less likely to converge on a local maxima. +""" +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +Lets print the result info and plot the fit to the dataset to confirm the priors have provided a better model-fit. +""" +print(result.info) + +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + linestyle="", +) +plt.plot(range(data.shape[0]), model_data, color="r") +for model_data_1d_individual in model_data_list: + plt.plot(range(data.shape[0]), model_data_1d_individual, "--") +plt.title(f"Fit (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.clf() +plt.close() + +residual_map = data - model_data +normalized_residual_map = residual_map / noise_map +plt.plot(xvalues, normalized_residual_map, color="k") +plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Normalized Residuals ($\sigma$)") +plt.show() +plt.clf() +plt.close() + +""" +Lets consider the advantages and disadvantages of prior tuning: + +**Advantages:** + +- Higher likelihood of finding the global maximum log likelihood solutions in parameter space. + +- Faster search times, as the non-linear search explores less of the parameter space. + +**Disadvantages:** + +- Incorrectly specified priors could lead the non-linear search to an incorrect solution. + +- It is not always clear how the priors should be tuned, especially for complex models. + +- Priors tuning must be applied to each dataset fitted. For large datasets, this process would be very time-consuming. + +__Reducing Complexity__ + +The non-linear search may fail because the model is too complex, making its parameter space too difficult to +sample accurately consistent. To address this, we may be able to simplify the model while ensuring it remains +realistic enough for our scientific study. By making certain assumptions, we can reduce the number of model +parameters, thereby lowering the dimensionality of the parameter space and improving the search's performance. + +For example, we may know that the `Gaussian`'s in our model are aligned at the same `centre`. We can therefore +compose a model that assumes that the `centre` of each `Gaussian` is the same, reducing the dimensionality of the +model from N=15 to N=11. + +The code below shows how we can customize the model components to ensure the `centre` of each `Gaussian` is the same: +""" +gaussian_0 = af.Model(Gaussian) +gaussian_1 = af.Model(Gaussian) +gaussian_2 = af.Model(Gaussian) +gaussian_3 = af.Model(Gaussian) +gaussian_4 = af.Model(Gaussian) + +gaussian_1.centre = gaussian_0.centre +gaussian_2.centre = gaussian_0.centre +gaussian_3.centre = gaussian_0.centre +gaussian_4.centre = gaussian_0.centre + +model = af.Collection( + gaussian_0=gaussian_0, + gaussian_1=gaussian_1, + gaussian_2=gaussian_2, + gaussian_3=gaussian_3, + gaussian_4=gaussian_4, +) + +""" +The `info` attribute shows the model is now using the same `centre` for all `Gaussian`'s and has 11 free parameters. +""" +print(model.info) + +""" +We now repeat the model-fit using this updated model where the `centre` of each `Gaussian` is the same. + +You should again note that the run time of the fit is significantly faster than the previous fits +and that it consistently produces a good model-fit more often. This is because the model is less complex, +non-linear parameter space is less difficult to sample and the search is less likely to converge on a local maxima. +""" +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +Lets print the result info and plot the fit to the dataset to confirm the reduced model complexity has +provided a better model-fit. +""" +print(result.info) + +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + linestyle="", +) +plt.plot(range(data.shape[0]), model_data, color="r") +for model_data_1d_individual in model_data_list: + plt.plot(range(data.shape[0]), model_data_1d_individual, "--") +plt.title(f"Fit (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.clf() +plt.close() + +residual_map = data - model_data +normalized_residual_map = residual_map / noise_map +plt.plot(xvalues, normalized_residual_map, color="k") +plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Normalized Residuals ($\sigma$)") +plt.show() +plt.clf() +plt.close() + +""" +Let’s consider the advantages and disadvantages of simplifying the model: + +Advantages: + +- By reducing the complexity of the parameter space, we increase the chances of finding the global maximum log +likelihood, and the search requires less time to do so. + +- Unlike with tuned priors, this approach is not specific to a single dataset, allowing us to use it on many datasets. + +Disadvantages: + +- Our model is less realistic, which may negatively impact the accuracy of our fit and the scientific results we +derive from it. + +__Search More Thoroughly__ + +In approaches 1 and 2, we assisted our non-linear search to find the highest log likelihood regions of parameter +space. In approach 3, we're simply going to tell the search to look more thoroughly through parameter space. + +Every non-linear search has settings that control how thoroughly it explores parameter space. For Dynesty, the +primary setting is the number of live points `nlive`. The more thoroughly the search examines the space, the more +likely it is to find the global maximum model. However, this also means the search will take longer! + +Below, we configure a more thorough Dynesty search with `nlive=500`. It is currently unclear what changing +this setting actually does and what the number of live points actually means. These will be covered in chapter 2 +of the **HowToFit** lectures, where we will also expand on how a non-linear search actually works and the different +types of methods that can be used to search parameter space. +""" +model = af.Collection( + gaussian_0=Gaussian, + gaussian_1=Gaussian, + gaussian_2=Gaussian, + gaussian_3=Gaussian, + gaussian_4=Gaussian, +) + +""" +The `model.info` confirms the model is the same model fitted initially, composed of 5 `Gaussian` profiles. +""" +print(model.info) + +""" +__Search__ + +We again use the nested sampling algorithm Dynesty to fit the model to the data, but now increase the number of live +points to 300 meaning it will search parameter space more thoroughly. +""" +search = af.DynestyStatic( + nlive=300, + sample="rwalk", # This makes dynesty run faster, don't worry about what it means for now! +) + +""" +__Model Fit__ + +Perform the fit using our five `Gaussian` model, which has 15 free parameters. +""" +analysis = Analysis(data=data, noise_map=noise_map) + +print( + """ + The non-linear search has begun running. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + + +""" +Lets print the result info and plot the fit to the dataset to confirm the more thorough search has provided a better +model-fit. +""" +print(result.info) + +plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + linestyle="", +) +plt.plot(range(data.shape[0]), model_data, color="r") +for model_data_1d_individual in model_data_list: + plt.plot(range(data.shape[0]), model_data_1d_individual, "--") +plt.title(f"Fit (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.clf() +plt.close() + +residual_map = data - model_data +normalized_residual_map = residual_map / noise_map +plt.plot(xvalues, normalized_residual_map, color="k") +plt.title(f"Normalized Residuals (log likelihood = {result.log_likelihood})") +plt.xlabel("x values of profile") +plt.ylabel("Normalized Residuals ($\sigma$)") +plt.show() +plt.clf() +plt.close() + +""" +If you repeat the fit multiple times, you will find that the model-fit is more likely to produce a good fit than +previously. + +However, the run-time of the search is noticeably longer, taking a few minutes to complete, owining +to the increased number of live points and fact it is searching parameter space more thoroughly. + +Let's list the advantages and disadvantages of simply adjusting the non-linear search: + +**Advantages:** + +- It’s easy to set up; just change the settings of the non-linear search. + +- It generalizes to any dataset. + +- We can retain a more complex model. + +**Disadvantage:** + +- It can be very expensive in terms of run time, producing run-times that are five, tens or even longer than the + original run-time. + +__Summary__ + +We have covered three strategies for ensuring the non-linear search estimates the correct solution: + +1) Prior Tuning: By setting our priors more accurately, we can help the search find the global solution instead of + getting stuck at a local maxima. + +2) Reducing Complexity: By making certain assumptions, we can reduce the number of model parameters, thereby lowering + the dimensionality of the parameter space and improving the search's performance. + +3) Search More Thoroughly: By increasing the number of live points, we can make the search explore parameter space + more thoroughly, increasing the likelihood of finding the global maximum model. + +Each of these strategies has its advantages and disadvantages, and your ability to fit models successfully will +likely rely on a combination of these strategies. Which one works best depends on the specific model, dataset and +scientific question you are trying to answer. Therefore, when you begin your own model-fitting, it is a good idea to +try each of these strategies and assess which works best for your model-fit. + +__Run Times__ + +One challenging aspect of model-fitting which was not properly covered in this tutorial is the run-time of a model-fit. +This example fits simple 1D datasets, which are computationally inexpensive to fit. That is, the `log_likelihood_function` +is evaluated in a fraction of a second, meaning the non-linear search fitted the model in mere minutes. + +Many model-fitting tasks are not as fast. For example, when fitting a model to a 2D image, the `log_likelihood_function` +may take of order seconds, or longer, because it comprises a number of expensive calculations (e.g. a Fourier transform, +2D convolution, etc.). Depending on the model complexity, this means that the non-linear search may take hours, days +or even weeks to fit the model. + +Run times are also dictated by the complexity of the model and the nature of the log likelihood function. For models +with many more dimensions than the simple 1D model used in this tutorial (e.g. hundreds or thousands of free parameters), +non-linear search may take tens or hundreds of more iterations to converge on a solution. This is because the parameter +space is significantly more complex and difficult to sample accurately. More iterations mean longer run times, +which in combination with a slow likelihood function can make model-fitting infeasible. + +Whether or not run times will pose a challenge to your model-fitting task depends on the complexity of the model and +nature of the log likelihood function. If your problem is computationally expensive, **PyAutoFit** provides many +tools to help, which will be the topic of tutorials in chapter 2 of the **HowToFit** lectures. + +__Model Mismatch__ + +In this example, interpreting how well the model fitted the data, and whether it found the global maxima, was +relatively straightforward. This is because the same model was used to simulate the data and fit it, meaning the +global maxima fit corresponded to one where the normalized residuals were minimized and consistent with the noise +(e.g. they went to sigma values below 3.0 or so). + +In many scientific studies, the data that one is fitting may have come from an instrument or simulation where the +exact physical processes that generate the data are not perfectly known. This then means that the model is +not a perfect representation of the data, and it may not ever be possible to fit the data perfectly. In this case, +we might infer a fit with significant residuals, but it may still correspond to the global maxima solution, +at least for that particular model. + +This makes it even more difficult to be certain if the non-linear search is sampling parameter space correctly, +and therefore requires even more care and attention to the strategies we have discussed above. + +Whether or not this is the case for your model-fitting task is something you will have to determine yourself. +**PyAutoFit** provides many tools to help assess the quality of a model-fit, which will be the topic of tutorials +in chapter 2 of the **HowToFit** lectures. + +__Astronomy Example__ + +At the end of chapter 1, we will fit a complex model to a real astronomical dataset in order to quantify +the distribution of stars in 2D images of galaxies. + +This example will illustrate many of the challenges discussed in this tutorial, including: + +- Fits using more complex models consisting of 15-20 parameters often infer local maxima, unless we assist the + non-linear search with tuned priors, reduced complexity or a more thorough search. + +- Fitting 2D imaging data requires a 2D convolution, which is somewhat computationally expensive and means run times + become something we must balance with model complexity. + +- The model is not a perfect representation of the data. For example, the model assumes the galaxy is elliptically + symmetric, whereas the real galaxy may not be. In certain examples, this means that the global maxima solution + actually leaves significant residuals, above 3.0 $\sigma$, in the data. + +__Wrap Up__ + +Now is a good time to assess how straightforward or difficult you think your model-fitting task will be. + +Are the models you will be fitting made up of tens of parameters? or thousands? Are there ways you can simplify +the model parameter or tune priors to make the model-fitting task more feasible? Will run times be an issue, or is +your likelihood function computationally cheap? And how confident are you that the model you are fitting is a good +representation of the data? + +These are all questions you should be asking yourself before beginning your model-fitting task, but they will +become easier to answer as you gain experience with model-fitting and **PyAutoFit**. +""" diff --git a/scripts/chapter_1_introduction/tutorial_5_results_and_samples.py b/scripts/chapter_1_introduction/tutorial_5_results_and_samples.py index 467c515..c6785fc 100644 --- a/scripts/chapter_1_introduction/tutorial_5_results_and_samples.py +++ b/scripts/chapter_1_introduction/tutorial_5_results_and_samples.py @@ -1,702 +1,702 @@ -""" -Tutorial 5: Results And Samples -=============================== - -In this tutorial, we'll cover all of the output that comes from a non-linear search's `Result` object. - -We used this object at various points in the chapter. The bulk of material covered here is described in the example -script `autofit_workspace/overview/simple/result.py`. Nevertheless, it is a good idea to refresh ourselves about how -results in **PyAutoFit** work before covering more advanced material. - -__Contents__ - -This tutorial is split into the following sections: - -- **Data**: Load the dataset from the HowToFit/dataset folder. -- **Reused Functions**: Reuse the `plot_profile_1d` and `Analysis` classes from the previous tutorial. -- **Model Fit**: Run a non-linear search to generate a `Result` object. -- **Result**: Examine the `Result` object and its info attribute. -- **Samples**: Introduce the `Samples` object containing the non-linear search samples. -- **Parameters**: Access parameter values from the samples. -- **Figures of Merit**: Examine log likelihood, log prior, and log posterior values. -- **Instances**: Return results as model instances from samples. -- **Vectors**: Return results as 1D parameter vectors. -- **Labels**: Access the paths, names, and labels for model parameters. -- **Posterior / PDF**: Access median PDF estimates for the model parameters. -- **Plot**: Visualize model fit results using instances. -- **Errors**: Compute parameter error estimates at specified sigma confidence limits. -- **PDF**: Plot Probability Density Functions using corner.py. -- **Other Results**: Access maximum log posterior and other sample statistics. -- **Sample Instance**: Create instances from individual samples in the sample list. -- **Bayesian Evidence**: Access the log evidence for nested sampling searches. -- **Derived Errors (PDF from samples)**: Compute errors on derived quantities from sample PDFs. -- **Samples Filtering**: Filter samples by parameter paths for specific parameter analysis. -- **Latex**: Generate LaTeX table code for modeling results. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import autofit as af -import autofit.plot as aplt -import os -from os import path -import numpy as np -import matplotlib.pyplot as plt - -""" -__Data__ - -Load the dataset from the `HowToFit/dataset` folder. -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1__exponential_x1") - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists(dataset_path): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) -noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") -) - -""" -__Reused Functions__ - -We'll reuse the `plot_profile_1d` and `Analysis` classes of the previous tutorial. -""" - - -def plot_profile_1d( - xvalues, - profile_1d, - title=None, - ylabel=None, - errors=None, - color="k", - output_path=None, - output_filename=None, -): - plt.errorbar( - x=xvalues, - y=profile_1d, - yerr=errors, - linestyle="", - color=color, - ecolor="k", - elinewidth=1, - capsize=2, - ) - plt.title(title) - plt.xlabel("x value of profile") - plt.ylabel(ylabel) - if not path.exists(output_path): - os.makedirs(output_path) - plt.savefig(path.join(output_path, f"{output_filename}.png")) - plt.clf() - - -class Analysis(af.Analysis): - def __init__(self, data, noise_map): - super().__init__() - - self.data = data - self.noise_map = noise_map - - def log_likelihood_function(self, instance): - model_data = self.model_data_from_instance(instance=instance) - - residual_map = self.data - model_data - chi_squared_map = (residual_map / self.noise_map) ** 2.0 - chi_squared = sum(chi_squared_map) - noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) - log_likelihood = -0.5 * (chi_squared + noise_normalization) - - return log_likelihood - - def model_data_from_instance(self, instance): - """ - To create the summed profile of all individual profiles in an instance, we can use a dictionary comprehension - to iterate over all profiles in the instance. - """ - xvalues = np.arange(self.data.shape[0]) - - return sum([profile.model_data_from(xvalues=xvalues) for profile in instance]) - - def visualize(self, paths, instance, during_analysis): - """ - This method is identical to the previous tutorial, except it now uses the `model_data_from_instance` method - to create the profile. - """ - xvalues = np.arange(self.data.shape[0]) - - model_data = self.model_data_from_instance(instance=instance) - - residual_map = self.data - model_data - chi_squared_map = (residual_map / self.noise_map) ** 2.0 - - """The visualizer now outputs images of the best-fit results to hard-disk (checkout `visualizer.py`).""" - plot_profile_1d( - xvalues=xvalues, - profile_1d=self.data, - title="Data", - ylabel="Data Values", - color="k", - output_path=paths.image_path, - output_filename="data", - ) - - plot_profile_1d( - xvalues=xvalues, - profile_1d=model_data, - title="Model Data", - ylabel="Model Data Values", - color="k", - output_path=paths.image_path, - output_filename="model_data", - ) - - plot_profile_1d( - xvalues=xvalues, - profile_1d=residual_map, - title="Residual Map", - ylabel="Residuals", - color="k", - output_path=paths.image_path, - output_filename="residual_map", - ) - - plot_profile_1d( - xvalues=xvalues, - profile_1d=chi_squared_map, - title="Chi-Squared Map", - ylabel="Chi-Squareds", - color="k", - output_path=paths.image_path, - output_filename="chi_squared_map", - ) - - -""" -__Model Fit__ - -Now lets run the non-linear search to get ourselves a `Result`. -""" - - -class Gaussian: - def __init__( - self, - centre=30.0, # <- **PyAutoFit** recognises these constructor arguments - normalization=1.0, # <- are the Gaussian`s model parameters. - sigma=5.0, - ): - """ - Represents a 1D Gaussian profile. - - This is a model-component of example models in the **HowToFit** lectures and is used to fit example datasets - via a non-linear search. - - Parameters - ---------- - centre - The x coordinate of the profile centre. - normalization - Overall normalization of the profile. - sigma - The sigma value controlling the size of the Gaussian. - """ - self.centre = centre - self.normalization = normalization - self.sigma = sigma - - def model_data_from(self, xvalues: np.ndarray): - """ - - Returns a 1D Gaussian on an input list of Cartesian x coordinates. - - The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. - - The output is referred to as the `model_data` to signify that it is a representation of the data from the - model. - - Parameters - ---------- - xvalues - The x coordinates in the original reference frame of the data. - """ - transformed_xvalues = np.subtract(xvalues, self.centre) - return np.multiply( - np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), - np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), - ) - - -class Exponential: - def __init__( - self, - centre=30.0, # <- **PyAutoFit** recognises these constructor arguments - normalization=1.0, # <- are the Exponential`s model parameters. - rate=0.01, - ): - """ - Represents a 1D Exponential profile. - - This is a model-component of example models in the **HowToFit** lectures and is used to fit example datasets - via a non-linear search. - - Parameters - ---------- - centre - The x coordinate of the profile centre. - normalization - Overall normalization of the profile. - ratw - The decay rate controlling has fast the Exponential declines. - """ - self.centre = centre - self.normalization = normalization - self.rate = rate - - def model_data_from(self, xvalues: np.ndarray): - """ - Returns a 1D Gaussian on an input list of Cartesian x coordinates. - - The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. - - The output is referred to as the `model_data` to signify that it is a representation of the data from the - model. - - Parameters - ---------- - xvalues - The x coordinates in the original reference frame of the data. - """ - transformed_xvalues = np.subtract(xvalues, self.centre) - return self.normalization * np.multiply( - self.rate, np.exp(-1.0 * self.rate * abs(transformed_xvalues)) - ) - - -model = af.Collection(gaussian=af.Model(Gaussian), exponential=af.Model(Exponential)) - -analysis = Analysis(data=data, noise_map=noise_map) - -search = af.Emcee( - name="tutorial_5_results_and_samples", - path_prefix="chapter_1_introduction", -) - -print( - """ - The non-linear search has begun running. - Checkout the HowToFit/output/chapter_1_introduction/tutorial_5_results_and_samples - folder for live output of the results. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! - """ -) - -result = search.fit(model=model, analysis=analysis) - -print("The search has finished run - you may now continue the notebook.") - -""" -__Result__ - -Here, we'll look in detail at what information is contained in the `Result`. - -It contains an `info` attribute which prints the result in readable format. -""" -print(result.info) - -""" -__Samples__ - -The result contains a `Samples` object, which contains all of the non-linear search samples. - -Each sample corresponds to a set of model parameters that were evaluated and accepted by our non linear search, -in this example emcee. - -This also includes their log likelihoods, which are used for computing additional information about the model-fit, -for example the error on every parameter. - -Our model-fit used the MCMC algorithm Emcee, so the `Samples` object returned is a `SamplesMCMC` object. -""" -samples = result.samples - -print("MCMC Samples: \n") -print(samples) - -""" -__Parameters__ - -The parameters are stored as a list of lists, where: - - - The outer list is the size of the total number of samples. - - The inner list is the size of the number of free parameters in the fit. - -Below, we print the first sample — its second parameter (Gaussian -> normalization) and its third -parameter (Gaussian -> sigma). Any other sample index would work the same way; index `0` is used -here so the example remains valid regardless of how many samples the search produced. -""" -samples = result.samples -print("Sample 0's second parameter value (Gaussian -> normalization):") -print(samples.parameter_lists[0][1]) -print("Sample 0's third parameter value (Gaussian -> sigma)") -print(samples.parameter_lists[0][2], "\n") - -""" -__Figures of Merit__ - -The Samples class also contains the log likelihood, log prior, log posterior and weight_list of every accepted sample, -where: - -- The log likelihood is the value evaluated from the likelihood function (e.g. -0.5 * chi_squared + the noise -normalized). - -- The log prior encodes information on how the priors on the parameters maps the log likelihood value to the log -posterior value. - -- The log posterior is log_likelihood + log_prior. - -- The weight gives information on how samples should be combined to estimate the posterior. The weight values depend on -the sampler used, for MCMC samples they are all 1 (e.g. all weighted equally). - -Below, we inspect the first sample. Any sample index would work the same way; index `0` is used here -so the example is valid regardless of how many samples the search produced. -""" -print("log(likelihood), log(prior), log(posterior) and weight of the first sample.") -print(samples.log_likelihood_list[0]) -print(samples.log_prior_list[0]) -print(samples.log_posterior_list[0]) -print(samples.weight_list[0]) - -""" -__Instances__ - -The `Samples` contains many results which are returned as an instance of the model, using the Python class structure -of the model composition. - -For example, we can return the model parameters corresponding to the maximum log likelihood sample. -""" -max_lh_instance = samples.max_log_likelihood() - -print("Max Log Likelihood `Gaussian` Instance:") -print("Centre = ", max_lh_instance.gaussian.centre) -print("Normalization = ", max_lh_instance.gaussian.normalization) -print("Sigma = ", max_lh_instance.gaussian.sigma, "\n") - -print("Max Log Likelihood Exponential Instance:") -print("Centre = ", max_lh_instance.exponential.centre) -print("Normalization = ", max_lh_instance.exponential.normalization) -print("Sigma = ", max_lh_instance.exponential.rate, "\n") - -""" -__Vectors__ - -All results can alternatively be returned as a 1D vector of values, by passing `as_instance=False`: -""" -max_lh_vector = samples.max_log_likelihood(as_instance=False) -print("Max Log Likelihood Model Parameters: \n") -print(max_lh_vector, "\n\n") - -""" -__Labels__ - -Vectors return a lists of all model parameters, but do not tell us which values correspond to which parameters. - -The following quantities are available in the `Model`, where the order of their entries correspond to the parameters -in the `ml_vector` above: - - - `paths`: a list of tuples which give the path of every parameter in the `Model`. - - `parameter_names`: a list of shorthand parameter names derived from the `paths`. - - `parameter_labels`: a list of parameter labels used when visualizing non-linear search results (see below). - -""" -model = samples.model - -print(model.paths) -print(model.parameter_names) -print(model.parameter_labels) -print(model.model_component_and_parameter_names) -print("\n") - -""" -From here on, we will returned all results information as instances, but every method below can be returned as a -vector via the `as_instance=False` input. - -__Posterior / PDF__ - -The ``Result`` object contains the full posterior information of our non-linear search, which can be used for -parameter estimation. - -The median pdf vector is available from the `Samples` object, which estimates the every parameter via 1D -marginalization of their PDFs. -""" -median_pdf_instance = samples.median_pdf() - -print("Max Log Likelihood `Gaussian` Instance:") -print("Centre = ", median_pdf_instance.gaussian.centre) -print("Normalization = ", median_pdf_instance.gaussian.normalization) -print("Sigma = ", median_pdf_instance.gaussian.sigma, "\n") - -print("Max Log Likelihood Exponential Instance:") -print("Centre = ", median_pdf_instance.exponential.centre) -print("Normalization = ", median_pdf_instance.exponential.normalization) -print("Sigma = ", median_pdf_instance.exponential.rate, "\n") - -""" -__Plot__ - -Because results are returned as instances, it is straight forward to use them and their associated functionality -to make plots of the results: -""" -model_gaussian = max_lh_instance.gaussian.model_data_from( - xvalues=np.arange(data.shape[0]) -) -model_exponential = max_lh_instance.exponential.model_data_from( - xvalues=np.arange(data.shape[0]) -) -model_data = model_gaussian + model_exponential - -plt.plot(range(data.shape[0]), data) -plt.plot(range(data.shape[0]), model_data) -plt.plot(range(data.shape[0]), model_gaussian, "--") -plt.plot(range(data.shape[0]), model_exponential, "--") -plt.title("Illustrative model fit to 1D `Gaussian` + Exponential profile data.") -plt.xlabel("x values of profile") -plt.ylabel("Profile normalization") -plt.show() -plt.close() - -""" -__Errors__ - -The samples include methods for computing the error estimates of all parameters, via 1D marginalization at an -input sigma confidence limit. -""" -errors_at_upper_sigma_instance = samples.errors_at_upper_sigma(sigma=3.0) -errors_at_lower_sigma_instance = samples.errors_at_lower_sigma(sigma=3.0) - -print("Upper Error values of Gaussian (at 3.0 sigma confidence):") -print("Centre = ", errors_at_upper_sigma_instance.gaussian.centre) -print("Normalization = ", errors_at_upper_sigma_instance.gaussian.normalization) -print("Sigma = ", errors_at_upper_sigma_instance.gaussian.sigma, "\n") - -print("lower Error values of Gaussian (at 3.0 sigma confidence):") -print("Centre = ", errors_at_lower_sigma_instance.gaussian.centre) -print("Normalization = ", errors_at_lower_sigma_instance.gaussian.normalization) -print("Sigma = ", errors_at_lower_sigma_instance.gaussian.sigma, "\n") - -""" -They can also be returned at the values of the parameters at their error values: -""" -values_at_upper_sigma_instance = samples.values_at_upper_sigma(sigma=3.0) -values_at_lower_sigma_instance = samples.values_at_lower_sigma(sigma=3.0) - -print("Upper Parameter values w/ error of Gaussian (at 3.0 sigma confidence):") -print("Centre = ", values_at_upper_sigma_instance.gaussian.centre) -print("Normalization = ", values_at_upper_sigma_instance.gaussian.normalization) -print("Sigma = ", values_at_upper_sigma_instance.gaussian.sigma, "\n") - -print("lower Parameter values w/ errors of Gaussian (at 3.0 sigma confidence):") -print("Centre = ", values_at_lower_sigma_instance.gaussian.centre) -print("Normalization = ", values_at_lower_sigma_instance.gaussian.normalization) -print("Sigma = ", values_at_lower_sigma_instance.gaussian.sigma, "\n") - -""" -__PDF__ - -The Probability Density Functions (PDF's) of the results can be plotted using the Emcee's visualization -tool `corner.py`, which is wrapped via the `aplt.corner_cornerpy` function. -""" -aplt.corner_cornerpy(samples=result.samples) - -""" -__Other Results__ - -The samples contain many useful vectors, including the samples with the highest posterior values. -""" -max_log_posterior_instance = samples.max_log_posterior() - -print("Maximum Log Posterior Vector:") -print("Centre = ", max_log_posterior_instance.gaussian.centre) -print("Normalization = ", max_log_posterior_instance.gaussian.normalization) -print("Sigma = ", max_log_posterior_instance.gaussian.sigma, "\n") - - -""" -All methods above are available as a vector: -""" -median_pdf_instance = samples.median_pdf(as_instance=False) -values_at_upper_sigma = samples.values_at_upper_sigma(sigma=3.0, as_instance=False) -values_at_lower_sigma = samples.values_at_lower_sigma(sigma=3.0, as_instance=False) -errors_at_upper_sigma = samples.errors_at_upper_sigma(sigma=3.0, as_instance=False) -errors_at_lower_sigma = samples.errors_at_lower_sigma(sigma=3.0, as_instance=False) - -""" -__Sample Instance__ - -A non-linear search retains every model that is accepted during the model-fit. - -We can create an instance of any lens model -- below we create an instance of the last accepted model. -""" -instance = samples.from_sample_index(sample_index=-1) - -print("Gaussian Instance of last sample") -print("Centre = ", instance.gaussian.centre) -print("Normalization = ", instance.gaussian.normalization) -print("Sigma = ", instance.gaussian.sigma, "\n") - -""" -__Bayesian Evidence__ - -If a nested sampling `NonLinearSearch` is used, the evidence of the model is also available which enables Bayesian -model comparison to be performed (given we are using Emcee, which is not a nested sampling algorithm, the log evidence -is None).: -""" -log_evidence = samples.log_evidence - -""" -__Derived Errors (PDF from samples)__ - -Computing the errors of a quantity like the `sigma` of the Gaussian is simple, because it is sampled by the non-linear -search. Thus, to get their errors above we used the `Samples` object to simply marginalize over all over parameters -via the 1D Probability Density Function (PDF). - -Computing errors on derived quantities is more tricky, because they are not sampled directly by the non-linear search. -For example, what if we want the error on the full width half maximum (FWHM) of the Gaussian? In order to do this -we need to create the PDF of that derived quantity, which we can then marginalize over using the same function we -use to marginalize model parameters. - -Below, we compute the FWHM of every accepted model sampled by the non-linear search and use this determine the PDF -of the FWHM. When combining the FWHM's we weight each value by its `weight`. For Emcee, an MCMC algorithm, the -weight of every sample is 1, but weights may take different values for other non-linear searches. - -In order to pass these samples to the function `marginalize`, which marginalizes over the PDF of the FWHM to compute -its error, we also pass the weight list of the samples. - -(Computing the error on the FWHM could be done in much simpler ways than creating its PDF from the list of every -sample. We chose this example for simplicity, in order to show this functionality, which can easily be extended to more -complicated derived quantities.) -""" -fwhm_list = [] - -for sample in samples.sample_list: - instance = sample.instance_for_model(model=samples.model) - - sigma = instance.gaussian.sigma - - fwhm = 2 * np.sqrt(2 * np.log(2)) * sigma - - fwhm_list.append(fwhm) - -median_fwhm, lower_fwhm, upper_fwhm = af.marginalize( - parameter_list=fwhm_list, sigma=3.0, weight_list=samples.weight_list -) - -print(f"FWHM = {median_fwhm} ({upper_fwhm} {lower_fwhm}") - -""" -__Samples Filtering__ - -Our samples object has the results for all three parameters in our model. However, we might only be interested in the -results of a specific parameter. - -The basic form of filtering specifies parameters via their path, which was printed above via the model and is printed -again below. -""" -samples = result.samples - -print("Parameter paths in the model which are used for filtering:") -print(samples.model.paths) - -print("All parameters of the very first sample") -print(samples.parameter_lists[0]) - -samples = samples.with_paths([("gaussian", "centre")]) - -print("All parameters of the very first sample (containing only the Gaussian centre.") -print(samples.parameter_lists[0]) - -print("Maximum Log Likelihood Model Instances (containing only the Gaussian centre):\n") -print(samples.max_log_likelihood(as_instance=False)) - -""" -Above, we specified each path as a list of tuples of strings. - -This is how the source code internally stores the path to different components of the model, but it is not -in-profile_1d with the PyAutoFIT API used to compose a model. - -We can alternatively use the following API: -""" -samples = result.samples - -samples = samples.with_paths(["gaussian.centre"]) - -print("All parameters of the very first sample (containing only the Gaussian centre).") -print(samples.parameter_lists[0]) - -""" -Above, we filtered the `Samples` but asking for all parameters which included the path ("gaussian", "centre"). - -We can alternatively filter the `Samples` object by removing all parameters with a certain path. Below, we remove -the Gaussian's `centre` to be left with 2 parameters; the `normalization` and `sigma`. -""" -samples = result.samples - -print("Parameter paths in the model which are used for filtering:") -print(samples.model.paths) - -print("All parameters of the very first sample") -print(samples.parameter_lists[0]) - -samples = samples.without_paths(["gaussian.centre"]) - -print( - "All parameters of the very first sample (containing only the Gaussian normalization and sigma)." -) -print(samples.parameter_lists[0]) - -""" -__Latex__ - -If you are writing modeling results up in a paper, you can use inbuilt latex tools to create latex table -code which you can copy to your .tex document. - -By combining this with the filtering tools below, specific parameters can be included or removed from the latex. - -Remember that the superscripts of a parameter are loaded from the config file `notation/label.yaml`, providing high -levels of customization for how the parameter names appear in the latex table. This is especially useful if your model -uses the same model components with the same parameter, which therefore need to be distinguished via superscripts. -""" -latex = af.text.Samples.latex( - samples=result.samples, - median_pdf_model=True, - sigma=3.0, - name_to_label=True, - include_name=True, - include_quickmath=True, - prefix="Example Prefix ", - suffix=" \\[-2pt]", -) - -print(latex) - -""" -__Wrap Up__ - -This tutorial showed how to inspect the results of a model-fit: the maximum log likelihood instance, the full set of -samples, parameter estimates with errors at a given confidence, and how to output quantities to a LaTeX table. These -tools are the foundation for interpreting every model-fit you perform with **PyAutoFit**. -""" +""" +Tutorial 5: Results And Samples +=============================== + +In this tutorial, we'll cover all of the output that comes from a non-linear search's `Result` object. + +We used this object at various points in the chapter. The bulk of material covered here is described in the example +script `autofit_workspace/overview/simple/result.py`. Nevertheless, it is a good idea to refresh ourselves about how +results in **PyAutoFit** work before covering more advanced material. + +__Contents__ + +This tutorial is split into the following sections: + +- **Data**: Load the dataset from the HowToFit/dataset folder. +- **Reused Functions**: Reuse the `plot_profile_1d` and `Analysis` classes from the previous tutorial. +- **Model Fit**: Run a non-linear search to generate a `Result` object. +- **Result**: Examine the `Result` object and its info attribute. +- **Samples**: Introduce the `Samples` object containing the non-linear search samples. +- **Parameters**: Access parameter values from the samples. +- **Figures of Merit**: Examine log likelihood, log prior, and log posterior values. +- **Instances**: Return results as model instances from samples. +- **Vectors**: Return results as 1D parameter vectors. +- **Labels**: Access the paths, names, and labels for model parameters. +- **Posterior / PDF**: Access median PDF estimates for the model parameters. +- **Plot**: Visualize model fit results using instances. +- **Errors**: Compute parameter error estimates at specified sigma confidence limits. +- **PDF**: Plot Probability Density Functions using corner.py. +- **Other Results**: Access maximum log posterior and other sample statistics. +- **Sample Instance**: Create instances from individual samples in the sample list. +- **Bayesian Evidence**: Access the log evidence for nested sampling searches. +- **Derived Errors (PDF from samples)**: Compute errors on derived quantities from sample PDFs. +- **Samples Filtering**: Filter samples by parameter paths for specific parameter analysis. +- **Latex**: Generate LaTeX table code for modeling results. +""" + +# from autofit import setup_notebook; setup_notebook() + +import autofit as af +import autofit.plot as aplt +import os +from os import path +import numpy as np +import matplotlib.pyplot as plt + +""" +__Data__ + +Load the dataset from the `HowToFit/dataset` folder. +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1__exponential_x1") + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists(dataset_path): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) +noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") +) + +""" +__Reused Functions__ + +We'll reuse the `plot_profile_1d` and `Analysis` classes of the previous tutorial. +""" + + +def plot_profile_1d( + xvalues, + profile_1d, + title=None, + ylabel=None, + errors=None, + color="k", + output_path=None, + output_filename=None, +): + plt.errorbar( + x=xvalues, + y=profile_1d, + yerr=errors, + linestyle="", + color=color, + ecolor="k", + elinewidth=1, + capsize=2, + ) + plt.title(title) + plt.xlabel("x value of profile") + plt.ylabel(ylabel) + if not path.exists(output_path): + os.makedirs(output_path) + plt.savefig(path.join(output_path, f"{output_filename}.png")) + plt.clf() + + +class Analysis(af.Analysis): + def __init__(self, data, noise_map): + super().__init__() + + self.data = data + self.noise_map = noise_map + + def log_likelihood_function(self, instance): + model_data = self.model_data_from_instance(instance=instance) + + residual_map = self.data - model_data + chi_squared_map = (residual_map / self.noise_map) ** 2.0 + chi_squared = sum(chi_squared_map) + noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) + log_likelihood = -0.5 * (chi_squared + noise_normalization) + + return log_likelihood + + def model_data_from_instance(self, instance): + """ + To create the summed profile of all individual profiles in an instance, we can use a dictionary comprehension + to iterate over all profiles in the instance. + """ + xvalues = np.arange(self.data.shape[0]) + + return sum([profile.model_data_from(xvalues=xvalues) for profile in instance]) + + def visualize(self, paths, instance, during_analysis): + """ + This method is identical to the previous tutorial, except it now uses the `model_data_from_instance` method + to create the profile. + """ + xvalues = np.arange(self.data.shape[0]) + + model_data = self.model_data_from_instance(instance=instance) + + residual_map = self.data - model_data + chi_squared_map = (residual_map / self.noise_map) ** 2.0 + + """The visualizer now outputs images of the best-fit results to hard-disk (checkout `visualizer.py`).""" + plot_profile_1d( + xvalues=xvalues, + profile_1d=self.data, + title="Data", + ylabel="Data Values", + color="k", + output_path=paths.image_path, + output_filename="data", + ) + + plot_profile_1d( + xvalues=xvalues, + profile_1d=model_data, + title="Model Data", + ylabel="Model Data Values", + color="k", + output_path=paths.image_path, + output_filename="model_data", + ) + + plot_profile_1d( + xvalues=xvalues, + profile_1d=residual_map, + title="Residual Map", + ylabel="Residuals", + color="k", + output_path=paths.image_path, + output_filename="residual_map", + ) + + plot_profile_1d( + xvalues=xvalues, + profile_1d=chi_squared_map, + title="Chi-Squared Map", + ylabel="Chi-Squareds", + color="k", + output_path=paths.image_path, + output_filename="chi_squared_map", + ) + + +""" +__Model Fit__ + +Now lets run the non-linear search to get ourselves a `Result`. +""" + + +class Gaussian: + def __init__( + self, + centre=30.0, # <- **PyAutoFit** recognises these constructor arguments + normalization=1.0, # <- are the Gaussian`s model parameters. + sigma=5.0, + ): + """ + Represents a 1D Gaussian profile. + + This is a model-component of example models in the **HowToFit** lectures and is used to fit example datasets + via a non-linear search. + + Parameters + ---------- + centre + The x coordinate of the profile centre. + normalization + Overall normalization of the profile. + sigma + The sigma value controlling the size of the Gaussian. + """ + self.centre = centre + self.normalization = normalization + self.sigma = sigma + + def model_data_from(self, xvalues: np.ndarray): + """ + + Returns a 1D Gaussian on an input list of Cartesian x coordinates. + + The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. + + The output is referred to as the `model_data` to signify that it is a representation of the data from the + model. + + Parameters + ---------- + xvalues + The x coordinates in the original reference frame of the data. + """ + transformed_xvalues = np.subtract(xvalues, self.centre) + return np.multiply( + np.divide(self.normalization, self.sigma * np.sqrt(2.0 * np.pi)), + np.exp(-0.5 * np.square(np.divide(transformed_xvalues, self.sigma))), + ) + + +class Exponential: + def __init__( + self, + centre=30.0, # <- **PyAutoFit** recognises these constructor arguments + normalization=1.0, # <- are the Exponential`s model parameters. + rate=0.01, + ): + """ + Represents a 1D Exponential profile. + + This is a model-component of example models in the **HowToFit** lectures and is used to fit example datasets + via a non-linear search. + + Parameters + ---------- + centre + The x coordinate of the profile centre. + normalization + Overall normalization of the profile. + ratw + The decay rate controlling has fast the Exponential declines. + """ + self.centre = centre + self.normalization = normalization + self.rate = rate + + def model_data_from(self, xvalues: np.ndarray): + """ + Returns a 1D Gaussian on an input list of Cartesian x coordinates. + + The input xvalues are translated to a coordinate system centred on the Gaussian, via its `centre`. + + The output is referred to as the `model_data` to signify that it is a representation of the data from the + model. + + Parameters + ---------- + xvalues + The x coordinates in the original reference frame of the data. + """ + transformed_xvalues = np.subtract(xvalues, self.centre) + return self.normalization * np.multiply( + self.rate, np.exp(-1.0 * self.rate * abs(transformed_xvalues)) + ) + + +model = af.Collection(gaussian=af.Model(Gaussian), exponential=af.Model(Exponential)) + +analysis = Analysis(data=data, noise_map=noise_map) + +search = af.Emcee( + name="tutorial_5_results_and_samples", + path_prefix="chapter_1_introduction", +) + +print( + """ + The non-linear search has begun running. + Checkout the HowToFit/output/chapter_1_introduction/tutorial_5_results_and_samples + folder for live output of the results. + This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + """ +) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +__Result__ + +Here, we'll look in detail at what information is contained in the `Result`. + +It contains an `info` attribute which prints the result in readable format. +""" +print(result.info) + +""" +__Samples__ + +The result contains a `Samples` object, which contains all of the non-linear search samples. + +Each sample corresponds to a set of model parameters that were evaluated and accepted by our non linear search, +in this example emcee. + +This also includes their log likelihoods, which are used for computing additional information about the model-fit, +for example the error on every parameter. + +Our model-fit used the MCMC algorithm Emcee, so the `Samples` object returned is a `SamplesMCMC` object. +""" +samples = result.samples + +print("MCMC Samples: \n") +print(samples) + +""" +__Parameters__ + +The parameters are stored as a list of lists, where: + + - The outer list is the size of the total number of samples. + - The inner list is the size of the number of free parameters in the fit. + +Below, we print the first sample — its second parameter (Gaussian -> normalization) and its third +parameter (Gaussian -> sigma). Any other sample index would work the same way; index `0` is used +here so the example remains valid regardless of how many samples the search produced. +""" +samples = result.samples +print("Sample 0's second parameter value (Gaussian -> normalization):") +print(samples.parameter_lists[0][1]) +print("Sample 0's third parameter value (Gaussian -> sigma)") +print(samples.parameter_lists[0][2], "\n") + +""" +__Figures of Merit__ + +The Samples class also contains the log likelihood, log prior, log posterior and weight_list of every accepted sample, +where: + +- The log likelihood is the value evaluated from the likelihood function (e.g. -0.5 * chi_squared + the noise +normalized). + +- The log prior encodes information on how the priors on the parameters maps the log likelihood value to the log +posterior value. + +- The log posterior is log_likelihood + log_prior. + +- The weight gives information on how samples should be combined to estimate the posterior. The weight values depend on +the sampler used, for MCMC samples they are all 1 (e.g. all weighted equally). + +Below, we inspect the first sample. Any sample index would work the same way; index `0` is used here +so the example is valid regardless of how many samples the search produced. +""" +print("log(likelihood), log(prior), log(posterior) and weight of the first sample.") +print(samples.log_likelihood_list[0]) +print(samples.log_prior_list[0]) +print(samples.log_posterior_list[0]) +print(samples.weight_list[0]) + +""" +__Instances__ + +The `Samples` contains many results which are returned as an instance of the model, using the Python class structure +of the model composition. + +For example, we can return the model parameters corresponding to the maximum log likelihood sample. +""" +max_lh_instance = samples.max_log_likelihood() + +print("Max Log Likelihood `Gaussian` Instance:") +print("Centre = ", max_lh_instance.gaussian.centre) +print("Normalization = ", max_lh_instance.gaussian.normalization) +print("Sigma = ", max_lh_instance.gaussian.sigma, "\n") + +print("Max Log Likelihood Exponential Instance:") +print("Centre = ", max_lh_instance.exponential.centre) +print("Normalization = ", max_lh_instance.exponential.normalization) +print("Sigma = ", max_lh_instance.exponential.rate, "\n") + +""" +__Vectors__ + +All results can alternatively be returned as a 1D vector of values, by passing `as_instance=False`: +""" +max_lh_vector = samples.max_log_likelihood(as_instance=False) +print("Max Log Likelihood Model Parameters: \n") +print(max_lh_vector, "\n\n") + +""" +__Labels__ + +Vectors return a lists of all model parameters, but do not tell us which values correspond to which parameters. + +The following quantities are available in the `Model`, where the order of their entries correspond to the parameters +in the `ml_vector` above: + + - `paths`: a list of tuples which give the path of every parameter in the `Model`. + - `parameter_names`: a list of shorthand parameter names derived from the `paths`. + - `parameter_labels`: a list of parameter labels used when visualizing non-linear search results (see below). + +""" +model = samples.model + +print(model.paths) +print(model.parameter_names) +print(model.parameter_labels) +print(model.model_component_and_parameter_names) +print("\n") + +""" +From here on, we will returned all results information as instances, but every method below can be returned as a +vector via the `as_instance=False` input. + +__Posterior / PDF__ + +The ``Result`` object contains the full posterior information of our non-linear search, which can be used for +parameter estimation. + +The median pdf vector is available from the `Samples` object, which estimates the every parameter via 1D +marginalization of their PDFs. +""" +median_pdf_instance = samples.median_pdf() + +print("Max Log Likelihood `Gaussian` Instance:") +print("Centre = ", median_pdf_instance.gaussian.centre) +print("Normalization = ", median_pdf_instance.gaussian.normalization) +print("Sigma = ", median_pdf_instance.gaussian.sigma, "\n") + +print("Max Log Likelihood Exponential Instance:") +print("Centre = ", median_pdf_instance.exponential.centre) +print("Normalization = ", median_pdf_instance.exponential.normalization) +print("Sigma = ", median_pdf_instance.exponential.rate, "\n") + +""" +__Plot__ + +Because results are returned as instances, it is straight forward to use them and their associated functionality +to make plots of the results: +""" +model_gaussian = max_lh_instance.gaussian.model_data_from( + xvalues=np.arange(data.shape[0]) +) +model_exponential = max_lh_instance.exponential.model_data_from( + xvalues=np.arange(data.shape[0]) +) +model_data = model_gaussian + model_exponential + +plt.plot(range(data.shape[0]), data) +plt.plot(range(data.shape[0]), model_data) +plt.plot(range(data.shape[0]), model_gaussian, "--") +plt.plot(range(data.shape[0]), model_exponential, "--") +plt.title("Illustrative model fit to 1D `Gaussian` + Exponential profile data.") +plt.xlabel("x values of profile") +plt.ylabel("Profile normalization") +plt.show() +plt.close() + +""" +__Errors__ + +The samples include methods for computing the error estimates of all parameters, via 1D marginalization at an +input sigma confidence limit. +""" +errors_at_upper_sigma_instance = samples.errors_at_upper_sigma(sigma=3.0) +errors_at_lower_sigma_instance = samples.errors_at_lower_sigma(sigma=3.0) + +print("Upper Error values of Gaussian (at 3.0 sigma confidence):") +print("Centre = ", errors_at_upper_sigma_instance.gaussian.centre) +print("Normalization = ", errors_at_upper_sigma_instance.gaussian.normalization) +print("Sigma = ", errors_at_upper_sigma_instance.gaussian.sigma, "\n") + +print("lower Error values of Gaussian (at 3.0 sigma confidence):") +print("Centre = ", errors_at_lower_sigma_instance.gaussian.centre) +print("Normalization = ", errors_at_lower_sigma_instance.gaussian.normalization) +print("Sigma = ", errors_at_lower_sigma_instance.gaussian.sigma, "\n") + +""" +They can also be returned at the values of the parameters at their error values: +""" +values_at_upper_sigma_instance = samples.values_at_upper_sigma(sigma=3.0) +values_at_lower_sigma_instance = samples.values_at_lower_sigma(sigma=3.0) + +print("Upper Parameter values w/ error of Gaussian (at 3.0 sigma confidence):") +print("Centre = ", values_at_upper_sigma_instance.gaussian.centre) +print("Normalization = ", values_at_upper_sigma_instance.gaussian.normalization) +print("Sigma = ", values_at_upper_sigma_instance.gaussian.sigma, "\n") + +print("lower Parameter values w/ errors of Gaussian (at 3.0 sigma confidence):") +print("Centre = ", values_at_lower_sigma_instance.gaussian.centre) +print("Normalization = ", values_at_lower_sigma_instance.gaussian.normalization) +print("Sigma = ", values_at_lower_sigma_instance.gaussian.sigma, "\n") + +""" +__PDF__ + +The Probability Density Functions (PDF's) of the results can be plotted using the Emcee's visualization +tool `corner.py`, which is wrapped via the `aplt.corner_cornerpy` function. +""" +aplt.corner_cornerpy(samples=result.samples) + +""" +__Other Results__ + +The samples contain many useful vectors, including the samples with the highest posterior values. +""" +max_log_posterior_instance = samples.max_log_posterior() + +print("Maximum Log Posterior Vector:") +print("Centre = ", max_log_posterior_instance.gaussian.centre) +print("Normalization = ", max_log_posterior_instance.gaussian.normalization) +print("Sigma = ", max_log_posterior_instance.gaussian.sigma, "\n") + + +""" +All methods above are available as a vector: +""" +median_pdf_instance = samples.median_pdf(as_instance=False) +values_at_upper_sigma = samples.values_at_upper_sigma(sigma=3.0, as_instance=False) +values_at_lower_sigma = samples.values_at_lower_sigma(sigma=3.0, as_instance=False) +errors_at_upper_sigma = samples.errors_at_upper_sigma(sigma=3.0, as_instance=False) +errors_at_lower_sigma = samples.errors_at_lower_sigma(sigma=3.0, as_instance=False) + +""" +__Sample Instance__ + +A non-linear search retains every model that is accepted during the model-fit. + +We can create an instance of any lens model -- below we create an instance of the last accepted model. +""" +instance = samples.from_sample_index(sample_index=-1) + +print("Gaussian Instance of last sample") +print("Centre = ", instance.gaussian.centre) +print("Normalization = ", instance.gaussian.normalization) +print("Sigma = ", instance.gaussian.sigma, "\n") + +""" +__Bayesian Evidence__ + +If a nested sampling `NonLinearSearch` is used, the evidence of the model is also available which enables Bayesian +model comparison to be performed (given we are using Emcee, which is not a nested sampling algorithm, the log evidence +is None).: +""" +log_evidence = samples.log_evidence + +""" +__Derived Errors (PDF from samples)__ + +Computing the errors of a quantity like the `sigma` of the Gaussian is simple, because it is sampled by the non-linear +search. Thus, to get their errors above we used the `Samples` object to simply marginalize over all over parameters +via the 1D Probability Density Function (PDF). + +Computing errors on derived quantities is more tricky, because they are not sampled directly by the non-linear search. +For example, what if we want the error on the full width half maximum (FWHM) of the Gaussian? In order to do this +we need to create the PDF of that derived quantity, which we can then marginalize over using the same function we +use to marginalize model parameters. + +Below, we compute the FWHM of every accepted model sampled by the non-linear search and use this determine the PDF +of the FWHM. When combining the FWHM's we weight each value by its `weight`. For Emcee, an MCMC algorithm, the +weight of every sample is 1, but weights may take different values for other non-linear searches. + +In order to pass these samples to the function `marginalize`, which marginalizes over the PDF of the FWHM to compute +its error, we also pass the weight list of the samples. + +(Computing the error on the FWHM could be done in much simpler ways than creating its PDF from the list of every +sample. We chose this example for simplicity, in order to show this functionality, which can easily be extended to more +complicated derived quantities.) +""" +fwhm_list = [] + +for sample in samples.sample_list: + instance = sample.instance_for_model(model=samples.model) + + sigma = instance.gaussian.sigma + + fwhm = 2 * np.sqrt(2 * np.log(2)) * sigma + + fwhm_list.append(fwhm) + +median_fwhm, lower_fwhm, upper_fwhm = af.marginalize( + parameter_list=fwhm_list, sigma=3.0, weight_list=samples.weight_list +) + +print(f"FWHM = {median_fwhm} ({upper_fwhm} {lower_fwhm}") + +""" +__Samples Filtering__ + +Our samples object has the results for all three parameters in our model. However, we might only be interested in the +results of a specific parameter. + +The basic form of filtering specifies parameters via their path, which was printed above via the model and is printed +again below. +""" +samples = result.samples + +print("Parameter paths in the model which are used for filtering:") +print(samples.model.paths) + +print("All parameters of the very first sample") +print(samples.parameter_lists[0]) + +samples = samples.with_paths([("gaussian", "centre")]) + +print("All parameters of the very first sample (containing only the Gaussian centre.") +print(samples.parameter_lists[0]) + +print("Maximum Log Likelihood Model Instances (containing only the Gaussian centre):\n") +print(samples.max_log_likelihood(as_instance=False)) + +""" +Above, we specified each path as a list of tuples of strings. + +This is how the source code internally stores the path to different components of the model, but it is not +in-profile_1d with the PyAutoFIT API used to compose a model. + +We can alternatively use the following API: +""" +samples = result.samples + +samples = samples.with_paths(["gaussian.centre"]) + +print("All parameters of the very first sample (containing only the Gaussian centre).") +print(samples.parameter_lists[0]) + +""" +Above, we filtered the `Samples` but asking for all parameters which included the path ("gaussian", "centre"). + +We can alternatively filter the `Samples` object by removing all parameters with a certain path. Below, we remove +the Gaussian's `centre` to be left with 2 parameters; the `normalization` and `sigma`. +""" +samples = result.samples + +print("Parameter paths in the model which are used for filtering:") +print(samples.model.paths) + +print("All parameters of the very first sample") +print(samples.parameter_lists[0]) + +samples = samples.without_paths(["gaussian.centre"]) + +print( + "All parameters of the very first sample (containing only the Gaussian normalization and sigma)." +) +print(samples.parameter_lists[0]) + +""" +__Latex__ + +If you are writing modeling results up in a paper, you can use inbuilt latex tools to create latex table +code which you can copy to your .tex document. + +By combining this with the filtering tools below, specific parameters can be included or removed from the latex. + +Remember that the superscripts of a parameter are loaded from the config file `notation/label.yaml`, providing high +levels of customization for how the parameter names appear in the latex table. This is especially useful if your model +uses the same model components with the same parameter, which therefore need to be distinguished via superscripts. +""" +latex = af.text.Samples.latex( + samples=result.samples, + median_pdf_model=True, + sigma=3.0, + name_to_label=True, + include_name=True, + include_quickmath=True, + prefix="Example Prefix ", + suffix=" \\[-2pt]", +) + +print(latex) + +""" +__Wrap Up__ + +This tutorial showed how to inspect the results of a model-fit: the maximum log likelihood instance, the full set of +samples, parameter estimates with errors at a given confidence, and how to output quantities to a LaTeX table. These +tools are the foundation for interpreting every model-fit you perform with **PyAutoFit**. +""" diff --git a/scripts/chapter_3_graphical_models/tutorial_1_individual_models.py b/scripts/chapter_3_graphical_models/tutorial_1_individual_models.py index f1b41a6..c146255 100644 --- a/scripts/chapter_3_graphical_models/tutorial_1_individual_models.py +++ b/scripts/chapter_3_graphical_models/tutorial_1_individual_models.py @@ -1,319 +1,319 @@ -""" -Tutorial 1: Individual Models -============================= - -In many examples, we fit one model to one dataset. For many problems, we may have a large dataset and are not -interested in how well the model fits each individual dataset. Instead, we want to know how the model fits the full -dataset, so that we can determine "global" trends of how the model fits the data. - -These tutorials show you how to compose and fit hierarchical models to large datasets, which fit many individual -models to each dataset. However, all parameters in the model are linked together, enabling global inference of the -model over the full dataset. This can extract a significant amount of extra information from large datasets, which -fitting each dataset individually does not. - -Fitting a hierarchical model uses a "graphical model", which is a model that is simultaneously fitted to every -dataset simultaneously. The graph expresses how the parameters of every individual model is paired with each dataset -and how they are linked to every other model parameter. Complex graphical models fitting a diversity of different -datasets and non-trivial model parameter linking is possible and common. - -This chapter will start by fitting a simple graphical model to a dataset of noisy 1D Gaussians. The Gaussians all -share the same `centre`, meaning that a graphical model can be composed where there is only a single global `centre` -shared by all Gaussians. - -However, before fitting a graphical model, we will first fit each Gaussian individually and combine the inference -on the `centre` after every fit is complete. This will give us an estimate of the `centre` that we can compare to -the result of the graphical model in tutorial 2. - -__Real World Example__ - -Hierarchical models are often used to determine effective drug treatments across a sample of patients distributed over -many hospitals. Trying to do this on each individual hospital dataset is not ideal, as the number of patients in each -hospital is small and the treatment may be more or less effective in some hospitals than others. Hierarchical models -can extract the global trends of how effective the treatment is across the full population of patients. - -In healthcare, there may also be many datasets available, with different formats that require slightly different models -to fit them. The high levels of customization possible in model composition and defining the analysis class mean -that fitting diverse datasets with hierarchical models is feasible. This also means that a common problem in healthcare -data, missing data, can be treated in a statistically robust manner. - -__Contents__ - -This tutorial is split into the following sections: - -- **Real World Example**: A healthcare example illustrating the value of hierarchical models. -- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. -- **Model**: Define a simple Gaussian model in a Collection. -- **Data**: Load and set up 5 noisy 1D Gaussian datasets. -- **Model Fits (one-by-one)**: Fit each dataset individually using a separate non-linear search. -- **Results**: Analyze the fit results and error estimates for each dataset. -- **Estimating the Centre**: Combine centre estimates using a weighted average approach. -- **Posterior Multiplication**: Discuss KDE-based posterior multiplication as an alternative method. -- **Wrap Up**: Summary and transition to the graphical model approach in the next tutorial. -""" - -# from autoconf import setup_notebook; setup_notebook() - -from os import path -import numpy as np - -import autofit as af -import autofit.plot as aplt - -""" -__Example Source Code (`af.ex`)__ - -The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: - - - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and - `visualize` functions. - - - `Gaussian`: a model component representing a 1D Gaussian profile. - - - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. - -These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have -seen and used elsewhere throughout the workspace. - -__Model__ - -Our model is a single `Gaussian`. - -We put this in a `Collection` so that when we extend the model in later tutorials we use the same API throughout -all tutorials. -""" -model = af.Collection(gaussian=af.ex.Gaussian) - -""" -__Data__ - -We quickly set up the name of each dataset, which is used below for loading the datasets. - -The dataset contains 10 Gaussians, but for speed we'll fit just 5. You can change this to 10 to see how the result -changes with more datasets. -""" -total_datasets = 5 - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists( - path.join("dataset", "example_1d", "gaussian_x1__low_snr", "dataset_0") -): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -dataset_name_list = [] - -for dataset_index in range(total_datasets): - dataset_name_list.append(f"dataset_{dataset_index}") - -""" -For each 1D Gaussian dataset we now set up the correct path, load it, and plot it. - -Notice how much lower the signal-to-noise is than you are used too, you probably find it difficult to estimate -the centre of some of the Gaussians by eye! -""" -for dataset_name in dataset_name_list: - """ - Load the dataset from the `HowToFit/dataset` folder. - """ - - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - af.ex.plot_profile_1d( - xvalues=np.arange(data.shape[0]), - profile_1d=data, - title=dataset_name, - ylabel="Data Values", - color="k", - ) - -""" -__Model Fits (one-by-one)__ - -For every dataset we now create an `Analysis` and fit it with a `Gaussian`. - -The `Result` is stored in the list `result_list`. -""" -result_list = [] - -for dataset_name in dataset_name_list: - """ - Load the dataset from the `HowToFit/dataset` folder. - """ - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - """ - For each dataset create a corresponding `Analysis` class. - """ - analysis = af.ex.Analysis(data=data, noise_map=noise_map) - - """ - Create the `DynestyStatic` non-linear search and use it to fit the data. - - We use custom dynesty settings which ensure the posterior is explored fully and that our error estimates are robust. - """ - dynesty = af.DynestyStatic( - name="global_model", - path_prefix=path.join( - "chapter_3_graphical_models", "tutorial_1_individual_models" - ), - unique_tag=dataset_name, - nlive=200, - dlogz=1e-4, - sample="rwalk", - walks=10, - ) - - print( - f"The non-linear search has begun running, checkout \n" - f"HowToFit/output/chapter_3_graphical_models/tutorial_1_individual_models/{dataset_name} for live \n" - f"output of the results. This Jupyter notebook cell with progress once search has completed, this could take a \n" - f"few minutes!" - ) - - result_list.append(dynesty.fit(model=model, analysis=analysis)) - - -""" -__Results__ - -Checkout the output folder, you should see five new sets of results corresponding to our Gaussian datasets. - -In the `model.results` file of each fit, it will be clear that the `centre` value of every fit (and the other -parameters) have much larger errors than other **PyAutoFit** examples due to the low signal to noise of the data. - -The `result_list` allows us to plot the median PDF value and 3.0 confidence intervals of the `centre` estimate from -the model-fit to each dataset. -""" -import matplotlib.pyplot as plt - -samples_list = [result.samples for result in result_list] - -mp_instances = [samps.median_pdf() for samps in samples_list] -ue3_instances = [samp.errors_at_upper_sigma(sigma=3.0) for samp in samples_list] -le3_instances = [samp.errors_at_lower_sigma(sigma=3.0) for samp in samples_list] - -mp_centres = [instance.gaussian.centre for instance in mp_instances] -ue3_centres = [instance.gaussian.centre for instance in ue3_instances] -le3_centres = [instance.gaussian.centre for instance in le3_instances] - -plt.errorbar( - x=[f"Gaussian {index}" for index in range(total_datasets)], - y=mp_centres, - marker=".", - linestyle="", - yerr=[le3_centres, ue3_centres], -) -plt.xticks(rotation=90) -plt.show() -plt.close() - -""" -These model-fits are consistent with a range of `centre` values. - -We can show this by plotting the 1D and 2D PDF's of each model fit -""" - -for samples in samples_list: - aplt.corner_cornerpy(samples=samples) - -""" -We can also print the values of each centre estimate, including their estimates at 3.0 sigma. - -Note that above we used the samples to estimate the size of the errors on the parameters. Below, we use the samples to -get the value of the parameter at these sigma confidence intervals. -""" -u1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] -l1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] - -u1_centres = [instance.gaussian.centre for instance in u1_instances] -l1_centres = [instance.gaussian.centre for instance in l1_instances] - -u3_instances = [samp.values_at_upper_sigma(sigma=3.0) for samp in samples_list] -l3_instances = [samp.values_at_lower_sigma(sigma=3.0) for samp in samples_list] - -u3_centres = [instance.gaussian.centre for instance in u3_instances] -l3_centres = [instance.gaussian.centre for instance in l3_instances] - -for index in range(total_datasets): - print(f"Centre estimate of Gaussian dataset {index}:\n") - print( - f"{mp_centres[index]} ({l1_centres[index]} {u1_centres[index]}) [1.0 sigma confidence interval]" - ) - print( - f"{mp_centres[index]} ({l3_centres[index]} {u3_centres[index]}) [3.0 sigma confidence interval] \n" - ) - - -""" -__Estimating the Centre__ - -So how might we estimate our global `centre` value? - -A simple approach takes the weighted average of the value inferred by all five fits above. -""" -ue1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] -le1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] - -ue1_centres = [instance.gaussian.centre for instance in ue1_instances] -le1_centres = [instance.gaussian.centre for instance in le1_instances] - -error_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres, le1_centres)] - -values = np.asarray(mp_centres) -sigmas = np.asarray(error_list) - -weights = 1 / sigmas**2.0 -weight_averaged = np.sum(1.0 / sigmas**2) - -weighted_centre = np.sum(values * weights) / np.sum(weights, axis=0) -weighted_error = 1.0 / np.sqrt(weight_averaged) - -print( - f"Weighted Average Centre Estimate = {weighted_centre} ({weighted_error}) [1.0 sigma confidence intervals]" -) - -""" -__Posterior Multiplication__ - -An alternative and more accurate way to combine each individual inferred centre is multiply their posteriors together. - -In order to do this, a smooth 1D profile must be fit to the posteriors via a Kernel Density Estimator (KDE). - -[There is currently no support for posterior multiplication and an example illustrating this is currently missing -from this tutorial. However, I will discuss KDE multiplication throughout these tutorials to give the reader context -for how this approach to parameter estimation compares to graphical models.] - -__Wrap Up__ - -Lets wrap up the tutorial. The methods used above combine the results of different fits and estimate a global -value of `centre` alongside estimates of its error. - -In this tutorial, we fitted just 5 datasets. Of course, we could easily fit more datasets, and we would find that -as we added more datasets our estimate of the global centre would become more precise. - -In the next tutorial, we will compare this result to one inferred via a graphical model. -""" +""" +Tutorial 1: Individual Models +============================= + +In many examples, we fit one model to one dataset. For many problems, we may have a large dataset and are not +interested in how well the model fits each individual dataset. Instead, we want to know how the model fits the full +dataset, so that we can determine "global" trends of how the model fits the data. + +These tutorials show you how to compose and fit hierarchical models to large datasets, which fit many individual +models to each dataset. However, all parameters in the model are linked together, enabling global inference of the +model over the full dataset. This can extract a significant amount of extra information from large datasets, which +fitting each dataset individually does not. + +Fitting a hierarchical model uses a "graphical model", which is a model that is simultaneously fitted to every +dataset simultaneously. The graph expresses how the parameters of every individual model is paired with each dataset +and how they are linked to every other model parameter. Complex graphical models fitting a diversity of different +datasets and non-trivial model parameter linking is possible and common. + +This chapter will start by fitting a simple graphical model to a dataset of noisy 1D Gaussians. The Gaussians all +share the same `centre`, meaning that a graphical model can be composed where there is only a single global `centre` +shared by all Gaussians. + +However, before fitting a graphical model, we will first fit each Gaussian individually and combine the inference +on the `centre` after every fit is complete. This will give us an estimate of the `centre` that we can compare to +the result of the graphical model in tutorial 2. + +__Real World Example__ + +Hierarchical models are often used to determine effective drug treatments across a sample of patients distributed over +many hospitals. Trying to do this on each individual hospital dataset is not ideal, as the number of patients in each +hospital is small and the treatment may be more or less effective in some hospitals than others. Hierarchical models +can extract the global trends of how effective the treatment is across the full population of patients. + +In healthcare, there may also be many datasets available, with different formats that require slightly different models +to fit them. The high levels of customization possible in model composition and defining the analysis class mean +that fitting diverse datasets with hierarchical models is feasible. This also means that a common problem in healthcare +data, missing data, can be treated in a statistically robust manner. + +__Contents__ + +This tutorial is split into the following sections: + +- **Real World Example**: A healthcare example illustrating the value of hierarchical models. +- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. +- **Model**: Define a simple Gaussian model in a Collection. +- **Data**: Load and set up 5 noisy 1D Gaussian datasets. +- **Model Fits (one-by-one)**: Fit each dataset individually using a separate non-linear search. +- **Results**: Analyze the fit results and error estimates for each dataset. +- **Estimating the Centre**: Combine centre estimates using a weighted average approach. +- **Posterior Multiplication**: Discuss KDE-based posterior multiplication as an alternative method. +- **Wrap Up**: Summary and transition to the graphical model approach in the next tutorial. +""" + +# from autofit import setup_notebook; setup_notebook() + +from os import path +import numpy as np + +import autofit as af +import autofit.plot as aplt + +""" +__Example Source Code (`af.ex`)__ + +The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: + + - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and + `visualize` functions. + + - `Gaussian`: a model component representing a 1D Gaussian profile. + + - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. + +These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have +seen and used elsewhere throughout the workspace. + +__Model__ + +Our model is a single `Gaussian`. + +We put this in a `Collection` so that when we extend the model in later tutorials we use the same API throughout +all tutorials. +""" +model = af.Collection(gaussian=af.ex.Gaussian) + +""" +__Data__ + +We quickly set up the name of each dataset, which is used below for loading the datasets. + +The dataset contains 10 Gaussians, but for speed we'll fit just 5. You can change this to 10 to see how the result +changes with more datasets. +""" +total_datasets = 5 + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists( + path.join("dataset", "example_1d", "gaussian_x1__low_snr", "dataset_0") +): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +dataset_name_list = [] + +for dataset_index in range(total_datasets): + dataset_name_list.append(f"dataset_{dataset_index}") + +""" +For each 1D Gaussian dataset we now set up the correct path, load it, and plot it. + +Notice how much lower the signal-to-noise is than you are used too, you probably find it difficult to estimate +the centre of some of the Gaussians by eye! +""" +for dataset_name in dataset_name_list: + """ + Load the dataset from the `HowToFit/dataset` folder. + """ + + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + af.ex.plot_profile_1d( + xvalues=np.arange(data.shape[0]), + profile_1d=data, + title=dataset_name, + ylabel="Data Values", + color="k", + ) + +""" +__Model Fits (one-by-one)__ + +For every dataset we now create an `Analysis` and fit it with a `Gaussian`. + +The `Result` is stored in the list `result_list`. +""" +result_list = [] + +for dataset_name in dataset_name_list: + """ + Load the dataset from the `HowToFit/dataset` folder. + """ + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + """ + For each dataset create a corresponding `Analysis` class. + """ + analysis = af.ex.Analysis(data=data, noise_map=noise_map) + + """ + Create the `DynestyStatic` non-linear search and use it to fit the data. + + We use custom dynesty settings which ensure the posterior is explored fully and that our error estimates are robust. + """ + dynesty = af.DynestyStatic( + name="global_model", + path_prefix=path.join( + "chapter_3_graphical_models", "tutorial_1_individual_models" + ), + unique_tag=dataset_name, + nlive=200, + dlogz=1e-4, + sample="rwalk", + walks=10, + ) + + print( + f"The non-linear search has begun running, checkout \n" + f"HowToFit/output/chapter_3_graphical_models/tutorial_1_individual_models/{dataset_name} for live \n" + f"output of the results. This Jupyter notebook cell with progress once search has completed, this could take a \n" + f"few minutes!" + ) + + result_list.append(dynesty.fit(model=model, analysis=analysis)) + + +""" +__Results__ + +Checkout the output folder, you should see five new sets of results corresponding to our Gaussian datasets. + +In the `model.results` file of each fit, it will be clear that the `centre` value of every fit (and the other +parameters) have much larger errors than other **PyAutoFit** examples due to the low signal to noise of the data. + +The `result_list` allows us to plot the median PDF value and 3.0 confidence intervals of the `centre` estimate from +the model-fit to each dataset. +""" +import matplotlib.pyplot as plt + +samples_list = [result.samples for result in result_list] + +mp_instances = [samps.median_pdf() for samps in samples_list] +ue3_instances = [samp.errors_at_upper_sigma(sigma=3.0) for samp in samples_list] +le3_instances = [samp.errors_at_lower_sigma(sigma=3.0) for samp in samples_list] + +mp_centres = [instance.gaussian.centre for instance in mp_instances] +ue3_centres = [instance.gaussian.centre for instance in ue3_instances] +le3_centres = [instance.gaussian.centre for instance in le3_instances] + +plt.errorbar( + x=[f"Gaussian {index}" for index in range(total_datasets)], + y=mp_centres, + marker=".", + linestyle="", + yerr=[le3_centres, ue3_centres], +) +plt.xticks(rotation=90) +plt.show() +plt.close() + +""" +These model-fits are consistent with a range of `centre` values. + +We can show this by plotting the 1D and 2D PDF's of each model fit +""" + +for samples in samples_list: + aplt.corner_cornerpy(samples=samples) + +""" +We can also print the values of each centre estimate, including their estimates at 3.0 sigma. + +Note that above we used the samples to estimate the size of the errors on the parameters. Below, we use the samples to +get the value of the parameter at these sigma confidence intervals. +""" +u1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] +l1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] + +u1_centres = [instance.gaussian.centre for instance in u1_instances] +l1_centres = [instance.gaussian.centre for instance in l1_instances] + +u3_instances = [samp.values_at_upper_sigma(sigma=3.0) for samp in samples_list] +l3_instances = [samp.values_at_lower_sigma(sigma=3.0) for samp in samples_list] + +u3_centres = [instance.gaussian.centre for instance in u3_instances] +l3_centres = [instance.gaussian.centre for instance in l3_instances] + +for index in range(total_datasets): + print(f"Centre estimate of Gaussian dataset {index}:\n") + print( + f"{mp_centres[index]} ({l1_centres[index]} {u1_centres[index]}) [1.0 sigma confidence interval]" + ) + print( + f"{mp_centres[index]} ({l3_centres[index]} {u3_centres[index]}) [3.0 sigma confidence interval] \n" + ) + + +""" +__Estimating the Centre__ + +So how might we estimate our global `centre` value? + +A simple approach takes the weighted average of the value inferred by all five fits above. +""" +ue1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] +le1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] + +ue1_centres = [instance.gaussian.centre for instance in ue1_instances] +le1_centres = [instance.gaussian.centre for instance in le1_instances] + +error_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres, le1_centres)] + +values = np.asarray(mp_centres) +sigmas = np.asarray(error_list) + +weights = 1 / sigmas**2.0 +weight_averaged = np.sum(1.0 / sigmas**2) + +weighted_centre = np.sum(values * weights) / np.sum(weights, axis=0) +weighted_error = 1.0 / np.sqrt(weight_averaged) + +print( + f"Weighted Average Centre Estimate = {weighted_centre} ({weighted_error}) [1.0 sigma confidence intervals]" +) + +""" +__Posterior Multiplication__ + +An alternative and more accurate way to combine each individual inferred centre is multiply their posteriors together. + +In order to do this, a smooth 1D profile must be fit to the posteriors via a Kernel Density Estimator (KDE). + +[There is currently no support for posterior multiplication and an example illustrating this is currently missing +from this tutorial. However, I will discuss KDE multiplication throughout these tutorials to give the reader context +for how this approach to parameter estimation compares to graphical models.] + +__Wrap Up__ + +Lets wrap up the tutorial. The methods used above combine the results of different fits and estimate a global +value of `centre` alongside estimates of its error. + +In this tutorial, we fitted just 5 datasets. Of course, we could easily fit more datasets, and we would find that +as we added more datasets our estimate of the global centre would become more precise. + +In the next tutorial, we will compare this result to one inferred via a graphical model. +""" diff --git a/scripts/chapter_3_graphical_models/tutorial_2_graphical_model.py b/scripts/chapter_3_graphical_models/tutorial_2_graphical_model.py index 969226b..5eed016 100644 --- a/scripts/chapter_3_graphical_models/tutorial_2_graphical_model.py +++ b/scripts/chapter_3_graphical_models/tutorial_2_graphical_model.py @@ -1,283 +1,283 @@ -""" -Tutorial 2: Graphical Models -============================ - -We have fitted a dataset containing 5 noisy 1D Gaussian which had a shared `centre` value. We estimated -the `centre` by fitting each dataset individually and combining the value of the `centre` inferred by each fit into -an overall estimate, using a weighted average. - -Graphical models use a different approach. They are a single model that is fitted to the entire dataset simultaneously. -The model includes specific model component for every individual 1D Gaussian in the sample. However, the graphical -model also has shared parameters between these individual model components. - -This example fits a graphical model using the same sample fitted in the previous tutorial, consisting of many 1D -Gaussians. However, whereas previously the `centre` of each Gaussian was a free parameter in each fit, in the graphical -model there is only a single parameter for the `centre` shared by all 1D Gaussians. - -This graphical model creates a non-linear parameter space with parameters for every Gaussian in our sample. For 5 -Gaussians each with their own model parameters but a single shared centre: - - - Each Gaussian has 2 free parameters from the components that are not shared (`normalization`, `sigma`). - - There is one additional free parameter, which is the `centre` shared by all 5 Gaussians. - -__Contents__ - -This tutorial is split into the following sections: - -- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. -- **Dataset**: Load the 5 noisy 1D Gaussian datasets for simultaneous fitting. -- **Analysis**: Create Analysis objects for each dataset. -- **Model**: Set up the graphical model with a shared prior for the centre parameter. -- **Analysis Factors**: Pair each model with its corresponding Analysis class at factor graph nodes. -- **Factor Graph**: Combine the Analysis Factors into a factor graph representing the graphical model. -- **Search**: Configure and run the non-linear search to fit the factor graph. -- **Result**: Inspect and compare the graphical model results to the individual fits. -- **Wrap Up**: Summary and discussion of the benefits of graphical models. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import numpy as np -from os import path - -import autofit as af - -""" -__Example Source Code (`af.ex`)__ - -The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: - - - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and - `visualize` functions. - - - `Gaussian`: a model component representing a 1D Gaussian profile. - - - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. - -These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you -have seen and used elsewhere throughout the workspace. - -__Dataset__ - -For each dataset we now set up the correct path and load it. - -Whereas in the previous tutorial we fitted each dataset one-by-one, in this tutorial we instead store each dataset -in a list so that we can set up a single model-fit that fits the 5 datasets simultaneously. -""" -total_datasets = 5 - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists( - path.join("dataset", "example_1d", "gaussian_x1__low_snr", "dataset_0") -): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -dataset_name_list = [] -data_list = [] -noise_map_list = [] - -for dataset_index in range(total_datasets): - dataset_name = f"dataset_{dataset_index}" - - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - dataset_name_list.append(dataset_name) - data_list.append(data) - noise_map_list.append(noise_map) - -""" -By plotting the Gaussians we can remind ourselves that determining their centres by eye is difficult. -""" -for dataset_name, data in zip(dataset_name_list, data_list): - af.ex.plot_profile_1d( - xvalues=np.arange(data.shape[0]), - profile_1d=data, - title=dataset_name, - ylabel="Data Values", - color="k", - ) - -""" -__Analysis__ - -For each dataset we now create a corresponding `Analysis` class. -""" -analysis_list = [] - -for data, noise_map in zip(data_list, noise_map_list): - analysis = af.ex.Analysis(data=data, noise_map=noise_map) - - analysis_list.append(analysis) - -""" -__Model__ - -We now compose the graphical model that we fit, using the `Model` object you are now familiar with. - -We begin by setting up a shared prior for `centre`. - -We set up this up as a single `UniformPrior` which is passed to separate `Model`'s for each `Gaussian` below. -""" -centre_shared_prior = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) - -""" -We now set up a list of `Model`'s, each of which contain a `Gaussian` that is used to fit each of the datasets -loaded above. - -All of these models use the `centre_shared_prior`, meaning that all model-components use the same value of `centre` -for every individual model component. - -For a fit using five Gaussians, this reduces the dimensionality of parameter space from N=15 (e.g. 3 parameters per -Gaussian) to N=11 (e.g. 5 `sigma`'s 5 `normalizations` and 1 `centre`). -""" -model_list = [] - -for model_index in range(len(data_list)): - gaussian = af.Model(af.ex.Gaussian) - - gaussian.centre = centre_shared_prior # This prior is used by all 3 Gaussians! - gaussian.normalization = af.LogUniformPrior(lower_limit=1e-6, upper_limit=1e6) - gaussian.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=25.0) - - model_list.append(gaussian) - -""" -__Analysis Factors__ - -Above, we composed a model consisting of three `Gaussian`'s with a shared `centre` prior. We also loaded three datasets -which we intend to fit with each of these `Gaussians`, setting up each in an `Analysis` class that defines how the -model is used to fit the data. - -We now simply pair each model-component to each `Analysis` class, so that: - -- `gaussian_0` fits `data_0` via `analysis_0`. -- `gaussian_1` fits `data_1` via `analysis_1`. -- `gaussian_2` fits `data_2` via `analysis_2`. - -The point where a `Model` and `Analysis` class meet is called an `AnalysisFactor`. - -This term denotes that we are composing a graphical model, which is commonly called a 'factor graph'. A factor -defines a node on this graph where we have some data, a model, and we fit the two together. The 'links' between these -different nodes then define the global model we are fitting. -""" -analysis_factor_list = [] - -for model, analysis in zip(model_list, analysis_list): - analysis_factor = af.AnalysisFactor(prior_model=model, analysis=analysis) - - analysis_factor_list.append(analysis_factor) - -""" -__Factor Graph__ - -We now combine our `AnalysisFactor`'s to compose a factor graph. - -What is a factor graph? A factor graph defines the graphical model's graph. For example, it defines the different -model components that make up our model (e.g. the individual `Gaussian` classes) and how their parameters are linked or -shared (e.g. that each `Gaussian` has its own unique `normalization` and `sigma`, but a shared `centre` parameter). - -This is what our factor graph looks like (visualization of graphs not implemented yet): - -The factor graph above is made up of two components: - -- Nodes: these are points on the graph where we have a unique set of data and a model that is made up of a subset of -our overall graphical model. This is effectively the `AnalysisFactor` objects we created above. - -- Links: these define the model components and parameters that are shared across different nodes and thus retain the -same values when fitting different datasets. -""" -factor_graph = af.FactorGraphModel(*analysis_factor_list) - -""" -The fit will use the factor graph's `global_prior_model`, which uses the models contained in every analysis factor -to contrast the overall global model that is fitted. - -Printing the `info` attribute of this model reveals the overall structure of the model, which is grouped in terms -of the analysis factors and therefore datasets. -""" -print(factor_graph.global_prior_model.info) - -""" -__Search__ - -We can now create a non-linear search and use it to the fit the factor graph, using its `global_prior_model` property. -""" -search = af.DynestyStatic( - path_prefix=path.join("chapter_3_graphical_models"), - name="tutorial_2_graphical_model", - nlive=200, - dlogz=1e-4, - sample="rwalk", - walks=10, -) - -result = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) - -""" -__Result__ - -The result's `info` attribute shows that the result is expressed following the same struture of analysis factors -that the `global_prior_model.info` attribute revealed above. -""" -print(result.info) - -""" -We can now inspect the inferred value of `centre`, and compare this to the value we estimated in the previous tutorial -via a weighted average or posterior multiplicaition using KDE.(feature missing currently). - -(The errors of the weighted average and KDE below is what was estimated for a run on my PC, yours may be slightly -different!) -""" -print( - f"Weighted Average Centre Estimate = 48.535531422571886 (4.139907734505303) [1.0 sigma confidence intervals] \n" -) - -centre = result.samples.median_pdf()[0].centre - -u1_error = result.samples.values_at_upper_sigma(sigma=1.0)[0].centre -l1_error = result.samples.values_at_lower_sigma(sigma=1.0)[0].centre - -u3_error = result.samples.values_at_upper_sigma(sigma=3.0)[0].centre -l3_error = result.samples.values_at_lower_sigma(sigma=3.0)[0].centre - -print("Inferred value of the shared centre via a graphical model fit: \n") -print(f"{centre} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") -print(f"{centre} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") - -""" -The graphical model's centre estimate and errors are pretty much exactly the same as the weighted average or KDE! - -Whats the point of fitting a graphical model if the much simpler approach of the previous tutorial gives the -same answer? - -The answer, is model complexity. Graphical models become more powerful as we make our model more complex, -our non-linear parameter space higher dimensionality and the degeneracies between different parameters on the graph -more significant. - -We will demonstrate this in the next tutorial. - -__Wrap Up__ - -In this tutorial, we showed that for our extremely simple model the graphical model gives pretty much the -same estimate of the 1D Gaussian centre's as simpler approaches followed in the previous tutorial. - -We will next show the strengths of graphical models by fitting more complex models. -""" +""" +Tutorial 2: Graphical Models +============================ + +We have fitted a dataset containing 5 noisy 1D Gaussian which had a shared `centre` value. We estimated +the `centre` by fitting each dataset individually and combining the value of the `centre` inferred by each fit into +an overall estimate, using a weighted average. + +Graphical models use a different approach. They are a single model that is fitted to the entire dataset simultaneously. +The model includes specific model component for every individual 1D Gaussian in the sample. However, the graphical +model also has shared parameters between these individual model components. + +This example fits a graphical model using the same sample fitted in the previous tutorial, consisting of many 1D +Gaussians. However, whereas previously the `centre` of each Gaussian was a free parameter in each fit, in the graphical +model there is only a single parameter for the `centre` shared by all 1D Gaussians. + +This graphical model creates a non-linear parameter space with parameters for every Gaussian in our sample. For 5 +Gaussians each with their own model parameters but a single shared centre: + + - Each Gaussian has 2 free parameters from the components that are not shared (`normalization`, `sigma`). + - There is one additional free parameter, which is the `centre` shared by all 5 Gaussians. + +__Contents__ + +This tutorial is split into the following sections: + +- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. +- **Dataset**: Load the 5 noisy 1D Gaussian datasets for simultaneous fitting. +- **Analysis**: Create Analysis objects for each dataset. +- **Model**: Set up the graphical model with a shared prior for the centre parameter. +- **Analysis Factors**: Pair each model with its corresponding Analysis class at factor graph nodes. +- **Factor Graph**: Combine the Analysis Factors into a factor graph representing the graphical model. +- **Search**: Configure and run the non-linear search to fit the factor graph. +- **Result**: Inspect and compare the graphical model results to the individual fits. +- **Wrap Up**: Summary and discussion of the benefits of graphical models. +""" + +# from autofit import setup_notebook; setup_notebook() + +import numpy as np +from os import path + +import autofit as af + +""" +__Example Source Code (`af.ex`)__ + +The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: + + - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and + `visualize` functions. + + - `Gaussian`: a model component representing a 1D Gaussian profile. + + - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. + +These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you +have seen and used elsewhere throughout the workspace. + +__Dataset__ + +For each dataset we now set up the correct path and load it. + +Whereas in the previous tutorial we fitted each dataset one-by-one, in this tutorial we instead store each dataset +in a list so that we can set up a single model-fit that fits the 5 datasets simultaneously. +""" +total_datasets = 5 + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists( + path.join("dataset", "example_1d", "gaussian_x1__low_snr", "dataset_0") +): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +dataset_name_list = [] +data_list = [] +noise_map_list = [] + +for dataset_index in range(total_datasets): + dataset_name = f"dataset_{dataset_index}" + + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + dataset_name_list.append(dataset_name) + data_list.append(data) + noise_map_list.append(noise_map) + +""" +By plotting the Gaussians we can remind ourselves that determining their centres by eye is difficult. +""" +for dataset_name, data in zip(dataset_name_list, data_list): + af.ex.plot_profile_1d( + xvalues=np.arange(data.shape[0]), + profile_1d=data, + title=dataset_name, + ylabel="Data Values", + color="k", + ) + +""" +__Analysis__ + +For each dataset we now create a corresponding `Analysis` class. +""" +analysis_list = [] + +for data, noise_map in zip(data_list, noise_map_list): + analysis = af.ex.Analysis(data=data, noise_map=noise_map) + + analysis_list.append(analysis) + +""" +__Model__ + +We now compose the graphical model that we fit, using the `Model` object you are now familiar with. + +We begin by setting up a shared prior for `centre`. + +We set up this up as a single `UniformPrior` which is passed to separate `Model`'s for each `Gaussian` below. +""" +centre_shared_prior = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) + +""" +We now set up a list of `Model`'s, each of which contain a `Gaussian` that is used to fit each of the datasets +loaded above. + +All of these models use the `centre_shared_prior`, meaning that all model-components use the same value of `centre` +for every individual model component. + +For a fit using five Gaussians, this reduces the dimensionality of parameter space from N=15 (e.g. 3 parameters per +Gaussian) to N=11 (e.g. 5 `sigma`'s 5 `normalizations` and 1 `centre`). +""" +model_list = [] + +for model_index in range(len(data_list)): + gaussian = af.Model(af.ex.Gaussian) + + gaussian.centre = centre_shared_prior # This prior is used by all 3 Gaussians! + gaussian.normalization = af.LogUniformPrior(lower_limit=1e-6, upper_limit=1e6) + gaussian.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=25.0) + + model_list.append(gaussian) + +""" +__Analysis Factors__ + +Above, we composed a model consisting of three `Gaussian`'s with a shared `centre` prior. We also loaded three datasets +which we intend to fit with each of these `Gaussians`, setting up each in an `Analysis` class that defines how the +model is used to fit the data. + +We now simply pair each model-component to each `Analysis` class, so that: + +- `gaussian_0` fits `data_0` via `analysis_0`. +- `gaussian_1` fits `data_1` via `analysis_1`. +- `gaussian_2` fits `data_2` via `analysis_2`. + +The point where a `Model` and `Analysis` class meet is called an `AnalysisFactor`. + +This term denotes that we are composing a graphical model, which is commonly called a 'factor graph'. A factor +defines a node on this graph where we have some data, a model, and we fit the two together. The 'links' between these +different nodes then define the global model we are fitting. +""" +analysis_factor_list = [] + +for model, analysis in zip(model_list, analysis_list): + analysis_factor = af.AnalysisFactor(prior_model=model, analysis=analysis) + + analysis_factor_list.append(analysis_factor) + +""" +__Factor Graph__ + +We now combine our `AnalysisFactor`'s to compose a factor graph. + +What is a factor graph? A factor graph defines the graphical model's graph. For example, it defines the different +model components that make up our model (e.g. the individual `Gaussian` classes) and how their parameters are linked or +shared (e.g. that each `Gaussian` has its own unique `normalization` and `sigma`, but a shared `centre` parameter). + +This is what our factor graph looks like (visualization of graphs not implemented yet): + +The factor graph above is made up of two components: + +- Nodes: these are points on the graph where we have a unique set of data and a model that is made up of a subset of +our overall graphical model. This is effectively the `AnalysisFactor` objects we created above. + +- Links: these define the model components and parameters that are shared across different nodes and thus retain the +same values when fitting different datasets. +""" +factor_graph = af.FactorGraphModel(*analysis_factor_list) + +""" +The fit will use the factor graph's `global_prior_model`, which uses the models contained in every analysis factor +to contrast the overall global model that is fitted. + +Printing the `info` attribute of this model reveals the overall structure of the model, which is grouped in terms +of the analysis factors and therefore datasets. +""" +print(factor_graph.global_prior_model.info) + +""" +__Search__ + +We can now create a non-linear search and use it to the fit the factor graph, using its `global_prior_model` property. +""" +search = af.DynestyStatic( + path_prefix=path.join("chapter_3_graphical_models"), + name="tutorial_2_graphical_model", + nlive=200, + dlogz=1e-4, + sample="rwalk", + walks=10, +) + +result = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) + +""" +__Result__ + +The result's `info` attribute shows that the result is expressed following the same struture of analysis factors +that the `global_prior_model.info` attribute revealed above. +""" +print(result.info) + +""" +We can now inspect the inferred value of `centre`, and compare this to the value we estimated in the previous tutorial +via a weighted average or posterior multiplicaition using KDE.(feature missing currently). + +(The errors of the weighted average and KDE below is what was estimated for a run on my PC, yours may be slightly +different!) +""" +print( + f"Weighted Average Centre Estimate = 48.535531422571886 (4.139907734505303) [1.0 sigma confidence intervals] \n" +) + +centre = result.samples.median_pdf()[0].centre + +u1_error = result.samples.values_at_upper_sigma(sigma=1.0)[0].centre +l1_error = result.samples.values_at_lower_sigma(sigma=1.0)[0].centre + +u3_error = result.samples.values_at_upper_sigma(sigma=3.0)[0].centre +l3_error = result.samples.values_at_lower_sigma(sigma=3.0)[0].centre + +print("Inferred value of the shared centre via a graphical model fit: \n") +print(f"{centre} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") +print(f"{centre} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") + +""" +The graphical model's centre estimate and errors are pretty much exactly the same as the weighted average or KDE! + +Whats the point of fitting a graphical model if the much simpler approach of the previous tutorial gives the +same answer? + +The answer, is model complexity. Graphical models become more powerful as we make our model more complex, +our non-linear parameter space higher dimensionality and the degeneracies between different parameters on the graph +more significant. + +We will demonstrate this in the next tutorial. + +__Wrap Up__ + +In this tutorial, we showed that for our extremely simple model the graphical model gives pretty much the +same estimate of the 1D Gaussian centre's as simpler approaches followed in the previous tutorial. + +We will next show the strengths of graphical models by fitting more complex models. +""" diff --git a/scripts/chapter_3_graphical_models/tutorial_3_graphical_benefits.py b/scripts/chapter_3_graphical_models/tutorial_3_graphical_benefits.py index 5931fc7..60709a3 100644 --- a/scripts/chapter_3_graphical_models/tutorial_3_graphical_benefits.py +++ b/scripts/chapter_3_graphical_models/tutorial_3_graphical_benefits.py @@ -1,481 +1,481 @@ -""" -Tutorial 3: Graphical Benefits -============================== - -In the previous tutorials, we fitted a dataset containing 5 noisy 1D Gaussian which had a shared `centre` value and -compared different approaches to estimate the shared `centre`. This included a simple approach fitting each dataset -one-by-one and estimating the centre via a weighted average or posterior multiplication and a more complicated -approach using a graphical model. - -The estimates were consistent with one another, making it hard to justify the use of the more complicated graphical -model. However, the model fitted in the previous tutorial was extremely simple, and by making it slightly more complex -we will show the benefits of the graphical model. - -__The Model__ - -In this tutorial, each dataset now contains two Gaussians, and they all have the same shared centres, located at -pixels 40 and 60. - -__Contents__ - -This tutorial is split into the following sections: - -- **The Model**: Describe the two-Gaussian model fitted in this tutorial. -- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. -- **Dataset**: Load datasets where each contains two Gaussians with shared centres. -- **Analysis**: Create Analysis objects for each dataset. -- **Model (one-by-one)**: Set up individual models with two Gaussians for one-by-one fitting. -- **Model Fits (one-by-one)**: Fit each dataset individually using separate non-linear searches. -- **Centre Estimates (Weighted Average)**: Compute centre estimates and errors using a weighted average. -- **Discussion**: Analyze the limitations of the one-by-one fitting approach. -- **Model (Graphical)**: Set up the graphical model with shared centre priors across datasets. -- **Analysis Factors**: Create Analysis Factors pairing models with Analysis objects. -- **Factor Graph**: Combine Analysis Factors into a factor graph. -- **Search**: Configure and run the non-linear search for the graphical model. -- **Result**: Inspect the graphical model results and compare to individual fits. -- **Discussion**: Discuss the benefits of graphical models over one-by-one fitting. -- **Posterior Multiplication**: Discuss KDE-based posterior multiplication as an alternative method. -- **Wrap Up**: Summary comparing the different methods and transition to hierarchical models. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import numpy as np -from os import path - -import autofit as af -import autofit.plot as aplt - -""" -__Example Source Code (`af.ex`)__ - -The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: - - - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and - `visualize` functions. - - - `Gaussian`: a model component representing a 1D Gaussian profile. - - - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. - -These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have seen -and used elsewhere throughout the workspace. - -__Dataset__ - -For each dataset we now set up the correct path and load it. - -Note that we are loading a new dataset called `gaussian_x2__offset_centres`. -""" -total_datasets = 5 - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists( - path.join("dataset", "example_1d", "gaussian_x2__offset_centres", "dataset_0") -): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -dataset_name_list = [] -data_list = [] -noise_map_list = [] - -for dataset_index in range(total_datasets): - dataset_name = f"dataset_{dataset_index}" - - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x2__offset_centres", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - dataset_name_list.append(dataset_name) - data_list.append(data) - noise_map_list.append(noise_map) - -""" -By plotting the datasets we see that each dataset contains two Gaussians. - -Their centres are offset from one another and not located at pixel 50, like in the previous tutorials. - -As discussed above, the Gaussians in every dataset are in facted centred at pixels 40 and 60. -""" -for dataset_name, data in zip(dataset_name_list, data_list): - af.ex.plot_profile_1d( - xvalues=np.arange(data.shape[0]), - profile_1d=data, - title=dataset_name, - ylabel="Data Values", - color="k", - ) - -""" -__Analysis__ - -For each dataset we now create a corresponding `Analysis` class. -""" -analysis_list = [] - -for data, noise_map in zip(data_list, noise_map_list): - analysis = af.ex.Analysis(data=data, noise_map=noise_map) - - analysis_list.append(analysis) - -""" -__Model (one-by-one)__ - -We are first going to fit each dataset one by one. - -Our model therefore now has two 1D `Gaussian`'s. - -To remove solutions where the Gaussians flip locations and fit the other Gaussian, we set uniform priors on the -`centre`'s which ensures one Gaussian stays on the left side of the data (fitting the Gaussian at pixel 40) -whilst the other stays on the right (fitting the Gaussian at pixel 60). -""" -gaussian_0 = af.Model(af.ex.Gaussian) - -gaussian_0.centre = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) - -gaussian_1 = af.Model(af.ex.Gaussian) - -gaussian_1.centre = af.UniformPrior(lower_limit=50.0, upper_limit=100.0) - -model = af.Collection(gaussian_0=gaussian_0, gaussian_1=gaussian_1) - -""" -__Model Fits (one-by-one)__ - -For every dataset we now create an `Analysis` and fit it with a `Gaussian`. - -The `Result` is stored in the list `result_list`. -""" -result_list = [] - -for i, analysis in enumerate(analysis_list): - """ - Create the `DynestyStatic` non-linear search and use it to fit the data. - - We use custom dynesty settings which ensure the posterior is explored fully and that our error estimates are robust. - """ - search = af.DynestyStatic( - name=f"individual_fit_{i}", - path_prefix=path.join( - "chapter_3_graphical_models", "tutorial_3_graphical_benefits" - ), - nlive=200, - dlogz=1e-4, - sample="rwalk", - walks=10, - ) - - print( - f"The non-linear search has begun running, checkout \n" - f"HowToFit/output/chapter_3_graphical_models/tutorial_3_graphical_benefits/{dataset_name} for live \n" - f"output of the results. This Jupyter notebook cell with progress once search has completed, this could take a \n" - f"few minutes!" - ) - - result_list.append(search.fit(model=model, analysis=analysis)) - -""" -__Centre Estimates (Weighted Average)__ - -We can now compute the centre estimate of both Gaussians, including their errors, from the individual model fits -performed above. -""" -samples_list = [result.samples for result in result_list] - -mp_instances = [samps.median_pdf() for samps in samples_list] - -mp_centres_0 = [instance.gaussian_0.centre for instance in mp_instances] -mp_centres_1 = [instance.gaussian_1.centre for instance in mp_instances] - -ue1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] -le1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] - -ue1_centres_0 = [instance.gaussian_0.centre for instance in ue1_instances] -le1_centres_0 = [instance.gaussian_0.centre for instance in le1_instances] - -error_0_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres_0, le1_centres_0)] - -values_0 = np.asarray(mp_centres_0) -sigmas_0 = np.asarray(error_0_list) - -weights_0 = 1 / sigmas_0**2.0 -weight_averaged_0 = np.sum(1.0 / sigmas_0**2) - -weighted_centre_0 = np.sum(values_0 * weights_0) / np.sum(weights_0, axis=0) -weighted_error_0 = 1.0 / np.sqrt(weight_averaged_0) - -ue1_centres_1 = [instance.gaussian_1.centre for instance in ue1_instances] -le1_centres_1 = [instance.gaussian_1.centre for instance in le1_instances] - -error_1_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres_1, le1_centres_1)] - -values_1 = np.asarray(mp_centres_1) -sigmas_1 = np.asarray(error_1_list) - -weights_1 = 1 / sigmas_1**2.0 -weight_averaged_1 = np.sum(1.0 / sigmas_1**2) - -weighted_centre_1 = np.sum(values_1 * weights_1) / np.sum(weights_1, axis=0) -weighted_error_1 = 1.0 / np.sqrt(weight_averaged_1) - - -print( - f"Centre 0 via Weighted Average: {weighted_centre_0} ({weighted_error_0}) [1.0 sigma confidence intervals]" -) -print( - f"Centre 1 via Weighted Average: {weighted_centre_1} ({weighted_error_1}) [1.0 sigma confidence intervals]" -) - -""" -The estimate of the centres is not accurate, with both estimates well offset from the input values of 40 and 60. - -We will next show that the graphical model offers a notable improvement, but first lets consider why this -approach is suboptimal. - -The most important difference between this model and the model fitted in the previous tutorial is that there are now -two shared parameters we are trying to estimate, which are degenerate with one another. - -We can see this by inspecting the probability distribution function (PDF) of the fit, placing particular focus on the -2D degeneracy between the Gaussians centres. -""" -aplt.corner_cornerpy(samples=result_list[0].samples) - -""" -The problem is that the simple approach of taking a weighted average does not capture the curved banana-like shape -of the PDF between the two centres. This leads to significant error over estimation and biased inferences on the centre. - -__Discussion__ - -Let us now consider other downsides of fitting each dataset one-by-one, from a statistical perspective. We -will contrast these to the graphical model later in the tutorial: - -1) By fitting each dataset one-by-one this means that each model-fit fails to fully exploit the information we know -about the global model. We know that there are only two single shared values of `centre` across the full dataset -that we want to estimate. However, each individual fit has its own `centre` value which is able to assume different -values than the `centre` values used to fit the other datasets. This means that large degeneracies between the two -centres are present in each model-fit. - -By not fitting our model as a global model, we do not maximize the amount of information that we can extract from the -dataset as a whole. If a model fits dataset 1 poorly, this should be reflected in how we interpret how well the model -fits datasets 2 and 3. Our non-linear search should have a global view of how well the model fits the whole dataset. -This is the crucial aspect of fitting each dataset individually that we miss, and what a graphical model addresses. - -2) When we combined the result to estimate the global `centre` value via a weighted average, we marginalized over -the samples in 1D. As showed above, when there are strong degeneracies between models parameters the information on -the covariance between these parameters is lost when computing the global `centre`. This increases the inferred -uncertainties. A graphical model performs no such 1D marginalization and therefore fully samples the -parameter covariances. - -3) In Bayesian inference it is important that we define priors on all of the model parameters. By estimating the -global `centre` after the model-fits are completed it is unclear what prior the global `centre` actually has! We -actually defined the prior five times -- once for each fit -- which is not a well defined prior. In a graphical model -the prior is clearly defined. - -What would have happened if we had estimate the shared centres via 2D posterior multiplication using a KDE? We -will discuss this at the end of the tutorial after fitting a graphical model. - -__Model (Graphical)__ - -We now compose a graphical model and fit it. - -Our model now consists of two Gaussians with two `centre_shared_prior` variables, such that the same centres are -used for each Gaussians across all datasets. - -We again restrict one Gaussian's centre between pixels 0 -> 50 and the other 50 -> 100 to remove solutions where -the Gaussians flip location. -""" -centre_0_shared_prior = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) -centre_1_shared_prior = af.UniformPrior(lower_limit=50.0, upper_limit=100.0) - -""" -We now set up a list of `Model`'s, each of which contain two `Gaussian`'s that are used to fit each of the datasets -loaded above. - -All of these `Model`'s use the `centre_shared_prior`'s abpve. This means all model-components use the same value -of `centre` for every model composed and fitted. - -For a fit to five datasets (each using two Gaussians), this reduces the dimensionality of parameter space -from N=30 (e.g. 6 parameters per pair of Gaussians) to N=22 (e.g. 10 `sigma`'s 10 `normalizations` and 2 `centre`'s). -""" -model_list = [] - -for model_index in range(len(data_list)): - gaussian_0 = af.Model(af.ex.Gaussian) - gaussian_1 = af.Model(af.ex.Gaussian) - - gaussian_0.centre = centre_0_shared_prior # This prior is used by all Gaussians! - gaussian_1.centre = centre_1_shared_prior # This prior is used by all Gaussians! - - model = af.Collection(gaussian_0=gaussian_0, gaussian_1=gaussian_1) - - model_list.append(model) - -""" -__Analysis Factors__ - -We again create the graphical model using `AnalysisFactor` objects. -""" -analysis_factor_list = [] - -for model, analysis in zip(model_list, analysis_list): - analysis_factor = af.AnalysisFactor(prior_model=model, analysis=analysis) - - analysis_factor_list.append(analysis_factor) - -""" -__Factor Graph__ - -The analysis factors are then used to create the factor graph. -""" -factor_graph = af.FactorGraphModel(*analysis_factor_list) - -""" -The factor graph model can again be printed via the `info` attribute, which shows that there are two shared -parameters across the datasets. -""" -print(factor_graph.global_prior_model.info) - -""" -__Search__ - -We can now create a non-linear search and use it to the fit the factor graph, again using its `global_prior_model` -property. -""" -search = af.DynestyStatic( - path_prefix=path.join("chapter_3_graphical_models"), - name="tutorial_3_graphical_benefits", - sample="rwalk", -) - -result = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) - -""" -__Result__ - -The result's `info` attribute shows that the result is expressed following the same structure of analysis factors -that the `global_prior_model.info` attribute revealed above. -""" -print(result.info) - -""" -We can now inspect the inferred `centre` values and compare this to the values estimated above via a weighted average. - -(The errors of the weighted average is what was estimated for a run on my PC, yours may be slightly different!) -""" -centre_0 = result.samples.median_pdf()[0].gaussian_0.centre - -u1_error_0 = result.samples.values_at_upper_sigma(sigma=1.0)[0].gaussian_0.centre -l1_error_0 = result.samples.values_at_lower_sigma(sigma=1.0)[0].gaussian_0.centre - -u3_error_0 = result.samples.values_at_upper_sigma(sigma=3.0)[0].gaussian_0.centre -l3_error_0 = result.samples.values_at_lower_sigma(sigma=3.0)[0].gaussian_0.centre - -centre_1 = result.samples.median_pdf()[0].gaussian_1.centre - -u1_error_1 = result.samples.values_at_upper_sigma(sigma=1.0)[0].gaussian_1.centre -l1_error_1 = result.samples.values_at_lower_sigma(sigma=1.0)[0].gaussian_1.centre - -u3_error_1 = result.samples.values_at_upper_sigma(sigma=3.0)[0].gaussian_1.centre -l3_error_1 = result.samples.values_at_lower_sigma(sigma=3.0)[0].gaussian_1.centre - - -print( - f"Centre 0 via Weighted Average: 29.415828686393333 (15.265325182888517) [1.0 sigma confidence intervals] \n" -) -print( - f"Centre 1 via Weighted Average: 54.13825075629124 (2.3460686758693234) [1.0 sigma confidence intervals] \n" -) - -print( - f"Inferred value of Gaussian 0's shared centre via a graphical fit to {total_datasets} datasets: \n" -) -print( - f"{centre_0} ({l1_error_0} {u1_error_0}) ({u1_error_0 - l1_error_0}) [1.0 sigma confidence intervals]" -) -print( - f"{centre_0} ({l3_error_0} {u3_error_0}) ({u3_error_0 - l3_error_0}) [3.0 sigma confidence intervals]" -) - -print( - f"Inferred value of Gaussian 1's shared centre via a graphical fit to {total_datasets} datasets: \n" -) -print( - f"{centre_1} ({l1_error_1} {u1_error_1}) ({u1_error_1 - l1_error_1}) [1.0 sigma confidence intervals]" -) -print( - f"{centre_1} ({l3_error_1} {u3_error_1}) ({u3_error_1 - l3_error_1}) [3.0 sigma confidence intervals]" -) - -""" -As expected, using a graphical model allows us to infer a more precise and accurate model. - -__Discussion__ - -Unlike a fit to each dataset one-by-one, the graphical model: - -1) Infers a PDF on the global centre that fully accounts for the degeneracies between the models fitted to different -datasets. This reduces significantly the large 2D degeneracies between the two centres we saw when inspecting the PDFs -of each individual fit. - -2) Fully exploits the information we know about the global model, for example that the centre of every Gaussian in every -dataset is aligned. Now, the fit of the Gaussian in dataset 1 informs the fits in datasets 2 and 3, and visa versa. - -3) Has a well defined prior on the global centre, instead of 5 independent priors on the centre of each dataset. - -__Posterior Multiplication__ - -What if we had combined the results of the individual model fits using 2D posterior multiplication via a KDE? - -This would produce an inaccurate estimate of the error, because each posterior contains the prior on the centre five -times which given the properties of this model should not be repeated. - -However, it is possible to convert each posterior to a likelihood (by dividing by its prior), combining these 5 -likelihoods to form a joint likelihood via 2D KDE multiplication and then insert just one prior back (again using a 2D -KDE) at the end to get a posterior which does not have repeated priors. - -This posterior, in theory, should be equivalent to the graphical model, giving the same accurate estimates of the -centres with precise errors. The process extracts the same information, fully accounting for the 2D structure of the -PDF between the two centres for each fit. - -However, in practise, this will likely not work well. Every time we use a KDE to represent and multiply a posterior, we -make an approximation which will impact our inferred errors. The removal of the prior before combining the likelihood -and reinserting it after also introduces approximations, especially because the fit performed by the non-linear search -is informed by the prior. - -Crucially, whilst posterior multiplication can work in two dimensions, for models with many more dimensions and -degeneracies between parameters that are in 3D, 4D of more dimensions it will introduce more and more numerical -inaccuracies. - -A graphical model fully samples all of the information a large dataset contains about the model, without making -such large approximation. Therefore, irrespective of how complex the model gets, it extracts significantly more -information contained in the dataset. - -__Wrap Up__ - -In this tutorial, we demonstrated the strengths of a graphical model over fitting each dataset one-by-one. - -We argued that irrespective of how one may try to combine the results of many individual fits, the approximations that -are made will always lead to a suboptimal estimation of the model parameters and fail to fully extract all information -from the dataset. - -We argued that for high dimensional complex models a graphical model is the only way to fully extract all of the -information contained in the dataset. - -In the next tutorial, we will consider a natural extension of a graphical model called a hierarchical model. -""" +""" +Tutorial 3: Graphical Benefits +============================== + +In the previous tutorials, we fitted a dataset containing 5 noisy 1D Gaussian which had a shared `centre` value and +compared different approaches to estimate the shared `centre`. This included a simple approach fitting each dataset +one-by-one and estimating the centre via a weighted average or posterior multiplication and a more complicated +approach using a graphical model. + +The estimates were consistent with one another, making it hard to justify the use of the more complicated graphical +model. However, the model fitted in the previous tutorial was extremely simple, and by making it slightly more complex +we will show the benefits of the graphical model. + +__The Model__ + +In this tutorial, each dataset now contains two Gaussians, and they all have the same shared centres, located at +pixels 40 and 60. + +__Contents__ + +This tutorial is split into the following sections: + +- **The Model**: Describe the two-Gaussian model fitted in this tutorial. +- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. +- **Dataset**: Load datasets where each contains two Gaussians with shared centres. +- **Analysis**: Create Analysis objects for each dataset. +- **Model (one-by-one)**: Set up individual models with two Gaussians for one-by-one fitting. +- **Model Fits (one-by-one)**: Fit each dataset individually using separate non-linear searches. +- **Centre Estimates (Weighted Average)**: Compute centre estimates and errors using a weighted average. +- **Discussion**: Analyze the limitations of the one-by-one fitting approach. +- **Model (Graphical)**: Set up the graphical model with shared centre priors across datasets. +- **Analysis Factors**: Create Analysis Factors pairing models with Analysis objects. +- **Factor Graph**: Combine Analysis Factors into a factor graph. +- **Search**: Configure and run the non-linear search for the graphical model. +- **Result**: Inspect the graphical model results and compare to individual fits. +- **Discussion**: Discuss the benefits of graphical models over one-by-one fitting. +- **Posterior Multiplication**: Discuss KDE-based posterior multiplication as an alternative method. +- **Wrap Up**: Summary comparing the different methods and transition to hierarchical models. +""" + +# from autofit import setup_notebook; setup_notebook() + +import numpy as np +from os import path + +import autofit as af +import autofit.plot as aplt + +""" +__Example Source Code (`af.ex`)__ + +The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: + + - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and + `visualize` functions. + + - `Gaussian`: a model component representing a 1D Gaussian profile. + + - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. + +These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have seen +and used elsewhere throughout the workspace. + +__Dataset__ + +For each dataset we now set up the correct path and load it. + +Note that we are loading a new dataset called `gaussian_x2__offset_centres`. +""" +total_datasets = 5 + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists( + path.join("dataset", "example_1d", "gaussian_x2__offset_centres", "dataset_0") +): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +dataset_name_list = [] +data_list = [] +noise_map_list = [] + +for dataset_index in range(total_datasets): + dataset_name = f"dataset_{dataset_index}" + + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x2__offset_centres", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + dataset_name_list.append(dataset_name) + data_list.append(data) + noise_map_list.append(noise_map) + +""" +By plotting the datasets we see that each dataset contains two Gaussians. + +Their centres are offset from one another and not located at pixel 50, like in the previous tutorials. + +As discussed above, the Gaussians in every dataset are in facted centred at pixels 40 and 60. +""" +for dataset_name, data in zip(dataset_name_list, data_list): + af.ex.plot_profile_1d( + xvalues=np.arange(data.shape[0]), + profile_1d=data, + title=dataset_name, + ylabel="Data Values", + color="k", + ) + +""" +__Analysis__ + +For each dataset we now create a corresponding `Analysis` class. +""" +analysis_list = [] + +for data, noise_map in zip(data_list, noise_map_list): + analysis = af.ex.Analysis(data=data, noise_map=noise_map) + + analysis_list.append(analysis) + +""" +__Model (one-by-one)__ + +We are first going to fit each dataset one by one. + +Our model therefore now has two 1D `Gaussian`'s. + +To remove solutions where the Gaussians flip locations and fit the other Gaussian, we set uniform priors on the +`centre`'s which ensures one Gaussian stays on the left side of the data (fitting the Gaussian at pixel 40) +whilst the other stays on the right (fitting the Gaussian at pixel 60). +""" +gaussian_0 = af.Model(af.ex.Gaussian) + +gaussian_0.centre = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) + +gaussian_1 = af.Model(af.ex.Gaussian) + +gaussian_1.centre = af.UniformPrior(lower_limit=50.0, upper_limit=100.0) + +model = af.Collection(gaussian_0=gaussian_0, gaussian_1=gaussian_1) + +""" +__Model Fits (one-by-one)__ + +For every dataset we now create an `Analysis` and fit it with a `Gaussian`. + +The `Result` is stored in the list `result_list`. +""" +result_list = [] + +for i, analysis in enumerate(analysis_list): + """ + Create the `DynestyStatic` non-linear search and use it to fit the data. + + We use custom dynesty settings which ensure the posterior is explored fully and that our error estimates are robust. + """ + search = af.DynestyStatic( + name=f"individual_fit_{i}", + path_prefix=path.join( + "chapter_3_graphical_models", "tutorial_3_graphical_benefits" + ), + nlive=200, + dlogz=1e-4, + sample="rwalk", + walks=10, + ) + + print( + f"The non-linear search has begun running, checkout \n" + f"HowToFit/output/chapter_3_graphical_models/tutorial_3_graphical_benefits/{dataset_name} for live \n" + f"output of the results. This Jupyter notebook cell with progress once search has completed, this could take a \n" + f"few minutes!" + ) + + result_list.append(search.fit(model=model, analysis=analysis)) + +""" +__Centre Estimates (Weighted Average)__ + +We can now compute the centre estimate of both Gaussians, including their errors, from the individual model fits +performed above. +""" +samples_list = [result.samples for result in result_list] + +mp_instances = [samps.median_pdf() for samps in samples_list] + +mp_centres_0 = [instance.gaussian_0.centre for instance in mp_instances] +mp_centres_1 = [instance.gaussian_1.centre for instance in mp_instances] + +ue1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] +le1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] + +ue1_centres_0 = [instance.gaussian_0.centre for instance in ue1_instances] +le1_centres_0 = [instance.gaussian_0.centre for instance in le1_instances] + +error_0_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres_0, le1_centres_0)] + +values_0 = np.asarray(mp_centres_0) +sigmas_0 = np.asarray(error_0_list) + +weights_0 = 1 / sigmas_0**2.0 +weight_averaged_0 = np.sum(1.0 / sigmas_0**2) + +weighted_centre_0 = np.sum(values_0 * weights_0) / np.sum(weights_0, axis=0) +weighted_error_0 = 1.0 / np.sqrt(weight_averaged_0) + +ue1_centres_1 = [instance.gaussian_1.centre for instance in ue1_instances] +le1_centres_1 = [instance.gaussian_1.centre for instance in le1_instances] + +error_1_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres_1, le1_centres_1)] + +values_1 = np.asarray(mp_centres_1) +sigmas_1 = np.asarray(error_1_list) + +weights_1 = 1 / sigmas_1**2.0 +weight_averaged_1 = np.sum(1.0 / sigmas_1**2) + +weighted_centre_1 = np.sum(values_1 * weights_1) / np.sum(weights_1, axis=0) +weighted_error_1 = 1.0 / np.sqrt(weight_averaged_1) + + +print( + f"Centre 0 via Weighted Average: {weighted_centre_0} ({weighted_error_0}) [1.0 sigma confidence intervals]" +) +print( + f"Centre 1 via Weighted Average: {weighted_centre_1} ({weighted_error_1}) [1.0 sigma confidence intervals]" +) + +""" +The estimate of the centres is not accurate, with both estimates well offset from the input values of 40 and 60. + +We will next show that the graphical model offers a notable improvement, but first lets consider why this +approach is suboptimal. + +The most important difference between this model and the model fitted in the previous tutorial is that there are now +two shared parameters we are trying to estimate, which are degenerate with one another. + +We can see this by inspecting the probability distribution function (PDF) of the fit, placing particular focus on the +2D degeneracy between the Gaussians centres. +""" +aplt.corner_cornerpy(samples=result_list[0].samples) + +""" +The problem is that the simple approach of taking a weighted average does not capture the curved banana-like shape +of the PDF between the two centres. This leads to significant error over estimation and biased inferences on the centre. + +__Discussion__ + +Let us now consider other downsides of fitting each dataset one-by-one, from a statistical perspective. We +will contrast these to the graphical model later in the tutorial: + +1) By fitting each dataset one-by-one this means that each model-fit fails to fully exploit the information we know +about the global model. We know that there are only two single shared values of `centre` across the full dataset +that we want to estimate. However, each individual fit has its own `centre` value which is able to assume different +values than the `centre` values used to fit the other datasets. This means that large degeneracies between the two +centres are present in each model-fit. + +By not fitting our model as a global model, we do not maximize the amount of information that we can extract from the +dataset as a whole. If a model fits dataset 1 poorly, this should be reflected in how we interpret how well the model +fits datasets 2 and 3. Our non-linear search should have a global view of how well the model fits the whole dataset. +This is the crucial aspect of fitting each dataset individually that we miss, and what a graphical model addresses. + +2) When we combined the result to estimate the global `centre` value via a weighted average, we marginalized over +the samples in 1D. As showed above, when there are strong degeneracies between models parameters the information on +the covariance between these parameters is lost when computing the global `centre`. This increases the inferred +uncertainties. A graphical model performs no such 1D marginalization and therefore fully samples the +parameter covariances. + +3) In Bayesian inference it is important that we define priors on all of the model parameters. By estimating the +global `centre` after the model-fits are completed it is unclear what prior the global `centre` actually has! We +actually defined the prior five times -- once for each fit -- which is not a well defined prior. In a graphical model +the prior is clearly defined. + +What would have happened if we had estimate the shared centres via 2D posterior multiplication using a KDE? We +will discuss this at the end of the tutorial after fitting a graphical model. + +__Model (Graphical)__ + +We now compose a graphical model and fit it. + +Our model now consists of two Gaussians with two `centre_shared_prior` variables, such that the same centres are +used for each Gaussians across all datasets. + +We again restrict one Gaussian's centre between pixels 0 -> 50 and the other 50 -> 100 to remove solutions where +the Gaussians flip location. +""" +centre_0_shared_prior = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) +centre_1_shared_prior = af.UniformPrior(lower_limit=50.0, upper_limit=100.0) + +""" +We now set up a list of `Model`'s, each of which contain two `Gaussian`'s that are used to fit each of the datasets +loaded above. + +All of these `Model`'s use the `centre_shared_prior`'s abpve. This means all model-components use the same value +of `centre` for every model composed and fitted. + +For a fit to five datasets (each using two Gaussians), this reduces the dimensionality of parameter space +from N=30 (e.g. 6 parameters per pair of Gaussians) to N=22 (e.g. 10 `sigma`'s 10 `normalizations` and 2 `centre`'s). +""" +model_list = [] + +for model_index in range(len(data_list)): + gaussian_0 = af.Model(af.ex.Gaussian) + gaussian_1 = af.Model(af.ex.Gaussian) + + gaussian_0.centre = centre_0_shared_prior # This prior is used by all Gaussians! + gaussian_1.centre = centre_1_shared_prior # This prior is used by all Gaussians! + + model = af.Collection(gaussian_0=gaussian_0, gaussian_1=gaussian_1) + + model_list.append(model) + +""" +__Analysis Factors__ + +We again create the graphical model using `AnalysisFactor` objects. +""" +analysis_factor_list = [] + +for model, analysis in zip(model_list, analysis_list): + analysis_factor = af.AnalysisFactor(prior_model=model, analysis=analysis) + + analysis_factor_list.append(analysis_factor) + +""" +__Factor Graph__ + +The analysis factors are then used to create the factor graph. +""" +factor_graph = af.FactorGraphModel(*analysis_factor_list) + +""" +The factor graph model can again be printed via the `info` attribute, which shows that there are two shared +parameters across the datasets. +""" +print(factor_graph.global_prior_model.info) + +""" +__Search__ + +We can now create a non-linear search and use it to the fit the factor graph, again using its `global_prior_model` +property. +""" +search = af.DynestyStatic( + path_prefix=path.join("chapter_3_graphical_models"), + name="tutorial_3_graphical_benefits", + sample="rwalk", +) + +result = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) + +""" +__Result__ + +The result's `info` attribute shows that the result is expressed following the same structure of analysis factors +that the `global_prior_model.info` attribute revealed above. +""" +print(result.info) + +""" +We can now inspect the inferred `centre` values and compare this to the values estimated above via a weighted average. + +(The errors of the weighted average is what was estimated for a run on my PC, yours may be slightly different!) +""" +centre_0 = result.samples.median_pdf()[0].gaussian_0.centre + +u1_error_0 = result.samples.values_at_upper_sigma(sigma=1.0)[0].gaussian_0.centre +l1_error_0 = result.samples.values_at_lower_sigma(sigma=1.0)[0].gaussian_0.centre + +u3_error_0 = result.samples.values_at_upper_sigma(sigma=3.0)[0].gaussian_0.centre +l3_error_0 = result.samples.values_at_lower_sigma(sigma=3.0)[0].gaussian_0.centre + +centre_1 = result.samples.median_pdf()[0].gaussian_1.centre + +u1_error_1 = result.samples.values_at_upper_sigma(sigma=1.0)[0].gaussian_1.centre +l1_error_1 = result.samples.values_at_lower_sigma(sigma=1.0)[0].gaussian_1.centre + +u3_error_1 = result.samples.values_at_upper_sigma(sigma=3.0)[0].gaussian_1.centre +l3_error_1 = result.samples.values_at_lower_sigma(sigma=3.0)[0].gaussian_1.centre + + +print( + f"Centre 0 via Weighted Average: 29.415828686393333 (15.265325182888517) [1.0 sigma confidence intervals] \n" +) +print( + f"Centre 1 via Weighted Average: 54.13825075629124 (2.3460686758693234) [1.0 sigma confidence intervals] \n" +) + +print( + f"Inferred value of Gaussian 0's shared centre via a graphical fit to {total_datasets} datasets: \n" +) +print( + f"{centre_0} ({l1_error_0} {u1_error_0}) ({u1_error_0 - l1_error_0}) [1.0 sigma confidence intervals]" +) +print( + f"{centre_0} ({l3_error_0} {u3_error_0}) ({u3_error_0 - l3_error_0}) [3.0 sigma confidence intervals]" +) + +print( + f"Inferred value of Gaussian 1's shared centre via a graphical fit to {total_datasets} datasets: \n" +) +print( + f"{centre_1} ({l1_error_1} {u1_error_1}) ({u1_error_1 - l1_error_1}) [1.0 sigma confidence intervals]" +) +print( + f"{centre_1} ({l3_error_1} {u3_error_1}) ({u3_error_1 - l3_error_1}) [3.0 sigma confidence intervals]" +) + +""" +As expected, using a graphical model allows us to infer a more precise and accurate model. + +__Discussion__ + +Unlike a fit to each dataset one-by-one, the graphical model: + +1) Infers a PDF on the global centre that fully accounts for the degeneracies between the models fitted to different +datasets. This reduces significantly the large 2D degeneracies between the two centres we saw when inspecting the PDFs +of each individual fit. + +2) Fully exploits the information we know about the global model, for example that the centre of every Gaussian in every +dataset is aligned. Now, the fit of the Gaussian in dataset 1 informs the fits in datasets 2 and 3, and visa versa. + +3) Has a well defined prior on the global centre, instead of 5 independent priors on the centre of each dataset. + +__Posterior Multiplication__ + +What if we had combined the results of the individual model fits using 2D posterior multiplication via a KDE? + +This would produce an inaccurate estimate of the error, because each posterior contains the prior on the centre five +times which given the properties of this model should not be repeated. + +However, it is possible to convert each posterior to a likelihood (by dividing by its prior), combining these 5 +likelihoods to form a joint likelihood via 2D KDE multiplication and then insert just one prior back (again using a 2D +KDE) at the end to get a posterior which does not have repeated priors. + +This posterior, in theory, should be equivalent to the graphical model, giving the same accurate estimates of the +centres with precise errors. The process extracts the same information, fully accounting for the 2D structure of the +PDF between the two centres for each fit. + +However, in practise, this will likely not work well. Every time we use a KDE to represent and multiply a posterior, we +make an approximation which will impact our inferred errors. The removal of the prior before combining the likelihood +and reinserting it after also introduces approximations, especially because the fit performed by the non-linear search +is informed by the prior. + +Crucially, whilst posterior multiplication can work in two dimensions, for models with many more dimensions and +degeneracies between parameters that are in 3D, 4D of more dimensions it will introduce more and more numerical +inaccuracies. + +A graphical model fully samples all of the information a large dataset contains about the model, without making +such large approximation. Therefore, irrespective of how complex the model gets, it extracts significantly more +information contained in the dataset. + +__Wrap Up__ + +In this tutorial, we demonstrated the strengths of a graphical model over fitting each dataset one-by-one. + +We argued that irrespective of how one may try to combine the results of many individual fits, the approximations that +are made will always lead to a suboptimal estimation of the model parameters and fail to fully extract all information +from the dataset. + +We argued that for high dimensional complex models a graphical model is the only way to fully extract all of the +information contained in the dataset. + +In the next tutorial, we will consider a natural extension of a graphical model called a hierarchical model. +""" diff --git a/scripts/chapter_3_graphical_models/tutorial_4_hierachical_models.py b/scripts/chapter_3_graphical_models/tutorial_4_hierachical_models.py index 23b4ee0..28a8fe2 100644 --- a/scripts/chapter_3_graphical_models/tutorial_4_hierachical_models.py +++ b/scripts/chapter_3_graphical_models/tutorial_4_hierachical_models.py @@ -1,349 +1,349 @@ -""" -Tutorial 4: Hierarchical -======================== - -In the previous tutorial, we fitted a graphical model with the aim of determining an estimate of shared parameters, -the `centre`'s of a dataset of 1D Gaussians. We did this by fitting all datasets simultaneously. When there are shared -parameters in a model, this is a powerful and effective tool, but things may not always be so simple. - -A common extension to the problem is one where we expect that the shared parameter(s) of the model do not have exactly -the same value in every dataset. Instead, our expectation is that the parameter(s) are drawn from a common -parent distribution (e.g. a Gaussian distribution). It is the parameters of this distribution that we consider shared -across the dataset (e.g. the means and scatter of the Gaussian distribution). These are the parameters we ultimately -wish to infer to understand the global behaviour of our model. - -This is called a hierarchical model, which we fit in this tutorial. The `centre` of each 1D Gaussian is now no -longer the same in each dataset and they are instead drawn from a shared parent Gaussian distribution -(with `mean=50.0` and `sigma=10.0`). The hierarchical model will recover the `mean` and `sigma` values of the parent -distribution'. - -__Contents__ - -This tutorial is split into the following sections: - -- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. -- **Dataset**: Load the hierarchical Gaussian datasets with variable centres. -- **Analysis**: Create Analysis objects for each dataset. -- **Model Individual Factors**: Set up individual Gaussian models with independent priors. -- **Analysis Factors**: Compose Analysis Factors pairing models with Analysis objects. -- **Model**: Create a HierarchicalFactor with a parent Gaussian distribution for the centres. -- **Factor Graph**: Compose the factor graph including the hierarchical factor. -- **Search**: Configure and run the non-linear search for the hierarchical model. -- **Result**: Inspect the inferred hierarchical distribution parameters. -- **Comparison to One-by-One Fits**: Compare the hierarchical model results to simpler individual fits. -- **Benefits of Graphical Model**: Discuss how datasets inform one another through the hierarchical model. -- **Wrap Up**: Summary and introduction to expectation propagation. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import numpy as np -from os import path - -import autofit as af - -""" -__Example Source Code (`af.ex`)__ - -The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: - - - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and - `visualize` functions. - - - `Gaussian`: a model component representing a 1D Gaussian profile. - - - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. - -These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you -have seen and used elsewhere throughout the workspace. - -__Dataset__ - -For each dataset we now set up the correct path and load it. - -We are loading a new Gaussian dataset, where the Gaussians have different centres which were drawn from a parent -Gaussian distribution with a mean centre value of 50.0 and sigma of 10.0. -""" -total_datasets = 5 - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists( - path.join("dataset", "example_1d", "gaussian_x1__hierarchical", "dataset_0") -): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -dataset_name_list = [] -data_list = [] -noise_map_list = [] - -for dataset_index in range(total_datasets): - dataset_name = f"dataset_{dataset_index}" - - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x1__hierarchical", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - dataset_name_list.append(dataset_name) - data_list.append(data) - noise_map_list.append(noise_map) - -""" -By plotting the Gaussians we can just about make out that their centres are not all at pixel 50, and are spread out -around it (albeit its difficult to be sure, due to the low signal-to-noise of the data). -""" -for dataset_name, data in zip(dataset_name_list, data_list): - af.ex.plot_profile_1d( - xvalues=np.arange(data.shape[0]), - profile_1d=data, - title=dataset_name, - ylabel="Data Values", - color="k", - ) - -""" -__Analysis__ - -For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. -""" -analysis_list = [] - -for data, noise_map in zip(data_list, noise_map_list): - analysis = af.ex.Analysis(data=data, noise_map=noise_map) - - analysis_list.append(analysis) - - -""" -__Model Individual Factors__ - -We first set up a model for each `Gaussian` which is individually fitted to each 1D dataset, which forms the -factors on the factor graph we compose. - -This uses a nearly identical for loop to the previous tutorials, however a shared `centre` is no longer used and each -`Gaussian` is given its own prior for the `centre`. - -We will see next how this `centre` is used to construct the hierarchical model. -""" -model_list = [] - -for model_index in range(len(data_list)): - gaussian = af.Model(af.ex.Gaussian) - - gaussian.centre = af.TruncatedGaussianPrior( - mean=50.0, sigma=20.0, lower_limit=0.0, upper_limit=100.0 - ) - gaussian.normalization = af.TruncatedGaussianPrior( - mean=3.0, sigma=5.0, lower_limit=0.0 - ) - gaussian.sigma = af.TruncatedGaussianPrior(mean=10.0, sigma=10.0, lower_limit=0.0) - - model_list.append(gaussian) - -""" -__Analysis Factors__ - -Now we have our `Analysis` classes and model components, we can compose our `AnalysisFactor`'s. - -These are composed in the same way as for the graphical model in the previous tutorial. -""" -analysis_factor_list = [] - -for model, analysis in zip(model_list, analysis_list): - analysis_factor = af.AnalysisFactor(prior_model=model, analysis=analysis) - - analysis_factor_list.append(analysis_factor) - -""" -__Model__ - -We now compose the hierarchical model that we fit, using the individual Gaussian model components created above. - -We first create a `HierarchicalFactor`, which represents the parent Gaussian distribution from which we will assume -that the `centre` of each individual `Gaussian` dataset is drawn. - -For this parent `Gaussian`, we have to place priors on its `mean` and `sigma`, given that they are parameters in our -model we are ultimately fitting for. -""" -hierarchical_factor = af.HierarchicalFactor( - af.GaussianPrior, - mean=af.TruncatedGaussianPrior( - mean=50.0, sigma=10, lower_limit=0.0, upper_limit=100.0 - ), - sigma=af.TruncatedGaussianPrior( - mean=10.0, sigma=5.0, lower_limit=0.0, upper_limit=100.0 - ), -) - -""" -We now add each of the individual model `Gaussian`'s `centre` parameters to the `hierarchical_factor`. - -This composes the hierarchical model whereby the individual `centre` of every `Gaussian` in our dataset is now assumed -to be drawn from a shared parent distribution. It is the `mean` and `sigma` of this distribution we are hoping to -estimate. -""" -for model in model_list: - hierarchical_factor.add_drawn_variable(model.centre) - -""" -__Factor Graph__ - -We now create the factor graph for this model, using the list of `AnalysisFactor`'s and the hierarchical factor. - -Note that the `hierarchical_factor` is passed in below, which was not the case in previous tutorials. -""" -factor_graph = af.FactorGraphModel(*analysis_factor_list, hierarchical_factor) - -""" -The factor graph model `info` attribute shows that the hierarchical factor's parameters are included in the model. -""" -print(factor_graph.global_prior_model.info) - -""" -__Search__ - -We can now create a non-linear search and used it to the fit the hierarchical model, again using -its `global_prior_model` property. -""" -search = af.DynestyStatic( - path_prefix=path.join("chapter_3_graphical_models"), - name="tutorial_4_hierarchical", - sample="rwalk", -) - -result = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) - -""" -__Result__ - -The result's `info` attribute shows the result, including the hierarchical factor's parameters. -""" -print(result.info) - -""" -We can now inspect the inferred value of hierarchical factor's mean and sigma. - -We see that they are consistent with the input values of `mean=50.0` and `sigma=10.0`. - -The hierarchical factor results are at the end of the samples list, hence why we extract them using `[-1]` and [-2]` -below. -""" -samples = result.samples - -mean = samples.median_pdf(as_instance=False)[-2] - -u1_error = samples.values_at_upper_sigma(sigma=1.0)[-2] -l1_error = samples.values_at_lower_sigma(sigma=1.0)[-2] - -u3_error = samples.values_at_upper_sigma(sigma=3.0)[-2] -l3_error = samples.values_at_lower_sigma(sigma=3.0)[-2] - -print( - "Inferred value of the mean of the parent hierarchical distribution for the centre: \n" -) -print(f"{mean} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") -print(f"{mean} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") - -scatter = samples.median_pdf(as_instance=False)[-1] - -u1_error = samples.values_at_upper_sigma(sigma=1.0)[-1] -l1_error = samples.values_at_lower_sigma(sigma=1.0)[-1] - -u3_error = samples.values_at_upper_sigma(sigma=3.0)[-1] -l3_error = samples.values_at_lower_sigma(sigma=3.0)[-1] - -print( - "Inferred value of the scatter (the sigma value of the Gassuain) of the parent hierarchical distribution for the centre: \n" -) -print(f"{scatter} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") -print(f"{scatter} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") - -""" -__Comparison to One-by-One Fits__ - -We can compare the inferred values above to the values inferred for individual fits in the -tutorial `tutorial_optional_hierarchical_individual.py`. - -This fits the hierarchical model is a much simpler way -- fitting each dataset one-by-one and then fitting the -parent Gaussian distribution to those results. - -For the results below, inferred on my laptop, we can see that the correct mean and scatter of the parent Gaussian is -inferred but the errors are much larger than the graphical model fit above. -""" -print( - "Inferred value of the mean of the parent hierarchical distribution for one-by-one fits: \n" -) -print( - "50.00519854538594 (35.825675441265815 65.56274024242403) [1.0 sigma confidence intervals]" -) -print( - "50.00519854538594 (1.3226539914914734 96.92151898283811) [3.0 sigma confidence intervals]" -) - -print( - "Inferred value of the scatter of the parent hierarchical distribution for one-by-one fits: \n" -) -print( - "15.094393493747617 (4.608862348173649 31.346751522582483) [1.0 sigma confidence intervals]" -) -print( - "15.094393493747617 (0.060533647989089806 49.05537884440667) [3.0 sigma confidence intervals]" -) - -""" -__Benefits of Graphical Model__ - -We compared the results inferred in this script via a graphical model to a simpler approach which fits each dataset -one-by-one and infers the hierarchical parent distribution's parameters afterwards. - -The graphical model provides a more accurate and precise estimate of the parent distribution's parameters. This is -because the fit to each dataset informs the hierarchical distribution's parameters, which in turn improves -constraints on the other datasets. In a hierarchical fit, we describe this as "the datasets talking to one another". - -For example, by itself, dataset_0 may give weak constraints on the centre spanning the range 20 -> 85 at 1 sigma -confidence. Now, consider if simultaneously all of the other datasets provide strong constraints on the -hierarchical's distribution's parameters, such that its `mean = 50 +- 5.0` and `sigma = 10.0 +- 2.0` at 1 sigma -confidence. - -This will significantly change our inferred parameters for dataset 0, as the other datasets inform us -that solutions where the centre is well below approximately 40 are less likely, because they are inconsistent with -the parent hierarchical distribution's parameters! - -For complex graphical models with many hierarchical factors, this phenomena of the "datasets talking to one another" -is crucial in breaking degeneracies between parameters and maximally extracting information from extremely large -datasets. - -__Wrap Up__ - -By composing and fitting hierarchical models in the graphical modeling framework we can fit for global trends -within large datasets. The tools applied in this tutorial and the previous tutorial can be easily extended to -compose complex graphical models, with multiple shared parameters and hierarchical factors. - -However, there is a clear challenge scaling the graphical modeling framework up in this way: model complexity. As the -model becomes more complex, an inadequate sampling of parameter space will lead one to infer local maxima. Furthermore, -one will soon hit computational limits on how many datasets can feasibly be fitted simultaneously, both in terms of -CPU time and memory limitations. - -Therefore, the next tutorial introduces expectation propagation, a framework that inspects the factor graph of a -graphical model and partitions the model-fit into many separate fits on each graph node. When a fit is complete, -it passes the information learned about the model to neighboring nodes. - -Therefore, graphs comprising hundreds of model components (and tens of thousands of parameters) can be fitted as -many bite-sized model fits, where the model fitted at each node consists of just tens of parameters. This makes -graphical models scalable to largest datasets and most complex models! -""" +""" +Tutorial 4: Hierarchical +======================== + +In the previous tutorial, we fitted a graphical model with the aim of determining an estimate of shared parameters, +the `centre`'s of a dataset of 1D Gaussians. We did this by fitting all datasets simultaneously. When there are shared +parameters in a model, this is a powerful and effective tool, but things may not always be so simple. + +A common extension to the problem is one where we expect that the shared parameter(s) of the model do not have exactly +the same value in every dataset. Instead, our expectation is that the parameter(s) are drawn from a common +parent distribution (e.g. a Gaussian distribution). It is the parameters of this distribution that we consider shared +across the dataset (e.g. the means and scatter of the Gaussian distribution). These are the parameters we ultimately +wish to infer to understand the global behaviour of our model. + +This is called a hierarchical model, which we fit in this tutorial. The `centre` of each 1D Gaussian is now no +longer the same in each dataset and they are instead drawn from a shared parent Gaussian distribution +(with `mean=50.0` and `sigma=10.0`). The hierarchical model will recover the `mean` and `sigma` values of the parent +distribution'. + +__Contents__ + +This tutorial is split into the following sections: + +- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. +- **Dataset**: Load the hierarchical Gaussian datasets with variable centres. +- **Analysis**: Create Analysis objects for each dataset. +- **Model Individual Factors**: Set up individual Gaussian models with independent priors. +- **Analysis Factors**: Compose Analysis Factors pairing models with Analysis objects. +- **Model**: Create a HierarchicalFactor with a parent Gaussian distribution for the centres. +- **Factor Graph**: Compose the factor graph including the hierarchical factor. +- **Search**: Configure and run the non-linear search for the hierarchical model. +- **Result**: Inspect the inferred hierarchical distribution parameters. +- **Comparison to One-by-One Fits**: Compare the hierarchical model results to simpler individual fits. +- **Benefits of Graphical Model**: Discuss how datasets inform one another through the hierarchical model. +- **Wrap Up**: Summary and introduction to expectation propagation. +""" + +# from autofit import setup_notebook; setup_notebook() + +import numpy as np +from os import path + +import autofit as af + +""" +__Example Source Code (`af.ex`)__ + +The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: + + - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and + `visualize` functions. + + - `Gaussian`: a model component representing a 1D Gaussian profile. + + - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. + +These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you +have seen and used elsewhere throughout the workspace. + +__Dataset__ + +For each dataset we now set up the correct path and load it. + +We are loading a new Gaussian dataset, where the Gaussians have different centres which were drawn from a parent +Gaussian distribution with a mean centre value of 50.0 and sigma of 10.0. +""" +total_datasets = 5 + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists( + path.join("dataset", "example_1d", "gaussian_x1__hierarchical", "dataset_0") +): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +dataset_name_list = [] +data_list = [] +noise_map_list = [] + +for dataset_index in range(total_datasets): + dataset_name = f"dataset_{dataset_index}" + + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x1__hierarchical", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + dataset_name_list.append(dataset_name) + data_list.append(data) + noise_map_list.append(noise_map) + +""" +By plotting the Gaussians we can just about make out that their centres are not all at pixel 50, and are spread out +around it (albeit its difficult to be sure, due to the low signal-to-noise of the data). +""" +for dataset_name, data in zip(dataset_name_list, data_list): + af.ex.plot_profile_1d( + xvalues=np.arange(data.shape[0]), + profile_1d=data, + title=dataset_name, + ylabel="Data Values", + color="k", + ) + +""" +__Analysis__ + +For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. +""" +analysis_list = [] + +for data, noise_map in zip(data_list, noise_map_list): + analysis = af.ex.Analysis(data=data, noise_map=noise_map) + + analysis_list.append(analysis) + + +""" +__Model Individual Factors__ + +We first set up a model for each `Gaussian` which is individually fitted to each 1D dataset, which forms the +factors on the factor graph we compose. + +This uses a nearly identical for loop to the previous tutorials, however a shared `centre` is no longer used and each +`Gaussian` is given its own prior for the `centre`. + +We will see next how this `centre` is used to construct the hierarchical model. +""" +model_list = [] + +for model_index in range(len(data_list)): + gaussian = af.Model(af.ex.Gaussian) + + gaussian.centre = af.TruncatedGaussianPrior( + mean=50.0, sigma=20.0, lower_limit=0.0, upper_limit=100.0 + ) + gaussian.normalization = af.TruncatedGaussianPrior( + mean=3.0, sigma=5.0, lower_limit=0.0 + ) + gaussian.sigma = af.TruncatedGaussianPrior(mean=10.0, sigma=10.0, lower_limit=0.0) + + model_list.append(gaussian) + +""" +__Analysis Factors__ + +Now we have our `Analysis` classes and model components, we can compose our `AnalysisFactor`'s. + +These are composed in the same way as for the graphical model in the previous tutorial. +""" +analysis_factor_list = [] + +for model, analysis in zip(model_list, analysis_list): + analysis_factor = af.AnalysisFactor(prior_model=model, analysis=analysis) + + analysis_factor_list.append(analysis_factor) + +""" +__Model__ + +We now compose the hierarchical model that we fit, using the individual Gaussian model components created above. + +We first create a `HierarchicalFactor`, which represents the parent Gaussian distribution from which we will assume +that the `centre` of each individual `Gaussian` dataset is drawn. + +For this parent `Gaussian`, we have to place priors on its `mean` and `sigma`, given that they are parameters in our +model we are ultimately fitting for. +""" +hierarchical_factor = af.HierarchicalFactor( + af.GaussianPrior, + mean=af.TruncatedGaussianPrior( + mean=50.0, sigma=10, lower_limit=0.0, upper_limit=100.0 + ), + sigma=af.TruncatedGaussianPrior( + mean=10.0, sigma=5.0, lower_limit=0.0, upper_limit=100.0 + ), +) + +""" +We now add each of the individual model `Gaussian`'s `centre` parameters to the `hierarchical_factor`. + +This composes the hierarchical model whereby the individual `centre` of every `Gaussian` in our dataset is now assumed +to be drawn from a shared parent distribution. It is the `mean` and `sigma` of this distribution we are hoping to +estimate. +""" +for model in model_list: + hierarchical_factor.add_drawn_variable(model.centre) + +""" +__Factor Graph__ + +We now create the factor graph for this model, using the list of `AnalysisFactor`'s and the hierarchical factor. + +Note that the `hierarchical_factor` is passed in below, which was not the case in previous tutorials. +""" +factor_graph = af.FactorGraphModel(*analysis_factor_list, hierarchical_factor) + +""" +The factor graph model `info` attribute shows that the hierarchical factor's parameters are included in the model. +""" +print(factor_graph.global_prior_model.info) + +""" +__Search__ + +We can now create a non-linear search and used it to the fit the hierarchical model, again using +its `global_prior_model` property. +""" +search = af.DynestyStatic( + path_prefix=path.join("chapter_3_graphical_models"), + name="tutorial_4_hierarchical", + sample="rwalk", +) + +result = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) + +""" +__Result__ + +The result's `info` attribute shows the result, including the hierarchical factor's parameters. +""" +print(result.info) + +""" +We can now inspect the inferred value of hierarchical factor's mean and sigma. + +We see that they are consistent with the input values of `mean=50.0` and `sigma=10.0`. + +The hierarchical factor results are at the end of the samples list, hence why we extract them using `[-1]` and [-2]` +below. +""" +samples = result.samples + +mean = samples.median_pdf(as_instance=False)[-2] + +u1_error = samples.values_at_upper_sigma(sigma=1.0)[-2] +l1_error = samples.values_at_lower_sigma(sigma=1.0)[-2] + +u3_error = samples.values_at_upper_sigma(sigma=3.0)[-2] +l3_error = samples.values_at_lower_sigma(sigma=3.0)[-2] + +print( + "Inferred value of the mean of the parent hierarchical distribution for the centre: \n" +) +print(f"{mean} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") +print(f"{mean} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") + +scatter = samples.median_pdf(as_instance=False)[-1] + +u1_error = samples.values_at_upper_sigma(sigma=1.0)[-1] +l1_error = samples.values_at_lower_sigma(sigma=1.0)[-1] + +u3_error = samples.values_at_upper_sigma(sigma=3.0)[-1] +l3_error = samples.values_at_lower_sigma(sigma=3.0)[-1] + +print( + "Inferred value of the scatter (the sigma value of the Gassuain) of the parent hierarchical distribution for the centre: \n" +) +print(f"{scatter} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") +print(f"{scatter} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") + +""" +__Comparison to One-by-One Fits__ + +We can compare the inferred values above to the values inferred for individual fits in the +tutorial `tutorial_optional_hierarchical_individual.py`. + +This fits the hierarchical model is a much simpler way -- fitting each dataset one-by-one and then fitting the +parent Gaussian distribution to those results. + +For the results below, inferred on my laptop, we can see that the correct mean and scatter of the parent Gaussian is +inferred but the errors are much larger than the graphical model fit above. +""" +print( + "Inferred value of the mean of the parent hierarchical distribution for one-by-one fits: \n" +) +print( + "50.00519854538594 (35.825675441265815 65.56274024242403) [1.0 sigma confidence intervals]" +) +print( + "50.00519854538594 (1.3226539914914734 96.92151898283811) [3.0 sigma confidence intervals]" +) + +print( + "Inferred value of the scatter of the parent hierarchical distribution for one-by-one fits: \n" +) +print( + "15.094393493747617 (4.608862348173649 31.346751522582483) [1.0 sigma confidence intervals]" +) +print( + "15.094393493747617 (0.060533647989089806 49.05537884440667) [3.0 sigma confidence intervals]" +) + +""" +__Benefits of Graphical Model__ + +We compared the results inferred in this script via a graphical model to a simpler approach which fits each dataset +one-by-one and infers the hierarchical parent distribution's parameters afterwards. + +The graphical model provides a more accurate and precise estimate of the parent distribution's parameters. This is +because the fit to each dataset informs the hierarchical distribution's parameters, which in turn improves +constraints on the other datasets. In a hierarchical fit, we describe this as "the datasets talking to one another". + +For example, by itself, dataset_0 may give weak constraints on the centre spanning the range 20 -> 85 at 1 sigma +confidence. Now, consider if simultaneously all of the other datasets provide strong constraints on the +hierarchical's distribution's parameters, such that its `mean = 50 +- 5.0` and `sigma = 10.0 +- 2.0` at 1 sigma +confidence. + +This will significantly change our inferred parameters for dataset 0, as the other datasets inform us +that solutions where the centre is well below approximately 40 are less likely, because they are inconsistent with +the parent hierarchical distribution's parameters! + +For complex graphical models with many hierarchical factors, this phenomena of the "datasets talking to one another" +is crucial in breaking degeneracies between parameters and maximally extracting information from extremely large +datasets. + +__Wrap Up__ + +By composing and fitting hierarchical models in the graphical modeling framework we can fit for global trends +within large datasets. The tools applied in this tutorial and the previous tutorial can be easily extended to +compose complex graphical models, with multiple shared parameters and hierarchical factors. + +However, there is a clear challenge scaling the graphical modeling framework up in this way: model complexity. As the +model becomes more complex, an inadequate sampling of parameter space will lead one to infer local maxima. Furthermore, +one will soon hit computational limits on how many datasets can feasibly be fitted simultaneously, both in terms of +CPU time and memory limitations. + +Therefore, the next tutorial introduces expectation propagation, a framework that inspects the factor graph of a +graphical model and partitions the model-fit into many separate fits on each graph node. When a fit is complete, +it passes the information learned about the model to neighboring nodes. + +Therefore, graphs comprising hundreds of model components (and tens of thousands of parameters) can be fitted as +many bite-sized model fits, where the model fitted at each node consists of just tens of parameters. This makes +graphical models scalable to largest datasets and most complex models! +""" diff --git a/scripts/chapter_3_graphical_models/tutorial_5_expectation_propagation.py b/scripts/chapter_3_graphical_models/tutorial_5_expectation_propagation.py index f190e54..6c71032 100644 --- a/scripts/chapter_3_graphical_models/tutorial_5_expectation_propagation.py +++ b/scripts/chapter_3_graphical_models/tutorial_5_expectation_propagation.py @@ -1,368 +1,368 @@ -""" -Tutorial 5: Expectation Propagation -=================================== - -In the previous tutorials, we fitted graphical models to dataset comprising many noisy 1D Gaussians. These had a shared -and global value of their `centre`, or assumed their centres were hierarchically drawn from a parent Gaussian -distribution. This provides the basis of composing and fitting complex graphical models to large datasets. - -We concluded by discussing that there is a ceiling scaling these graphical models up to extremely large datasets. One -would soon find that the parameter space is too complex to sample, and computational limits would ultimately cap how -many datasets one could feasibly fit. - -This tutorial introduces expectation propagation (EP), the solution to this problem, which inspects a factor graph -and partitions the model-fit into many simpler fits of sub-components of the graph to individual datasets. This -overcomes the challenge of model complexity, and mitigates computational restrictions that may occur if one tries to -fit every dataset simultaneously. - -This tutorial fits a global model with a shared parameter and does not use a hierarchical model. The optional tutorial -`tutorial_optional_hierarchical_ep` shows an example fit of a hierarchical model with EP. - -__Contents__ - -This tutorial is split into the following sections: - -- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. -- **Dataset**: Load the noisy 1D Gaussian datasets with a shared centre. -- **Analysis**: Create Analysis objects for each dataset. -- **Model**: Set up the model with a shared centre prior across all datasets. -- **Analysis Factors**: Create Analysis Factors with individual searches for each dataset. -- **Factor Graph**: Compose the factor graph for the EP framework. -- **Expectation Propagation**: Explain the EP message passing algorithm. -- **Cyclic Fitting**: Describe the iterative EP convergence process. -- **Result**: Access the result of the EP fit. -- **Output**: Describe the output directory structure and files generated by the EP fit. -- **Results**: Use the MeanField object to infer parameter estimates and errors. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import numpy as np -from os import path - -import autofit as af - -""" -__Example Source Code (`af.ex`)__ - -The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: - - - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and - `visualize` functions. - - - `Gaussian`: a model component representing a 1D Gaussian profile. - - - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. - -These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have seen -and used elsewhere throughout the workspace. - -__Dataset__ - -For each dataset we now set up the correct path and load it. - -We first fit the 1D Gaussians which all share the same centre, thus not requiring a hierarchical model. - -An example for fitting the hierarchical model with EP is given at the end of this tutorial. -""" -total_datasets = 3 - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists( - path.join("dataset", "example_1d", "gaussian_x1__low_snr", "dataset_0") -): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -dataset_name_list = [] -data_list = [] -noise_map_list = [] - -for dataset_index in range(total_datasets): - dataset_name = f"dataset_{dataset_index}" - - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - dataset_name_list.append(dataset_name) - data_list.append(data) - noise_map_list.append(noise_map) - -""" -By plotting the Gaussians we can remind ourselves that determining their centres by eye is difficult. -""" -for dataset_name, data in zip(dataset_name_list, data_list): - af.ex.plot_profile_1d( - xvalues=np.arange(data.shape[0]), - profile_1d=data, - title=dataset_name, - ylabel="Data Values", - color="k", - ) - -""" -__Analysis__ - -For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. -""" -analysis_list = [] - -for data, noise_map in zip(data_list, noise_map_list): - analysis = af.ex.Analysis(data=data, noise_map=noise_map) - - analysis_list.append(analysis) - -""" -__Model__ - -We now compose the graphical model that we fit, using the `Model` and `Collection` objects you are now familiar with. - -We will assume all Gaussians share the same centre, therefore we set up a shared prior for `centre`. -""" -centre_shared_prior = af.GaussianPrior(mean=50.0, sigma=30.0) - -model_list = [] - -for model_index in range(len(data_list)): - gaussian = af.Model(af.ex.Gaussian) - - gaussian.centre = af.GaussianPrior(mean=50.0, sigma=30.0) - gaussian.normalization = af.TruncatedGaussianPrior( - mean=3.0, sigma=5.0, lower_limit=0.0 - ) - gaussian.sigma = af.TruncatedGaussianPrior(mean=10.0, sigma=10.0, lower_limit=0.0) - - model = af.Collection(gaussian=gaussian) - - model_list.append(model) - -""" -__Analysis Factors__ - -Now we have our `Analysis` classes and graphical model, we can compose our `AnalysisFactor`'s. - -However, unlike the previous tutorials, each `AnalysisFactor` is now assigned its own `search`. This is because the EP -framework performs a model-fit to each node on the factor graph (e.g. each `AnalysisFactor`). Therefore, each node -requires its own non-linear search, and in this tutorial we use `dynesty`. For complex graphs consisting of many -nodes, one could easily use different searches for different nodes on the factor graph. - -Each `AnalysisFactor` is also given a `name`, corresponding to the name of the dataset it fits. These names are used -to name the folders containing the results in the output directory. -""" -paths = af.DirectoryPaths( - name=path.join("chapter_3_graphical_models", "tutorial_5_expectation_propagation") -) - -search = af.DynestyStatic(paths=paths, nlive=100, sample="rwalk") - -analysis_factor_list = [] - -dataset_index = 0 - -for model, analysis in zip(model_list, analysis_list): - dataset_name = f"dataset_{dataset_index}" - dataset_index += 1 - - analysis_factor = af.AnalysisFactor( - prior_model=model, analysis=analysis, optimiser=search, name=dataset_name - ) - - analysis_factor_list.append(analysis_factor) - - -class LinearRegressionAnalysis(af.Analysis): - def log_likelihood_function(self, instance, xp=np): - return -1 - - -fwhm_list = [2 * np.sqrt(2 * np.log(2)) * model.gaussian.sigma for model in model_list] - -linear_regression_factor = af.AnalysisFactor( - prior_model=af.Collection( - m=af.GaussianPrior(mean=0.0, sigma=1.0), - c=af.GaussianPrior(mean=0.0, sigma=1.0), - *fwhm_list, - ), - analysis=LinearRegressionAnalysis(), - optimiser=search, - name="linear_regression", -) - -""" -__Factor Graph__ - -We combine our `AnalysisFactors` into one, to compose the factor graph. -""" -factor_graph = af.FactorGraphModel( - *analysis_factor_list, - linear_regression_factor, -) - -""" -The factor graph model `info` attribute shows the model which we fit via expectaton propagation (note that we do -not use `global_prior_model` below when performing the fit). -""" -print(factor_graph.global_prior_model.info) - -""" -__Expectation Propagation__ - -In the previous tutorials, we used the `global_prior_model` of the `factor_graph` to fit the global model. In this -tutorial, we instead fit the `factor_graph` using the EP framework, which fits the graphical model composed in this -tutorial as follows: - -1) Go to the first node on the factor graph (e.g. `analysis_factor_list[0]`) and fit its model to its dataset. This is -simply a fit of the `Gaussian` model to the first 1D Gaussian dataset, the model-fit we are used to performing by now. - -2) Once the model-fit is complete, inspect the model for parameters that are shared with other nodes on the factor -graph. In this example, the `centre` of the `Gaussian` fitted to the first dataset is global, and therefore connects -to the other nodes on the factor graph (the `AnalysisFactor`'s) of the second and first `Gaussian` datasets. - -3) The EP framework now creates a 'message' that is to be passed to the connecting nodes on the factor graph. This -message informs them of the results of the model-fit, so they can update their priors on the `Gaussian`'s centre -accordingly and, more importantly, update their posterior inference and therefore estimate of the global centre. - -For example, the model fitted to the first Gaussian dataset includes the global centre. Therefore, after the model is -fitted, the EP framework creates a 'message' informing the factor graph about its inference on that Gaussians's centre, -thereby updating our overall inference on this shared parameter. This is termed 'message passing'. - -__Cyclic Fitting__ - -After every `AnalysisFactor` has been fitted (e.g. after each fit to each of the 5 datasets in this example), we have a -new estimate of the shared parameter `centre`. This updates our priors on the shared parameter `centre`, which needs -to be reflected in each model-fit we perform on each `AnalysisFactor`. - -The EP framework therefore performs a second iteration of model-fits. It again cycles through each `AnalysisFactor` -and refits the model, using updated priors on shared parameters like the `centre`. At the end of each fit, we again -create messages that update our knowledge about other parameters on the graph. - -This process is repeated multiple times, until a convergence criteria is met whereby continued cycles are expected to -produce the same estimate of the shared parameter `centre`. - -When we fit the factor graph a `name` is passed, which determines the folder all results of the factor graph are -stored in. -""" -laplace = af.LaplaceOptimiser() - -factor_graph_result = factor_graph.optimise( - optimiser=laplace, paths=paths, ep_history=af.EPHistory(kl_tol=0.05), max_steps=5 -) - -""" -__Result__ - -An `info` attribute for the result of a factor graph fitted via EP does not exist yet, its on the to do list! - -The result can be seen in the `graph.result` file output to hard-disk. -""" -### print(factor_graph_result.info)## - -""" -__Output__ - -The results of the factor graph, using the EP framework and message passing, are contained in the folder -`output/chapter_3_graphical_models/tutorial_5_expectation_propagation`. - -The following folders and files are worth of note: - - - `graph.info`: this provides an overall summary of the graphical model that is fitted, including every parameter, - how parameters are shared across `AnalysisFactor`'s and the priors associated to each individual parameter. - - - The 3 folders titled `gaussian_x1_#__low_snr` correspond to the three `AnalysisFactor`'s and therefore signify - repeated non-linear searches that are performed to fit each dataset. - - - Inside each of these folders are `optimization_#` folders, corresponding to each model-fit performed over cycles of - the EP fit. A careful inspection of the `model.info` files inside each folder reveals how the priors are updated - over each cycle, whereas the `model.results` file should indicate the improved estimate of model parameters over each - cycle. - -__Results__ - -The `MeanField` object represent the posterior of the entire factor graph and is used to infer estimates of the -values and error of each parameter in the graph. -""" -mean_field = factor_graph_result.updated_ep_mean_field.mean_field -print(mean_field) -print() - -""" -The object has a `variables` property which lists every variable in the factor graph, which is essentially all of the -free parameters on the graph. - -This includes the parameters specific to each data (E.g. each node on the graph) as well as the shared centre. -""" -print(mean_field.variables) -print() - -# """ -# The variables above use the priors on each parameter as their key. -# -# Therefore to estimate mean-field quantities of the shared centre, we can simply use the `centre_shared_prior` defined -# above. -# -# Each parameter estimate is given by the mean of its value in the `MeanField`. Below, we use the `centred_shared_prior` -# as a key to the `MeanField.mean` dictionary to print the estimated value of the shared centre. -# """ -# print(f"Centre Mean Parameter Estimate = {mean_field.mean[centre_shared_prior]}") -# print() -# -# """ -# If we want the parameter estimate of another parameter in the model, we can use the `model_list` that we composed -# above to pass a parameter prior to the mean field dictionary. -# """ -# print( -# f"Normalization Gaussian Dataset 0 Mean = {mean_field.mean[model_list[0].gaussian.normalization]}" -# ) -# -# """ -# The mean-field mean dictionary contains the estimate value of every parameter. -# """ -# print(f"All Parameter Estimates = {mean_field.mean}") -# print() -# -# """ -# The mean-field also contains a `variance` dictionary, which has the same keys as the `mean` dictionary above. -# -# This is the easier way to estimate the error on every parameter, for example that of the shared centre. -# """ -# print(f"Centre Variance = {mean_field.variance[centre_shared_prior]}") -# print() -# -# """ -# The standard deviation (or error at one sigma confidence interval) is given by the square root of the variance. -# """ -# print(f"Centre 1 Sigma = {np.sqrt(mean_field.variance[centre_shared_prior])}") -# print() -# -# """ -# The mean field object also contains a dictionary of the s.d./variance**0.5. -# """ -# print(f"Centre SD/sqrt(variance) = {mean_field.scale[centre_shared_prior]}") -# print() - -""" -__Wrap Up__ - -This tutorial introduced Expectation Propagation (EP), which fits a graphical model by passing messages between its -factors rather than sampling the full joint parameter space at once. This makes inference scalable to graphs with many -datasets and shared parameters. - -The commented-out code above shows optional further inspection of the `MeanField` result (per-parameter means, -variances and standard deviations) which you can uncomment to explore. -""" +""" +Tutorial 5: Expectation Propagation +=================================== + +In the previous tutorials, we fitted graphical models to dataset comprising many noisy 1D Gaussians. These had a shared +and global value of their `centre`, or assumed their centres were hierarchically drawn from a parent Gaussian +distribution. This provides the basis of composing and fitting complex graphical models to large datasets. + +We concluded by discussing that there is a ceiling scaling these graphical models up to extremely large datasets. One +would soon find that the parameter space is too complex to sample, and computational limits would ultimately cap how +many datasets one could feasibly fit. + +This tutorial introduces expectation propagation (EP), the solution to this problem, which inspects a factor graph +and partitions the model-fit into many simpler fits of sub-components of the graph to individual datasets. This +overcomes the challenge of model complexity, and mitigates computational restrictions that may occur if one tries to +fit every dataset simultaneously. + +This tutorial fits a global model with a shared parameter and does not use a hierarchical model. The optional tutorial +`tutorial_optional_hierarchical_ep` shows an example fit of a hierarchical model with EP. + +__Contents__ + +This tutorial is split into the following sections: + +- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. +- **Dataset**: Load the noisy 1D Gaussian datasets with a shared centre. +- **Analysis**: Create Analysis objects for each dataset. +- **Model**: Set up the model with a shared centre prior across all datasets. +- **Analysis Factors**: Create Analysis Factors with individual searches for each dataset. +- **Factor Graph**: Compose the factor graph for the EP framework. +- **Expectation Propagation**: Explain the EP message passing algorithm. +- **Cyclic Fitting**: Describe the iterative EP convergence process. +- **Result**: Access the result of the EP fit. +- **Output**: Describe the output directory structure and files generated by the EP fit. +- **Results**: Use the MeanField object to infer parameter estimates and errors. +""" + +# from autofit import setup_notebook; setup_notebook() + +import numpy as np +from os import path + +import autofit as af + +""" +__Example Source Code (`af.ex`)__ + +The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: + + - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and + `visualize` functions. + + - `Gaussian`: a model component representing a 1D Gaussian profile. + + - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. + +These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have seen +and used elsewhere throughout the workspace. + +__Dataset__ + +For each dataset we now set up the correct path and load it. + +We first fit the 1D Gaussians which all share the same centre, thus not requiring a hierarchical model. + +An example for fitting the hierarchical model with EP is given at the end of this tutorial. +""" +total_datasets = 3 + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists( + path.join("dataset", "example_1d", "gaussian_x1__low_snr", "dataset_0") +): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +dataset_name_list = [] +data_list = [] +noise_map_list = [] + +for dataset_index in range(total_datasets): + dataset_name = f"dataset_{dataset_index}" + + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x1__low_snr", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + dataset_name_list.append(dataset_name) + data_list.append(data) + noise_map_list.append(noise_map) + +""" +By plotting the Gaussians we can remind ourselves that determining their centres by eye is difficult. +""" +for dataset_name, data in zip(dataset_name_list, data_list): + af.ex.plot_profile_1d( + xvalues=np.arange(data.shape[0]), + profile_1d=data, + title=dataset_name, + ylabel="Data Values", + color="k", + ) + +""" +__Analysis__ + +For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. +""" +analysis_list = [] + +for data, noise_map in zip(data_list, noise_map_list): + analysis = af.ex.Analysis(data=data, noise_map=noise_map) + + analysis_list.append(analysis) + +""" +__Model__ + +We now compose the graphical model that we fit, using the `Model` and `Collection` objects you are now familiar with. + +We will assume all Gaussians share the same centre, therefore we set up a shared prior for `centre`. +""" +centre_shared_prior = af.GaussianPrior(mean=50.0, sigma=30.0) + +model_list = [] + +for model_index in range(len(data_list)): + gaussian = af.Model(af.ex.Gaussian) + + gaussian.centre = af.GaussianPrior(mean=50.0, sigma=30.0) + gaussian.normalization = af.TruncatedGaussianPrior( + mean=3.0, sigma=5.0, lower_limit=0.0 + ) + gaussian.sigma = af.TruncatedGaussianPrior(mean=10.0, sigma=10.0, lower_limit=0.0) + + model = af.Collection(gaussian=gaussian) + + model_list.append(model) + +""" +__Analysis Factors__ + +Now we have our `Analysis` classes and graphical model, we can compose our `AnalysisFactor`'s. + +However, unlike the previous tutorials, each `AnalysisFactor` is now assigned its own `search`. This is because the EP +framework performs a model-fit to each node on the factor graph (e.g. each `AnalysisFactor`). Therefore, each node +requires its own non-linear search, and in this tutorial we use `dynesty`. For complex graphs consisting of many +nodes, one could easily use different searches for different nodes on the factor graph. + +Each `AnalysisFactor` is also given a `name`, corresponding to the name of the dataset it fits. These names are used +to name the folders containing the results in the output directory. +""" +paths = af.DirectoryPaths( + name=path.join("chapter_3_graphical_models", "tutorial_5_expectation_propagation") +) + +search = af.DynestyStatic(paths=paths, nlive=100, sample="rwalk") + +analysis_factor_list = [] + +dataset_index = 0 + +for model, analysis in zip(model_list, analysis_list): + dataset_name = f"dataset_{dataset_index}" + dataset_index += 1 + + analysis_factor = af.AnalysisFactor( + prior_model=model, analysis=analysis, optimiser=search, name=dataset_name + ) + + analysis_factor_list.append(analysis_factor) + + +class LinearRegressionAnalysis(af.Analysis): + def log_likelihood_function(self, instance, xp=np): + return -1 + + +fwhm_list = [2 * np.sqrt(2 * np.log(2)) * model.gaussian.sigma for model in model_list] + +linear_regression_factor = af.AnalysisFactor( + prior_model=af.Collection( + m=af.GaussianPrior(mean=0.0, sigma=1.0), + c=af.GaussianPrior(mean=0.0, sigma=1.0), + *fwhm_list, + ), + analysis=LinearRegressionAnalysis(), + optimiser=search, + name="linear_regression", +) + +""" +__Factor Graph__ + +We combine our `AnalysisFactors` into one, to compose the factor graph. +""" +factor_graph = af.FactorGraphModel( + *analysis_factor_list, + linear_regression_factor, +) + +""" +The factor graph model `info` attribute shows the model which we fit via expectaton propagation (note that we do +not use `global_prior_model` below when performing the fit). +""" +print(factor_graph.global_prior_model.info) + +""" +__Expectation Propagation__ + +In the previous tutorials, we used the `global_prior_model` of the `factor_graph` to fit the global model. In this +tutorial, we instead fit the `factor_graph` using the EP framework, which fits the graphical model composed in this +tutorial as follows: + +1) Go to the first node on the factor graph (e.g. `analysis_factor_list[0]`) and fit its model to its dataset. This is +simply a fit of the `Gaussian` model to the first 1D Gaussian dataset, the model-fit we are used to performing by now. + +2) Once the model-fit is complete, inspect the model for parameters that are shared with other nodes on the factor +graph. In this example, the `centre` of the `Gaussian` fitted to the first dataset is global, and therefore connects +to the other nodes on the factor graph (the `AnalysisFactor`'s) of the second and first `Gaussian` datasets. + +3) The EP framework now creates a 'message' that is to be passed to the connecting nodes on the factor graph. This +message informs them of the results of the model-fit, so they can update their priors on the `Gaussian`'s centre +accordingly and, more importantly, update their posterior inference and therefore estimate of the global centre. + +For example, the model fitted to the first Gaussian dataset includes the global centre. Therefore, after the model is +fitted, the EP framework creates a 'message' informing the factor graph about its inference on that Gaussians's centre, +thereby updating our overall inference on this shared parameter. This is termed 'message passing'. + +__Cyclic Fitting__ + +After every `AnalysisFactor` has been fitted (e.g. after each fit to each of the 5 datasets in this example), we have a +new estimate of the shared parameter `centre`. This updates our priors on the shared parameter `centre`, which needs +to be reflected in each model-fit we perform on each `AnalysisFactor`. + +The EP framework therefore performs a second iteration of model-fits. It again cycles through each `AnalysisFactor` +and refits the model, using updated priors on shared parameters like the `centre`. At the end of each fit, we again +create messages that update our knowledge about other parameters on the graph. + +This process is repeated multiple times, until a convergence criteria is met whereby continued cycles are expected to +produce the same estimate of the shared parameter `centre`. + +When we fit the factor graph a `name` is passed, which determines the folder all results of the factor graph are +stored in. +""" +laplace = af.LaplaceOptimiser() + +factor_graph_result = factor_graph.optimise( + optimiser=laplace, paths=paths, ep_history=af.EPHistory(kl_tol=0.05), max_steps=5 +) + +""" +__Result__ + +An `info` attribute for the result of a factor graph fitted via EP does not exist yet, its on the to do list! + +The result can be seen in the `graph.result` file output to hard-disk. +""" +### print(factor_graph_result.info)## + +""" +__Output__ + +The results of the factor graph, using the EP framework and message passing, are contained in the folder +`output/chapter_3_graphical_models/tutorial_5_expectation_propagation`. + +The following folders and files are worth of note: + + - `graph.info`: this provides an overall summary of the graphical model that is fitted, including every parameter, + how parameters are shared across `AnalysisFactor`'s and the priors associated to each individual parameter. + + - The 3 folders titled `gaussian_x1_#__low_snr` correspond to the three `AnalysisFactor`'s and therefore signify + repeated non-linear searches that are performed to fit each dataset. + + - Inside each of these folders are `optimization_#` folders, corresponding to each model-fit performed over cycles of + the EP fit. A careful inspection of the `model.info` files inside each folder reveals how the priors are updated + over each cycle, whereas the `model.results` file should indicate the improved estimate of model parameters over each + cycle. + +__Results__ + +The `MeanField` object represent the posterior of the entire factor graph and is used to infer estimates of the +values and error of each parameter in the graph. +""" +mean_field = factor_graph_result.updated_ep_mean_field.mean_field +print(mean_field) +print() + +""" +The object has a `variables` property which lists every variable in the factor graph, which is essentially all of the +free parameters on the graph. + +This includes the parameters specific to each data (E.g. each node on the graph) as well as the shared centre. +""" +print(mean_field.variables) +print() + +# """ +# The variables above use the priors on each parameter as their key. +# +# Therefore to estimate mean-field quantities of the shared centre, we can simply use the `centre_shared_prior` defined +# above. +# +# Each parameter estimate is given by the mean of its value in the `MeanField`. Below, we use the `centred_shared_prior` +# as a key to the `MeanField.mean` dictionary to print the estimated value of the shared centre. +# """ +# print(f"Centre Mean Parameter Estimate = {mean_field.mean[centre_shared_prior]}") +# print() +# +# """ +# If we want the parameter estimate of another parameter in the model, we can use the `model_list` that we composed +# above to pass a parameter prior to the mean field dictionary. +# """ +# print( +# f"Normalization Gaussian Dataset 0 Mean = {mean_field.mean[model_list[0].gaussian.normalization]}" +# ) +# +# """ +# The mean-field mean dictionary contains the estimate value of every parameter. +# """ +# print(f"All Parameter Estimates = {mean_field.mean}") +# print() +# +# """ +# The mean-field also contains a `variance` dictionary, which has the same keys as the `mean` dictionary above. +# +# This is the easier way to estimate the error on every parameter, for example that of the shared centre. +# """ +# print(f"Centre Variance = {mean_field.variance[centre_shared_prior]}") +# print() +# +# """ +# The standard deviation (or error at one sigma confidence interval) is given by the square root of the variance. +# """ +# print(f"Centre 1 Sigma = {np.sqrt(mean_field.variance[centre_shared_prior])}") +# print() +# +# """ +# The mean field object also contains a dictionary of the s.d./variance**0.5. +# """ +# print(f"Centre SD/sqrt(variance) = {mean_field.scale[centre_shared_prior]}") +# print() + +""" +__Wrap Up__ + +This tutorial introduced Expectation Propagation (EP), which fits a graphical model by passing messages between its +factors rather than sampling the full joint parameter space at once. This makes inference scalable to graphs with many +datasets and shared parameters. + +The commented-out code above shows optional further inspection of the `MeanField` result (per-parameter means, +variances and standard deviations) which you can uncomment to explore. +""" diff --git a/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.py b/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.py index e32c2b9..19fde22 100644 --- a/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.py +++ b/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_ep.py @@ -1,241 +1,241 @@ -""" -Tutorial Optional: Hierarchical Expectation Propagation (EP) -============================================================ - -This optional tutorial gives an example of fitting a hierarchical model using EP. - -The API is a straightforward combination of tutorials 3 and 4. - -__Contents__ - -This tutorial is split into the following sections: - -- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. -- **Dataset**: Load the hierarchical Gaussian datasets with variable centres. -- **Analysis**: Create Analysis objects for each dataset. -- **Model Individual Factors**: Set up individual Gaussian models with independent priors. -- **Analysis Factors**: Compose Analysis Factors with individual searches for each dataset. -- **Model**: Create a HierarchicalFactor with a parent Gaussian distribution. -- **Factor Graph**: Compose the factor graph including the hierarchical factor. -- **Model Fit**: Run the EP fit of the hierarchical model. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import numpy as np -from os import path - -import autofit as af - -""" -__Example Source Code (`af.ex`)__ - -The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: - - - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and - `visualize` functions. - - - `Gaussian`: a model component representing a 1D Gaussian profile. - - - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. - -These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have seen -and used elsewhere throughout the workspace. - -__Dataset__ - -For each dataset we now set up the correct path and load it. - -In this example, the three Gaussians have different centres, which are drawn from a parent Gaussian distribution -whose mean and scatter we aim to estimate. -""" -total_datasets = 5 - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists( - path.join("dataset", "example_1d", "gaussian_x1__hierarchical", "dataset_0") -): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -dataset_name_list = [] -data_list = [] -noise_map_list = [] - -for dataset_index in range(total_datasets): - dataset_name = f"dataset_{dataset_index}" - - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x1__hierarchical", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - dataset_name_list.append(dataset_name) - data_list.append(data) - noise_map_list.append(noise_map) - -""" -By plotting the Gaussians we can just about make out that their centres are not all at pix 50, and are spreasd out -around it (albeit its difficult to be sure, due to the low signal-to-noise of the data). -""" -for dataset_name, data in zip(dataset_name_list, data_list): - af.ex.plot_profile_1d( - xvalues=np.arange(data.shape[0]), - profile_1d=data, - title=dataset_name, - ylabel="Data Values", - color="k", - ) - -""" -__Analysis__ - -For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. -""" -analysis_list = [] - -for data, noise_map in zip(data_list, noise_map_list): - analysis = af.ex.Analysis(data=data, noise_map=noise_map) - - analysis_list.append(analysis) - - -""" -__Model Individual Factors__ - -We first set up a model for each `Gaussian` which is individually fitted to each 1D dataset, which forms the -factors on the factor graph we compose. - -This uses a nearly identical for loop to the previous tutorial, however a shared `centre` is no longer used and each -`Gaussian` is given its own prior for the `centre`. We will see next how this `centre` is used to construct the -hierachical model. -""" - -model_list = [] - -for model_index in range(len(data_list)): - gaussian = af.Model(af.ex.Gaussian) - - # gaussian.centre = af.UniformPrior(lower_limit=0.0, upper_limit=1e2) - # gaussian.normalization = af.UniformPrior(lower_limit=0.0, upper_limit=1e2) - # gaussian.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=25.0) - - gaussian.centre = af.TruncatedGaussianPrior( - mean=50.0, sigma=20.0, lower_limit=0.0, upper_limit=100.0 - ) - gaussian.normalization = af.TruncatedGaussianPrior( - mean=3.0, sigma=5.0, lower_limit=0.0 - ) - gaussian.sigma = af.TruncatedGaussianPrior(mean=10.0, sigma=10.0, lower_limit=0.0) - - model_list.append(gaussian) - -""" -__Analysis Factors__ - -Now we have our `Analysis` classes and model components, we can compose our `AnalysisFactor`'s. - -The hierarchical model fit uses EP, therefore we again supply each `AnalysisFactor` its own `search` and `name`. -""" -dynesty = af.DynestyStatic(nlive=100, sample="rwalk") - -analysis_factor_list = [] - -dataset_index = 0 - -for model, analysis in zip(model_list, analysis_list): - dataset_name = f"dataset_{dataset_index}" - dataset_index += 1 - - analysis_factor = af.AnalysisFactor( - prior_model=model, analysis=analysis, optimiser=dynesty, name=dataset_name - ) - - analysis_factor_list.append(analysis_factor) - - -""" -__Model__ - -We now compose the hierarchical model that we fit, using the individual Gaussian model components we created above. - -We first create a `HierarchicalFactor`, which represents the parent Gaussian distribution from which we will assume -that the `centre` of each individual `Gaussian` dataset is drawn. - -For this parent `Gaussian`, we have to place priors on its `mean` and `sigma`, given that they are parameters in our -model we are ultimately fitting for. -""" -hierarchical_factor = af.HierarchicalFactor( - af.GaussianPrior, - mean=af.TruncatedGaussianPrior( - mean=50.0, sigma=10, lower_limit=0.0, upper_limit=100.0 - ), - sigma=af.TruncatedGaussianPrior( - mean=10.0, sigma=5.0, lower_limit=0.0, upper_limit=100.0 - ), -) - -""" -We now add each of the individual model `Gaussian`'s `centre` parameters to the `hierarchical_factor`. - -This composes the hierarchical model whereby the individual `centre` of every `Gaussian` in our dataset is now assumed -to be drawn from a shared parent distribution. It is the `mean` and `sigma` of this distribution we are hoping to -estimate. -""" -for model in model_list: - hierarchical_factor.add_drawn_variable(model.centre) - -""" -__Factor Graph__ - -We now create the factor graph for this model, using the list of `AnalysisFactor`'s and the hierarchical factor. - -Note that in previous tutorials, when we created the `FactorGraphModel` we only passed the list of `AnalysisFactor`'s, -which contained the necessary information on the model create the factor graph that was fitted. The `AnalysisFactor`'s -were created before we composed the `HierachicalFactor`, which is why we need to pass it separate when composing the -factor graph. -""" -factor_graph = af.FactorGraphModel(*analysis_factor_list, hierarchical_factor) - -""" -__Model Fit__ - - -""" -laplace = af.LaplaceOptimiser() - -# ep_result = factor_graph.optimise( -# laplace, -# paths=af.DirectoryPaths( -# name=path.join( -# "chapter_3_graphical_models", "tutorial_4_hierarchical" -# ) -# ), -# ep_history=af.EPHistory(kl_tol=1.0), -# max_steps=5, -# ) - -""" -__Wrap Up__ - -This optional tutorial showed how a hierarchical model can be fitted using Expectation Propagation (EP). The -`factor_graph.optimise` call above is commented out because a full EP fit is computationally expensive; uncomment it -to run the fit yourself. - -For the concepts behind hierarchical models and EP, see the main chapter 3 tutorials -(`tutorial_4_hierarchical_model` and `tutorial_5_expectation_propagation`). -""" +""" +Tutorial Optional: Hierarchical Expectation Propagation (EP) +============================================================ + +This optional tutorial gives an example of fitting a hierarchical model using EP. + +The API is a straightforward combination of tutorials 3 and 4. + +__Contents__ + +This tutorial is split into the following sections: + +- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. +- **Dataset**: Load the hierarchical Gaussian datasets with variable centres. +- **Analysis**: Create Analysis objects for each dataset. +- **Model Individual Factors**: Set up individual Gaussian models with independent priors. +- **Analysis Factors**: Compose Analysis Factors with individual searches for each dataset. +- **Model**: Create a HierarchicalFactor with a parent Gaussian distribution. +- **Factor Graph**: Compose the factor graph including the hierarchical factor. +- **Model Fit**: Run the EP fit of the hierarchical model. +""" + +# from autofit import setup_notebook; setup_notebook() + +import numpy as np +from os import path + +import autofit as af + +""" +__Example Source Code (`af.ex`)__ + +The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: + + - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and + `visualize` functions. + + - `Gaussian`: a model component representing a 1D Gaussian profile. + + - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. + +These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you have seen +and used elsewhere throughout the workspace. + +__Dataset__ + +For each dataset we now set up the correct path and load it. + +In this example, the three Gaussians have different centres, which are drawn from a parent Gaussian distribution +whose mean and scatter we aim to estimate. +""" +total_datasets = 5 + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists( + path.join("dataset", "example_1d", "gaussian_x1__hierarchical", "dataset_0") +): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +dataset_name_list = [] +data_list = [] +noise_map_list = [] + +for dataset_index in range(total_datasets): + dataset_name = f"dataset_{dataset_index}" + + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x1__hierarchical", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + dataset_name_list.append(dataset_name) + data_list.append(data) + noise_map_list.append(noise_map) + +""" +By plotting the Gaussians we can just about make out that their centres are not all at pix 50, and are spreasd out +around it (albeit its difficult to be sure, due to the low signal-to-noise of the data). +""" +for dataset_name, data in zip(dataset_name_list, data_list): + af.ex.plot_profile_1d( + xvalues=np.arange(data.shape[0]), + profile_1d=data, + title=dataset_name, + ylabel="Data Values", + color="k", + ) + +""" +__Analysis__ + +For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. +""" +analysis_list = [] + +for data, noise_map in zip(data_list, noise_map_list): + analysis = af.ex.Analysis(data=data, noise_map=noise_map) + + analysis_list.append(analysis) + + +""" +__Model Individual Factors__ + +We first set up a model for each `Gaussian` which is individually fitted to each 1D dataset, which forms the +factors on the factor graph we compose. + +This uses a nearly identical for loop to the previous tutorial, however a shared `centre` is no longer used and each +`Gaussian` is given its own prior for the `centre`. We will see next how this `centre` is used to construct the +hierachical model. +""" + +model_list = [] + +for model_index in range(len(data_list)): + gaussian = af.Model(af.ex.Gaussian) + + # gaussian.centre = af.UniformPrior(lower_limit=0.0, upper_limit=1e2) + # gaussian.normalization = af.UniformPrior(lower_limit=0.0, upper_limit=1e2) + # gaussian.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=25.0) + + gaussian.centre = af.TruncatedGaussianPrior( + mean=50.0, sigma=20.0, lower_limit=0.0, upper_limit=100.0 + ) + gaussian.normalization = af.TruncatedGaussianPrior( + mean=3.0, sigma=5.0, lower_limit=0.0 + ) + gaussian.sigma = af.TruncatedGaussianPrior(mean=10.0, sigma=10.0, lower_limit=0.0) + + model_list.append(gaussian) + +""" +__Analysis Factors__ + +Now we have our `Analysis` classes and model components, we can compose our `AnalysisFactor`'s. + +The hierarchical model fit uses EP, therefore we again supply each `AnalysisFactor` its own `search` and `name`. +""" +dynesty = af.DynestyStatic(nlive=100, sample="rwalk") + +analysis_factor_list = [] + +dataset_index = 0 + +for model, analysis in zip(model_list, analysis_list): + dataset_name = f"dataset_{dataset_index}" + dataset_index += 1 + + analysis_factor = af.AnalysisFactor( + prior_model=model, analysis=analysis, optimiser=dynesty, name=dataset_name + ) + + analysis_factor_list.append(analysis_factor) + + +""" +__Model__ + +We now compose the hierarchical model that we fit, using the individual Gaussian model components we created above. + +We first create a `HierarchicalFactor`, which represents the parent Gaussian distribution from which we will assume +that the `centre` of each individual `Gaussian` dataset is drawn. + +For this parent `Gaussian`, we have to place priors on its `mean` and `sigma`, given that they are parameters in our +model we are ultimately fitting for. +""" +hierarchical_factor = af.HierarchicalFactor( + af.GaussianPrior, + mean=af.TruncatedGaussianPrior( + mean=50.0, sigma=10, lower_limit=0.0, upper_limit=100.0 + ), + sigma=af.TruncatedGaussianPrior( + mean=10.0, sigma=5.0, lower_limit=0.0, upper_limit=100.0 + ), +) + +""" +We now add each of the individual model `Gaussian`'s `centre` parameters to the `hierarchical_factor`. + +This composes the hierarchical model whereby the individual `centre` of every `Gaussian` in our dataset is now assumed +to be drawn from a shared parent distribution. It is the `mean` and `sigma` of this distribution we are hoping to +estimate. +""" +for model in model_list: + hierarchical_factor.add_drawn_variable(model.centre) + +""" +__Factor Graph__ + +We now create the factor graph for this model, using the list of `AnalysisFactor`'s and the hierarchical factor. + +Note that in previous tutorials, when we created the `FactorGraphModel` we only passed the list of `AnalysisFactor`'s, +which contained the necessary information on the model create the factor graph that was fitted. The `AnalysisFactor`'s +were created before we composed the `HierachicalFactor`, which is why we need to pass it separate when composing the +factor graph. +""" +factor_graph = af.FactorGraphModel(*analysis_factor_list, hierarchical_factor) + +""" +__Model Fit__ + + +""" +laplace = af.LaplaceOptimiser() + +# ep_result = factor_graph.optimise( +# laplace, +# paths=af.DirectoryPaths( +# name=path.join( +# "chapter_3_graphical_models", "tutorial_4_hierarchical" +# ) +# ), +# ep_history=af.EPHistory(kl_tol=1.0), +# max_steps=5, +# ) + +""" +__Wrap Up__ + +This optional tutorial showed how a hierarchical model can be fitted using Expectation Propagation (EP). The +`factor_graph.optimise` call above is commented out because a full EP fit is computationally expensive; uncomment it +to run the fit yourself. + +For the concepts behind hierarchical models and EP, see the main chapter 3 tutorials +(`tutorial_4_hierarchical_model` and `tutorial_5_expectation_propagation`). +""" diff --git a/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.py b/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.py index 91e8ef9..bd7b360 100644 --- a/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.py +++ b/scripts/chapter_3_graphical_models/tutorial_optional_hierarchical_individual.py @@ -1,342 +1,342 @@ -""" -Tutorial Optional: Hierarchical Individual -========================================== - -In tutorial 4, we fit a hierarchical model using a graphical model, whereby all datasets are fitted simultaneously -and the hierarchical parameters are fitted for simultaneously with the model parameters of each 1D Gaussian in each -dataset. - -This script illustrates how the hierarchical parameters can be estimated using a simpler approach, which fits -each dataset one-by-one and estimates the hierarchical parameters afterwards by fitting the inferred `centres` -with a Gaussian distribution. - -__Contents__ - -This tutorial is split into the following sections: - -- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. -- **Dataset**: Load the hierarchical Gaussian datasets with variable centres. -- **Analysis**: Create Analysis objects for each dataset. -- **Model**: Define a simple Gaussian model with uniform priors. -- **Model Fits (one-by-one)**: Fit each dataset individually using separate non-linear searches. -- **Results**: Analyze and plot the results of each individual fit. -- **Overall Gaussian Parent Distribution**: Fit a parent Gaussian distribution to the inferred centres. -- **Model**: Set up a ParentGaussian model for the parent distribution fitting. -- **Analysis + Search**: Create the analysis and search for the parent distribution fit. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import numpy as np -from os import path - -import autofit as af - -""" -__Example Source Code (`af.ex`)__ - -The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: - - - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and - `visualize` functions. - - - `Gaussian`: a model component representing a 1D Gaussian profile. - - - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. - -These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you -have seen and used elsewhere throughout the workspace. - -__Dataset__ - -For each dataset we now set up the correct path and load it. - -We are loading a new Gaussian dataset, where the Gaussians have different centres which were drawn from a parent -Gaussian distribution with a mean centre value of 50.0 and sigma of 10.0. -""" -total_datasets = 5 - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if not path.exists( - path.join("dataset", "example_1d", "gaussian_x1__hierarchical", "dataset_0") -): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/simulators.py"], - check=True, - ) - -dataset_name_list = [] -data_list = [] -noise_map_list = [] - -for dataset_index in range(total_datasets): - dataset_name = f"dataset_{dataset_index}" - - dataset_path = path.join( - "dataset", "example_1d", "gaussian_x1__hierarchical", dataset_name - ) - - data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) - noise_map = af.util.numpy_array_from_json( - file_path=path.join(dataset_path, "noise_map.json") - ) - - dataset_name_list.append(dataset_name) - data_list.append(data) - noise_map_list.append(noise_map) - -""" -By plotting the Gaussians we can just about make out that their centres are not all at pix 50, and are spread out -around it (albeit its difficult to be sure, due to the low signal-to-noise of the data). -""" -for dataset_name, data in zip(dataset_name_list, data_list): - af.ex.plot_profile_1d( - xvalues=np.arange(data.shape[0]), - profile_1d=data, - title=dataset_name, - ylabel="Data Values", - color="k", - ) - -""" -__Analysis__ - -For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. -""" -analysis_list = [] - -for data, noise_map in zip(data_list, noise_map_list): - analysis = af.ex.Analysis(data=data, noise_map=noise_map) - - analysis_list.append(analysis) - - -""" -__Model__ - -The model we fit to each dataset, which is a simple 1D Gaussian with all 3 parameters free. -""" -gaussian = af.Model(af.ex.Gaussian) - -gaussian.centre = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) -gaussian.normalization = af.UniformPrior(lower_limit=0.0, upper_limit=10.0) -gaussian.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) - -model = af.Collection(gaussian=gaussian) - - -""" -__Model Fits (one-by-one)__ - -For every dataset we now create an `Analysis` class using it and use `Dynesty` to fit it with a `Gaussian`. - -The `Result` is stored in the list `results`. -""" -result_list = [] - -for dataset_name, analysis in zip(dataset_name_list, analysis_list): - """ - Create the `DynestyStatic` non-linear search and use it to fit the data. - """ - dynesty = af.DynestyStatic( - name="tutorial_optional_hierarchical_individual", - unique_tag=dataset_name, - nlive=200, - dlogz=1e-4, - sample="rwalk", - walks=10, - ) - - result_list.append(dynesty.fit(model=model, analysis=analysis)) - -""" -__Results__ - -Checkout the output folder, you should see three new sets of results corresponding to our 3 `Gaussian` datasets. - -The `result_list` allows us to plot the median PDF value and 3.0 confidence intervals of the `centre` estimate from -the model-fit to each dataset. -""" -samples_list = [result.samples for result in result_list] - -mp_instances = [samps.median_pdf() for samps in samples_list] -ue3_instances = [samp.errors_at_upper_sigma(sigma=3.0) for samp in samples_list] -le3_instances = [samp.errors_at_lower_sigma(sigma=3.0) for samp in samples_list] - -mp_centres = [instance.gaussian.centre for instance in mp_instances] -ue3_centres = [instance.gaussian.centre for instance in ue3_instances] -le3_centres = [instance.gaussian.centre for instance in le3_instances] - -print(f"Median PDF inferred centre values") -print(mp_centres) -print() - -""" -__Overall Gaussian Parent Distribution__ - -Fit the inferred `centre`'s from the fits performed above with a Gaussian distribution, in order to -estimate the mean and scatter of the Gaussian from which the centres were drawn. - -We first extract the inferred median PDF centre values and their 1 sigma errors below, which will be the inputs -to our fit for the parent Gaussian. -""" -ue1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] -le1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] - -ue1_centres = [instance.gaussian.centre for instance in ue1_instances] -le1_centres = [instance.gaussian.centre for instance in le1_instances] - -error_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres, le1_centres)] - -""" -The `Analysis` class below fits a Gaussian distribution to the inferred `centre` values from each of the fits above, -where the inferred error values are used as the errors. -""" - - -class Analysis(af.Analysis): - def __init__(self, data: np.ndarray, errors: np.ndarray): - super().__init__() - - self.data = np.array(data) - self.errors = np.array(errors) - - def log_likelihood_function(self, instance: af.ModelInstance) -> float: - """ - Fits a set of 1D data points with a 1D Gaussian distribution, in order to determine from what Gaussian - distribution the analysis classes `data` were drawn. - - In this example, this function determines from what parent Gaussian disrtribution the inferred centres - of each 1D Gaussian were drawn. - """ - log_likelihood_term_1 = np.sum( - -np.divide( - (self.data - instance.median) ** 2, - 2 * (instance.scatter**2 + self.errors**2), - ) - ) - log_likelihood_term_2 = -np.sum( - 0.5 * np.log(instance.scatter**2 + self.errors**2) - ) - - return log_likelihood_term_1 + log_likelihood_term_2 - - -""" -The `ParentGaussian` class is the model-component which used to fit the parent Gaussian to the inferred `centre` values. -""" - - -class ParentGaussian: - def __init__(self, median: float = 0.0, scatter: float = 0.01): - """ - A model component which represents a parent Gaussian distribution, which can be fitted to a 1D set of - measurments with errors in order to determine the probabilty they were drawn from this Gaussian. - - Parameters - ---------- - median - The median value of the parent Gaussian distribution. - scatter - The scatter (E.g. the sigma value) of the Gaussian. - """ - - self.median = median - self.scatter = scatter - - def probability_from_values(self, values: np.ndarray) -> float: - """ - For a set of 1D values, determine the probability that they were random drawn from this parent Gaussian - based on its `median` and `scatter` attributes. - - Parameters - ---------- - values - A set of 1D values from which we will determine the probability they were drawn from the parent Gaussian. - """ - values = np.sort(np.array(values)) - transformed_values = np.subtract(values, self.median) - - return np.multiply( - np.divide(1, self.scatter * np.sqrt(2.0 * np.pi)), - np.exp(-0.5 * np.square(np.divide(transformed_values, self.scatter))), - ) - - -""" -__Model__ - -The `ParentGaussian` is the model component we fit in order to determine the probability the inferred centres were -drawn from the distribution. - -This will be fitted via a non-linear search and therefore is created as a model component using `af.Model()` as per -usual in **PyAutoFit**. -""" -model = af.Model(ParentGaussian) - -model.median = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) -model.scatter = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) - -""" -__Analysis + Search__ - -We now create the Analysis class above which fits a parent 1D gaussian and create a dynesty search in order to fit -it to the 1D inferred list of `centres`. -""" -analysis = Analysis(data=mp_centres, errors=error_list) -search = af.DynestyStatic(nlive=100) - -result = search.fit(model=model, analysis=analysis) - -""" -The results of this fit tell us the most probably values for the `median` and `scatter` of the 1D parent Gaussian fit. -""" -samples = result.samples - -median = samples.median_pdf().median - -u1_error = samples.values_at_upper_sigma(sigma=1.0).median -l1_error = samples.values_at_lower_sigma(sigma=1.0).median - -u3_error = samples.values_at_upper_sigma(sigma=3.0).median -l3_error = samples.values_at_lower_sigma(sigma=3.0).median - -print( - f"Inferred value of the hierarchical median via simple fit to {total_datasets} datasets: \n " -) -print(f"{median} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") -print(f"{median} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") -print() - -scatter = samples.median_pdf().scatter - -u1_error = samples.values_at_upper_sigma(sigma=1.0).scatter -l1_error = samples.values_at_lower_sigma(sigma=1.0).scatter - -u3_error = samples.values_at_upper_sigma(sigma=3.0).scatter -l3_error = samples.values_at_lower_sigma(sigma=3.0).scatter - -print( - f"Inferred value of the hierarchical scatter via simple fit to {total_datasets} datasets: \n " -) -print(f"{scatter} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") -print(f"{scatter} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") -print() - -""" -__Wrap Up__ - -We can compare these values to those inferred in `tutorial_4_hierarchical_model`, which fits all datasets and the -hierarchical values of the parent Gaussian simultaneously. - -The errors for the fit performed in this tutorial are much larger. This is because of how in a graphical model -the "datasets talk to one another", which is described fully in that tutorials subsection "Benefits of Graphical Model". -""" +""" +Tutorial Optional: Hierarchical Individual +========================================== + +In tutorial 4, we fit a hierarchical model using a graphical model, whereby all datasets are fitted simultaneously +and the hierarchical parameters are fitted for simultaneously with the model parameters of each 1D Gaussian in each +dataset. + +This script illustrates how the hierarchical parameters can be estimated using a simpler approach, which fits +each dataset one-by-one and estimates the hierarchical parameters afterwards by fitting the inferred `centres` +with a Gaussian distribution. + +__Contents__ + +This tutorial is split into the following sections: + +- **Example Source Code (`af.ex`)**: The example objects used in this tutorial. +- **Dataset**: Load the hierarchical Gaussian datasets with variable centres. +- **Analysis**: Create Analysis objects for each dataset. +- **Model**: Define a simple Gaussian model with uniform priors. +- **Model Fits (one-by-one)**: Fit each dataset individually using separate non-linear searches. +- **Results**: Analyze and plot the results of each individual fit. +- **Overall Gaussian Parent Distribution**: Fit a parent Gaussian distribution to the inferred centres. +- **Model**: Set up a ParentGaussian model for the parent distribution fitting. +- **Analysis + Search**: Create the analysis and search for the parent distribution fit. +""" + +# from autofit import setup_notebook; setup_notebook() + +import numpy as np +from os import path + +import autofit as af + +""" +__Example Source Code (`af.ex`)__ + +The **PyAutoFit** source code has the following example objects (accessed via `af.ex`) used in this tutorial: + + - `Analysis`: an analysis object which fits noisy 1D datasets, including `log_likelihood_function` and + `visualize` functions. + + - `Gaussian`: a model component representing a 1D Gaussian profile. + + - `plot_profile_1d`: a function for plotting 1D profile datasets including their noise. + +These are functionally identical to the `Analysis`, `Gaussian` and `plot_profile_1d` objects and functions you +have seen and used elsewhere throughout the workspace. + +__Dataset__ + +For each dataset we now set up the correct path and load it. + +We are loading a new Gaussian dataset, where the Gaussians have different centres which were drawn from a parent +Gaussian distribution with a mean centre value of 50.0 and sigma of 10.0. +""" +total_datasets = 5 + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if not path.exists( + path.join("dataset", "example_1d", "gaussian_x1__hierarchical", "dataset_0") +): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simulators.py"], + check=True, + ) + +dataset_name_list = [] +data_list = [] +noise_map_list = [] + +for dataset_index in range(total_datasets): + dataset_name = f"dataset_{dataset_index}" + + dataset_path = path.join( + "dataset", "example_1d", "gaussian_x1__hierarchical", dataset_name + ) + + data = af.util.numpy_array_from_json(file_path=path.join(dataset_path, "data.json")) + noise_map = af.util.numpy_array_from_json( + file_path=path.join(dataset_path, "noise_map.json") + ) + + dataset_name_list.append(dataset_name) + data_list.append(data) + noise_map_list.append(noise_map) + +""" +By plotting the Gaussians we can just about make out that their centres are not all at pix 50, and are spread out +around it (albeit its difficult to be sure, due to the low signal-to-noise of the data). +""" +for dataset_name, data in zip(dataset_name_list, data_list): + af.ex.plot_profile_1d( + xvalues=np.arange(data.shape[0]), + profile_1d=data, + title=dataset_name, + ylabel="Data Values", + color="k", + ) + +""" +__Analysis__ + +For each dataset we now create a corresponding `Analysis` class, like in the previous tutorial. +""" +analysis_list = [] + +for data, noise_map in zip(data_list, noise_map_list): + analysis = af.ex.Analysis(data=data, noise_map=noise_map) + + analysis_list.append(analysis) + + +""" +__Model__ + +The model we fit to each dataset, which is a simple 1D Gaussian with all 3 parameters free. +""" +gaussian = af.Model(af.ex.Gaussian) + +gaussian.centre = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) +gaussian.normalization = af.UniformPrior(lower_limit=0.0, upper_limit=10.0) +gaussian.sigma = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) + +model = af.Collection(gaussian=gaussian) + + +""" +__Model Fits (one-by-one)__ + +For every dataset we now create an `Analysis` class using it and use `Dynesty` to fit it with a `Gaussian`. + +The `Result` is stored in the list `results`. +""" +result_list = [] + +for dataset_name, analysis in zip(dataset_name_list, analysis_list): + """ + Create the `DynestyStatic` non-linear search and use it to fit the data. + """ + dynesty = af.DynestyStatic( + name="tutorial_optional_hierarchical_individual", + unique_tag=dataset_name, + nlive=200, + dlogz=1e-4, + sample="rwalk", + walks=10, + ) + + result_list.append(dynesty.fit(model=model, analysis=analysis)) + +""" +__Results__ + +Checkout the output folder, you should see three new sets of results corresponding to our 3 `Gaussian` datasets. + +The `result_list` allows us to plot the median PDF value and 3.0 confidence intervals of the `centre` estimate from +the model-fit to each dataset. +""" +samples_list = [result.samples for result in result_list] + +mp_instances = [samps.median_pdf() for samps in samples_list] +ue3_instances = [samp.errors_at_upper_sigma(sigma=3.0) for samp in samples_list] +le3_instances = [samp.errors_at_lower_sigma(sigma=3.0) for samp in samples_list] + +mp_centres = [instance.gaussian.centre for instance in mp_instances] +ue3_centres = [instance.gaussian.centre for instance in ue3_instances] +le3_centres = [instance.gaussian.centre for instance in le3_instances] + +print(f"Median PDF inferred centre values") +print(mp_centres) +print() + +""" +__Overall Gaussian Parent Distribution__ + +Fit the inferred `centre`'s from the fits performed above with a Gaussian distribution, in order to +estimate the mean and scatter of the Gaussian from which the centres were drawn. + +We first extract the inferred median PDF centre values and their 1 sigma errors below, which will be the inputs +to our fit for the parent Gaussian. +""" +ue1_instances = [samp.values_at_upper_sigma(sigma=1.0) for samp in samples_list] +le1_instances = [samp.values_at_lower_sigma(sigma=1.0) for samp in samples_list] + +ue1_centres = [instance.gaussian.centre for instance in ue1_instances] +le1_centres = [instance.gaussian.centre for instance in le1_instances] + +error_list = [ue1 - le1 for ue1, le1 in zip(ue1_centres, le1_centres)] + +""" +The `Analysis` class below fits a Gaussian distribution to the inferred `centre` values from each of the fits above, +where the inferred error values are used as the errors. +""" + + +class Analysis(af.Analysis): + def __init__(self, data: np.ndarray, errors: np.ndarray): + super().__init__() + + self.data = np.array(data) + self.errors = np.array(errors) + + def log_likelihood_function(self, instance: af.ModelInstance) -> float: + """ + Fits a set of 1D data points with a 1D Gaussian distribution, in order to determine from what Gaussian + distribution the analysis classes `data` were drawn. + + In this example, this function determines from what parent Gaussian disrtribution the inferred centres + of each 1D Gaussian were drawn. + """ + log_likelihood_term_1 = np.sum( + -np.divide( + (self.data - instance.median) ** 2, + 2 * (instance.scatter**2 + self.errors**2), + ) + ) + log_likelihood_term_2 = -np.sum( + 0.5 * np.log(instance.scatter**2 + self.errors**2) + ) + + return log_likelihood_term_1 + log_likelihood_term_2 + + +""" +The `ParentGaussian` class is the model-component which used to fit the parent Gaussian to the inferred `centre` values. +""" + + +class ParentGaussian: + def __init__(self, median: float = 0.0, scatter: float = 0.01): + """ + A model component which represents a parent Gaussian distribution, which can be fitted to a 1D set of + measurments with errors in order to determine the probabilty they were drawn from this Gaussian. + + Parameters + ---------- + median + The median value of the parent Gaussian distribution. + scatter + The scatter (E.g. the sigma value) of the Gaussian. + """ + + self.median = median + self.scatter = scatter + + def probability_from_values(self, values: np.ndarray) -> float: + """ + For a set of 1D values, determine the probability that they were random drawn from this parent Gaussian + based on its `median` and `scatter` attributes. + + Parameters + ---------- + values + A set of 1D values from which we will determine the probability they were drawn from the parent Gaussian. + """ + values = np.sort(np.array(values)) + transformed_values = np.subtract(values, self.median) + + return np.multiply( + np.divide(1, self.scatter * np.sqrt(2.0 * np.pi)), + np.exp(-0.5 * np.square(np.divide(transformed_values, self.scatter))), + ) + + +""" +__Model__ + +The `ParentGaussian` is the model component we fit in order to determine the probability the inferred centres were +drawn from the distribution. + +This will be fitted via a non-linear search and therefore is created as a model component using `af.Model()` as per +usual in **PyAutoFit**. +""" +model = af.Model(ParentGaussian) + +model.median = af.UniformPrior(lower_limit=0.0, upper_limit=100.0) +model.scatter = af.UniformPrior(lower_limit=0.0, upper_limit=50.0) + +""" +__Analysis + Search__ + +We now create the Analysis class above which fits a parent 1D gaussian and create a dynesty search in order to fit +it to the 1D inferred list of `centres`. +""" +analysis = Analysis(data=mp_centres, errors=error_list) +search = af.DynestyStatic(nlive=100) + +result = search.fit(model=model, analysis=analysis) + +""" +The results of this fit tell us the most probably values for the `median` and `scatter` of the 1D parent Gaussian fit. +""" +samples = result.samples + +median = samples.median_pdf().median + +u1_error = samples.values_at_upper_sigma(sigma=1.0).median +l1_error = samples.values_at_lower_sigma(sigma=1.0).median + +u3_error = samples.values_at_upper_sigma(sigma=3.0).median +l3_error = samples.values_at_lower_sigma(sigma=3.0).median + +print( + f"Inferred value of the hierarchical median via simple fit to {total_datasets} datasets: \n " +) +print(f"{median} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") +print(f"{median} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") +print() + +scatter = samples.median_pdf().scatter + +u1_error = samples.values_at_upper_sigma(sigma=1.0).scatter +l1_error = samples.values_at_lower_sigma(sigma=1.0).scatter + +u3_error = samples.values_at_upper_sigma(sigma=3.0).scatter +l3_error = samples.values_at_lower_sigma(sigma=3.0).scatter + +print( + f"Inferred value of the hierarchical scatter via simple fit to {total_datasets} datasets: \n " +) +print(f"{scatter} ({l1_error} {u1_error}) [1.0 sigma confidence intervals]") +print(f"{scatter} ({l3_error} {u3_error}) [3.0 sigma confidence intervals]") +print() + +""" +__Wrap Up__ + +We can compare these values to those inferred in `tutorial_4_hierarchical_model`, which fits all datasets and the +hierarchical values of the parent Gaussian simultaneously. + +The errors for the fit performed in this tutorial are much larger. This is because of how in a graphical model +the "datasets talk to one another", which is described fully in that tutorials subsection "Benefits of Graphical Model". +""" diff --git a/scripts/simulators/simulators.py b/scripts/simulators/simulators.py index 9d54d0b..8f1c099 100644 --- a/scripts/simulators/simulators.py +++ b/scripts/simulators/simulators.py @@ -1,224 +1,224 @@ -""" -__Simulators__ - -These scripts simulate the 1D Gaussian datasets used to demonstrate model-fitting. - -__Contents__ - -This script is split into the following sections: - -- **Gaussian x1**: Simulate a single 1D Gaussian dataset. -- **Gaussian x1 (0)**: Simulate a single Gaussian with sigma=1.0. -- **Gaussian x1 (1)**: Simulate a single Gaussian with sigma=5.0. -- **Gaussian x1 (2)**: Simulate a single Gaussian with sigma=10.0. -- **Gaussian x1 (Identical 0)**: Simulate an identical single Gaussian dataset (copy 0). -- **Gaussian x1 (Identical 1)**: Simulate an identical single Gaussian dataset (copy 1). -- **Gaussian x1 (Identical 2)**: Simulate an identical single Gaussian dataset (copy 2). -- **Gaussian x1 + Exponential x1**: Simulate a dataset with one Gaussian and one Exponential. -- **Gaussian x2 + Exponential x1**: Simulate a dataset with two Gaussians and one Exponential. -- **Gaussian x2**: Simulate a dataset with two Gaussians. -- **Gaussian x3**: Simulate a dataset with three Gaussians. -- **Gaussian x5**: Simulate a dataset with five Gaussians. -- **Gaussian x1 unconvolved**: Simulate a single Gaussian without convolution. -- **Gaussian x1 convolved**: Simulate a single Gaussian with kernel convolution. -- **Gaussian x1 with feature**: Simulate a Gaussian with a small feature bump. -- **Gaussian x2 split**: Simulate two separated Gaussians. -- **Gaussian x1 time**: Simulate time-varying Gaussian datasets. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import util -from os import path - -import autofit as af - -""" -__Gaussian x1__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 (0)__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_0") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=1.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 (1)__ -""" -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=5.0) -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_1") -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 (2)__ -""" -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_2") -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 (Identical 0)__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_identical_0") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 (Identical 1)__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_identical_1") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 (Identical 2)__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_identical_2") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 + Exponential x1__ -""" -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -exponential = af.ex.Exponential(centre=50.0, normalization=40.0, rate=0.05) -dataset_path = path.join("dataset", "example_1d", "gaussian_x1__exponential_x1") -util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian, exponential], dataset_path=dataset_path -) - -""" -__Gaussian x2 + Exponential x1__ -""" -gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -gaussian_1 = af.ex.Gaussian(centre=20.0, normalization=30.0, sigma=5.0) -exponential = af.ex.Exponential(centre=70.0, normalization=40.0, rate=0.005) -dataset_path = path.join("dataset", "example_1d", "gaussian_x2__exponential_x1") -util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian_0, gaussian_1, exponential], dataset_path=dataset_path -) - -""" -__Gaussian x2__ -""" -gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=20.0, sigma=1.0) -gaussian_1 = af.ex.Gaussian(centre=50.0, normalization=40.0, sigma=5.0) -dataset_path = path.join("dataset", "example_1d", "gaussian_x2") -util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian_0, gaussian_1], dataset_path=dataset_path -) - -""" -__Gaussian x3__ -""" -gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=20.0, sigma=1.0) -gaussian_1 = af.ex.Gaussian(centre=50.0, normalization=40.0, sigma=5.0) -gaussian_2 = af.ex.Gaussian(centre=50.0, normalization=60.0, sigma=10.0) -dataset_path = path.join("dataset", "example_1d", "gaussian_x3") -util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian_0, gaussian_1, gaussian_2], dataset_path=dataset_path -) - -""" -__Gaussian x5__ -""" -gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=20.0, sigma=1.0) -gaussian_1 = af.ex.Gaussian(centre=50.0, normalization=40.0, sigma=5.0) -gaussian_2 = af.ex.Gaussian(centre=50.0, normalization=60.0, sigma=10.0) -gaussian_3 = af.ex.Gaussian(centre=50.0, normalization=80.0, sigma=15.0) -gaussian_4 = af.ex.Gaussian(centre=50.0, normalization=100.0, sigma=20.0) -dataset_path = path.join("dataset", "example_1d", "gaussian_x5") -util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian_0, gaussian_1, gaussian_2, gaussian_3, gaussian_4], - dataset_path=dataset_path, -) - -""" -__Gaussian x1 unconvolved__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_unconvolved") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=3.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Gaussian x1 convolved__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_convolved") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=3.0) -util.simulate_data_1d_with_kernel_via_gaussian_from( - gaussian=gaussian, dataset_path=dataset_path -) - -""" -__Gaussian x1 with feature__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_with_feature") -gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) -gaussian_feature = af.ex.Gaussian(centre=70.0, normalization=0.3, sigma=0.5) -util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian, gaussian_feature], dataset_path=dataset_path -) - -""" -__Gaussian x2 split__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x2_split") -gaussian_0 = af.ex.Gaussian(centre=25.0, normalization=50.0, sigma=12.5) -gaussian_1 = af.ex.Gaussian(centre=75.0, normalization=50.0, sigma=12.5) -util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian_0, gaussian_1], dataset_path=dataset_path -) - - -""" -__Gaussian x1 time__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_time", "time_0") -gaussian = af.ex.Gaussian(centre=40.0, normalization=50.0, sigma=20.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_time", "time_1") -gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=20.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_time", "time_2") -gaussian = af.ex.Gaussian(centre=60.0, normalization=50.0, sigma=20.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - - -""" -__Gaussian x1 time__ -""" -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_variable", "sigma_0") -gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=10.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_variable", "sigma_1") -gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=20.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -dataset_path = path.join("dataset", "example_1d", "gaussian_x1_variable", "sigma_2") -gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=30.0) -util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) - -""" -__Sample Datasets (Chapter 3)__ - -Chain-run the sample simulator so every tutorial's `simulators.py` subprocess -call also produces the `dataset_*/` sample folders used by chapter 3. -""" -import runpy - -runpy.run_path( - path.join(path.dirname(path.abspath(__file__)), "simulators_sample.py"), - run_name="__main__", -) - -""" -Finish. -""" +""" +__Simulators__ + +These scripts simulate the 1D Gaussian datasets used to demonstrate model-fitting. + +__Contents__ + +This script is split into the following sections: + +- **Gaussian x1**: Simulate a single 1D Gaussian dataset. +- **Gaussian x1 (0)**: Simulate a single Gaussian with sigma=1.0. +- **Gaussian x1 (1)**: Simulate a single Gaussian with sigma=5.0. +- **Gaussian x1 (2)**: Simulate a single Gaussian with sigma=10.0. +- **Gaussian x1 (Identical 0)**: Simulate an identical single Gaussian dataset (copy 0). +- **Gaussian x1 (Identical 1)**: Simulate an identical single Gaussian dataset (copy 1). +- **Gaussian x1 (Identical 2)**: Simulate an identical single Gaussian dataset (copy 2). +- **Gaussian x1 + Exponential x1**: Simulate a dataset with one Gaussian and one Exponential. +- **Gaussian x2 + Exponential x1**: Simulate a dataset with two Gaussians and one Exponential. +- **Gaussian x2**: Simulate a dataset with two Gaussians. +- **Gaussian x3**: Simulate a dataset with three Gaussians. +- **Gaussian x5**: Simulate a dataset with five Gaussians. +- **Gaussian x1 unconvolved**: Simulate a single Gaussian without convolution. +- **Gaussian x1 convolved**: Simulate a single Gaussian with kernel convolution. +- **Gaussian x1 with feature**: Simulate a Gaussian with a small feature bump. +- **Gaussian x2 split**: Simulate two separated Gaussians. +- **Gaussian x1 time**: Simulate time-varying Gaussian datasets. +""" + +# from autofit import setup_notebook; setup_notebook() + +import util +from os import path + +import autofit as af + +""" +__Gaussian x1__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 (0)__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_0") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=1.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 (1)__ +""" +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=5.0) +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_1") +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 (2)__ +""" +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_2") +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 (Identical 0)__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_identical_0") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 (Identical 1)__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_identical_1") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 (Identical 2)__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_identical_2") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 + Exponential x1__ +""" +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +exponential = af.ex.Exponential(centre=50.0, normalization=40.0, rate=0.05) +dataset_path = path.join("dataset", "example_1d", "gaussian_x1__exponential_x1") +util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian, exponential], dataset_path=dataset_path +) + +""" +__Gaussian x2 + Exponential x1__ +""" +gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +gaussian_1 = af.ex.Gaussian(centre=20.0, normalization=30.0, sigma=5.0) +exponential = af.ex.Exponential(centre=70.0, normalization=40.0, rate=0.005) +dataset_path = path.join("dataset", "example_1d", "gaussian_x2__exponential_x1") +util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian_0, gaussian_1, exponential], dataset_path=dataset_path +) + +""" +__Gaussian x2__ +""" +gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=20.0, sigma=1.0) +gaussian_1 = af.ex.Gaussian(centre=50.0, normalization=40.0, sigma=5.0) +dataset_path = path.join("dataset", "example_1d", "gaussian_x2") +util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian_0, gaussian_1], dataset_path=dataset_path +) + +""" +__Gaussian x3__ +""" +gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=20.0, sigma=1.0) +gaussian_1 = af.ex.Gaussian(centre=50.0, normalization=40.0, sigma=5.0) +gaussian_2 = af.ex.Gaussian(centre=50.0, normalization=60.0, sigma=10.0) +dataset_path = path.join("dataset", "example_1d", "gaussian_x3") +util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian_0, gaussian_1, gaussian_2], dataset_path=dataset_path +) + +""" +__Gaussian x5__ +""" +gaussian_0 = af.ex.Gaussian(centre=50.0, normalization=20.0, sigma=1.0) +gaussian_1 = af.ex.Gaussian(centre=50.0, normalization=40.0, sigma=5.0) +gaussian_2 = af.ex.Gaussian(centre=50.0, normalization=60.0, sigma=10.0) +gaussian_3 = af.ex.Gaussian(centre=50.0, normalization=80.0, sigma=15.0) +gaussian_4 = af.ex.Gaussian(centre=50.0, normalization=100.0, sigma=20.0) +dataset_path = path.join("dataset", "example_1d", "gaussian_x5") +util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian_0, gaussian_1, gaussian_2, gaussian_3, gaussian_4], + dataset_path=dataset_path, +) + +""" +__Gaussian x1 unconvolved__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_unconvolved") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=3.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Gaussian x1 convolved__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_convolved") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=3.0) +util.simulate_data_1d_with_kernel_via_gaussian_from( + gaussian=gaussian, dataset_path=dataset_path +) + +""" +__Gaussian x1 with feature__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_with_feature") +gaussian = af.ex.Gaussian(centre=50.0, normalization=25.0, sigma=10.0) +gaussian_feature = af.ex.Gaussian(centre=70.0, normalization=0.3, sigma=0.5) +util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian, gaussian_feature], dataset_path=dataset_path +) + +""" +__Gaussian x2 split__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x2_split") +gaussian_0 = af.ex.Gaussian(centre=25.0, normalization=50.0, sigma=12.5) +gaussian_1 = af.ex.Gaussian(centre=75.0, normalization=50.0, sigma=12.5) +util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian_0, gaussian_1], dataset_path=dataset_path +) + + +""" +__Gaussian x1 time__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_time", "time_0") +gaussian = af.ex.Gaussian(centre=40.0, normalization=50.0, sigma=20.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_time", "time_1") +gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=20.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_time", "time_2") +gaussian = af.ex.Gaussian(centre=60.0, normalization=50.0, sigma=20.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + + +""" +__Gaussian x1 time__ +""" +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_variable", "sigma_0") +gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=10.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_variable", "sigma_1") +gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=20.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +dataset_path = path.join("dataset", "example_1d", "gaussian_x1_variable", "sigma_2") +gaussian = af.ex.Gaussian(centre=50.0, normalization=50.0, sigma=30.0) +util.simulate_dataset_1d_via_gaussian_from(gaussian=gaussian, dataset_path=dataset_path) + +""" +__Sample Datasets (Chapter 3)__ + +Chain-run the sample simulator so every tutorial's `simulators.py` subprocess +call also produces the `dataset_*/` sample folders used by chapter 3. +""" +import runpy + +runpy.run_path( + path.join(path.dirname(path.abspath(__file__)), "simulators_sample.py"), + run_name="__main__", +) + +""" +Finish. +""" diff --git a/scripts/simulators/simulators_sample.py b/scripts/simulators/simulators_sample.py index 8951dc8..1a7621f 100644 --- a/scripts/simulators/simulators_sample.py +++ b/scripts/simulators/simulators_sample.py @@ -1,109 +1,109 @@ -""" -__Simulators__ - -These scripts simulates many 1D Gaussian datasets with a low signal to noise ratio, which are used to demonstrate -model-fitting. - -__Contents__ - -This script is split into the following sections: - -- **Gaussian x1 low snr (centre fixed to 50.0)**: Simulate low signal-to-noise Gaussian datasets with a fixed centre. -- **Gaussian x1 low snr (centre drawn from parent Gaussian distribution to 50.0)**: Simulate hierarchical Gaussian datasets with centres drawn from a parent distribution. -- **Gaussian x2 offset centre**: Simulate datasets with two Gaussians with offset centres for graphical model demonstrations. -""" - -# from autoconf import setup_notebook; setup_notebook() - -import numpy as np -from os import path - -import autofit as af -import util - -""" -__Gaussian x1 low snr (centre fixed to 50.0)__ - -This is used for demonstrating expectation propagation, whereby a shared `centre` parameter is inferred from a sample -of `total_datasets` 1D Gaussian datasets. -""" -total_datasets = 50 - -for i in range(total_datasets): - dataset_path = path.join( - "dataset", "example_1d", f"gaussian_x1__low_snr", f"dataset_{i}" - ) - gaussian = af.ex.Gaussian(centre=50.0, normalization=0.5, sigma=5.0) - util.simulate_dataset_1d_via_gaussian_from( - gaussian=gaussian, dataset_path=dataset_path - ) - -""" -__Gaussian x1 low snr (centre drawn from parent Gaussian distribution to 50.0)__ - -This is used for demonstrating expectation propagation and hierachical modeling, whereby a the `centre` parameters -of a sample of `total_datasets` 1D Gaussian datasets are drawn from a Gaussian distribution. -""" - -total_datasets = 10 - -gaussian_parent_model = af.Model( - af.ex.Gaussian, - centre=af.TruncatedGaussianPrior( - mean=50.0, sigma=10.0, lower_limit=0.0, upper_limit=100.0 - ), - normalization=0.5, - sigma=5.0, -) - -for i in range(total_datasets): - dataset_path = path.join( - "dataset", "example_1d", f"gaussian_x1__hierarchical", f"dataset_{i}" - ) - - gaussian = gaussian_parent_model.random_instance() - - util.simulate_dataset_1d_via_gaussian_from( - gaussian=gaussian, dataset_path=dataset_path - ) - - -""" -__Gaussian x2 offset centre__ - -This is used for demonstrating the benefits of graphical models over fitting one-by-one, because it creates a -degeneracy in the offset of the centres of the two Gaussians. -""" -total_datasets = 10 - -for i in range(total_datasets): - dataset_path = path.join( - "dataset", "example_1d", f"gaussian_x2__offset_centres", f"dataset_{i}" - ) - - sigma_0_prior = af.TruncatedGaussianPrior( - lower_limit=0.0, upper_limit=20.0, mean=10.0, sigma=10.0 - ) - while True: - try: - sigma_0_value = sigma_0_prior.value_for(unit=np.random.random(1)) - break - except af.exc.PriorLimitException: - continue - - sigma_1_prior = af.TruncatedGaussianPrior( - lower_limit=0.0, upper_limit=20.0, mean=10.0, sigma=10.0 - ) - while True: - try: - sigma_1_value = sigma_1_prior.value_for(unit=np.random.random(1)) - break - except af.exc.PriorLimitException: - continue - - gaussian_0 = af.ex.Gaussian(centre=40.0, normalization=1.0, sigma=sigma_0_value) - gaussian_1 = af.ex.Gaussian(centre=60.0, normalization=1.0, sigma=sigma_1_value) - - util.simulate_dataset_1d_via_profile_1d_list_from( - profile_1d_list=[gaussian_0, gaussian_1], dataset_path=dataset_path - ) +""" +__Simulators__ + +These scripts simulates many 1D Gaussian datasets with a low signal to noise ratio, which are used to demonstrate +model-fitting. + +__Contents__ + +This script is split into the following sections: + +- **Gaussian x1 low snr (centre fixed to 50.0)**: Simulate low signal-to-noise Gaussian datasets with a fixed centre. +- **Gaussian x1 low snr (centre drawn from parent Gaussian distribution to 50.0)**: Simulate hierarchical Gaussian datasets with centres drawn from a parent distribution. +- **Gaussian x2 offset centre**: Simulate datasets with two Gaussians with offset centres for graphical model demonstrations. +""" + +# from autofit import setup_notebook; setup_notebook() + +import numpy as np +from os import path + +import autofit as af +import util + +""" +__Gaussian x1 low snr (centre fixed to 50.0)__ + +This is used for demonstrating expectation propagation, whereby a shared `centre` parameter is inferred from a sample +of `total_datasets` 1D Gaussian datasets. +""" +total_datasets = 50 + +for i in range(total_datasets): + dataset_path = path.join( + "dataset", "example_1d", f"gaussian_x1__low_snr", f"dataset_{i}" + ) + gaussian = af.ex.Gaussian(centre=50.0, normalization=0.5, sigma=5.0) + util.simulate_dataset_1d_via_gaussian_from( + gaussian=gaussian, dataset_path=dataset_path + ) + +""" +__Gaussian x1 low snr (centre drawn from parent Gaussian distribution to 50.0)__ + +This is used for demonstrating expectation propagation and hierachical modeling, whereby a the `centre` parameters +of a sample of `total_datasets` 1D Gaussian datasets are drawn from a Gaussian distribution. +""" + +total_datasets = 10 + +gaussian_parent_model = af.Model( + af.ex.Gaussian, + centre=af.TruncatedGaussianPrior( + mean=50.0, sigma=10.0, lower_limit=0.0, upper_limit=100.0 + ), + normalization=0.5, + sigma=5.0, +) + +for i in range(total_datasets): + dataset_path = path.join( + "dataset", "example_1d", f"gaussian_x1__hierarchical", f"dataset_{i}" + ) + + gaussian = gaussian_parent_model.random_instance() + + util.simulate_dataset_1d_via_gaussian_from( + gaussian=gaussian, dataset_path=dataset_path + ) + + +""" +__Gaussian x2 offset centre__ + +This is used for demonstrating the benefits of graphical models over fitting one-by-one, because it creates a +degeneracy in the offset of the centres of the two Gaussians. +""" +total_datasets = 10 + +for i in range(total_datasets): + dataset_path = path.join( + "dataset", "example_1d", f"gaussian_x2__offset_centres", f"dataset_{i}" + ) + + sigma_0_prior = af.TruncatedGaussianPrior( + lower_limit=0.0, upper_limit=20.0, mean=10.0, sigma=10.0 + ) + while True: + try: + sigma_0_value = sigma_0_prior.value_for(unit=np.random.random(1)) + break + except af.exc.PriorLimitException: + continue + + sigma_1_prior = af.TruncatedGaussianPrior( + lower_limit=0.0, upper_limit=20.0, mean=10.0, sigma=10.0 + ) + while True: + try: + sigma_1_value = sigma_1_prior.value_for(unit=np.random.random(1)) + break + except af.exc.PriorLimitException: + continue + + gaussian_0 = af.ex.Gaussian(centre=40.0, normalization=1.0, sigma=sigma_0_value) + gaussian_1 = af.ex.Gaussian(centre=60.0, normalization=1.0, sigma=sigma_1_value) + + util.simulate_dataset_1d_via_profile_1d_list_from( + profile_1d_list=[gaussian_0, gaussian_1], dataset_path=dataset_path + ) diff --git a/scripts/simulators/util.py b/scripts/simulators/util.py index 5c1c8c8..f68b6bd 100644 --- a/scripts/simulators/util.py +++ b/scripts/simulators/util.py @@ -1,346 +1,346 @@ -from autoconf.dictable import to_dict - -import autofit as af - -import json -from os import path -import numpy as np -import matplotlib.pyplot as plt - - -def simulate_dataset_1d_via_gaussian_from(gaussian, dataset_path): - """ - Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and - thus defining the number of data-points in our data. - """ - pixels = 100 - xvalues = np.arange(pixels) - - """ - Evaluate this `Gaussian` model instance at every xvalues to create its model profile. - """ - model_data_1d = gaussian.model_data_from(xvalues=xvalues) - - """ - Determine the noise (at a specified signal to noise level) in every pixel of our model profile. - """ - signal_to_noise_ratio = 25.0 - noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) - - """ - Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute - noise-map of our data which is required when evaluating the chi-squared value of the likelihood. - """ - data = model_data_1d + noise - noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) - - """ - Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used - in other example scripts. - """ - af.util.numpy_array_to_json( - array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True - ) - af.util.numpy_array_to_json( - array=noise_map, - file_path=path.join(dataset_path, "noise_map.json"), - overwrite=True, - ) - plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - ) - plt.title("1D Gaussian Dataset.") - plt.xlabel("x values of profile") - plt.ylabel("Profile normalization") - plt.savefig(path.join(dataset_path, "image.png")) - plt.close() - - """ - __Model Json__ - - Output the model to a .json file so we can refer to its parameters in the future. - """ - model_file = path.join(dataset_path, "model.json") - - with open(model_file, "w+") as f: - try: - json.dump(to_dict(gaussian), f, indent=4) - except (TypeError, ValueError): - pass - - -def simulate_data_1d_with_kernel_via_gaussian_from(gaussian, dataset_path): - """ - Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and - thus defining the number of data-points in our data. - """ - pixels = 100 - xvalues = np.arange(pixels) - - """ - Evaluate this `Gaussian` model instance at every xvalues to create its model profile. - """ - model_data_1d = gaussian.model_data_from(xvalues=xvalues) - - """ - Determine the noise (at a specified signal to noise level) in every pixel of our model profile. - """ - kernel_pixels = 21 - kernel_xvalues = np.arange(kernel_pixels) - kernel_sigma = 5.0 - kernel_centre = 10.0 - kernel_xvalues = np.subtract(kernel_xvalues, kernel_centre) - kernel = np.multiply( - np.divide(1.0, kernel_sigma * np.sqrt(2.0 * np.pi)), - np.exp(-0.5 * np.square(np.divide(kernel_xvalues, kernel_sigma))), - ) - kernel = kernel / np.sum(kernel) - - """ - Convolve the model line with this kernel. - """ - blurred_model_data_1d = np.convolve(model_data_1d, kernel, mode="same") - - """ - Create a Gaussian kernel which the model line will be convolved with. - """ - signal_to_noise_ratio = 25.0 - noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) - - """ - Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute - noise-map of our data which is required when evaluating the chi-squared value of the likelihood. - """ - data = blurred_model_data_1d + noise - noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) - - """ - Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used - in other example scripts. - """ - af.util.numpy_array_to_json( - array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True - ) - af.util.numpy_array_to_json( - array=noise_map, - file_path=path.join(dataset_path, "noise_map.json"), - overwrite=True, - ) - af.util.numpy_array_to_json( - array=kernel, file_path=path.join(dataset_path, "kernel.json"), overwrite=True - ) - plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - ) - plt.title("1D Gaussian Dataset with Convolver Blurring.") - plt.xlabel("x values of profile") - plt.ylabel("Profile normalization") - plt.savefig(path.join(dataset_path, "image.png")) - plt.close() - - """ - __Model Json__ - - Output the model to a .json file so we can refer to its parameters in the future. - """ - model_file = path.join(dataset_path, "model.json") - - with open(model_file, "w+") as f: - json.dump(to_dict(gaussian), f, indent=4) - - -def simulate_dataset_1d_via_profile_1d_list_from(profile_1d_list, dataset_path): - """ - Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and - thus defining the number of data-points in our data. - """ - pixels = 100 - xvalues = np.arange(pixels) - - """ - Evaluate the `Gaussian` and Exponential model instances at every xvalues to create their model profile and sum - them together to create the overall model profile. - """ - model_data_1d_list = [ - profile_1d.model_data_from(xvalues=xvalues) for profile_1d in profile_1d_list - ] - - model_data_1d = sum(model_data_1d_list) - - """ - Determine the noise (at a specified signal to noise level) in every pixel of our model profile. - """ - signal_to_noise_ratio = 25.0 - noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) - - """ - Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute - noise-map of our data which is required when evaluating the chi-squared value of the likelihood. - """ - data = model_data_1d + noise - noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) - - """ - Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used - in other example scripts. - """ - af.util.numpy_array_to_json( - array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True - ) - af.util.numpy_array_to_json( - array=noise_map, - file_path=path.join(dataset_path, "noise_map.json"), - overwrite=True, - ) - plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - ) - plt.plot(range(data.shape[0]), model_data_1d, color="r") - for model_data_1d_individual in model_data_1d_list: - plt.plot(range(data.shape[0]), model_data_1d_individual, "--") - plt.title("1D Profiles Dataset.") - plt.xlabel("x values of profile") - plt.ylabel("Profile normalization") - plt.savefig(path.join(dataset_path, "image.png")) - plt.close() - - """ - __Model Json__ - - Output the model to a .json file so we can refer to its parameters in the future. - """ - for i, profile in enumerate(profile_1d_list): - model_file = path.join(dataset_path, f"model_{i}.json") - - with open(model_file, "w+") as f: - try: - json.dump(to_dict(profile), f, indent=4) - except (TypeError, ValueError): - pass - - """ - __Max Log Likelihood__ - """ - chi_squared = np.sum(((data - model_data_1d) / noise_map) ** 2) - noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) - log_likelihood = -0.5 * (chi_squared + noise_normalization) - - with open(path.join(dataset_path, "max_log_likelihood.json"), "w+") as f: - json.dump({"log_likelihood": log_likelihood}, f, indent=4) - - -def simulate_data_1d_with_kernel_via_profile_1d_list_from( - profile_1d_list, dataset_path -): - """ - Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and - thus defining the number of data-points in our data. - """ - pixels = 100 - xvalues = np.arange(pixels) - - """ - Evaluate the `Gaussian` and Exponential model instances at every xvalues to create their model profile and sum - them together to create the overall model profile. - """ - model_data_1d = np.zeros(shape=pixels) - - for profile in profile_1d_list: - model_data_1d += profile.model_data_from(xvalues=xvalues) - - """ - Create a Gaussian kernel which the model line will be convolved with. - """ - kernel_pixels = 21 - kernel_xvalues = np.arange(kernel_pixels) - kernel_sigma = 5.0 - kernel_centre = 10.0 - kernel_xvalues = np.subtract(kernel_xvalues, kernel_centre) - kernel = np.multiply( - np.divide(1.0, kernel_sigma * np.sqrt(2.0 * np.pi)), - np.exp(-0.5 * np.square(np.divide(kernel_xvalues, kernel_sigma))), - ) - kernel = kernel / np.sum(kernel) - - """ - Convolve the model line with this kernel. - """ - - blurred_model_data_1d = np.convolve(model_data_1d, kernel, mode="same") - - """ - Determine the noise (at a specified signal to noise level) in every pixel of our model profile. - """ - signal_to_noise_ratio = 25.0 - noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) - - """ - Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute - noise-map of our data which is required when evaluating the chi-squared value of the likelihood. - """ - data = blurred_model_data_1d + noise - noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) - - """ - Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used - in other example scripts. - """ - af.util.numpy_array_to_json( - array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True - ) - af.util.numpy_array_to_json( - array=noise_map, - file_path=path.join(dataset_path, "noise_map.json"), - overwrite=True, - ) - af.util.numpy_array_to_json( - array=kernel, file_path=path.join(dataset_path, "kernel.json"), overwrite=True - ) - - plt.errorbar( - x=xvalues, - y=data, - yerr=noise_map, - linestyle="", - color="k", - ecolor="k", - elinewidth=1, - capsize=2, - ) - plt.title("1D Profiles Dataset with Convolver Blurring.") - plt.xlabel("x values of profile") - plt.ylabel("Profile normalization") - plt.savefig(path.join(dataset_path, "image.png")) - plt.close() - - """ - __Model Json__ - - Output the model to a .json file so we can refer to its parameters in the future. - """ - for i, profile in enumerate(profile_1d_list): - model_file = path.join(dataset_path, f"model_{i}.json") - - with open(model_file, "w+") as f: - json.dump(to_dict(profile), f, indent=4) +from autofit import to_dict + +import autofit as af + +import json +from os import path +import numpy as np +import matplotlib.pyplot as plt + + +def simulate_dataset_1d_via_gaussian_from(gaussian, dataset_path): + """ + Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and + thus defining the number of data-points in our data. + """ + pixels = 100 + xvalues = np.arange(pixels) + + """ + Evaluate this `Gaussian` model instance at every xvalues to create its model profile. + """ + model_data_1d = gaussian.model_data_from(xvalues=xvalues) + + """ + Determine the noise (at a specified signal to noise level) in every pixel of our model profile. + """ + signal_to_noise_ratio = 25.0 + noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) + + """ + Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute + noise-map of our data which is required when evaluating the chi-squared value of the likelihood. + """ + data = model_data_1d + noise + noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) + + """ + Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used + in other example scripts. + """ + af.util.numpy_array_to_json( + array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True + ) + af.util.numpy_array_to_json( + array=noise_map, + file_path=path.join(dataset_path, "noise_map.json"), + overwrite=True, + ) + plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + ) + plt.title("1D Gaussian Dataset.") + plt.xlabel("x values of profile") + plt.ylabel("Profile normalization") + plt.savefig(path.join(dataset_path, "image.png")) + plt.close() + + """ + __Model Json__ + + Output the model to a .json file so we can refer to its parameters in the future. + """ + model_file = path.join(dataset_path, "model.json") + + with open(model_file, "w+") as f: + try: + json.dump(to_dict(gaussian), f, indent=4) + except (TypeError, ValueError): + pass + + +def simulate_data_1d_with_kernel_via_gaussian_from(gaussian, dataset_path): + """ + Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and + thus defining the number of data-points in our data. + """ + pixels = 100 + xvalues = np.arange(pixels) + + """ + Evaluate this `Gaussian` model instance at every xvalues to create its model profile. + """ + model_data_1d = gaussian.model_data_from(xvalues=xvalues) + + """ + Determine the noise (at a specified signal to noise level) in every pixel of our model profile. + """ + kernel_pixels = 21 + kernel_xvalues = np.arange(kernel_pixels) + kernel_sigma = 5.0 + kernel_centre = 10.0 + kernel_xvalues = np.subtract(kernel_xvalues, kernel_centre) + kernel = np.multiply( + np.divide(1.0, kernel_sigma * np.sqrt(2.0 * np.pi)), + np.exp(-0.5 * np.square(np.divide(kernel_xvalues, kernel_sigma))), + ) + kernel = kernel / np.sum(kernel) + + """ + Convolve the model line with this kernel. + """ + blurred_model_data_1d = np.convolve(model_data_1d, kernel, mode="same") + + """ + Create a Gaussian kernel which the model line will be convolved with. + """ + signal_to_noise_ratio = 25.0 + noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) + + """ + Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute + noise-map of our data which is required when evaluating the chi-squared value of the likelihood. + """ + data = blurred_model_data_1d + noise + noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) + + """ + Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used + in other example scripts. + """ + af.util.numpy_array_to_json( + array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True + ) + af.util.numpy_array_to_json( + array=noise_map, + file_path=path.join(dataset_path, "noise_map.json"), + overwrite=True, + ) + af.util.numpy_array_to_json( + array=kernel, file_path=path.join(dataset_path, "kernel.json"), overwrite=True + ) + plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + ) + plt.title("1D Gaussian Dataset with Convolver Blurring.") + plt.xlabel("x values of profile") + plt.ylabel("Profile normalization") + plt.savefig(path.join(dataset_path, "image.png")) + plt.close() + + """ + __Model Json__ + + Output the model to a .json file so we can refer to its parameters in the future. + """ + model_file = path.join(dataset_path, "model.json") + + with open(model_file, "w+") as f: + json.dump(to_dict(gaussian), f, indent=4) + + +def simulate_dataset_1d_via_profile_1d_list_from(profile_1d_list, dataset_path): + """ + Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and + thus defining the number of data-points in our data. + """ + pixels = 100 + xvalues = np.arange(pixels) + + """ + Evaluate the `Gaussian` and Exponential model instances at every xvalues to create their model profile and sum + them together to create the overall model profile. + """ + model_data_1d_list = [ + profile_1d.model_data_from(xvalues=xvalues) for profile_1d in profile_1d_list + ] + + model_data_1d = sum(model_data_1d_list) + + """ + Determine the noise (at a specified signal to noise level) in every pixel of our model profile. + """ + signal_to_noise_ratio = 25.0 + noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) + + """ + Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute + noise-map of our data which is required when evaluating the chi-squared value of the likelihood. + """ + data = model_data_1d + noise + noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) + + """ + Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used + in other example scripts. + """ + af.util.numpy_array_to_json( + array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True + ) + af.util.numpy_array_to_json( + array=noise_map, + file_path=path.join(dataset_path, "noise_map.json"), + overwrite=True, + ) + plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + ) + plt.plot(range(data.shape[0]), model_data_1d, color="r") + for model_data_1d_individual in model_data_1d_list: + plt.plot(range(data.shape[0]), model_data_1d_individual, "--") + plt.title("1D Profiles Dataset.") + plt.xlabel("x values of profile") + plt.ylabel("Profile normalization") + plt.savefig(path.join(dataset_path, "image.png")) + plt.close() + + """ + __Model Json__ + + Output the model to a .json file so we can refer to its parameters in the future. + """ + for i, profile in enumerate(profile_1d_list): + model_file = path.join(dataset_path, f"model_{i}.json") + + with open(model_file, "w+") as f: + try: + json.dump(to_dict(profile), f, indent=4) + except (TypeError, ValueError): + pass + + """ + __Max Log Likelihood__ + """ + chi_squared = np.sum(((data - model_data_1d) / noise_map) ** 2) + noise_normalization = np.sum(np.log(2 * np.pi * noise_map**2.0)) + log_likelihood = -0.5 * (chi_squared + noise_normalization) + + with open(path.join(dataset_path, "max_log_likelihood.json"), "w+") as f: + json.dump({"log_likelihood": log_likelihood}, f, indent=4) + + +def simulate_data_1d_with_kernel_via_profile_1d_list_from( + profile_1d_list, dataset_path +): + """ + Specify the number of pixels used to create the xvalues on which the 1D line of the profile is generated using and + thus defining the number of data-points in our data. + """ + pixels = 100 + xvalues = np.arange(pixels) + + """ + Evaluate the `Gaussian` and Exponential model instances at every xvalues to create their model profile and sum + them together to create the overall model profile. + """ + model_data_1d = np.zeros(shape=pixels) + + for profile in profile_1d_list: + model_data_1d += profile.model_data_from(xvalues=xvalues) + + """ + Create a Gaussian kernel which the model line will be convolved with. + """ + kernel_pixels = 21 + kernel_xvalues = np.arange(kernel_pixels) + kernel_sigma = 5.0 + kernel_centre = 10.0 + kernel_xvalues = np.subtract(kernel_xvalues, kernel_centre) + kernel = np.multiply( + np.divide(1.0, kernel_sigma * np.sqrt(2.0 * np.pi)), + np.exp(-0.5 * np.square(np.divide(kernel_xvalues, kernel_sigma))), + ) + kernel = kernel / np.sum(kernel) + + """ + Convolve the model line with this kernel. + """ + + blurred_model_data_1d = np.convolve(model_data_1d, kernel, mode="same") + + """ + Determine the noise (at a specified signal to noise level) in every pixel of our model profile. + """ + signal_to_noise_ratio = 25.0 + noise = np.random.normal(0.0, 1.0 / signal_to_noise_ratio, pixels) + + """ + Add this noise to the model line to create the line data that is fitted, using the signal-to-noise ratio to compute + noise-map of our data which is required when evaluating the chi-squared value of the likelihood. + """ + data = blurred_model_data_1d + noise + noise_map = (1.0 / signal_to_noise_ratio) * np.ones(pixels) + + """ + Output the data and noise-map to the `HowToFit/dataset` folder so they can be loaded and used + in other example scripts. + """ + af.util.numpy_array_to_json( + array=data, file_path=path.join(dataset_path, "data.json"), overwrite=True + ) + af.util.numpy_array_to_json( + array=noise_map, + file_path=path.join(dataset_path, "noise_map.json"), + overwrite=True, + ) + af.util.numpy_array_to_json( + array=kernel, file_path=path.join(dataset_path, "kernel.json"), overwrite=True + ) + + plt.errorbar( + x=xvalues, + y=data, + yerr=noise_map, + linestyle="", + color="k", + ecolor="k", + elinewidth=1, + capsize=2, + ) + plt.title("1D Profiles Dataset with Convolver Blurring.") + plt.xlabel("x values of profile") + plt.ylabel("Profile normalization") + plt.savefig(path.join(dataset_path, "image.png")) + plt.close() + + """ + __Model Json__ + + Output the model to a .json file so we can refer to its parameters in the future. + """ + for i, profile in enumerate(profile_1d_list): + model_file = path.join(dataset_path, f"model_{i}.json") + + with open(model_file, "w+") as f: + json.dump(to_dict(profile), f, indent=4)