From d1a27a4bfa0616d61e120b2ab5794aed318750e7 Mon Sep 17 00:00:00 2001 From: amyheather Date: Thu, 12 Feb 2026 15:17:17 +0000 Subject: [PATCH 01/17] fix(mathematical): remove r function that I'd accidentally pasted in! (#158) --- .../verification_validation/mathematical.qmd | 48 +------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/pages/guide/verification_validation/mathematical.qmd b/pages/guide/verification_validation/mathematical.qmd index ba7bbbd..a9ab772 100644 --- a/pages/guide/verification_validation/mathematical.qmd +++ b/pages/guide/verification_validation/mathematical.qmd @@ -437,53 +437,7 @@ First, we compute P0 which is the probability that the system is completely empt """ # Sum for n = 0 to s-1 sum_part = sum( - (self.lambda_over_mu**n) / math.factorial(n)# Validates a discrete event simulation of a healthcare M/M/S queue by -# comparing simulation results to analytical queueing theory. -# -# Metrics (using standard queueing theory notation): -# - ρ (rho): utilisation -# - Lq: mean queue length -# - W: mean time in system -# - Wq: mean waiting -# -# Results must match theory with a 15% tolerance (accomodates stochasticity). -# Tests are run across diverse parameter combinations and utilisation levels. -# System stability requires arrival rate < number_of_servers * service_rate. - - -#' Run simulation and return key performance indicators using standard queueing -#' theory notation. -#' -#' The warm-up period should be sufficiently long to allow the system to reach -#' steady-state before data collection begins. - -run_simulation_model <- function( - patient_inter, mean_n_consult_time, number_of_nurses -) { - # Run simulation - param <- parameters( - patient_inter = patient_inter, - mean_n_consult_time = mean_n_consult_time, - number_of_nurses = number_of_nurses, - warm_up_period = 500L, - data_collection_period = 1500L, - number_of_runs = 100L, - scenario_name = 0L, - cores = 1L - ) - run_results <- runner(param)[["run_results"]] - - # Get overall results, using queueing theory notation in column names - results <- run_results |> - summarise( - RO = mean(.data[["utilisation_nurse"]]), - Lq = mean(.data[["mean_queue_length_nurse"]]), - W = mean(.data[["mean_time_in_system"]]), - Wq = mean(.data[["mean_waiting_time_nurse"]]) - ) - results -} - + (self.lambda_over_mu**n) / math.factorial(n) for n in range(self.num_servers) ) From 7006e88eec91e9bf24e4a2ae485700747271c946 Mon Sep 17 00:00:00 2001 From: amyheather Date: Thu, 12 Feb 2026 15:19:34 +0000 Subject: [PATCH 02/17] feat(mathematical): addressing nav's comments (#158) --- pages/guide/verification_validation/mathematical.qmd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pages/guide/verification_validation/mathematical.qmd b/pages/guide/verification_validation/mathematical.qmd index a9ab772..3081241 100644 --- a/pages/guide/verification_validation/mathematical.qmd +++ b/pages/guide/verification_validation/mathematical.qmd @@ -20,7 +20,7 @@ date: "2025-10-23T16:24:23+01:00" This page implements mathematical proof of correctness implemented on the [verification and validation page](verification_validation.qmd). It does so within a test ([tests](tests.qmd) page). -The test is run on the model from the [parallel processing](../output_analysis/parallel.qmd) page (likewise, also used on the [scenario and sensitivity analysis](../experiments/scenarios.qmd) page and [tests](tests.qmd) pages). +The test is run on the model from the [parallel processing](../output_analysis/parallel.qmd) page. > *Entity generation → Entity processing → Initialisation bias → Performance measures → Replications → Parallel processing → Mathematical proof of correctness* @@ -90,11 +90,11 @@ A mathematical model is a set of equations or formulas that describe how systems Real-world systems are often too complex for neat equations (e.g., variable arrivals, different service rules or other complex features). In these cases, discrete-event simulation allows us to better mimic the system. However, if your simulation is simple enough - or if you start with a basic model to verify your code (and later build in complexity) - you can use mathematical proof of correctness. -The small example we've been building throughout this book is an instance of an **M/M/s queue model** (as is the nurse visit simulation example). For a reminder on what an M/M/s queue model is, see the section "*Find out more about M/M/s models*" on the [example conceptual models](/pages/example_models/example_models.qmd) page. In the following sections, we'll show how to verify our small example by comparing it to a mathematical model. +The simple example we've been building throughout this book is an instance of an **M/M/s queue model** (as is the nurse visit simulation example). For a reminder on what an M/M/s queue model is, see the section "*Find out more about M/M/s models*" on the [example conceptual models](/pages/example_models/example_models.qmd) page. In the following sections, we'll show how to verify our simple example by comparing it to a mathematical model. ## Simulation code -The model used is the same as that on the [parallel processing](../output_analysis/parallel.qmd) page - as also used on the [scenario and sensitivity analysis](../experiments/scenarios.qmd) and [tests](tests.qmd) pages. +The model used is the same as that on the [parallel processing](../output_analysis/parallel.qmd) page. ```{python} #| file: "tests_resources/simulation.py" From fd0e52da71cde50ea63491f3bfcca5f09f762433 Mon Sep 17 00:00:00 2001 From: amyheather Date: Thu, 12 Feb 2026 15:41:20 +0000 Subject: [PATCH 03/17] build(env): add nbconvert to this too --- pages/guide/environment-full.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/guide/environment-full.yaml b/pages/guide/environment-full.yaml index e2dbca3..ad38cbc 100644 --- a/pages/guide/environment-full.yaml +++ b/pages/guide/environment-full.yaml @@ -6,7 +6,7 @@ dependencies: - flake8=7.3.0 - itables=2.5.2 - jupyter=1.1.1 - - nbconvert-core=7.16.6 + - nbconvert=7.17.0 - openssl=3.3.0 - pandas=2.3.1 - plotly=6.3.0 From 576c9f324771c114c9112c523c7341bf2d67a715 Mon Sep 17 00:00:00 2001 From: amyheather Date: Thu, 12 Feb 2026 16:16:31 +0000 Subject: [PATCH 04/17] feat(tests): addressing nav's feedback (#158) --- pages/guide/verification_validation/tests.qmd | 40 +++++++++++-------- .../tests_resources/test_back.py | 1 + .../tests_resources/test_example_param.py | 9 +++-- .../tests_resources/test_example_simple.py | 2 +- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/pages/guide/verification_validation/tests.qmd b/pages/guide/verification_validation/tests.qmd index 29c3065..579dd16 100644 --- a/pages/guide/verification_validation/tests.qmd +++ b/pages/guide/verification_validation/tests.qmd @@ -21,7 +21,7 @@ date: "2025-11-05T11:21:31+00:00" **Pre-reading:** -This page will run tests on the model from the [parallel processing](../output_analysis/parallel.qmd) page (likewise, also used on the [scenario and sensitivity analysis](../experiments/scenarios.qmd) page). +This page will run tests on the model from the [parallel processing](../output_analysis/parallel.qmd) page. > *Entity generation → Entity processing → Initialisation bias → Performance measures → Replications → Parallel processing → Tests* @@ -60,9 +60,11 @@ def test_positive(): Confirm that the number is positive. """ number = 5 - assert number > 0, "The number should be positive" + assert number > 0 ``` +This is a minimal example of a test. In this test, we hard‑code `number = 5` inside the function and then use an `assert` to check that it is positive. This test is just for illustration and should always pass. In the examples below, we will do more interesting things than this. + ::: ::: {.r-content} @@ -87,6 +89,8 @@ test_that("Confirm that the number is positive", { }) ``` +This is a minimal example of a test. In this test, we hard‑code `number <- 5L` inside the function and then use `expect_gt()` to check that it is positive. This test is just for illustration and should always pass. In the examples below, we will do more interesting things than this. + ### Setting up a testing structure with testthat If you model is [structured as a package](/pages/guide/setup/package.qmd), the easiest way to get started with tests is to let `usethis` create the standard `testthat` layout for you: @@ -125,6 +129,8 @@ pytest.main(["tests_resources/test_example_simple.py"]) ::: +In this book, we have to run `pytest` from within Python, so you also see a final line like ``. When you run `pytest` directly in your own terminal, you will just see the usual summary line (for example, `1 passed in 0.01s`) and not this extra ExitCode line. + :::: ::: {.r-content} @@ -139,6 +145,8 @@ When you run a test, you'll see output in the console indicating whether tests p ::: +If you try changing the number to a negative value, you should find that the test returns an error. + ### Parametrised tests :::: {.python-content} @@ -192,7 +200,7 @@ There are many different ways of categorising tests. We will focus on three type * **Unit testing** * **Back testing** -As mentioned above, we are running these tests on the model from the [parallel processing](../output_analysis/parallel.qmd) page (which is also the one used on the [scenario and sensitivity analysis](../experiments/scenarios.qmd) page). +As mentioned above, we are running these tests on the model from the [parallel processing](../output_analysis/parallel.qmd) pag. :::: {.python-content} @@ -230,6 +238,10 @@ For example, we expect that the number of arrivals should decrease if: :::: {.python-content} +In your test file, you need to import the model classes you want to test If your code is split across files, you might use something like: `from simulation.parameters import Parameters` and `from simulation.runner import Runner`. If your model lives in a single file (for example `simulation.py`), you can import directly as: `from simulation import Parameters, Runner`. + +Make sure you run `pytest` from the project root (the folder that contains the `tests/` directory). For example: `pytest tests/test_functional.py`. This lets Python find your simulation package or module correctly when the tests import it. + ```{python} {{< include tests_resources/test_functional.py >}} ``` @@ -275,7 +287,7 @@ cases( ::: -As another example, we expect that our model should fail if the number of doctors or the patient inter-arrival time were set to 0. This is tested using `test_zero_inputs`: +Here's another example of a functional test. In this case, we **expect the model to raise an error when we give it clearly invalid inputs** (for example, setting the number of doctors or the inter‑arrival time to 0). The test will **pass** if creating the model with these zero values raises the expected error with the right message, and it will **fail** only if no error (or the wrong error/message) is raised. :::: {.python-content} @@ -434,33 +446,29 @@ testthat::test_file(file.path("tests_resources", "test_back.R")) ::: -::: {.python-content} +::: {.callout-tip title="Break the test!"} -We generate the expected results for our backtest in a **seperate Python file or Jupyter notebook**, rather than within the test itself. +Want to see this back test fail? Change one of the parameters in the test (for example, set the inter-arrival time to 4 instead of 5). The model will now produce different results from the saved `.csv` files, and the test should fail. ::: -::: {.r-content} - -We generate the expected results for our backtest in a **seperate R file or R markdown file**, rather than within the test itself. - -::: - -We then would generally run tests using the same pre-generated `.csv` files, without regenerating them. However, the test will **fail if the model logic is intentionally changed**, leading to different results from the same parameters. +#### Generating the expected results for our back test ::: {.python-content} -In that case, if we are **certain** that these changes are the reason for differing results, we should re-run the Python file or notebook to regenerate the `.csv`. +We generate the expected results for our backtest in a **seperate Python script or Jupyter notebook**, rather than within the test itself. ::: ::: {.r-content} -In that case, if we are **certain** that these changes are the reason for differing results, we should re-run the R file or R markdown file to regenerate the `.csv`. +We generate the expected results for our backtest in a **seperate R script or R markdown file**, rather than within the test itself. ::: -It is crucial to exercise caution when doing this, to avoid unintentionally overwriting correct expected results. +We save these results as `.csv` files and then write a test that re-runs the model with the same parameters and compares the new results against the saved `.csv` files. The test passes if the new results exactly match the stored ones, and fails if they differ. + +If we intentionally change the model logic, we expect this back test to start failing, because the model now produces different results from before. In that situation, and only when we are confident the changes are correct, we rerun the code to regenerate the expected `.csv` files. We need to be careful when doing this, so we do not accidentally overwrite correct expected results with incorrect ones. ## Other things you can do when testing code diff --git a/pages/guide/verification_validation/tests_resources/test_back.py b/pages/guide/verification_validation/tests_resources/test_back.py index 0c3b05a..02b1881 100644 --- a/pages/guide/verification_validation/tests_resources/test_back.py +++ b/pages/guide/verification_validation/tests_resources/test_back.py @@ -1,6 +1,7 @@ from pathlib import Path import pandas as pd +import pytest from simulation import Parameters, Runner diff --git a/pages/guide/verification_validation/tests_resources/test_example_param.py b/pages/guide/verification_validation/tests_resources/test_example_param.py index 1070f8e..8a585a8 100644 --- a/pages/guide/verification_validation/tests_resources/test_example_param.py +++ b/pages/guide/verification_validation/tests_resources/test_example_param.py @@ -6,8 +6,9 @@ def test_positive_param(number): """ Confirm that the number is positive. - Arguments: - number (float): - Number to check. + Parameters + ---------- + number : float + Number to check. """ - assert number > 0, f"The number {number} is not positive." + assert number > 0 diff --git a/pages/guide/verification_validation/tests_resources/test_example_simple.py b/pages/guide/verification_validation/tests_resources/test_example_simple.py index 25dafd4..45ce271 100644 --- a/pages/guide/verification_validation/tests_resources/test_example_simple.py +++ b/pages/guide/verification_validation/tests_resources/test_example_simple.py @@ -3,4 +3,4 @@ def test_positive(): Confirm that the number is positive. """ number = 5 - assert number > 0, "The number should be positive" + assert number > 0 From 6425908922d58669f5e77e6ea833d4396e55cbb5 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 11:05:04 +0000 Subject: [PATCH 05/17] feat(tests): addressing nav's feedback (#158) --- pages/guide/verification_validation/tests.qmd | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/pages/guide/verification_validation/tests.qmd b/pages/guide/verification_validation/tests.qmd index 579dd16..c70786f 100644 --- a/pages/guide/verification_validation/tests.qmd +++ b/pages/guide/verification_validation/tests.qmd @@ -596,29 +596,54 @@ package_coverage(type = "examples") ::: {.python-content} +A range of unit, functional, and back tests are available for both example models. These are available in the `tests/` folder. Data for back tests are generated within `notebooks/generate_exp_results.ipynb`. + +Tests are automatically run on pushes to `main` via GitHub actions (`.github/workflows/tests.yaml`). Test coverage is calculated as described in the coverage section above and the GitHub Actions page (with a coverage badge updated via `.github/workflows/tests.yaml`). + {{< include /html/pydesrapmms.html >}} -{{< include /html/pydesrapstroke.html >}} +Test files include: -A wide variety of tests are available in the `tests/` folder. +* `tests/test_backtest.py` +* `tests/test_functionaltest.py` +* `tests/test_mms.py` +* `tests/test_unittest.py` + +{{< include /html/pydesrapstroke.html >}} -Data for back tests are generated within `notebooks/generate_exp_results.ipynb`. +Test files include: -Tests are run and coverage are calculated on pushes to main via GitHub actions (`.github/workflows/tests.yaml`). +* `tests/test_backtest.py` +* `tests/test_functionaltest.py` +* `tests/test_unittest.py` ::: ::: {.r-content} +A range of unit, functional, and back tests are available for both example models. These are available in the `tests/` folder. Data for back tests are generated within `notebooks/generate_exp_results.Rmd`. + +Tests are automatically run on pushes to `main` via GitHub actions (`.github/workflows/R-CMD-check.yaml`). Test coverage is calculated as described in the coverage section above and the GitHub Actions page (with a coverage badge updated via `.github/workflows/test-coverage.yaml`). + {{< include /html/rdesrapmms.html >}} -{{< include /html/rdesrapstroke.html >}} +Test files include: -A wide variety of tests are available in the `tests/` folder. +* `tests/testthat/test-backtest-replications.R` +* `tests/testthat/test-backtest.R` +* `tests/testthat/test-functionaltest-replications.R` +* `tests/testthat/test-functionaltest.R` +* `tests/testthat/test-mms.R` +* `tests/testthat/test-unittest-replications.R` +* `tests/testthat/test-unittest.R` + +{{< include /html/rdesrapstroke.html >}} -Data for back tests are generated within `notebooks/generate_exp_results.Rmd`. +Test files include: -Tests are run via GitHub actions (`.github/workflows/R-CMD-check.yaml`) and a coverage badge is updated (`.github/workflows/test-coverage.yaml`). +* `tests/testthat/test-backtest.R` +* `tests/testthat/test-functionaltest.R` +* `tests/testthat/test-unittest.R` ::: From d6c31ae6a3f968d2103f11b16d4027d561c95ff2 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 11:16:09 +0000 Subject: [PATCH 06/17] feat(qa): addressing nav's feedback (#158) --- .../verification_validation/quality_assurance.qmd | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pages/guide/verification_validation/quality_assurance.qmd b/pages/guide/verification_validation/quality_assurance.qmd index a36caaf..e387c7e 100644 --- a/pages/guide/verification_validation/quality_assurance.qmd +++ b/pages/guide/verification_validation/quality_assurance.qmd @@ -21,6 +21,8 @@ date: "2025-11-25T14:57:15+00:00" * **Why do QA?** It helps build trust in the work by ensuring the right processes are followed, and that analysis and results are checked. + * [Harper et al. (2021)](https://doi.org/10.1016/j.ejor.2020.06.043) show that trust in a simulation does not come from a single thing, but from many elements working together. These include (among other things): the credibility of the modellers, transparency of assumptions, suitability of methods, and how well the model answers the decision question. The QA activities on this page provide a structured way to address those facets systematically (for example through clear roles, documentation, and independent assurance). + * **When to do QA?** It spans the entire project lifecycle: from initial scoping, through design and analysis, to final delivery and sign-off. * **How does QA relate to verification and validation?** Verification and validation are specific checks within QA that focus on whether the analysis has been built correctly and answers the right question; QA is broader and also covers how the work is planned, documented, reviewed, and signed off. @@ -50,6 +52,8 @@ For smaller projects, you might be the analyst, your supervisor the commissioner For larger or higher-risk projects, these roles are often held by different people or groups, with more formal assurance and sign-off. +In summary, the commissioner and approver mainly shape institutional and decision‑level trust (is this suitable to base decisions on?), while the analyst and assurer focus on technical and process trust (is the model well‑built, transparent, and appropriately checked?). Making these roles explicit helps you design QA that supports trust in the model and its results across different stakeholders. + ### Quality assurance when scoping the project When scoping a project, QA mainly means: @@ -68,7 +72,7 @@ When scoping a project, QA mainly means: ::: -It is up to you where you record this information. A practical option is to keep it in your repository so everything is in one place and under version control - for example in a `scoping.md` file, as part of an analysis plan, or in a project protocol (which might also be archived on a platform like [OSF](https://osf.io/)). The important thing is that it is written down, easy to find, and shared with the people involved. +It is up to you where you record this information. A practical option is to keep it in your repository so everything is in one place and under version control - for example in a scoping document (such as `scoping.md`), as part of an analysis plan, or in a project protocol (which might also be archived on a platform like [OSF](https://osf.io/)). The important thing is that it is written down, easy to find, and shared with the people involved. ::: {.callout-note title="Suggestion on what your plan should include" collapse="true"} @@ -225,4 +229,6 @@ Key resources to check out when planning your quality assurance: * The [Quality Assurance (QA) tools and guidance](https://www.gov.uk/government/publications/energy-security-and-net-zero-modelling-quality-assurance-qa-tools-and-guidance) from the Department for Energy Security and Net Zero modelling. -* The [Quality assurance of code for analysis and research](https://best-practice-and-impact.github.io/qa-of-code-guidance/intro.html) book from the Government Analysis Function +* The [Quality assurance of code for analysis and research](https://best-practice-and-impact.github.io/qa-of-code-guidance/intro.html) book from the Government Analysis Function. + +* [Facets of trust in simulation studies](https://doi.org/10.1016/j.ejor.2020.06.043) - a framework for thinking about different dimensions of trust in simulation work and how they evolve over a study’s lifecycle. From 16e8d3396cee75f654bad09cb8d87f578e5a90b1 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 11:41:11 +0000 Subject: [PATCH 07/17] feat(linting): addressing nav's feedback (#158) --- pages/guide/style_docs/linting.qmd | 60 ++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/pages/guide/style_docs/linting.qmd b/pages/guide/style_docs/linting.qmd index 3c7762b..ca8121f 100644 --- a/pages/guide/style_docs/linting.qmd +++ b/pages/guide/style_docs/linting.qmd @@ -61,13 +61,13 @@ Popular R style guides include: ::: -You are not required to follow style guides, but they are recommended for writing clear, maintainable, and consistent code. The easiest way to enforce their requirements is using linters or code formatters. +You are not required to follow style guides, but they are recommended for writing clear, maintainable, and consistent code. The guides themselves are written documents; tools like **linters** and **code formatters** help you check your code against them automatically. ## Linters **Linters** are tools that analyse code for: -* **Possible errors** - looking for issues like unused variables and code complexity. +* **Possible errors** - looking for issues like unused variables, unreachable code, and code complexity. * **Style issues** - enforcing requirements from style guides. They are often run from the terminal/command line, and return a list of suggestions for you to address. Sometimes, they might flag code that you have intentionally written, and it's okay to disagree with their suggestion to change it! @@ -80,20 +80,15 @@ They are handy for "desk checking" as part of your model [verification](../verif Popular linters in Python include: -* [**pylint**](https://pypi.org/project/pylint/) - detailed linter that detects errors, bugs, variable naming issues, and other code problems. +* [**pylint**](https://pypi.org/project/pylint/) - detailed linter that detects potential bugs (for example, undefined variables or unreachable code) as well as style and naming issues. * [**flake8**](https://pypi.org/project/flake8/) - lightweight tool focused on PEP-8 style, basic errors, and code complexity. -* [**ruff**](https://pypi.org/project/ruff/) - modern, ultra-fast linter that implements checks from Flake8 and some other popular plugins. It is also a code formatter (see below) - though note: the linter does not report every issue that can be fixed by the formatter. Stylistic issues are handled silently by the formatter and not surfaced as lint errors - so the linter is intentionally more restricted in scope than the formatter. +* [**ruff**](https://pypi.org/project/ruff/) - modern, ultra-fast linter that re-implements many of Flake8's checks (and popular Flake8 plugins) inside Ruff itself, so it can often be used as a drop‑in replacement. It is also a code formatter (see below) - though note: the linter does not report every issue that can be fixed by the formatter. Stylistic issues are handled silently by the formatter and not surfaced as lint errors - so the linter is intentionally more restricted in scope than the formatter. You can use several linters, or choose one that suits you best. -To better understand their differences, let's run them on a some poor quality code. We can see that: - -* Both identify the trailing whitespace. -* Pylint identifies the missing module and function docstrings, and the function name. -* Flake8 identifies the missing whitespace around an operator. -* Ruff did not identify any issues! This is because the linter incorporates some - but not all - of Flake8's checks - and it doesn't check whitespace by default. +To better understand their differences, let's run them on a some poor quality code. `code.py`: @@ -149,6 +144,27 @@ result = subprocess.run( print(result.stdout) ``` +We can see that: + +* Pylint and Flake8 both identify the trailing whitespace. +* Pylint identifies the missing module and function docstrings, and the function name. +* Flake8 identifies the missing whitespace around an operator. +* Ruff did not identify any issues. This is because, by default, Ruff only enables Flake8's "error" rules (F and a subset of E) and omits many purely stylistic checks, such as whitespace‑only issues, which are instead handled by formatters like `ruff format` or Black. + +
+ +::: {.callout-tip title="Final newline"} + +If you try running this example locally, you may get another error too: + +`C0304: Final newline missing (missing-final-newline)` + +This will be present if you don't include a blank line at the end of your `.py` file. + +**Note:** In editors like VS Code, you can avoid this by enabling the "Insert Final newline" setting, which automatically adds a newline at the end of the file when you save. On VS Code, go to File -> Preferences -> Settings then search `insertfinalnewline` and click the box by "Files: Insert Final Newline". + +::: +
::: {.callout-note title="Side note: Type hints"} @@ -208,7 +224,9 @@ Popular code formatters in Python include: * [**ruff**](https://github.com/astral-sh/ruff) - designed to be a drop-in replacement for `black` that should have a near identical output, but runs faster. -As an example, if we ran either of these code formatters on our example by executing `black linting_resources/code.py` or `ruff format linting_resources/code.py`, they would alter the file as follows: +As an example, if we ran either of these code formatters on our example by executing `black linting_resources/code.py` or `ruff format linting_resources/code.py`, they would update the file in place. + +Below, we show the changes as a unified diff for illustration: ```{python} #| echo: false @@ -241,6 +259,8 @@ html_output = re.sub(r'font-family:.*?;', '', html_output) HTML(html_output) ``` +When you run `black` or `ruff format` normally, they just rewrite the file and either print nothing or a short "reformatted file.py" message. Using a `--diff` option or version control tools like `git diff` is what produces the `---`, `+++`, and `+`/`-` markers shown above. + ::: ::: {.r-content} @@ -479,21 +499,21 @@ Linters will sometimes flag things that you don't want to change. For example: If you want to ignore these warnings, you can disable them in your code **using comments**. For example: ```{.python} -# pylint: disable=wrong-import-position +# pylint: disable=too-many-arguments ``` -You can place this comment: - -* At the top of a code cell or file to disable the warning for everything below it. -* Directory above a line or block of code to apply it only there. -* At the end of a specific line to disable it just for that line. - You can also disable multiple warnings at once by listing them, separated by commas: ```{.python} # pylint: disable=too-many-arguments,missing-module-docstring ``` +You can place this comment: + +* At the top of a code cell or file to disable the warning for everything below it. +* Directory above a line or block of code to apply it only there. +* At the end of a specific line to disable it just for that line. + ::: ::: {.r-content} @@ -660,7 +680,7 @@ lint.sh {{< include /html/pydesrapstroke.html >}} -These repositories use `pylint` for linting, and can run it on model code, tests and notebooks with a single command using `lint.sh`. They have a GitHub action described in `lint.yaml` which runs the script on pushes to main. +These repositories use `pylint` for linting, and can run it on model code, tests and notebooks with a single command using `lint.sh`. They have a GitHub actions workflow (`.github/workflows/lint.yaml`) that is configured to run this script automatically whenever new commits are pushed to the `main` branch, so linting happens on every update to the main version of the code. ::: @@ -670,7 +690,7 @@ These repositories use `pylint` for linting, and can run it on model code, tests {{< include /html/rdesrapstroke.html >}} -These repositories use `lintr`. They have a GitHub action described in `lint.yaml` which runs the linter on pushes to main. Settings are provided in `.lintr`. +These repositories use `lintr`. They have a GitHub actions workflow (`.github/workflows/lint.yaml`) that is configured to run `lintr` automatically whenever new commits are pushed to the `main` branch, so linting happens on every update to the main version of the code. Settings are provided in `.lintr`. ::: From fde4cfcc1e837baa57bd4b1f7e0cd2935dc900ee Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 11:43:58 +0000 Subject: [PATCH 08/17] feat(docstrings): addressing nav's feedback (#161) --- pages/guide/style_docs/docstrings.qmd | 108 +++++++++++++------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/pages/guide/style_docs/docstrings.qmd b/pages/guide/style_docs/docstrings.qmd index 0769b3d..c02a7a7 100644 --- a/pages/guide/style_docs/docstrings.qmd +++ b/pages/guide/style_docs/docstrings.qmd @@ -166,7 +166,7 @@ In‑line comments should be concise and avoid clutter; more is not necessarily Import SimPy and load the documentation for the `Environment()` class. ```{.python} - library(simpy) + import simpy help(simpy.core.Environment) ``` @@ -201,46 +201,24 @@ Two popular ones include: In our example models, we have used NumPy style. Below, we illustrate both approaches for comparison. -### Function - -**NumPy style:** - -```{python} -def estimate_wait_time(queue_length, avg_service_time): - """ - Estimate the total wait time for all customers in the queue. +### Module - Parameters - ---------- - queue_length : int or float - Number of customers currently in the queue. - avg_service_time : int or float - Average time taken to serve one customer. +A module is a `.py` file within a package containing package code, like functions or classes. Each `.py` file is considered a module, and modules can be used to group related code together. - Returns - ------- - float - Estimated total wait time. - """ - return queue_length * avg_service_time -``` +Module docstrings - similar to tests - are usually brief, giving an overview of the file's purpose and main contents, without necessarily following a strict style guide. -**Google style:** +**Example:** -```{python} -def estimate_wait_time(queue_length, avg_service_time): - """Estimate the total wait time for all customers in the queue. +```{.python} +""" +hospital.py - Args: - queue_length (int or float): Number of customers currently in the - queue. - avg_service_time (int or float): Average time taken to serve one - customer. +A simple healthcare utility module. - Returns: - float: Estimated total wait time. - """ - return queue_length * avg_service_time +Provides: +- A function to estimate patient wait times in a queue. +- A class to represent a patient and manage their admission status. +""" ``` ### Class @@ -322,36 +300,58 @@ class Patient: ``` -### Test - -Tests are a bit different. They usually just have a short, one‑line description without formal parameters or return sections, and are not based on any specific style guide. +### Function -**Example:** +**NumPy style:** ```{python} -def test_estimate_wait_time(): - """Test estimate_wait_time with sample input and expected output.""" - assert estimate_wait_time(5, 2) == 10 +def estimate_wait_time(queue_length, avg_service_time): + """ + Estimate the total wait time for all customers in the queue. + + Parameters + ---------- + queue_length : int or float + Number of customers currently in the queue. + avg_service_time : int or float + Average time taken to serve one customer. + + Returns + ------- + float + Estimated total wait time. + """ + return queue_length * avg_service_time ``` -### Module +**Google style:** -A module is a `.py` file within a package containing package code, like functions or classes. Each `.py` file is considered a module, and modules can be used to group related code together. +```{python} +def estimate_wait_time(queue_length, avg_service_time): + """Estimate the total wait time for all customers in the queue. -Module docstrings - similar to tests - are usually brief, giving an overview of the file's purpose and main contents, without necessarily following a strict style guide. + Args: + queue_length (int or float): Number of customers currently in the + queue. + avg_service_time (int or float): Average time taken to serve one + customer. -**Example:** + Returns: + float: Estimated total wait time. + """ + return queue_length * avg_service_time +``` -```{.python} -""" -hospital.py +### Test -A simple healthcare utility module. +Tests are a bit different. They usually just have a short, one‑line description without formal parameters or return sections, and are not based on any specific style guide. -Provides: -- A function to estimate patient wait times in a queue. -- A class to represent a patient and manage their admission status. -""" +**Example:** + +```{python} +def test_estimate_wait_time(): + """Test estimate_wait_time with sample input and expected output.""" + assert estimate_wait_time(5, 2) == 10 ``` ::: From d488588f3af43fa76a62e18c7710316565b8cc11 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 11:51:23 +0000 Subject: [PATCH 09/17] feat(github actions): addressing nav's feedback (#161) --- pages/guide/style_docs/github_actions.qmd | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pages/guide/style_docs/github_actions.qmd b/pages/guide/style_docs/github_actions.qmd index 1f05fd1..09c3563 100644 --- a/pages/guide/style_docs/github_actions.qmd +++ b/pages/guide/style_docs/github_actions.qmd @@ -9,7 +9,7 @@ date: "2025-11-03T10:48:30+00:00" **Learning objectives:** -* Recognise the utility of CI/CD for RAP. +* Recognise the utility of CI/CD (Continuous Integration and Continuous Delivery/Deployment) for RAP. * Understand the basic structure and set-up of GitHub actions. * Explore some example actions that could be relevant for your research. @@ -23,13 +23,13 @@ This page assumes your simulation project is in a GitHub repository - set-up gui :::: -## Continuous Integration, Continuous Delivery, and Continuous Deployment (CI/CD) +## Continuous Integration and Continuous Delivery/Deployment (CI/CD)
### Definition -CI/CD processes (Continuous Integration, Continuous Delivery, and Continuous Deployment) are a valuable tool for reproducible analytical pipelines. These processes are typically defined as follows: +CI/CD processes (Continuous Integration and Continuous Delivery/Deployment) are a valuable tool for reproducible analytical pipelines. These processes are typically defined as follows: ::: {.blue} @@ -39,6 +39,8 @@ CI/CD processes (Continuous Integration, Continuous Delivery, and Continuous Dep **Continuous deployment (CD):** Extends CD by releasing changes automatically to production as soon as all tests pass, with no manual approval required. +*It may seem a little odd that CD is used for both continuous delivery and continuous deployment, but this dual use is the standard convention in most CI/CD tooling and documentation.* + ::: **Further information** @@ -47,7 +49,7 @@ CI/CD processes (Continuous Integration, Continuous Delivery, and Continuous Dep ### Relevance for your simulation RAP -CI will be **relevant for all projects**. CI automates checks on your code and outputs to ensure reliability and code quality. This can include tests, linting, formatting and other checks. +CI will be **relevant for all projects**. CI automates checks on your code and outputs to ensure reliability and code quality. This can include tests, linting, formatting and other checks. It can also support collaborative work by running these checks automatically whenever team members merge or push changes into a shared branch (for example, using GitHub Actions on the `main` branch). CD **may not be needed for every project**, but it can be useful: @@ -101,9 +103,9 @@ Creating a GitHub action to run your model tests is really helpful, as it helps ::: {.python-content} -This simple action will install Python 3.13 and dependencies in a `requirements.txt` file, then run `pytest()`. +This simple GitHub Actions workflow will install Python 3.13 and dependencies in a `requirements.txt` file, then run `pytest()`. -> **Note:** We'd recommend using `requirements.txt` for GitHub actions as it is much simpler than working with conda for these. We typically maintain an `environment.yaml` file for working with conda locally, and then an equivalent `requirements.txt` for GitHub actions. +> **Note:** We'd recommend using `requirements.txt` for GitHub actions as it is much simpler than working with conda inside CI. Locally, we typically maintain a full `environment.yaml` file for conda (which includes the Python version and all dependencies), and then create a corresponding `requirements.txt` listing the Python packages needed to run the project in GitHub Actions. In other words, `requirements.txt` is a pip-style dependency list derived from the same environment as `environment.yaml`, tailored for use in CI. ```{.yaml} name: tests From fc5aa88658be6196ae098641015bca4f9a1f3053 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 13:20:59 +0000 Subject: [PATCH 10/17] feat(peer_review): addressing nav's feedback (#161) --- pages/guide/sharing/peer_review.qmd | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pages/guide/sharing/peer_review.qmd b/pages/guide/sharing/peer_review.qmd index ce574c8..6997dd4 100644 --- a/pages/guide/sharing/peer_review.qmd +++ b/pages/guide/sharing/peer_review.qmd @@ -36,7 +36,7 @@ When **team members** review code, it increases shared understanding of the code **Regularly.** If you do it little and often, review is usually quicker and can go into more detail than if reviewers are faced with the entire codebase all at once. -That said, this approach is often more ideal than realistic. In research, work doesn't always follow tidy steps, and it's common to end up with just one or two big review moments instead of regular check-ins. Even so, any review - whether early, late, or somewhere in between - helps strengthen your code. +That said, this kind of regular, incremental review is often more of an ideal than a reality in research projects, where work can be exploratory and fast‑moving, and code may only be reviewed once or twice at key moments rather than through frequent check‑ins. Even so, any review - whether early, late, or somewhere in between - helps strengthen your code. ## Structures and tools for review @@ -52,7 +52,7 @@ It's also a good point to run automated tests with GitHub Actions, checking your ![Screenshot of requesting a reviewer in a GitHub pull request](peer_review_resources/pull_request_reviewer.png)\ -
+::: {.callout-note title="Click to learn more about branch structures for code review" collapse="true"} You can use pull requests with just main and dev branches, but that approach often means either not reviewing every merge, or letting branches grow large and unwieldy over time. @@ -62,7 +62,7 @@ A more flexible and common setup, inspired by **"Gitflow"**, uses main, dev, and ![](peer_review_resources/git_branch_feature.drawio.png){fig-alt="Diagram showing review of changes in feature branch before merge into dev."} -
+::: These are recommendations, and its rare to follow them perfectly in research. If you only have time for occasional peer review, you might: @@ -81,6 +81,8 @@ To start one, just click  [New issue]{style="background: #597341; color: #f Issues work well for more occasional reviews, letting you track comments and discussions over time. If review happens elsewhere (like email or in person), **writing a summary in an issue** helps keep a transparent record in one place, making it easy to refer back later. +::: {.callout-note title="Github issue templates" collapse="true"} + You can also create **templates** for GitHub issues. A template provides a pre-set structure, making reviews more consistent and easier to follow. You can set them up by going to the repository **⚙ Settings**, then scrolling down to **Features** and selecting  [Set up templates]{style="background: #597341; color: #fff; border-radius:4px; padding:2px 8px; font-family:monospace; font-weight:bold; font-size:90%;"} . ![](peer_review_resources/set_up_template.png){fig-alt="Screenshot of GitHub settings 'Set up templates'."} @@ -95,9 +97,9 @@ Now, when reviewers select  [New issue]{style="background: #597341; color: ![](peer_review_resources/new_issue_edit.png){fig-alt="Screenshot of GitHub creating an issue using the template."} -
+::: -#### Tip: Checklists (GitHub tasklists) +::: {.callout-note title="Checklists (GitHub tasklists)" collapse="true"} Checklists can be used to break down complex issues into actionable steps. To add a checklist using Markdown: @@ -113,9 +115,9 @@ If a task mentions an issue or pull request (e.g., `* [ ] #123`) the box will up [→ Find out more about tasklists](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/about-tasklists) -
+::: -#### Tip: Sub-issues +::: {.callout-note title="Sub-issues" collapse="true"} Sub-issues are great for more detailed actions. @@ -127,9 +129,9 @@ The new sub-issue will be nested as part of the larger task. [→ Find out more about sub-issues](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/adding-sub-issues) -
+::: -#### Tip: GitHub projects +::: {.callout-note title="GitHub projects" collapse="true"} GitHub Projects help organize issues, pull requests, checklists, and sub-issues visually. You can set-up a project using the **Project** tab: @@ -141,6 +143,8 @@ GitHub provide several templates for projects. A common example is a board with [→ Find out more about GitHub Projects.](https://docs.github.com/en/issues/planning-and-tracking-with-projects/learning-about-projects/about-projects) +::: + ### Conversations **Meeting in person or virtually** to talk through code can help clarify questions, resolve confusion, and share knowledge within the team. @@ -395,7 +399,7 @@ cat(longmcq(c( *Beginner friendly explanation on using pull requests for research code review.* -* [Developers perception of peer code review in research software development](https://doi.org/10.1007/s10664-021-10053-x) by Eisty and Carver 2022 +* Eisty, N.U. and Carver, J.C. (2022). Developers perception of peer code review in research software development. Empirical Software Engineering, 27(13). . *Study that interviewed research software developers about their code review practices.* From dde692355a5c92f26967071f3996c64302e3fc3c Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 13:37:06 +0000 Subject: [PATCH 11/17] feat(citation): addressing nav's feedback (#161) --- pages/guide/sharing/citation.qmd | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pages/guide/sharing/citation.qmd b/pages/guide/sharing/citation.qmd index 9970cfc..61450a9 100644 --- a/pages/guide/sharing/citation.qmd +++ b/pages/guide/sharing/citation.qmd @@ -49,7 +49,9 @@ This page explains why and how to provide citation instructions in your simulati > Explicitly list and acknowledge everyone who played a role in the project (including minor contributions). There are structured frameworks which can be used to specify roles, like the [Contributor Role Taxonomy (CRediT)](https://credit.niso.org/) for research projects. -For `CITATION.cff` files... +
+ +Many repositories use a CITATION.cff file to store structured citation metadata for software and datasets. This plain-text file is both human- and machine-readable and is supported by tools like GitHub, Zenodo, and Zotero. For repositories using `CITATION.cff` files, additional benefits are that it is... **Used when archiving** @@ -65,7 +67,7 @@ For `CITATION.cff` files... ### `CITATION.cff` -A `CITATION.cff` file is a plain text file with a standard structure to share citation metadata for software or datasets. +As mentioned above, a `CITATION.cff` file is a plain text file with a standard structure to share citation metadata for software or datasets. The easiest way to create this file is using the [cffinit](https://citation-file-format.github.io/cff-initializer-javascript/#/) web application. It will guide you through the required information, and generate a `CITATION.cff` file at the end. @@ -85,9 +87,15 @@ As an example, the `CITATION.cff` from the repository for this book: * **Zotero:** The Zotero reference manager uses `CITATION.cff` when importing the repository as a reference. -* **GitHub:** Adds a "Cite this repository" button to the sidebar, which provides APA and BibTeX citations (based on the `CITATION.cff` file), and links to the file. For example: +* **GitHub:** Adds a "Cite this repository" button to the sidebar, which provides APA and BibTeX citations (based on the `CITATION.cff` file), and links to the file. + +::: {.callout-tip title="Why Zenodo and Zotero matter for your model"} + +**Zenodo:** Archiving a release of your GitHub repo on Zenodo gives your software a DOI, and Zenodo can read your `CITATION.cff` to auto‑populate title, authors, and version. This means others can cite the exact version of your model reproducibly. + +**Zotero:** Many collaborators and reviewers manage references in Zotero. Having a DOI and structured metadata (via `CITATION.cff` or Zenodo) lets them add your software to their library and cite it like a paper or dataset. -![](citation_resources/cff_github.png){fig-alt="Screenshot of GitHub 'Cite this repository'."} +::: ### `README.md` @@ -120,11 +128,11 @@ Heather, A., Monks, T., Mustafee, N., Harper, A., Alidoost, F., Challen, R., & S If you have [structured your model as a package](../setup/package.qmd), then you can include some citation instructions within the package. :::{.python-content} -Depending on your package manager, you can include citation and author details in several ways... +Depending on your package manager, you can include citation and author details in several ways. A package manager is a tool that helps you build, install, and share your Python project as a package; it usually reads project metadata from a configuration file called `pyproject.toml`. This file can store information such as the package name, version, and authors, which users and tools can then use when citing your work.
-**Flit:** Our tutorial used this as it is a very basic minimalist package manager. It had `__init.py` and `pyproject.toml` files. We can amend the `pyproject.toml` to add some citation information as follows: +**Flit:** Our tutorial used Flit, a minimalist package manager that keeps configuration in a `pyproject.toml` file. You can add basic citation information (such as your name and email) to the `[project]` section so it is recorded as part of the package metadata: ```{.bash} [build-system] @@ -141,7 +149,7 @@ authors = [
-**Poetry:** Another popular manager, this has project details within `pyproject.toml` like so: +**Poetry:** Poetry is another popular tool for managing dependencies and packaging, which also uses `pyproject.toml` to store project details. Here, citation‑related information like the package name, version, and authors is defined under the `[tool.poetry]` section: ```{.bash} [tool.poetry] @@ -150,6 +158,11 @@ version = "0.1.0" authors = ["First Last "] readme = "README.md" ``` + +
+ +Other tools follow the same idea, but the exact section and format depend on the package manager you use. + ::: :::{.r-content} From 3598b8cbf45874aa0a0af2877fff93c1c9dc1e91 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 13:38:38 +0000 Subject: [PATCH 12/17] feat(archive): addressing nav's feedback (#161) --- pages/guide/sharing/archive.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/guide/sharing/archive.qmd b/pages/guide/sharing/archive.qmd index a8cf0c2..66455cd 100644 --- a/pages/guide/sharing/archive.qmd +++ b/pages/guide/sharing/archive.qmd @@ -137,7 +137,7 @@ Whilst platforms like GitHub are great for sharing code, they provide no guarant Open science archives will **provide long-term storage guarantees** ensuring future access to your work. They will create a **digital object identifier (DOI)** (a persistent identifier for your code). -Archives should adhere to recognised principles like [TRUST (Transparency, Responsibility, User focus, Sustainability, and Technology)](https://doi.org/10.1038/s41597-020-0486-7) and the [FORCE11 standards](https://doi.org/10.7717/peerj-cs.86) +Archives should adhere to recognised principles like [TRUST (Transparency, Responsibility, User focus, Sustainability, and Technology)](https://doi.org/10.1038/s41597-020-0486-7) and the [FORCE11 (Future of Research Communications and E-Scholarship) standards](https://doi.org/10.7717/peerj-cs.86) Archives can be generalist (for any research discipline), subject-specific, or institutional (hosted by a particular university or research institute). Examples include: From 2d01f4d0082d118abeed53948b3e5753dd6a4709 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 13 Feb 2026 13:41:20 +0000 Subject: [PATCH 13/17] feat(conclusion): addressing nav's feedback (#161) --- pages/guide/further_info/conclusion.qmd | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pages/guide/further_info/conclusion.qmd b/pages/guide/further_info/conclusion.qmd index bcafe83..5d1e848 100644 --- a/pages/guide/further_info/conclusion.qmd +++ b/pages/guide/further_info/conclusion.qmd @@ -3,13 +3,6 @@ title: Conclusion date: "2025-11-05T11:44:20+00:00" --- - - -
::: {.pale-blue} @@ -32,16 +25,32 @@ Remember, these are **examples**, not prescriptions. They're not perfect, and th **Nurse visit simulation:** +::: {.python-content} + {{< include /html/pydesrapmms.html >}} +::: + +::: {.r-content} + {{< include /html/rdesrapmms.html >}} +::: + **Stroke pathway simulation:** +::: {.python-content} + {{< include /html/pydesrapstroke.html >}} +::: + +::: {.r-content} + {{< include /html/rdesrapstroke.html >}} +::: + ### Make your own model The best way to solidify what you've learned is to apply it. When planning your model, remember that a good simulation starts with **conceptual modelling**. As defined in Robinson (2007): From 1c418c93f2236b217493758ba5372ac3c9e51f88 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 20 Feb 2026 13:54:49 +0000 Subject: [PATCH 14/17] feat(length_warmup): update description of run length + warm-up choice (#177) --- pages/guide/output_analysis/length_warmup.qmd | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pages/guide/output_analysis/length_warmup.qmd b/pages/guide/output_analysis/length_warmup.qmd index 82a864b..d147b62 100644 --- a/pages/guide/output_analysis/length_warmup.qmd +++ b/pages/guide/output_analysis/length_warmup.qmd @@ -1021,8 +1021,6 @@ If you've already built warm-up logic into your model (as we have done), make su As mentioned above, we have used multiple replications and a long run length. This run length is based on the assumption that our desired simulation run length is 10 days (14400 minutes). -> So far, the model we have been constructing in this book has intentionally had a short duration. This is handy when developing - for example, for looking over the log messages. However, this is unrealistic of an actual simulation of a healthcare system, and will produce great variability (requiring very long warm-up / large number of replications for stable results). From this point onwards, we increase the run time as mentioned. - ```{r} #| warning: false @@ -1079,11 +1077,11 @@ ggplotly(plots[[5L]]) :::: -The selection of a warm-up period is **subjective**. The dotted red line shows our suggestion, but this choice could be changed subject to discussion and evaluation of other metrics. +The selection of a warm-up period is **subjective**. The dotted red line shows our suggestion, but in practice, there is usually a range of sensible warm-up values that give very similar results, and small changes within a reasonable range will often only have a small effect on results, as long as the total run length is long enough. ::: {.callout-note} -In this example, we assumed a total run length of 10 days (14400 minutes) to illustrate how to determine an appropriate warm-up period. However, for the remainder of this book, we will return to using shorter parameters (i.e., 30 minute warm-up, 40 minute data collection period, 5 replications) to keep examples simple, quick to run, and easy to follow. +In this example, we assumed a total run length of 10 days (14400 minutes) to illustrate how to determine an appropriate warm-up period. Short model runs, like those we've used so far, are useful during development (e.g., when inspecting log messages), but they produce high variability and are unrealistic of an actual healthcare system. However, for the remainder of this book, we will return to using shorter parameters (i.e., 30 minute warm-up, 40 minute data collection period, 5 replications) to keep examples simple, quick to run, and easy to follow. ::: From f3b2a3d3ccfd9c332f38e634288f44c95e65392d Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 20 Feb 2026 13:58:17 +0000 Subject: [PATCH 15/17] chore(impact): fix line breaks --- pages/impact/impact.qmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/impact/impact.qmd b/pages/impact/impact.qmd index 959b860..2127033 100644 --- a/pages/impact/impact.qmd +++ b/pages/impact/impact.qmd @@ -22,9 +22,9 @@ The DES RAP Book is already being used and adapted in several teaching and train ## Research -* **NHS model reuse project (planned)** +* **NHS model reuse project (planned)** Collaboration between King’s, The Strategy Unit and the University of Exeter, where DES models are provided to NHS analysts. It explores whether they are able to reproduce and reuse them in their own context, and the model sample includes our [python stroke example model](https://github.com/pythonhealthdatascience/pydesrap_stroke). -* **Stroke model of Same Day Emergency Care (SDEC) and CT Perfusion (CTP) scanning** +* **Stroke model of Same Day Emergency Care (SDEC) and CT Perfusion (CTP) scanning** Hyperacute and acute stroke pathway model created by John Williams at Maidstone and Tunbridge Wells NHS Trust. Now being updated using DES RAP principles to add documentation, reproducible workflows, testing and other RAP infrastructure. \ No newline at end of file From e9a767bf8ae228e88cdd891f572b43a69211f886 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 20 Feb 2026 14:13:23 +0000 Subject: [PATCH 16/17] feat(n_reps): addressing tom's feedback (#177) --- pages/guide/output_analysis/n_reps.qmd | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pages/guide/output_analysis/n_reps.qmd b/pages/guide/output_analysis/n_reps.qmd index f62659b..8ac04cb 100644 --- a/pages/guide/output_analysis/n_reps.qmd +++ b/pages/guide/output_analysis/n_reps.qmd @@ -31,10 +31,10 @@ This page continues on from: [Replications](replications.qmd). The following imports are required. These should be available from environment setup on the [Structuring as a package](/pages/guide/setup/package.qmd) page. -Since [Replications](replications.qmd), we have several new imports: functions from `typing` and from `sim_tools.output_analysis`. - ::: {.python-content} +Since [Replications](replications.qmd), we have several new imports: functions from `typing` and from `sim_tools.output_analysis`. + ```{python} # For display purposes: from IPython.display import HTML @@ -181,7 +181,7 @@ The `desired_precision` parameter is set to 0.1, which means we are interested i ::: {.callout-note title="Stopping condition: `desired_precision` and confidence intervals"} -We construct a confidence interval for each performance measure at a chosen confidence level (typically 95%). The interval has a half‑width, which is the margin of error around the estimated mean. The `desired_precision` parameter sets how small this half‑width must be relative to the mean (for example, 0.1 means “within 10% of the mean”). +We construct a confidence interval for each performance measure at a chosen confidence level (typically 95%). The interval has a half‑width, which is the margin of error around the estimated mean. The `desired_precision` parameter sets how small this half‑width must be relative to the mean (for example, 0.1 means "within 10% of the mean"). The confidence interval method keeps increasing the number of replications until this relative half‑width falls below `desired_precision` (and is above `min_rep`), indicating that additional replications are unlikely to change the estimate by more than the chosen percentage. @@ -508,6 +508,20 @@ plot_replication_ci <- function( ::: +`confidence_interval_method()` runs the model for the specified number of replications, and after each replication it updates a single row that summarises what we know so far about the chosen performance measure. It records the most recent value of the metric, the running average across all completed replications, the standard deviation, and the 95% confidence interval around that average (lower and upper bounds). From this interval it also calculates a relative "half‑width" (how far the upper confidence limit is above the mean, as a proportion of the mean), which is compared to desired_precision to decide when you have run enough replications. The function returns a data frame containing this full sequence of per‑replication summaries, and prints a message when the desired precision is first achieved. + +We supply `confidence_interval_method` with three main inputs: the parameters, the performance measure to analyse, and the `desired_precision`. The `desired_precision` parameter is set to 0.1, which means we are interested in identifying when the percentage deviation of the confidence interval from the mean falls **below 10%**. + +::: {.callout-note title="Stopping condition: `desired_precision` and confidence intervals"} + +We construct a confidence interval for each performance measure at a chosen confidence level (typically 95%). The interval has a half‑width, which is the margin of error around the estimated mean. The `desired_precision` parameter sets how small this half‑width must be relative to the mean (for example, 0.1 means "within 10% of the mean"). + +The confidence interval method keeps increasing the number of replications until this relative half‑width falls below `desired_precision`, indicating that additional replications are unlikely to change the estimate by more than the chosen percentage. + +**Note:** The 10% threshold used is our default, but is relatively arbitrary. We are just using it to help us identify the point where results are stable enough. + +::: + ```{r} metrics <- c("mean_wait_time_doctor", "utilisation_doctor", "mean_queue_length_doctor", "mean_time_in_system", @@ -526,6 +540,8 @@ for (m in metrics) { } ``` +`plot_replication_ci()` then takes that data frame and produces a plot showing how the cumulative mean and its confidence interval change as you add more replications, optionally marking the recommended number of replications with a vertical dashed line so you can visually check that the estimates have stabilised. + Click the name of each metric below to view the relevant plot: ::: {.panel-tabset} @@ -1451,7 +1467,11 @@ res <- run_replications_algorithm( # View recommended number of replications res$nreps +``` +The algorithm returns the minimum number of replications required for each performance measure to reach the desired precision. In this example, the largest of these values is 19 (for `mean_queue_length_doctor`), so we would use approximately 19 replications for this model overall, to ensure all performance measures meet the target precision. + +```{r} # View full results table res$summary_table |> kable() |> From a877aed60ab58218d01114157ad202563eaedf31 Mon Sep 17 00:00:00 2001 From: amyheather Date: Fri, 20 Feb 2026 14:25:42 +0000 Subject: [PATCH 17/17] chore(v0.5.0): update changelog and citation.cff for release --- CHANGELOG.md | 22 +++++++++++++++++++++- CITATION.cff | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 623516e..09f7607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). Dates formatted as YYYY-MM-DD as per [ISO standard](https://www.iso.org/iso-8601-date-and-time-format.html). +## v0.5.0 - 2026-02-20 + +This release has lots and lots of changes based on peer review of the book from Nav Mustafee, Rob Challen, Tom Slater and Alison Harper. Other changes include addressing FAIRness requirements, switching R length of warm-up analysis to use intervals, and improving the docker action used to build the site. + +### Added + +* Addressed FAIRness requirements ([#170](https://github.com/pythonhealthdatascience/des_rap_book/issues/170)), with some larger changes including: + * Addition of action which checks all images have alt text. + * New "Impact" page. + +### Changed + +* R length of warm-up now uses intervals. +* Improved `docker_quarto.yaml` (shorter run-time as doesn't need to restore renv every time build book). +* Addressed peer review feedback from Nav Mustafee ([#149](https://github.com/pythonhealthdatascience/des_rap_book/issues/149), [#150](https://github.com/pythonhealthdatascience/des_rap_book/issues/150), [#158](https://github.com/pythonhealthdatascience/des_rap_book/issues/158), and [#161](https://github.com/pythonhealthdatascience/des_rap_book/issues/161)). +* Addressed peer review feedback from Rob Challen ([#152](https://github.com/pythonhealthdatascience/des_rap_book/issues/152) and sub-issues). +* Addressed peer review feedback from Tom Slater ([#172](https://github.com/pythonhealthdatascience/des_rap_book/issues/172) and sub-issues). +* Addressed peer review feedback from Alison Harper ([#169](https://github.com/pythonhealthdatascience/des_rap_book/issues/169) and sub-issues). +* Add all-contributors CLI instructions to `CONTRIBUTING.md`. + ## v0.4.0 - 2026-01-06 ### Added @@ -129,4 +149,4 @@ This release adds a large number of new pages. It also introduces pre-commits, c * Model building section covering: * Randomness * Entity generation - * Entity processing \ No newline at end of file + * Entity processing diff --git a/CITATION.cff b/CITATION.cff index 6c82bde..4886c5f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -53,5 +53,5 @@ keywords: - simmer - rap license: MIT -version: '0.4.0' -date-released: '2026-01-06' \ No newline at end of file +version: '0.5.0' +date-released: '2026-02-20'