update transformerImpute branch

.Rbuildignore

@@ -12,3 +12,7 @@ pkgdown
 ^revdep$
 ^\.github$
 ^codecov\.yml$
+^R/test\.R0$
+^src/.*\.(gcda|gcno|o|so)$
+^CLAUDE\.md$
+^\.claude$

.github/workflows/test-coverage.yaml

@@ -4,9 +4,12 @@ on:
   push:
     branches: [main, master]
   pull_request:
-    branches: [main, master]
 
-name: test-coverage
+name: test-coverage.yaml
+
+permissions:
+  contents: read
+  id-token: write
 
 jobs:
   test-coverage:
@@ -15,36 +18,46 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: r-lib/actions/setup-r@v2
         with:
           use-public-rspm: true
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          extra-packages: any::covr
+          extra-packages: any::covr, any::xml2
           needs: coverage
 
       - name: Test coverage
         run: |
-          covr::codecov(
+          cov <- covr::package_coverage(
             quiet = FALSE,
             clean = FALSE,
             install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package")
           )
+          print(cov)
+          covr::to_cobertura(cov)
         shell: Rscript {0}
 
+      - uses: codecov/codecov-action@v5
+        with:
+          fail_ci_if_error: true
+          files: ./cobertura.xml
+          plugins: noop
+          disable_search: true
+          use_oidc: true
+
       - name: Show testthat output
         if: always()
         run: |
           ## --------------------------------------------------------------------
-          find ${{ runner.temp }}/package -name 'testthat.Rout*' -exec cat '{}' \; || true
+          find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true
         shell: bash
 
       - name: Upload test results
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-test-failures
           path: ${{ runner.temp }}/package

.gitignore

@@ -1,4 +1,12 @@
+.DS_Store
+inst/.DS_Store
 .Rproj.user
 *.Rproj
 .Rhistory
 docs
+src/*.gcda
+
+CLAUDE.md
+.claude
+VIM.Rcheck/
+VIM_*.tar.gz

DESCRIPTION

@@ -1,13 +1,17 @@
 Package: VIM
-Version: 6.2.4
+Version: 7.1.0
 Title: Visualization and Imputation of Missing Values
 Authors@R: c(
   person("Matthias", "Templ", email = "matthias.templ@gmail.com", role = c("aut","cre")),
   person("Alexander", "Kowarik", email = "alexander.kowarik@statistik.gv.at", role = c("aut"), comment=c(ORCID="0000-0001-8598-4130")),
   person("Andreas", "Alfons", role = c("aut")),
+  person("Johannes", "Gussenbauer", role = c("aut")),
+  person("Nina", "Niederhametner", role = c("aut")),
+  person("Eileen", "Vattheuer", role = c("aut")),
   person("Gregor", "de Cillia", email = "gregor.decillia@statistik.gv.at", role = c("aut")),
   person("Bernd", "Prantner", role = c("ctb")),
-  person("Wolfgang", "Rannetbauer", role = c("aut")))
+  person("Wolfgang", "Rannetbauer", role = c("aut"))
+  )
 Depends:
     R (>= 4.1.0),colorspace,grid
 Imports:
@@ -28,33 +32,54 @@ Imports:
     MASS,
     xgboost,
     data.table(>= 1.9.4),
-    keras,
-    tensorflow,
-    stringr
+    mlr3,
+    mlr3pipelines,
+    R6,
+    paradox,
+    mlr3tuning,
+    mlr3learners,
+    future
 Suggests: 
     dplyr,
     tinytest,
     knitr,
+    mgcv,
     rmarkdown,
     reactable,
     covr,
-    withr
-Description: New tools for the visualization of missing and/or imputed values
-    are introduced, which can be used for exploring the data and the structure of
-    the missing and/or imputed values. Depending on this structure of the missing
-    values, the corresponding methods may help to identify the mechanism generating
-    the missing values and allows to explore the data including missing values.
-    In addition, the quality of imputation can be visually explored using various
-    univariate, bivariate, multiple and multivariate plot methods. A graphical user
-    interface available in the separate package VIMGUI allows an easy handling of
-    the implemented plot methods.
+    withr,
+    pdist,
+    enetLTS,
+    robmixglm,
+    stringr,
+    glmnet
+Description: Provides methods for imputation and visualization of 
+    missing values. It includes graphical tools to explore the amount, structure 
+    and patterns of missing and/or imputed values, supporting exploratory 
+    data analysis and helping to investigate potential missingness mechanisms
+    (details in Alfons, Templ and Filzmoser, <doi:10.1007/s11634-011-0102-y>. 
+    The quality of imputations can be assessed visually using a wide range of 
+    univariate, bivariate and multivariate plots. 
+    The package further provides several imputation methods, 
+    including efficient implementations of k-nearest neighbour and hot-deck 
+    imputation (Kowarik and Templ 2013, <doi:10.18637/jss.v074.i07>, 
+    iterative robust model-based multiple 
+    imputation (Templ 2011, <doi:10.1016/j.csda.2011.04.012>; 
+    Templ 2023, <doi:10.3390/math11122729>), and machine learning–based 
+    approaches such as robust GAM-based multiple imputation 
+    (Templ 2024, <doi:10.1007/s11222-024-10429-1>) as well as gradient boosting 
+    (XGBoost) and transformer-based methods 
+    (Niederhametner et al., <doi:10.1177/18747655251339401>). 
+    General background and practical guidance on imputation are provided in the 
+    Springer book by 
+    Templ (2023) <doi:10.1007/978-3-031-30073-8>.
 LazyData: TRUE
 ByteCompile: TRUE
 License: GPL (>= 2)
 URL: https://github.com/statistikat/VIM
 Repository: CRAN
 LinkingTo: Rcpp
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.3
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE)
 VignetteBuilder: knitr

NAMESPACE

@@ -1,5 +1,7 @@
 # Generated by roxygen2: do not edit by hand
 
+S3method(Inter,list)
+S3method(Unit,list)
 S3method(plot,aggr)
 S3method(print,aggr)
 S3method(print,summary.aggr)
@@ -22,6 +24,8 @@ export(growdotMiss)
 export(histMiss)
 export(hotdeck)
 export(impPCA)
+export(imputeRobust)
+export(imputeRobustChain)
 export(initialise)
 export(irmi)
 export(kNN)
@@ -50,7 +54,7 @@ export(scattMiss)
 export(scattmatrixMiss)
 export(spineMiss)
 export(tableMiss)
-export(transformerImpute)
+export(vimpute)
 export(xgboostImpute)
 import(Rcpp)
 import(colorspace)
@@ -93,11 +97,38 @@ importFrom(graphics,text)
 importFrom(graphics,title)
 importFrom(laeken,weightedMean)
 importFrom(laeken,weightedMedian)
+importFrom(mlr3,LearnerClassif)
+importFrom(mlr3,LearnerRegr)
+importFrom(mlr3,PredictionClassif)
+importFrom(mlr3,PredictionRegr)
+importFrom(mlr3,TaskClassif)
+importFrom(mlr3,TaskRegr)
+importFrom(mlr3,as_task_classif)
+importFrom(mlr3,as_task_regr)
+importFrom(mlr3,lrn)
+importFrom(mlr3,msr)
+importFrom(mlr3,resample)
+importFrom(mlr3,rsmp)
+importFrom(mlr3learners,LearnerRegrCVGlmnet)
+importFrom(mlr3pipelines,"%>>%")
+importFrom(mlr3pipelines,GraphLearner)
+importFrom(mlr3pipelines,PipeOpModelMatrix)
+importFrom(mlr3pipelines,po)
+importFrom(mlr3tuning,TuningInstanceBatchSingleCrit)
+importFrom(mlr3tuning,tnr)
+importFrom(mlr3tuning,trm)
+importFrom(paradox,p_dbl)
+importFrom(paradox,p_fct)
+importFrom(paradox,p_int)
+importFrom(paradox,ps)
 importFrom(ranger,importance)
 importFrom(ranger,ranger)
+importFrom(robustbase,lmrob)
 importFrom(utils,capture.output)
 importFrom(utils,flush.console)
 importFrom(utils,head)
+importFrom(utils,modifyList)
 importFrom(vcd,labeling_border)
 importFrom(vcd,mosaic)
+importFrom(xgboost,xgboost)
 useDynLib(VIM)

NEWS.md

@@ -1,7 +1,18 @@
-# VIM 6.x.x
+# VIM 7.1.0
+- improve `vimpute()` compatibility and validation
+- make `rangerImpute()`, `xgboostImpute()`, and `regressionImp()` delegate to `vimpute()`
+- vimpute: fall back from regularized to robust models when too few predictor columns remain after preprocessing
+- fix documentation and package check issues around `vimpute()`
+- OpenMP is used in gowerD
+
+# VIM 7.0.0
+ - new function vimpute that uses `mlr3` backend for a flexible imputation method.
+
+# VIM 6.2.4
  - fix infinite loop in matchImpute in case all observations of a variable are missing
  - remove parameter metric from kNN because it was not used
  - add function xgboostImpute for using a simple xgboostModel to impute
+ - add imputeRobust function to impute numeric variables with robust methods (linear and non-linear ones)
 
 # VIM 6.2.3
 - default robust regression method for irmi for numeric variables changes from rlm to lmrob.

R/RcppExports.R

@@ -13,15 +13,3 @@ gowerDind <- function(dataX, dataY, weights, ncolNUMFAC, levOrders, mixedConstan
     .Call('_VIM_gowerDind', PACKAGE = 'VIM', dataX, dataY, weights, ncolNUMFAC, levOrders, mixedConstants, nR, returnMinR)
 }
 
-tokenpred_to_string_cpp <- function(probs, target_tok, sample_tok_probs) {
-    .Call('_VIM_tokenpred_to_string_cpp', PACKAGE = 'VIM', probs, target_tok, sample_tok_probs)
-}
-
-parallel_tokenpred_to_string <- function(probs, target_tok, sample_tok_probs) {
-    .Call('_VIM_parallel_tokenpred_to_string', PACKAGE = 'VIM', probs, target_tok, sample_tok_probs)
-}
-
-training_seq_cpp <- function(len_target, train_tokenized) {
-    .Call('_VIM_training_seq_cpp', PACKAGE = 'VIM', len_target, train_tokenized)
-}
-

R/VIM-package.R

@@ -17,12 +17,17 @@
 #' @importFrom laeken weightedMean
 #' @importFrom graphics Axis abline axTicks axis barplot box hist boxplot layout lcm lines locator par plot.new plot.window points
 #' @importFrom graphics polygon rect strheight strwidth text title
-#' @importFrom utils capture.output flush.console head
+#' @importFrom utils capture.output flush.console head modifyList
 #' @importFrom ranger ranger importance
 #' @importFrom MASS stepAIC lqs polr rlm
+#' @importFrom mlr3 LearnerRegr PredictionRegr LearnerClassif PredictionClassif lrn TaskRegr TaskClassif as_task_regr as_task_classif rsmp msr resample
+#' @importFrom mlr3pipelines PipeOpModelMatrix %>>% GraphLearner po
+#' @importFrom paradox ps p_fct p_dbl p_int
+#' @importFrom mlr3tuning tnr trm TuningInstanceBatchSingleCrit
+#' @importFrom mlr3learners LearnerRegrCVGlmnet
 #' @useDynLib VIM
 NULL
-
+utils::globalVariables(c("self", "super"))
 #' Animals_na
 #'
 #' @description Average log brain and log body weights for 28 Species
@@ -596,21 +601,36 @@ NULL
 #' handling of the plot methods.  In addition, `VIM` can be used for data
 #' from essentially any field.
 #'
+#' @title The VIM Package: Visualization and Imputation of Missing Values
 #' @name VIM-package
 #' @aliases VIM-package VIM
-"_PACKAGE"
+#' @description
+#' VIM provides tools for visualization, imputation, and exploration of missing 
+#' and multivariate data.
+#' @details
+#' This package includes advanced imputation methods, robust statistics, 
+#' and tools for data preprocessing and diagnostics.
 #' @author Matthias Templ, Andreas Alfons, Alexander Kowarik, Bernd Prantner
 #'
-#' Maintainer: Matthias Templ <templ@@tuwien.ac.at>
-#' @references M. Templ, A. Alfons, P. Filzmoser (2012) Exploring incomplete
+#' Maintainer: Matthias Templ <matthias.templ@@gmail.com>
+#' @references 
+#' M. Templ (2023) *Visualization and Imputation of Missing Values*. Springer Publishing.
+#' Series in Computational Statistics. Cham. Switzerland. 463 pages. 
+#' DOI: 10.1007/978-3-031-30073-8
+#' 
+#' A. Kowarik, M. Templ (2016) Imputation with
+#' R package VIM.  *Journal of
+#' Statistical Software*, 74(7), 1-16.
+#' 
+#' M. Templ, A. Alfons, P. Filzmoser (2012) Exploring incomplete
 #' data using visualization tools.  *Journal of Advances in Data Analysis
 #' and Classification*, Online first. DOI: 10.1007/s11634-011-0102-y.
 #'
 #' M. Templ, A. Kowarik, P. Filzmoser (2011) Iterative stepwise regression
 #' imputation using standard and robust methods.  *Journal of
 #' Computational Statistics and Data Analysis*, Vol. 55, pp. 2793-2806.
-#' @keywords package
-NULL
+#' @keywords internal
+"_PACKAGE"
 
 
 

R/evaluation.R

@@ -15,7 +15,6 @@
 #' regression imputation using standard and robust methods.  *Journal of
 #' Computational Statistics and Data Analysis*, Vol. 55, pp. 2793-2806.
 #' 
-# seealso \code{\link{robCompositions::rdcm}}
 #' @details This function has been mainly written for procudures 
 #' that evaluate imputation or replacement of rounded zeros. The ni parameter can thus, e.g. be
 #' used for expressing the number of rounded zeros.

R/gowerD.R

@@ -54,7 +54,7 @@ gowerD <- function(data.x, data.y = data.x,
       warning("The number of unique values in the ordinal variables in data.x
               does not correspond to the values given in levOrders")
     }
-    levOrdersUniqueY <- sapply(orders,function(x)length(unique(data.x[[x]])))
+    levOrdersUniqueY <- sapply(orders,function(x)length(unique(data.y[[x]])))
     if(any(levOrdersUniqueY!=levOrders)){
       warning("The number of unique values in the ordinal variables in data.y
               does not correspond to the values given in levOrders")