diff --git a/.Rbuildignore b/.Rbuildignore index 43240d6..0b02d29 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,126 +1,202 @@ -^.*\.Rproj$ -^\.Rproj\.user$ -^\.git$ -^\.github$ -^\.gitignore$ -^\.gitattributes$ -^\.flox$ -^\.claude$ -^_bookdown_files$ -^rsconnect$ -^\.Rhistory$ -^\.RData$ -^.*_cache$ -^.*_files$ -^RPubs -^data-raw$ -^analysis$ - -# Large data files (not included in package) -^CohortData_.*\.csv$ -^CensusTractData\.csv$ -^.*\.geojson$ -^TRACT_ZIP\.csv$ -^energy_burden_zip.*\.csv$ -^data/ - -# Analysis scripts and outputs (keep in repo but not package) -^analysis/ -^research/ -^deprecated/ -^all_utilities_energy_burden\.R$ -^nc_all_utilities_energy_burden\.R$ -^nc_cooperatives_energy_burden\.R$ -^render_.*_tables\.R$ -^format_.*_table\.R$ -^all_utilities_.*\.(csv|html|tex|md)$ -^nc_all_utilities_.*\.(csv|html|tex|md)$ -^nc_cooperatives_.*\.(csv|html|tex|md)$ -^ANALYSIS_COMPLETE\.md$ - -# Rmd research files (not vignettes) -^.*\.Rmd$ -^.*\.qmd$ -^.*\.log$ -^.*\.nav$ -^.*\.snm$ -^.*\.toc$ -^.*\.tex$ -^preamble-latex\.tex$ - -# Generated outputs -^.*\.pdf$ -^.*\.html$ -^.*\.docx$ -^.*\.odt$ -^.*\.pptx$ -^.*\.svg$ -^.*\.png$ -^.*\.jpg$ - -# Shiny app (separate from package) -^ui\.R$ -^server\.R$ -^global\.R$ - -# Development/project files -^sources\.R$ -^render\.R$ -^libraries\.R$ -^comparison_table\.R$ -^paper_source\.R$ -^poster_.*\.R$ -^.*_munging\.R$ -^.*_slides\.R.*$ -^.*_poster\.R.*$ -^MarketFigures\.R$ -^methods\.R$ -^figures\.R$ -^usurdb_.*\.(R|py|db)$ -^energy_poverty_dashboard\.R$ - -# Config files -^config.*\.yml$ -^\.env$ -^\.private$ - -# Lua filters -^.*\.lua$ - -# Bibliography -^.*\.bib$ -^.*\.csl$ -^.*\.bst$ - -# Misc -^README\.md$ -^NEWS\.md$ -^cran-comments\.md$ -^pkgdown$ -^_pkgdown\.yml$ -^\.lintr$ -^codecov\.yml$ -^\.dev$ - -# Additional project files -^PACKAGE_TRANSFORMATION\.md$ -^QUICK_START\.md$ -^STATUS\.md$ -^cleanup_conflicts\.R$ -^helpers\.R$ -^ratios\.R$ -^comparison_table\.csv$ -^market_sums\.csv$ -^nerc_state_mapping\.(csv|ods)$ -^poster_results\.RData$ -^poster_test\.md$ -^svglov3\.clo$ -^svjour3\.cls$ -^usurdb\.db$ -^NatureComm_figure-latex/ - -# Package check fixes -^\.zenodo\.json$ -^CODE_OF_CONDUCT\.md$ -^CONTRIBUTING\.md$ -^docs$ +^.*\.Rproj$ +^\.Rproj\.user$ +^\.git$ +^\.github$ +^\.gitignore$ +^\.gitattributes$ +^\.flox$ +^\.claude$ +^_bookdown_files$ +^rsconnect$ +^\.Rhistory$ +^\.RData$ +^.*_cache$ +^.*_files$ +^RPubs +^data-raw$ +^analysis$ + +# Large data files (not included in package) +^CohortData_.*\.csv$ +^CensusTractData\.csv$ +^.*\.geojson$ +^TRACT_ZIP\.csv$ +^energy_burden_zip.*\.csv$ +^data/.*\.csv$ +^data/.*\.xlsx$ +^data/.*\.xls$ +^data/.*\.db$ +^data/.*\.sqlite$ +^data/.*\.gdb$ +^data/.*\.shp$ +^data/.*\.dbf$ +^data/.*\.prj$ +^data/.*\.sbn$ +^data/.*\.sbx$ +^data/.*\.shx$ +^data/.*\.xml$ +^data/.*\.cpg$ +^data/.*\.pdf$ +^data/.*\.json$ +^data/BIA_National_LAR_shp/ +^data/calenviroscreen40gdb_F_2021\.gdb/ +^data/dsire/ +^data/f8612018/ +^data/f8612022/ +^data/GRF21/ +^data/ICADisplay\.gdb/ +^data/nerc_regions\.gdb/ +^data/PGE_DIDF_Tables_Public/ +^data/PGE_POSTSR2A_3-1-2022\.gdb/ +^data/\.archived_bad_data/ + +# Analysis scripts and outputs (keep in repo but not package) +^analysis/ +^research/ +^deprecated/ +^all_utilities_energy_burden\.R$ +^nc_all_utilities_energy_burden\.R$ +^nc_cooperatives_energy_burden\.R$ +^render_.*_tables\.R$ +^format_.*_table\.R$ +^all_utilities_.*\.(csv|html|tex|md)$ +^nc_all_utilities_.*\.(csv|html|tex|md)$ +^nc_cooperatives_.*\.(csv|html|tex|md)$ +^ANALYSIS_COMPLETE\.md$ + +# Rmd research files (not vignettes) +# Exclude .Rmd files in specific directories, but NOT in vignettes/ +^research/.*\.Rmd$ +^analysis/.*\.Rmd$ +^\.dev/.*\.Rmd$ +^deprecated/.*\.Rmd$ +^.*\.qmd$ +^.*\.log$ +^.*\.nav$ +^.*\.snm$ +^.*\.toc$ +^.*\.tex$ +^preamble-latex\.tex$ + +# Generated outputs (but not in vignettes/) +^research/.*\.pdf$ +^analysis/.*\.pdf$ +^\.dev/.*\.pdf$ +^deprecated/.*\.pdf$ +^research/.*\.html$ +^analysis/.*\.html$ +^\.dev/.*\.html$ +^deprecated/.*\.html$ +^.*\.docx$ +^.*\.odt$ +^.*\.pptx$ +^research/.*\.svg$ +^analysis/.*\.svg$ +^\.dev/.*\.svg$ +^deprecated/.*\.svg$ +^research/.*\.png$ +^analysis/.*\.png$ +^\.dev/.*\.png$ +^deprecated/.*\.png$ +^research/.*\.jpg$ +^analysis/.*\.jpg$ +^\.dev/.*\.jpg$ +^deprecated/.*\.jpg$ + +# Shiny app (separate from package) +^ui\.R$ +^server\.R$ +^global\.R$ + +# Development/project files +^sources\.R$ +^render\.R$ +^libraries\.R$ +^comparison_table\.R$ +^paper_source\.R$ +^poster_.*\.R$ +^.*_munging\.R$ +^.*_slides\.R.*$ +^.*_poster\.R.*$ +^MarketFigures\.R$ +^methods\.R$ +^figures\.R$ +^usurdb_.*\.(R|py|db)$ +^energy_poverty_dashboard\.R$ + +# Config files +^config.*\.yml$ +^\.env$ +^\.private$ + +# Lua filters +^.*\.lua$ + +# Bibliography (exclude from non-vignette directories) +^research/.*\.bib$ +^analysis/.*\.bib$ +^deprecated/.*\.bib$ +^\.dev/.*\.bib$ +^.*\.csl$ +^.*\.bst$ + +# Misc +^README\.md$ +^NEWS\.md$ +^cran-comments\.md$ +^pkgdown$ +^_pkgdown\.yml$ +^\.lintr$ +^TESTING_REPORT\.md$ +^WORKFLOW-DEPLOYMENT-GUIDE\.md$ +^controlled-release-public\.yaml$ +^cran-check-output\.txt$ +^codecov\.yml$ +^\.dev$ + +# Additional project files +^PACKAGE_TRANSFORMATION\.md$ +^QUICK_START\.md$ +^STATUS\.md$ +^zenodo-upload$ +^zenodo-upload-nationwide$ +^data/zenodo-upload-nationwide$ +^cleanup_conflicts\.R$ +^helpers\.R$ +^ratios\.R$ +^comparison_table\.csv$ +^market_sums\.csv$ +^nerc_state_mapping\.(csv|ods)$ +^poster_results\.RData$ +^poster_test\.md$ +^svglov3\.clo$ +^svjour3\.cls$ +^usurdb\.db$ +^NatureComm_figure-latex/ + +# Package check fixes +^\.zenodo\.json$ +^CODE_OF_CONDUCT\.md$ +^CONTRIBUTING\.md$ +^docs$ + +# Top-level presentation/poster files (not part of package) +^.*_poster\.html$ +^.*_poster\.knit\.html$ +^.*_poster\.pdf$ +^.*_slides\.pdf$ +^.*_poster_map\.svg$ +^.*_poster_map\.pdf$ +^poster_.*\.svg$ +^.*_dashboard\.html$ +^.*_snapshot\.html$ +^net_energy_equity.*\.pdf$ +^UNC_logo.*\.(png|svg)$ +^qr\.svg$ +^Rplots\.pdf$ +^state_prosumer_ratios.*\.png$ +^.*_region_poster_map\.svg$ +^CRAN-SUBMISSION$ +^LICENSE$ +^doc$ +^Meta$ diff --git a/.dev/CRAN-SUBMISSION-GUIDE.md b/.dev/CRAN-SUBMISSION-GUIDE.md new file mode 100644 index 0000000..d321675 --- /dev/null +++ b/.dev/CRAN-SUBMISSION-GUIDE.md @@ -0,0 +1,358 @@ +# Complete CRAN Submission Workflow Guide + +This guide walks through the full process for submitting the `emburden` package to CRAN, from local validation to automated submission. + +## Overview + +The CRAN submission process has multiple validation layers: + +``` +Local Pre-flight → Git Push → Auto-tag → GitHub Actions → Win-builder → Manual Approval → Auto-submit +``` + +## Prerequisites + +### 1. GitHub Setup (One-time) + +- **GitHub Environment**: `cran-production` with manual approval requirement + - Go to repository **Settings** → **Environments** + - Create `cran-production` environment + - Add yourself as required reviewer + +- **GitHub Secrets**: + - `CRAN_EMAIL`: Your CRAN maintainer email address + - `PUBLIC_REPO_TOKEN`: Personal Access Token for triggering workflows + +### 2. Local Setup (One-time) + +- R packages: `devtools`, `rcmdcheck`, `usethis` +- Git configured with your name and email +- Optional: Set `CRAN_EMAIL` environment variable locally + +```bash +# In your ~/.bashrc or ~/.zshrc +export CRAN_EMAIL="your.email@example.com" +``` + +## Full CRAN Release Process + +### Step 1: Pre-flight Checks (Local) + +Before making any version changes, ensure your package is CRAN-ready: + +```bash +# Validate GitHub Actions workflow files +bash .dev/validate-workflows.sh + +# Run comprehensive local validation +Rscript .dev/pre-tag-cran-check.R + +# Optional: Also submit to Win-builder for Windows testing +Rscript .dev/pre-tag-cran-check.R --submit-winbuilder +``` + +This validation will: +- ✅ Validate GitHub Actions workflow YAML syntax (prevents tagging errors) +- ✅ Check version consistency across files +- ✅ Validate NEWS.md is updated +- ✅ Check git status +- ✅ Build source package +- ✅ Run R CMD check --as-cran +- ✅ Optionally submit to Win-builder + +**If any checks fail, fix them before proceeding!** + +**Note**: The workflow validation is critical - it prevents YAML syntax errors in GitHub Actions workflows from blocking the automated tagging and release process. The auto-tag workflow will also validate workflows before creating version tags as a safety gate. + +### Step 2: Update Version and Documentation + +Update three key files: + +#### a. DESCRIPTION +```r +Version: 0.5.8 # Increment version +``` + +#### b. NEWS.md +```markdown +# emburden 0.5.8 + +## New Features +- Added new functionality... + +## Bug Fixes +- Fixed issue with... + +## Documentation +- Updated vignette... +``` + +#### c. inst/CITATION +```r +note = "R package version 0.5.8", +``` + +### Step 3: Run Pre-tag Validation Again + +After version updates, validate everything again: + +```bash +Rscript .dev/pre-tag-cran-check.R --submit-winbuilder +``` + +If Win-builder is enabled, wait ~30 minutes for email results before proceeding. + +### Step 4: Commit and Push Version Bump + +```bash +git add DESCRIPTION NEWS.md inst/CITATION +git commit -m "Bump version to 0.5.8 for CRAN submission" +git push +``` + +**This will automatically trigger:** +1. **auto-tag-on-version-bump.yml** - Creates `v0.5.8` tag +2. **auto-release.yml** - Creates GitHub release +3. **publish-to-public.yml** - Syncs to public repository +4. **cran-release.yml** - Starts CRAN validation workflow + +### Step 5: Monitor GitHub Actions + +The automated workflow will: + +#### Phase 1: Validation (5-10 minutes) +```bash +# Check workflow status +gh run list --workflow=cran-release.yml + +# Watch live +gh run watch +``` + +The validation phase: +- ✅ Builds source package (.tar.gz) +- ✅ Runs R CMD check --as-cran +- ✅ Submits to Win-builder (optional, ~30 min for results) +- ✅ Uploads artifacts + +#### Phase 2: Manual Approval (Human Required) + +GitHub will notify you when validation completes. + +1. Go to **Actions** tab in GitHub +2. Click the running `CRAN Release` workflow +3. Review the check results +4. Click **Review deployments** +5. Select `cran-production` +6. Click **Approve and deploy** + +**Before approving, verify:** +- ✅ R CMD check passed (0 errors, 0 warnings) +- ✅ Win-builder results received (check email) +- ✅ Package tarball uploaded +- ✅ All files up to date + +#### Phase 3: Auto-submission (Automated) + +After approval, the workflow automatically: +- ✅ Downloads validated tarball +- ✅ Generates CRAN submission comments +- ✅ Submits to CRAN via `devtools::submit_cran()` +- ✅ Creates GitHub release with tarball + +### Step 6: CRAN Response + +Within minutes to hours, you'll receive an email from CRAN: + +**Possible responses:** + +1. **Auto-check success** → Package accepted, published within 1-3 days +2. **Auto-check issues** → Fix and resubmit +3. **Manual review required** → CRAN team will email feedback + +Monitor at: +- **CRAN Incoming**: https://cran.r-project.org/incoming/ +- **Package page**: https://cran.r-project.org/web/packages/emburden/ + +## Repository Structure + +This workflow works across your private and public repositories: + +``` +Private: ScheierVentures/emburden (working repo) + ↓ (publish-to-public.yml) +Public: ericscheier/emburden (CRAN submission happens here) +``` + +**Important**: The CRAN release workflow runs on the **public repository** after the tag is synced from private. + +## Quick Reference Commands + +```bash +# Validate GitHub Actions workflows (prevents tagging issues) +bash .dev/validate-workflows.sh + +# Pre-flight validation +Rscript .dev/pre-tag-cran-check.R --submit-winbuilder + +# Version consistency check only +Rscript .dev/check-version-consistency.R + +# Manual Win-builder submission +Rscript -e "devtools::check_win_release(email = Sys.getenv('CRAN_EMAIL'))" + +# Check workflow status +gh run list --workflow=cran-release.yml +gh run view + +# Check existing tags +git tag -l "v*" + +# Check existing releases +gh release list +``` + +## Triggering CRAN Submission for Existing Version + +If you already have a tagged version (e.g., v0.5.7) and want to submit it to CRAN: + +### Option 1: Manual Workflow Trigger (Safest) + +```bash +# Trigger the workflow manually on the public repo +gh workflow run cran-release.yml --repo ericscheier/emburden +``` + +Then approve when validation completes. + +### Option 2: Re-push Existing Tag + +```bash +# Delete and recreate tag (forces workflow to run) +git tag -d v0.5.7 +git push origin :refs/tags/v0.5.7 +git tag -a v0.5.7 -m "Release v0.5.7" +git push origin v0.5.7 +``` + +**Note**: Only do this if the tag hasn't been used for CRAN submission yet. + +### Option 3: Wait for Next Version + +If v0.5.7 has issues or you want to test the full workflow: +- Bump to v0.5.8 +- Go through the complete process + +## Troubleshooting + +### Workflow Not Triggered + +**Problem**: Version bump pushed but no tag created + +**Solution**: Check auto-tag-on-version-bump workflow: +```bash +gh run list --workflow=auto-tag-on-version-bump.yml +``` + +The workflow requires: +- Version changed in DESCRIPTION +- Push to main branch +- PUBLIC_REPO_TOKEN configured + +### CRAN Check Failures + +**Problem**: R CMD check fails with errors/warnings + +**Solution**: +1. Download check results artifact from GitHub Actions +2. Review `00check.log` and `00install.out` +3. Fix issues locally +4. Re-run pre-tag validation +5. Bump version and retry + +### Win-builder Issues + +**Problem**: Win-builder email shows errors + +**Solution**: +- Win-builder is optional for approval decision +- Common issues: Windows-specific path problems, missing system deps +- If critical, fix and resubmit with new version +- If minor, note in CRAN submission comments + +### Approval Timeout + +**Problem**: Didn't approve within timeout window + +**Solution**: +```bash +# Re-run the workflow +gh run rerun + +# Or create a new version tag +git push origin v0.5.8 --force # if same version +# OR +# Bump to v0.5.9 and push +``` + +### CRAN Submission Failed + +**Problem**: devtools::submit_cran() failed + +**Solution**: +1. Check error in workflow logs +2. Common causes: network issues, CRAN temporarily down +3. Manual submission: + ```bash + # Download tarball from GitHub release + wget https://github.com/ericscheier/emburden/releases/download/v0.5.8/emburden_0.5.8.tar.gz + + # Submit manually + Rscript -e "devtools::submit_cran('emburden_0.5.8.tar.gz', email = Sys.getenv('CRAN_EMAIL'))" + ``` + +## Best Practices + +1. **Test locally first**: Always run pre-tag validation before pushing +2. **Check Win-builder**: Use `--submit-winbuilder` at least once before submission +3. **Version consistently**: Update all three files (DESCRIPTION, NEWS.md, CITATION) +4. **Review before approval**: Don't auto-approve; check results +5. **Timing**: CRAN prefers submissions no more than once per 1-2 months +6. **Communication**: Respond promptly to CRAN reviewer feedback + +## Timeline Example + +``` +10:00 - Run pre-tag-cran-check.R locally +10:10 - Update version files (DESCRIPTION, NEWS.md, CITATION) +10:15 - Commit and push version bump +10:16 - Auto-tag creates v0.5.8 tag +10:17 - Publish-to-public syncs to public repo +10:18 - CRAN release workflow starts validation +10:25 - Validation complete (R CMD check passed) +10:25 - Win-builder submission sent (email arrives ~10:55) +10:55 - Review Win-builder results +11:00 - Approve deployment in GitHub +11:01 - Auto-submit to CRAN +11:02 - GitHub release created +11:05 - CRAN confirmation email received +11:30 - CRAN auto-check email (success/failure) +``` + +**Total time**: ~1-1.5 hours from start to CRAN submission + +## Related Documentation + +- `.github/workflows/CRAN-RELEASE.md` - Workflow technical details +- `.dev/pre-tag-cran-check.R` - Local validation script +- `.dev/check-version-consistency.R` - Version consistency checker +- [CRAN Repository Policy](https://cran.r-project.org/web/packages/policies.html) +- [devtools documentation](https://devtools.r-lib.org/) + +## Questions? + +If you encounter issues not covered here: +1. Check workflow logs in GitHub Actions +2. Review CRAN emails carefully +3. Consult CRAN policy documentation +4. For workflow issues, check `.github/workflows/` configuration diff --git a/.dev/FEATURE_INTEGRATION_CHECKLIST.md b/.dev/FEATURE_INTEGRATION_CHECKLIST.md new file mode 100644 index 0000000..07cdb3d --- /dev/null +++ b/.dev/FEATURE_INTEGRATION_CHECKLIST.md @@ -0,0 +1,214 @@ +# Feature Integration Checklist + +**Feature Name:** _[e.g., housing-dimensions, temporal-comparison, etc.]_ +**Branch:** _[e.g., feature/housing-dimensions]_ +**Target:** `dev` +**Date:** _[YYYY-MM-DD]_ +**Author:** _[Your name]_ + +--- + +## Overview + +Brief description of what this feature does and why it's being added: + +_[2-3 sentences describing the feature]_ + +--- + +## Phase 1: Testing + +### Test Coverage +- [ ] Created dedicated test file(s) in `tests/testthat/` + - [ ] File(s): `_______________________` + - [ ] Number of new test cases: `_______` +- [ ] Updated existing test files (if applicable) + - [ ] File(s): `_______________________` + - [ ] Number of new test cases: `_______` +- [ ] All tests pass locally (`devtools::test()`) + - [ ] Total tests: `_______` + - [ ] Pass: `_______` + - [ ] Fail: `_______` + - [ ] Skip: `_______` + +### Test Categories Covered +- [ ] Unit tests for new functions +- [ ] Integration tests for feature workflow +- [ ] Edge cases and error handling +- [ ] Backward compatibility tests +- [ ] Performance tests (if applicable) + +--- + +## Phase 2: Documentation + +### Function Documentation +- [ ] Added/updated roxygen2 documentation + - [ ] File(s): `_______________________` + - [ ] All parameters documented with `@param` + - [ ] Return values documented with `@return` + - [ ] Examples added with `@examples` + - [ ] Export status correct (`@export` or not) + +### Vignettes +- [ ] Updated existing vignette(s) + - [ ] Vignette(s): `_______________________` + - [ ] Added section(s): `_______________________` +- [ ] Created new vignette (if needed) + - [ ] Vignette: `_______________________` + +### Package Metadata +- [ ] Updated `NEWS.md` with changelog entry + - [ ] Version: `_______` + - [ ] Category: `_______` (New Features / Enhancements / Bug Fixes) + - [ ] Entry: `_______________________` +- [ ] Updated `DESCRIPTION` (if needed) + - [ ] Version bumped: `_______` + - [ ] Dependencies updated: `_______` +- [ ] Updated `inst/CITATION` (if needed) + - [ ] What changed: `_______________________` + +### Other Documentation +- [ ] Updated README.md (if needed) +- [ ] Created/updated technical documentation in `.dev/` +- [ ] Added code comments for complex logic + +--- + +## Phase 3: Validation + +### Local Checks +- [ ] `devtools::document()` completed successfully +- [ ] `devtools::test()` completed successfully + - [ ] 0 errors + - [ ] 0 failures + - [ ] Acceptable warnings/notes (document below) +- [ ] `devtools::check()` completed successfully + - [ ] 0 errors + - [ ] Warnings: `_______` (acceptable: qpdf, time verification) + - [ ] Notes: `_______` + +### Code Quality +- [ ] No hardcoded file paths (use system.file, tempdir, etc.) +- [ ] No browser() or debug statements left in code +- [ ] Consistent coding style with package conventions +- [ ] Appropriate error messages and warnings +- [ ] No security vulnerabilities (XSS, SQL injection, etc.) + +--- + +## Phase 4: Git Integration + +### Branch Management +- [ ] Current branch: `_______` +- [ ] Target branch: `dev` +- [ ] Branch is up to date with latest `dev` + - [ ] Ran: `git pull origin dev` + - [ ] Resolved any merge conflicts + +### Commit Preparation +- [ ] Reviewed all changed files + - [ ] Number of files changed: `_______` + - [ ] Files reviewed: YES / NO +- [ ] No unintended changes included +- [ ] No secrets or credentials in commits +- [ ] Removed debugging code and commented-out blocks + +### Commit +- [ ] Staged appropriate files +- [ ] Created commit with descriptive message + - [ ] Commit message: `_______________________` + - [ ] Follows convention: `feat:` / `fix:` / `docs:` / `refactor:` + - [ ] Commit SHA: `_______________________` + +--- + +## Phase 5: CI/CD Validation + +### Pre-Push +- [ ] Reviewed git log to verify commit history +- [ ] Confirmed no sensitive information in diff +- [ ] Ready to push to remote + +### Push and CI +- [ ] Pushed to remote: `git push origin [branch]` +- [ ] CI workflow triggered on GitHub Actions + - [ ] Workflow run ID: `_______________________` + - [ ] Workflow URL: `_______________________` + +### CI Results +- [ ] R CMD check passed on all platforms + - [ ] Ubuntu: PASS / FAIL + - [ ] macOS: PASS / FAIL + - [ ] Windows: PASS / FAIL +- [ ] Test coverage workflow passed + - [ ] Coverage %: `_______` + - [ ] Coverage change: `_______` +- [ ] All checks green + +### CI Issues (if any) +_[Document any CI failures and how they were resolved]_ + +--- + +## Phase 6: Staging Preparation + +### Ready for Staging +- [ ] All CI checks passing +- [ ] Feature complete and tested +- [ ] Documentation complete +- [ ] No known issues or TODOs +- [ ] Reviewed by team (if applicable) + +### Merge to Staging (when ready) +- [ ] Create PR: `dev` → `staging` + - [ ] PR number: `_______` + - [ ] PR URL: `_______________________` +- [ ] PR description includes feature summary and testing notes +- [ ] Linked to any relevant issues +- [ ] Ready for review + +--- + +## Additional Notes + +### Known Issues or Limitations +_[Document any known issues, limitations, or future improvements]_ + +### Dependencies +_[List any dependencies on other features or external packages]_ + +### Breaking Changes +_[Document any breaking changes and migration path]_ + +### Performance Impact +_[Note any performance implications, positive or negative]_ + +--- + +## Completion + +**Integration Status:** ⬜ In Progress / ⬜ Complete / ⬜ Blocked + +**Completed by:** _[Name]_ +**Date completed:** _[YYYY-MM-DD]_ +**Final commit SHA:** _[SHA]_ + +--- + +## Usage Example + +For future reference, run the integration protocol with: + +```bash +# Automated integration check +Rscript .dev/integrate-feature-to-dev.R --feature-name "your-feature-name" + +# Skip R CMD check for faster iteration +Rscript .dev/integrate-feature-to-dev.R --feature-name "your-feature-name" --skip-check +``` + +--- + +**Template Version:** 1.0 +**Last Updated:** 2025-11-25 diff --git a/.dev/NC-TO-NATIONWIDE-TRANSITION.md b/.dev/NC-TO-NATIONWIDE-TRANSITION.md new file mode 100644 index 0000000..914dda7 --- /dev/null +++ b/.dev/NC-TO-NATIONWIDE-TRANSITION.md @@ -0,0 +1,229 @@ +# NC → Nationwide Transition Plan + +**Goal**: Update package from NC-focused to nationwide US (51 states) + +**Status**: Zenodo nationwide data uploaded ✓, now updating code/docs + +--- + +## Phase 1: CRITICAL (Required for next release) + +### 1.1 Core Documentation +- [ ] `README.md` - Change primary examples from NC to nationwide + - Update intro paragraph + - Change example from `states = "NC"` to multi-state or no filter + - Update data availability statement + +- [ ] `R/zenodo.R` - Already updated ✓ + - Description changed from "NC" to "US Nationwide" ✓ + +### 1.2 Primary Tests +- [ ] `tests/testthat/test-zenodo-download.R` + - Currently tests NC data + - Change to test nationwide data availability + +- [ ] `tests/testthat/test-data-loaders.R` + - Update tests to use multiple states instead of just NC + - Keep NC as ONE example, but not the ONLY example + +### 1.3 Function Documentation +- [ ] `R/compare_burden.R` + `man/compare_energy_burden.Rd` + - Examples use `states = "NC"` + - Add examples with multiple states: `states = c("NC", "CA", "TX")` + +- [ ] `R/lead_data_loaders.R` + man files + - `load_cohort_data()` examples + - `load_census_tract_data()` examples + +--- + +## Phase 2: IMPORTANT (Should do soon) + +### 2.1 Vignettes +- [ ] `vignettes/getting-started.Rmd` + - Primary vignette - should showcase nationwide capability + - Keep NC examples but add nationwide examples + +- [ ] `vignettes/jss-emburden.Rmd` + - JSS manuscript - can stay NC-focused as case study + - Add note that nationwide data available + +### 2.2 Sample Data +- [ ] Keep `orange_county_sample` (NC) for offline demos +- [ ] Consider adding CA or TX sample data for diversity +- [ ] Update `R/data.R` documentation + +### 2.3 Development Docs +- [ ] `.dev/ZENODO_UPLOAD_GUIDE.md` - Update from NC to nationwide +- [ ] `.dev/TEST_ZENODO_DOWNLOAD.md` - Already updated +- [ ] `NEWS.md` - Add nationwide transition notes for v0.4.8 + +--- + +## Phase 3: OPTIONAL (Future releases) + +### 3.1 Analysis Scripts (in `analysis/`) +These are research outputs, can stay as-is: +- `nc_all_utilities_energy_burden.R` +- `nc_cooperatives_energy_burden.R` +- `nc_comparison_for_email.R` + +### 3.2 Research Materials (in `research/`) +These are historical, can stay as-is: +- Manuscripts +- Presentations +- Posters + +--- + +## Key Files to Update (Priority Order) + +### Must Update (v0.4.8): +1. `README.md` - Primary package introduction +2. `R/compare_burden.R` - Add nationwide examples +3. `R/lead_data_loaders.R` - Add nationwide examples +4. `man/*.Rd` - Regenerate with roxygen2 +5. `tests/testthat/test-data-loaders.R` - Expand tests +6. `NEWS.md` - Document transition + +### Should Update (v0.4.9): +7. `vignettes/getting-started.Rmd` - Primary tutorial +8. `vignettes/methodology.Rmd` - If it has examples + +### Can Keep as NC: +- `orange_county_sample` data +- JSS vignette (as case study) +- Analysis scripts +- Research materials + +--- + +## Implementation Strategy + +### Option A: Gradual Transition (RECOMMENDED) +- **v0.4.8**: Update core docs + examples to nationwide, keep NC as one example +- **v0.4.9**: Expand vignettes to showcase nationwide analysis +- **v0.5.0**: Full nationwide focus, NC is just one state among many + +### Option B: Immediate Transition +- All at once in v0.4.8 +- More risky, harder to review + +### Option C: Dual Focus +- Keep showing NC examples (familiar, small, fast) +- Add nationwide examples alongside +- Best of both worlds + +--- + +## Specific Changes Needed + +### README.md +```r +# BEFORE +data <- load_cohort_data("fpl", "2022", states = "NC") + +# AFTER (show both!) +# Example 1: Single state (fast, good for learning) +nc_data <- load_cohort_data("fpl", "2022", states = "NC") + +# Example 2: Multiple states +southeast <- load_cohort_data("fpl", "2022", states = c("NC", "SC", "GA", "FL")) + +# Example 3: Nationwide (all 51 states) +us_data <- load_cohort_data("fpl", "2022") # No filter = all states +``` + +### Test Files +```r +# BEFORE +test_that("can load NC data", { + data <- load_cohort_data("fpl", "2022", states = "NC") + expect_true(nrow(data) > 0) +}) + +# AFTER +test_that("can load nationwide data", { + # Test single state + nc <- load_cohort_data("fpl", "2022", states = "NC") + expect_true(nrow(nc) > 0) + + # Test multiple states + multi <- load_cohort_data("fpl", "2022", states = c("NC", "CA", "TX")) + expect_true(nrow(multi) > nrow(nc)) + expect_equal(length(unique(multi$state_abbr)), 3) + + # Test all states + all_states <- load_cohort_data("fpl", "2022") + expect_true(nrow(all_states) > 500000) # Should be ~588k + expect_equal(length(unique(all_states$state_abbr)), 51) +}) +``` + +--- + +## Regression Concerns + +### What Could Break? +1. **Orange County sample data** - Still works, no changes needed +2. **Existing user scripts** - Still work, just with more data available +3. **Vignettes** - May need data size notes for nationwide examples +4. **Tests** - Need to handle larger datasets in tests + +### Migration Path for Users +- **No breaking changes** - NC data still available +- **More options** - Can now access any state or nationwide +- **Same API** - `states =` parameter works same way + +--- + +## Timeline + +### v0.4.8 (Current Release) +- [x] Upload nationwide data to Zenodo +- [x] Update R/zenodo.R +- [ ] Update README primary examples +- [ ] Update function examples in R/*.R +- [ ] Update core tests +- [ ] Update NEWS.md + +### v0.4.9 (Next Release) +- [ ] Expand vignettes with nationwide examples +- [ ] Add multi-state comparison examples +- [ ] Performance guide for large queries + +### v0.5.0 (Future) +- [ ] Full nationwide focus in all documentation +- [ ] Remove "proof of concept" language +- [ ] CRAN submission ready + +--- + +## Questions to Resolve + +1. **Should we keep Orange County sample data?** + - YES - valuable for offline demos, small size + +2. **Should tests download nationwide data?** + - NO - too slow, use mock data or small subsets + - YES - for integration tests (mark as slow) + +3. **Should default be NC or nationwide?** + - Nationwide - shows full capability + - But provide NC examples for learning + +4. **How to handle large datasets in examples?** + - Use `states = c("NC", "CA")` for speed + - Note that nationwide is available + - Show how to filter results + +--- + +## Success Metrics + +- [ ] README mentions nationwide in first paragraph +- [ ] All function examples work with multiple states +- [ ] Tests cover nationwide data loading +- [ ] Vignettes show nationwide capability +- [ ] No references to "proof of concept" or "NC only" +- [ ] Users can discover nationwide data easily diff --git a/.dev/PER-STATE-CACHING-PROPOSAL.md b/.dev/PER-STATE-CACHING-PROPOSAL.md new file mode 100644 index 0000000..ca39b38 --- /dev/null +++ b/.dev/PER-STATE-CACHING-PROPOSAL.md @@ -0,0 +1,258 @@ +# Per-State Caching Architecture Proposal + +## Problem + +Currently, if a nationwide dataset is missing just 1-2 states (e.g., FPL 2022 missing HI and IL), we must re-download all 51 states (~12GB, 30-60 minutes) instead of just the missing states. + +**Root Cause**: Individual state ZIP files are not cached - they're downloaded, extracted, then deleted. + +## Proposed Architecture + +### 1. Cache Structure + +``` +~/.cache/emburden/ +├── lead_2022_fpl_AL.zip # Individual state ZIPs (kept!) +├── lead_2022_fpl_AK.zip +├── lead_2022_fpl_... +├── lead_2022_fpl_HI.zip # Missing state +├── lead_2022_fpl_IL.zip # Missing state +├── lead_2022_fpl_WY.zip +├── lead_2022_fpl.csv # Merged nationwide CSV +└── emburden_db.sqlite # Database with nationwide data +``` + +### 2. Smart Download Logic + +#### Before (Current): +```r +download_and_merge_states() { + for each state in all 51 states: + download ZIP + extract CSV + delete ZIP # ❌ Lost! + merge all CSVs + save merged CSV +} +``` + +#### After (Proposed): +```r +download_and_merge_states() { + # 1. Check which states are already cached + cached_states <- check_cached_state_files(dataset, vintage) + missing_states <- setdiff(all_states, cached_states) + + # 2. Only download missing states + for each state in missing_states: + download ZIP to state-specific file (e.g., lead_2022_fpl_HI.zip) + keep ZIP for future use # ✅ Cached! + + # 3. Load all states (cached + newly downloaded) + for each state in all 51 states: + if (state ZIP exists): + extract and load data + else: + skip (log warning) + + # 4. Merge and validate + merge all loaded states + if (missing states): + report which states are missing + save merged CSV +} +``` + +### 3. Validation & Self-Healing + +When validation detects corrupt/incomplete nationwide data: + +```r +# Current behavior: +clear_dataset_cache("fpl", "2022") # Deletes EVERYTHING +re-download all 51 states # 12GB download + +# Proposed behavior: +detect_missing_states(data) # Returns: ["HI", "IL"] +clear_state_cache("fpl", "2022", c("HI", "IL")) # Delete only corrupt states +re-download missing 2 states # 500MB download +merge with 49 cached states # 1-2 minutes +``` + +### 4. Functions to Implement + +#### `check_cached_state_files(dataset, vintage)` +Returns character vector of states that have valid cached ZIP files. + +```r +check_cached_state_files <- function(dataset, vintage) { + cache_dir <- get_cache_dir() + all_states <- get_all_states() + + cached <- character() + for (state in all_states) { + zip_file <- file.path(cache_dir, + sprintf("lead_%s_%s_%s.zip", vintage, dataset, state)) + if (file.exists(zip_file) && file.size(zip_file) > 10000) { # >10KB + cached <- c(cached, state) + } + } + + return(cached) +} +``` + +#### `clear_state_cache(dataset, vintage, states)` +Removes specific state ZIP files (for corrupted data). + +```r +clear_state_cache <- function(dataset, vintage, states, verbose = TRUE) { + cache_dir <- get_cache_dir() + + for (state in states) { + zip_file <- file.path(cache_dir, + sprintf("lead_%s_%s_%s.zip", vintage, dataset, state)) + if (file.exists(zip_file)) { + unlink(zip_file) + if (verbose) message(" ✓ Deleted: ", basename(zip_file)) + } + } +} +``` + +#### Modified `download_and_merge_states()` + +```r +download_and_merge_states <- function(dataset, vintage, states, verbose = TRUE) { + + # Check which states are already cached + cached_states <- check_cached_state_files(dataset, vintage) + missing_states <- setdiff(states, cached_states) + + if (verbose) { + message(sprintf("Cached states: %d, Missing states: %d", + length(cached_states), length(missing_states))) + if (length(missing_states) > 0) { + message("Will download: ", paste(missing_states, collapse = ", ")) + } + if (length(cached_states) > 0) { + message("Will load from cache: ", paste(cached_states, collapse = ", ")) + } + } + + # Download only missing states + if (length(missing_states) > 0) { + for (i in seq_along(missing_states)) { + state <- missing_states[i] + if (verbose) { + message(sprintf("[%d/%d] Downloading %s...", i, length(missing_states), state)) + } + download_single_state_cached(dataset, vintage, state, verbose = FALSE) + } + } + + # Load all states (cached + newly downloaded) + all_data <- list() + failed_states <- character() + + for (state in states) { + tryCatch({ + state_data <- load_state_from_cache(dataset, vintage, state, verbose = FALSE) + if (!is.null(state_data) && nrow(state_data) > 0) { + all_data[[state]] <- state_data + } else { + failed_states <- c(failed_states, state) + } + }, error = function(e) { + warning(sprintf("Failed to load %s: %s", state, e$message)) + failed_states <- c(failed_states, state) + }) + } + + # Merge and save + combined_data <- dplyr::bind_rows(all_data) + + # Save merged nationwide CSV + cache_dir <- get_cache_dir() + cache_file <- file.path(cache_dir, paste0("lead_", vintage, "_", dataset, ".csv")) + readr::write_csv(combined_data, cache_file) + + # Import to database + try_import_to_database(combined_data, dataset, vintage, verbose = verbose) + + return(combined_data) +} +``` + +### 5. Benefits + +✅ **Efficiency**: Download only missing states (minutes vs hours) +✅ **Resilience**: Individual state corruption doesn't require full re-download +✅ **Transparency**: Clear reporting of cached vs downloaded states +✅ **Storage**: ~13GB per dataset (51 states × ~250MB), but saves bandwidth +✅ **Debugging**: Can inspect individual state files + +### 6. Disk Space Considerations + +**Before**: ~50MB merged CSV per dataset +**After**: ~13GB state ZIPs + ~50MB merged CSV per dataset + +**Mitigation**: +- State ZIPs can be deleted after successful merge (optional) +- Add `clear_state_cache()` function for manual cleanup +- Add `--keep-state-cache` flag to regeneration script + +### 7. Implementation Priority + +1. **Phase 1** (For current regeneration): + - Modify `download_and_merge_states()` to cache state ZIPs + - Implement `check_cached_state_files()` + - Test with current FPL 2022 issue + +2. **Phase 2** (Post-CRAN): + - Add `clear_state_cache()` to `R/cache_utils.R` + - Update corruption detection to identify missing states + - Implement selective re-download + +3. **Phase 3** (Optional): + - Add cleanup options to regeneration script + - Implement automatic state cache expiration (30 days?) + +### 8. Migration Strategy + +Existing users with no cached state files will simply download as before. Once state caching is implemented, future downloads benefit from the per-state cache. + +No breaking changes to existing API. + +--- + +## Implementation Decision + +**Should we implement this now?** + +### Option A: Implement now (before completing current regeneration) +- ✅ PRO: Solves FPL 2022 issue efficiently (download just HI, IL) +- ✅ PRO: Future-proofs against similar issues +- ❌ CON: Delays Zenodo upload by 1-2 hours +- ❌ CON: Requires testing with active downloads + +### Option B: Implement after Zenodo upload (post-CRAN) +- ✅ PRO: Current regeneration completes sooner +- ✅ PRO: Can test thoroughly in development +- ✅ PRO: CRAN submission not delayed +- ❌ CON: Must re-download all 51 states for FPL 2022 now + +### Recommendation: **Option B** + +**Reason**: We're already 71% through AMI 2018 download. Implementing per-state caching now would require: +1. Stopping current regeneration +2. Implementing and testing new code +3. Re-running downloads (losing current progress) + +Better to: +1. Complete current regeneration +2. Get clean datasets to Zenodo +3. Implement per-state caching properly in next version +4. Include in v0.6.0 release notes as improvement + +This makes per-state caching a **v0.6.0 feature** rather than rushing it into v0.5.x. diff --git a/.dev/README.md b/.dev/README.md new file mode 100644 index 0000000..c815e4d --- /dev/null +++ b/.dev/README.md @@ -0,0 +1,66 @@ +# Development Scripts + +This directory contains helper scripts for package development and maintenance. + +## Setup Scripts + +### `install-tinytex.R` + +Installs TinyTeX for building PDF vignettes. Required for package development but **not** for end users. + +```r +# Install TinyTeX +Rscript .dev/install-tinytex.R +``` + +TinyTeX is a lightweight LaTeX distribution (~100MB) needed to build the JSS (Journal of Statistical Software) PDF vignette. End users get pre-built vignettes with the package and don't need LaTeX installed. + +## Version Management + +### `bump-version.R` + +Automatically bumps package version across all metadata files (DESCRIPTION, NEWS.md, inst/CITATION, .zenodo.json). + +```bash +# Bump to a specific version +Rscript .dev/bump-version.R 0.5.2 +``` + +## Data Management + +### `prepare-zenodo-data-nationwide.R` + +Prepares nationwide cohort data files for upload to Zenodo. + +```bash +# Prepare all 4 datasets (AMI/FPL for 2018/2022) +Rscript .dev/prepare-zenodo-data-nationwide.R --nationwide-only +``` + +### `zenodo-upload.sh` + +Uploads prepared datasets to Zenodo. Requires `ZENODO_TOKEN` environment variable. + +```bash +# Upload to Zenodo +export ZENODO_TOKEN="your_token_here" +bash .dev/zenodo-upload.sh +``` + +## Workflow Notes + +### Building Vignettes + +**For developers:** +1. Install TinyTeX once: `Rscript .dev/install-tinytex.R` +2. Build package normally: `R CMD build .` +3. Vignettes are built automatically + +**For end users:** +- Vignettes are pre-built and included in the package tarball +- No LaTeX installation required +- Just install the package: `install.packages("emburden")` + +### CI/CD + +GitHub Actions already has TinyTeX installed, so vignettes build automatically in CI. diff --git a/.dev/REGENERATION-DECISION-GUIDE.md b/.dev/REGENERATION-DECISION-GUIDE.md new file mode 100644 index 0000000..586a735 --- /dev/null +++ b/.dev/REGENERATION-DECISION-GUIDE.md @@ -0,0 +1,180 @@ +# Zenodo Dataset Regeneration Decision Guide + +This guide helps you decide whether to **post-process** existing datasets or **regenerate from scratch**. + +## Quick Decision Tree + +``` +Does the change affect... +│ +├─ Column names/renaming only? +│ └─ ✅ POST-PROCESS (fast, no download) +│ → Use: .dev/post-process-zenodo-data.R +│ +├─ Adding metadata/derived columns? +│ └─ ✅ POST-PROCESS (if source data unchanged) +│ → Use: .dev/post-process-zenodo-data.R +│ +├─ Data loading/downloading logic? +│ └─ ❌ REGENERATE (requires fresh download) +│ → Use: .dev/prepare-zenodo-data-nationwide.R --force-download +│ +├─ Aggregation/grouping/filtering during load? +│ └─ ❌ REGENERATE (requires re-processing) +│ → Use: .dev/prepare-zenodo-data-nationwide.R --force-download +│ +└─ New data sources or vintages? + └─ ❌ REGENERATE (new data required) + → Use: .dev/prepare-zenodo-data-nationwide.R --force-download +``` + +--- + +## Examples + +### ✅ POST-PROCESS (No Regeneration Needed) + +**Scenario**: AMI datasets use `AMI150` column but should use `income_bracket` + +**Why**: This is just a column rename on already-correct data + +**Command**: +```bash +Rscript .dev/post-process-zenodo-data.R --fix ami-column-rename +``` + +**Time**: ~1 minute (vs 2-3 hours for full regeneration) + +--- + +**Scenario**: Add a `burden_category` column based on existing `income_bracket` + +**Why**: Derived from existing data, no need to re-download + +**Command**: +```bash +# Edit post-process-zenodo-data.R to add the transformation +Rscript .dev/post-process-zenodo-data.R +``` + +--- + +### ❌ REGENERATE (Full Regeneration Required) + +**Scenario**: Change how data is aggregated by census tract + +**Why**: Requires re-processing raw OpenEI data + +**Command**: +```bash +Rscript .dev/prepare-zenodo-data-nationwide.R --force-download --nationwide-only +``` + +**Time**: 2-3 hours (downloads 30GB) + +--- + +**Scenario**: Fix a bug in `load_cohort_data()` that affects what data is downloaded + +**Why**: Need fresh data with corrected loading logic + +**Command**: +```bash +# After fixing R/lead_data_loaders.R: +Rscript .dev/prepare-zenodo-data-nationwide.R --force-download --nationwide-only +``` + +--- + +**Scenario**: Add 2020 vintage datasets + +**Why**: New data source, doesn't exist in cache + +**Command**: +```bash +# After updating datasets list in prepare script: +Rscript .dev/prepare-zenodo-data-nationwide.R --nationwide-only +``` + +--- + +## When to Trigger Version Bump + +Full regeneration should trigger version management workflow: + +### Automatic Version Bump Triggers: +- Changes to `R/lead_data_loaders.R` (data loading logic) +- Changes to dataset aggregation/filtering +- Adding new vintages or data sources +- Changes that affect dataset checksums + +### Manual Version Bump (Optional): +- Post-processing fixes (column renames, metadata) +- Documentation changes +- Non-data changes + +### Version Bump Workflow: + +```bash +# 1. After successful regeneration, bump version +Rscript .dev/bump-version.R --minor # or --major, --patch + +# 2. This automatically: +# - Updates DESCRIPTION version +# - Updates NEWS.md +# - Updates R/zenodo.R checksums +# - Creates git tag +# - Commits changes + +# 3. Then upload to Zenodo +bash .dev/upload-to-zenodo-nationwide.sh + +# 4. Push with tags +git push --follow-tags +``` + +--- + +## Cache Management + +### Use Cached Data (Recommended for Post-Processing) +```bash +Rscript .dev/prepare-zenodo-data-nationwide.R --nationwide-only --use-cache +``` +- Uses existing downloaded data +- Fast (minutes, not hours) +- Ideal when fixing processing bugs, not loading bugs + +### Force Fresh Download (Slow but Thorough) +```bash +Rscript .dev/prepare-zenodo-data-nationwide.R --nationwide-only --force-download +``` +- Clears cache and re-downloads all data +- Slow (~2-3 hours) +- Required when data source or loading logic changes + +### Default (Smart Caching) +```bash +Rscript .dev/prepare-zenodo-data-nationwide.R --nationwide-only +``` +- Uses cache if available, downloads if missing +- Good balance for most use cases + +--- + +## Summary + +**Post-processing** = Fast fixes to existing data (minutes) +- Column renames +- Adding derived fields +- Metadata updates + +**Regeneration** = Full re-download and re-process (hours) +- Data loading changes +- Aggregation changes +- New data sources + +**Version bump** = Automatic on regeneration, manual on post-processing +- Triggers when dataset output changes +- Updates DESCRIPTION, NEWS.md, git tags +- Coordinates with Zenodo upload diff --git a/.dev/RELEASE-AUTOMATION-GUIDE.md b/.dev/RELEASE-AUTOMATION-GUIDE.md new file mode 100644 index 0000000..57a57c1 --- /dev/null +++ b/.dev/RELEASE-AUTOMATION-GUIDE.md @@ -0,0 +1,413 @@ +# Release Automation Guide + +Comprehensive guide for using the automated version bump and release workflow. + +## Quick Start + +```bash +# Fully automated release (no prompts) +bash .dev/release-version.sh 0.5.11 --auto + +# Interactive release (recommended for first-time users) +bash .dev/release-version.sh 0.5.11 + +# Or manually with individual steps +Rscript .dev/bump-version.R 0.5.11 +``` + +## Scripts Overview + +### `release-version.sh` - Complete Release Automation (Recommended) + +**Location**: `.dev/release-version.sh` + +**What it does**: +1. ✅ Validates git repository state +2. ✅ Checks for uncommitted changes +3. ✅ Warns if not on main branch +4. ✅ Runs `bump-version.R` to update version files +5. ✅ Auto-creates NEWS.md template entry +6. ✅ Opens editor for NEWS.md editing +7. ✅ Shows git diff for review +8. ✅ Stages changes +9. ✅ Creates commit +10. ✅ Pushes to remote (optional) +11. ✅ Creates or updates pull request + +**Usage**: +```bash +bash .dev/release-version.sh 0.5.11 +``` + +**Features**: +- Interactive prompts with sensible defaults +- Color-coded output (success, warnings, errors) +- Comprehensive error handling +- Safe defaults (won't push without confirmation) +- Automatic NEWS.md template generation +- Editor integration (respects $EDITOR, falls back to nano/vi) +- **NEW: `--auto` flag for fully automated releases (no prompts)** + +### `bump-version.R` - Version File Updater + +**Location**: `.dev/bump-version.R` + +**What it does**: +- Updates `DESCRIPTION` +- Updates `inst/CITATION` (both version references) +- Updates `.zenodo.json` +- Validates semantic versioning format +- Shows summary of updated files + +**Usage**: +```bash +Rscript .dev/bump-version.R 0.5.11 +``` + +**Standalone mode** (when you want manual control): +```bash +# 1. Update version files +Rscript .dev/bump-version.R 0.5.11 + +# 2. Manually edit NEWS.md + +# 3. Review changes +git diff + +# 4. Stage and commit +git add DESCRIPTION inst/CITATION .zenodo.json NEWS.md +git commit -m "Bump version to 0.5.11" + +# 5. Push and create PR +git push scheier +gh pr create --base main --head + +# Note: Git tag will be created automatically by auto-tag-on-version-bump +# workflow after the PR is merged to main +``` + +## Versioning Format + +Follows **semantic versioning**: `MAJOR.MINOR.PATCH` + +Examples: +- `0.5.11` - Standard release +- `0.5.11.9001` - Development version (optional) + +Pattern validation: +- Must match: `^\d+\.\d+\.\d+(\.\d{4})?$` +- Valid: `0.5.11`, `1.0.0`, `0.5.11.9001` +- Invalid: `0.5`, `v0.5.11`, `0.5.11-beta` + +## NEWS.md Template + +When `release-version.sh` creates a NEWS.md entry, it uses this template: + +```markdown +# emburden 0.5.11 + +## Changes + +### New Features + +* (Add new features here) + +### Bug Fixes + +* (Add bug fixes here) + +### Enhancements + +* (Add enhancements here) + +--- +``` + +**Guidelines**: +- Use clear, concise bullet points +- Group related changes together +- Include PR/issue references if applicable +- Focus on user-facing changes +- Delete unused sections + +## Workflow Integration + +The release automation integrates with your CI/CD pipeline: + +``` +release-version.sh (local) + ↓ + [Push commit to branch] + ↓ + [Create/update PR] + ↓ + [Merge PR to main] + ↓ + auto-tag-on-version-bump.yml (creates git tag on main) + ↓ + auto-release.yml (creates GitHub release) + ↓ + publish-to-public.yml (syncs to public repo) + ↓ + cran-release.yml (public repo - manual approval) +``` + +## Safety Features + +### Pre-flight Checks +- ✅ Validates git repository exists +- ✅ Checks for DESCRIPTION and NEWS.md files +- ✅ Validates semantic versioning format +- ✅ Warns about uncommitted changes +- ✅ Warns if not on main branch + +### Interactive Confirmation +- Confirms changes before committing +- Asks before pushing to remote +- Shows full diff for review +- Allows custom commit messages +- Supports aborting at any step + +### Error Handling +- Uses `set -euo pipefail` for strict error checking +- Colored error messages +- Descriptive exit codes +- Clean failure modes + +## Environment Variables + +### `$EDITOR` +Controls which editor opens NEWS.md: +```bash +# Use VS Code +export EDITOR="code --wait" + +# Use Emacs +export EDITOR="emacs" + +# Use nano (default fallback) +export EDITOR="nano" +``` + +Falls back to: `nano` → `vi` → manual edit + +## Troubleshooting + +### "You have uncommitted changes" +```bash +# Option 1: Commit them first +git add . +git commit -m "Pre-release cleanup" + +# Option 2: Stash them +git stash + +# Then retry +bash .dev/release-version.sh 0.5.11 +``` + +### "Not on main branch" +```bash +# Switch to main +git checkout main + +# Or continue anyway (script will warn) +# The script allows this but warns you +``` + +### Script execution permission denied +```bash +chmod +x .dev/release-version.sh +``` + +## Automation Modes + +### Fully Automated Mode (`--auto`) + +**NEW FEATURE**: Use the `--auto` flag for completely automated releases with no interactive prompts. + +```bash +bash .dev/release-version.sh 0.5.11 --auto +``` + +**What it does automatically**: +- ✅ Continues despite uncommitted changes (with warning) +- ✅ Continues if not on main branch (with warning) +- ✅ Skips NEWS.md editor (uses template) +- ✅ Skips diff review +- ✅ Uses default commit message +- ✅ Automatically pushes to remote + +**When to use `--auto`**: +- CI/CD pipeline automation +- Rapid iteration during development +- When you trust the automated process +- When you've already reviewed changes manually + +**When NOT to use `--auto`**: +- First time using the script +- Major version releases +- When you need to write detailed NEWS.md entries +- When you're unsure about the changes + +### Interactive Mode (Default) + +```bash +bash .dev/release-version.sh 0.5.11 +``` + +Prompts for confirmation at each step. Recommended for: +- First-time releases +- Important releases +- When you want to review each step + +## Examples + +### Standard Release +```bash +# Interactive release (recommended for first use) +bash .dev/release-version.sh 0.5.11 + +# Script will: +# 1. Update version files +# 2. Open NEWS.md for editing +# 3. Show diff +# 4. Ask for confirmation +# 5. Commit and tag +# 6. Ask to push +``` + +### Fully Automated Release +```bash +# Zero-prompt release +bash .dev/release-version.sh 0.5.11 --auto + +# Script will: +# 1. Update version files +# 2. Auto-create NEWS.md template (no editor) +# 3. Show diff (no confirmation) +# 4. Commit with default message +# 5. Push automatically +# 6. Create/update PR automatically +``` + +### Development Version +```bash +# Create development version +bash .dev/release-version.sh 0.5.11.9001 +``` + +### Manual Control (Don't Push) +```bash +# Run script but decline push +bash .dev/release-version.sh 0.5.11 + +# At "Push to remote?" prompt, answer: n + +# Now you can: +# - Test locally +# - Make additional changes +# - Push manually later +``` + +### Custom Commit Message +```bash +bash .dev/release-version.sh 0.5.11 + +# At "Use default commit message?" prompt, answer: n +# Then enter custom message in editor +``` + +## Best Practices + +1. **Always run from project root** + ```bash + cd /path/to/net_energy_equity + bash .dev/release-version.sh 0.5.11 + ``` + +2. **Update NEWS.md thoughtfully** + - Document all user-facing changes + - Group related changes + - Reference issues/PRs + - Delete unused sections + +3. **Review the diff** + - Check version numbers are correct + - Verify CITATION date updates + - Ensure NEWS.md is complete + +4. **Test before pushing** + - Decline push option + - Run local tests + - Build package + - Then push manually if needed + +5. **Follow semantic versioning** + - Patch (0.5.X): Bug fixes + - Minor (0.X.0): New features (backwards compatible) + - Major (X.0.0): Breaking changes + +## Advanced Usage + +### Dry Run +```bash +# Update version files without committing +Rscript .dev/bump-version.R 0.5.11 +git diff +git checkout -- DESCRIPTION inst/CITATION .zenodo.json +``` + +### Batch Multiple Files +```bash +# If you need to update additional files +bash .dev/release-version.sh 0.5.11 + +# Before answering "y" to commit: +# Press Ctrl+C to abort +# Make additional changes +# Manually stage and commit all files together +``` + +### Skip NEWS.md Update +```bash +# Pre-populate NEWS.md before running +vim NEWS.md # Add entry manually +bash .dev/release-version.sh 0.5.11 +# Script will detect existing entry +``` + +## Comparison: Automated vs Manual + +### With `release-version.sh` (Recommended) +```bash +bash .dev/release-version.sh 0.5.11 +# → 8 steps automated, ~2 minutes +``` + +### Manual Process +```bash +Rscript .dev/bump-version.R 0.5.11 +vim NEWS.md +git diff DESCRIPTION inst/CITATION .zenodo.json NEWS.md +git add DESCRIPTION inst/CITATION .zenodo.json NEWS.md +git commit -m "Bump version to 0.5.11" +git push scheier +gh pr create --base main +# [Wait for merge, then workflow creates tag] +# → 7 manual steps, ~5 minutes, error-prone +``` + +## Related Documentation + +- [CRAN Submission Guide](.dev/CRAN-SUBMISSION-GUIDE.md) +- [Workflow Documentation](.github/workflows/README.md) +- [Version Consistency Checker](.dev/check-version-consistency.R) + +## Support + +For issues or questions: +1. Check this guide +2. Review `.dev/bump-version.R` comments +3. Inspect `.dev/release-version.sh` implementation +4. File an issue with clear reproduction steps diff --git a/.dev/RELEASE-PROCESS.md b/.dev/RELEASE-PROCESS.md new file mode 100644 index 0000000..6dfb185 --- /dev/null +++ b/.dev/RELEASE-PROCESS.md @@ -0,0 +1,287 @@ +# Release Process Documentation + +This document describes the automated and semi-automated release process for the emburden package. + +## Overview + +The release process consists of three main stages: + +1. **Development & Testing** (automatic via CI) +2. **Version Bumping & Tagging** (semi-automatic with helper scripts) +3. **Controlled Release** (automatic with manual approval gates) + +## Stage 1: Development & Testing (Automatic) + +When you push code or create a PR: + +- ✅ **Automatic**: R CMD check runs on multiple platforms +- ✅ **Automatic**: Test coverage calculated +- ✅ **Automatic**: Package documentation built (pkgdown) + +**No manual intervention required** - all checks run automatically via GitHub Actions. + +## Stage 2: Version Bumping & Tagging (Semi-Automatic) + +When you're ready to create a new release: + +### Option A: Use Helper Script (Recommended) + +1. **Bump version** (updates all metadata files consistently): + ```bash + Rscript .dev/bump-version.R 0.3.0 + ``` + + This updates: + - `DESCRIPTION` + - `inst/CITATION` + - `.zenodo.json` + - `NEWS.md` (adds template section) + +2. **Edit NEWS.md** to add release notes for the new version + +3. **Commit and push changes**: + ```bash + git add -A + git commit -m "Bump version to 0.3.0" + git push scheier main + ``` + +4. **Create release tag** (automatic validation + tagging): + ```bash + Rscript .dev/create-release-tag.R + ``` + + This script: + - ✅ Verifies version consistency across all files + - ✅ Checks that tag doesn't already exist + - ✅ Extracts release notes from NEWS.md + - ✅ Creates annotated git tag + - ✅ Pushes tag to trigger release workflow + - ✅ Provides instructions for monitoring progress + + **Dry run mode** (preview without creating tag): + ```bash + Rscript .dev/create-release-tag.R --dry-run + ``` + +### Option B: Manual Process + +If you prefer to do it manually: + +```bash +# 1. Verify version consistency +Rscript .dev/check-version-consistency.R + +# 2. Create annotated tag +git tag -a v0.3.0 -m "Release version 0.3.0 + +Your release notes here..." + +# 3. Push tag +git push scheier v0.3.0 +``` + +## Stage 3: Controlled Release (Automatic with Approval Gates) + +Once the tag is pushed, the **Controlled Release** workflow automatically: + +### 1. Validation (Automatic) + +- Runs R CMD check on all platforms +- Runs full test suite with coverage checks +- Builds package tarball +- Generates validation report + +### 2. Gate 1: Pre-Release Review (Manual Approval Required) + +⚠️ **Manual approval required** via GitHub Actions UI + +Review the validation report and approve if all checks pass. + +### 3. Create Draft Release (Automatic) + +- Creates draft GitHub release +- Uploads package tarball +- Uploads validation report +- Extracts release notes from NEWS.md + +### 4. Gate 2: Production Release Approval (Manual Approval Required) + +⚠️ **Manual approval required** via GitHub Actions UI + +Final review before publishing the release. + +### 5. Publish Release (Automatic) + +- Publishes GitHub release +- Triggers Zenodo archival (automatic DOI assignment) +- Provides instructions for optional CRAN submission + +## Monitoring Releases + +### Check workflow status: + +```bash +# List recent release workflows +gh run list --workflow="Controlled Release" --limit 5 + +# Watch current release in real-time +gh run watch --workflow="Controlled Release" + +# View specific run details +gh run view +``` + +### View releases: + +```bash +# List all releases +gh release list + +# View specific release +gh release view v0.2.0 +``` + +## Helper Scripts + +All helper scripts are in `.dev/` directory: + +| Script | Purpose | Usage | +|--------|---------|-------| +| `bump-version.R` | Update version across all metadata files | `Rscript .dev/bump-version.R 0.3.0` | +| `check-version-consistency.R` | Verify versions match across files | `Rscript .dev/check-version-consistency.R` | +| `create-release-tag.R` | Automated tag creation with validation | `Rscript .dev/create-release-tag.R` | +| `run-tests-locally.R` | Run full test suite locally | `Rscript .dev/run-tests-locally.R` | + +## Approval Gates Setup + +The workflow requires two GitHub Environments with required reviewers: + +### Environment: `pre-release-review` +- Required reviewers: 1-2 maintainers +- Reviews validation results before creating draft release + +### Environment: `public-release` +- Required reviewers: 1-2 different maintainers (for dual approval) +- Final approval before publishing release + +To configure environments: +1. Go to Settings → Environments → New environment +2. Add environment name +3. Enable "Required reviewers" +4. Add reviewers + +## CRAN Submission (Optional, Manual) + +CRAN submissions are **always manual** and done by the package maintainer: + +1. Download package tarball from GitHub release +2. Review CRAN submission checklist +3. Submit to https://cran.r-project.org/submit.html +4. Monitor email for CRAN automated checks +5. Respond to any reviewer feedback + +The workflow provides instructions after successful release publication. + +## What's Automated vs Manual + +| Task | Automation Level | +|------|-----------------| +| CI checks on PRs | ✅ Fully automatic | +| Test coverage reports | ✅ Fully automatic | +| Package documentation build | ✅ Fully automatic | +| Version bumping | ⚙️ Semi-automatic (helper script) | +| Release tag creation | ⚙️ Semi-automatic (helper script) | +| Validation stage | ✅ Fully automatic (triggered by tag) | +| Pre-release review | ⚠️ Manual approval required | +| Draft release creation | ✅ Fully automatic (after approval) | +| Production release approval | ⚠️ Manual approval required | +| Release publication | ✅ Fully automatic (after approval) | +| Zenodo archival | ✅ Fully automatic (triggered by release) | +| CRAN submission | ⚠️ Always manual | + +## Quick Reference: Full Release Workflow + +```bash +# 1. Bump version and update NEWS.md +Rscript .dev/bump-version.R 0.3.0 +# Edit NEWS.md to add release notes + +# 2. Commit and push +git add -A +git commit -m "Bump version to 0.3.0" +git push scheier main + +# 3. Create and push release tag (with automatic validation) +Rscript .dev/create-release-tag.R + +# 4. Monitor workflow +gh run watch --workflow="Controlled Release" + +# 5. Approve at Gate 1 (via GitHub UI) +# Review validation report, then approve in Actions tab + +# 6. Approve at Gate 2 (via GitHub UI) +# Final review before publication + +# 7. Release is published automatically +# Zenodo DOI assigned automatically + +# 8. (Optional) Submit to CRAN manually +# Download tarball from release and submit to CRAN +``` + +## Benefits of This Process + +1. **Consistency**: Version numbers always in sync across all metadata files +2. **Safety**: Dual approval gates prevent accidental releases +3. **Automation**: Reduces manual steps and potential errors +4. **Validation**: Comprehensive checks before any release +5. **Reproducibility**: All releases have associated DOIs via Zenodo +6. **Transparency**: Full audit trail in GitHub Actions + +## Troubleshooting + +### Tag already exists + +```bash +# Delete local tag +git tag -d v0.2.0 + +# Delete remote tag +git push scheier --delete v0.2.0 + +# Try again +Rscript .dev/create-release-tag.R +``` + +### Version mismatch errors + +```bash +# Check what's inconsistent +Rscript .dev/check-version-consistency.R + +# Use bump-version to fix +Rscript .dev/bump-version.R 0.2.0 +``` + +### Workflow fails + +```bash +# View failure details +gh run view --log-failed + +# Check specific job +gh run view --job= +``` + +## Future Improvements + +Potential automation enhancements: + +1. **Auto-create tags on version bumps**: Workflow that auto-tags when DESCRIPTION version changes on main branch +2. **Auto-CRAN submission**: Once stable, could automate CRAN submissions +3. **Release notes from commits**: Auto-generate release notes from commit messages +4. **Automated dependency updates**: Dependabot for R package dependencies + +These can be added incrementally as the release process matures. diff --git a/.dev/TEST_ZENODO_DOWNLOAD.md b/.dev/TEST_ZENODO_DOWNLOAD.md new file mode 100644 index 0000000..7791c53 --- /dev/null +++ b/.dev/TEST_ZENODO_DOWNLOAD.md @@ -0,0 +1,218 @@ +# Testing Zenodo Downloads Safely + +## Overview + +This guide shows how to test Zenodo downloads WITHOUT touching your production database. + +## Database Protection + +The package now has **TWO separate databases**: + +1. **Production Database** (`emburden_db.sqlite`) + - Contains your real data + - **PROTECTED** from accidental deletion + - Located at: `~/.cache/emburden/emburden_db.sqlite` + +2. **Test Database** (`emburden_test_db.sqlite`) + - Used for testing only + - Safe to delete anytime + - Located at: `~/.cache/emburden/emburden_test_db.sqlite` + +## Testing Zenodo Downloads + +### Option 1: Manual Test Script (Recommended) + +Create a test script that uses the test database: + +```r +# test-zenodo.R +devtools::load_all() + +# Clear test environment (SAFE - only touches test DB) +clear_test_environment() + +# Test loading with verbose output +message("Testing Zenodo download...") +data <- load_cohort_data('ami', states = 'NC', vintage = '2022', verbose = TRUE) + +message("\n✓ SUCCESS! Loaded ", nrow(data), " rows") +message("Sample data:") +print(head(data[, 1:5], 3)) +``` + +Run with: +```bash +Rscript test-zenodo.R +``` + +### Option 2: Interactive R Session + +```r +library(emburden) + +# Clear only test data (production untouched) +clear_test_environment() + +# Test a download +data <- load_cohort_data('fpl', 'NC', '2022', verbose = TRUE) +nrow(data) # Should see data from Zenodo +``` + +### Option 3: Automated Tests + +```r +devtools::test() # Runs all tests including Zenodo tests +``` + +## Database Safety Features + +### Protection Against Deletion + +```r +# This FAILS (good!) +delete_db(test = FALSE) +# Error: Cannot delete production database without confirmation! + +# This works (production DB deletion requires explicit confirm) +delete_db(test = FALSE, confirm = TRUE) # DANGER! + +# This is SAFE (test DB) +delete_db(test = TRUE) # OK - deletes test DB only +``` + +### Backup Production Database + +Before any risky operations: + +```r +# Create timestamped backup +backup_db() +# Database backed up successfully! +# Location: ~/.cache/emburden/backups/emburden_db_backup_20251114_001234.sqlite +# Size: 42.5 MB +``` + +### Clear Test Environment + +Safe function that NEVER touches production: + +```r +clear_test_environment() +# Clearing test environment... +# ✓ Deleted test database +# ✓ Deleted 0 test cache files +# Test environment cleared (production data untouched) +``` + +## Verifying Zenodo Infrastructure + +### Check Configuration + +```r +config <- get_zenodo_config() + +# View DOIs +config$concept_doi # "10.5281/zenodo.17604955" +config$version_doi # "10.5281/zenodo.17604956" + +# View file URLs +config$files$ami_2022$url +# "https://zenodo.org/records/17604956/files/lead_ami_cohorts_2022_us.csv.gz" +``` + +### Test URL Accessibility + +```r +# Check if Zenodo URL is reachable +url <- config$files$ami_2022$url +response <- httr::HEAD(url) +httr::status_code(response) # Should be 200 +``` + +### Test Download (Small File) + +```r +# Test with smallest file (AMI 2022 = 3.3 MB) +clear_test_environment() # Safe! + +data <- load_cohort_data('ami', 'NC', '2022', verbose = TRUE) +# Should see: "Downloading from Zenodo repository..." +``` + +## Current Zenodo Status + +- **Published**: ✅ Yes (2025-11-14) +- **Scope**: North Carolina only +- **Files**: 4 datasets (AMI/FPL 2018/2022) +- **Total Size**: 164 MB compressed +- **Public URL**: https://zenodo.org/records/17604956 + +## Development Workflow + +### Before Making Changes + +```r +# 1. Backup production database +backup_db() + +# 2. Make your changes +# ... + +# 3. Test with test database +clear_test_environment() +# ... run tests ... +``` + +### Testing New Features + +```r +# Always use test environment for development +withr::with_envvar( + c(EMBURDEN_TEST_MODE = "true"), + { + # Your tests here + data <- load_cohort_data('ami', 'NC', '2022') + } +) +``` + +## Troubleshooting + +### "Can't find production database" + +That's OK! It will be created automatically when you first download data. + +### "Test is using production database" + +Check that you're using `test = TRUE` in database functions: +```r +get_db_path(test = TRUE) # Test DB +get_db_path(test = FALSE) # Production DB +``` + +### "Want to completely reset" + +```r +# Clear test data (SAFE) +clear_test_environment() + +# Clear production data (REQUIRES BACKUP FIRST!) +backup_db() # Creates backup +delete_db(test = FALSE, confirm = TRUE) # Deletes production DB +``` + +## Summary + +✅ **Safe Operations**: +- `clear_test_environment()` - Always safe +- `delete_db(test = TRUE)` - Safe, only deletes test DB +- `backup_db()` - Safe, creates backup + +⚠️ **Dangerous Operations** (require explicit confirmation): +- `delete_db(test = FALSE, confirm = TRUE)` - Deletes production DB! +- Manual file deletion in `~/.cache/emburden/` + +🔒 **Protected**: +- Production database cannot be deleted without `confirm = TRUE` +- All test functions use separate test database +- Clear warnings before any destructive operations diff --git a/.dev/ZENODO_UPLOAD_GUIDE.md b/.dev/ZENODO_UPLOAD_GUIDE.md new file mode 100644 index 0000000..facda9f --- /dev/null +++ b/.dev/ZENODO_UPLOAD_GUIDE.md @@ -0,0 +1,305 @@ +# Zenodo Data Hosting Guide + +This guide documents how to prepare and upload emburden datasets to Zenodo for CRAN-compliant data hosting. + +## Overview + +**Problem**: CRAN has a 5MB package size limit, but our full US energy burden datasets exceed 500MB. + +**Solution**: Host processed datasets on Zenodo (free, permanent, citable), include small sample data in package, auto-download full data on first use. + +## Benefits of Zenodo + +1. **Free & Permanent**: Datasets get a DOI and are permanently archived +2. **Fast Downloads**: Better mirrors and CDN than OpenEI +3. **Versioned**: Each upload gets its own DOI for reproducibility +4. **Citable**: Proper academic citation with DOI +5. **CRAN-Friendly**: No package size restrictions + +--- + +## Upload Process + +### 1. Prepare PROCESSED Datasets + +**IMPORTANT**: Upload PROCESSED, analysis-ready data, NOT raw OpenEI data! + +Run the automated data preparation script: + +```bash +cd /path/to/emburden + +# This script: +# 1. Downloads raw data from OpenEI (if not cached) +# 2. Processes it into analysis-ready format +# 3. Saves processed CSV files ready for Zenodo upload +Rscript .dev/prepare-zenodo-data.R +``` + +This creates 4 processed datasets in `zenodo-upload/`: + +```bash +# 2022 PROCESSED Cohort Data (analysis-ready!) +lead_ami_cohorts_2022_us.csv # ~200 MB uncompressed, PROCESSED +lead_fpl_cohorts_2022_us.csv # ~300 MB uncompressed, PROCESSED + +# 2018 PROCESSED Cohort Data (analysis-ready!) +lead_ami_cohorts_2018_us.csv # ~180 MB uncompressed, PROCESSED +lead_fpl_cohorts_2018_us.csv # ~250 MB uncompressed, PROCESSED +``` + +**What makes these "processed"?** +- ✅ Aggregated by census tract + income bracket +- ✅ Includes computed energy burden metrics (EROI, NER, DEAR) +- ✅ Standardized column names +- ✅ Ready for immediate analysis (no processing needed) + +**Census Tract Data** (optional - can also be bundled in package): +```bash +census_tract_data.csv # ~40 MB uncompressed +``` + +### 2. Compress Datasets + +Zenodo supports gzip compression for faster transfers: + +```bash +cd zenodo-upload/ + +# Compress each PROCESSED file +gzip -9 -k lead_ami_cohorts_2022_us.csv # Creates .csv.gz +gzip -9 -k lead_fpl_cohorts_2022_us.csv +gzip -9 -k lead_ami_cohorts_2018_us.csv +gzip -9 -k lead_fpl_cohorts_2018_us.csv + +# Verify compression ratios +ls -lh *.csv.gz +``` + +Expected compression: 70-85% reduction in size. + +**Note**: These are PROCESSED files, so they're already smaller than raw data! + +### 3. Calculate MD5 Checksums + +For data integrity verification: + +```bash +md5sum lead_ami_cohorts_2022_us.csv.gz > checksums.txt +md5sum lead_fpl_cohorts_2022_us.csv.gz >> checksums.txt +md5sum lead_ami_cohorts_2018_us.csv.gz >> checksums.txt +md5sum lead_fpl_cohorts_2018_us.csv.gz >> checksums.txt +md5sum census_tract_data.csv.gz >> checksums.txt + +cat checksums.txt +``` + +### 4. Create Zenodo Record + +1. Go to https://zenodo.org/ and log in (or create account) +2. Click "New Upload" +3. Fill in metadata: + +**Basic Information:** +- **Title**: "emburden: Pre-processed Energy Burden Datasets" +- **Upload type**: Dataset +- **Publication date**: (today's date) +- **DOI**: (leave blank - Zenodo will assign) + +**Creators:** +- Name: Eric Scheier +- Affiliation: Emergi Foundation, UNC Chapel Hill +- ORCID: 0000-0001-9849-9089 + +**Description:** +``` +PROCESSED, analysis-ready household energy burden datasets from the DOE Low-Income +Energy Affordability Data (LEAD) Tool, formatted for the emburden R package. + +**IMPORTANT**: These are PRE-PROCESSED datasets, not raw OpenEI data. They have been: +- Aggregated by census tract + income bracket +- Enriched with computed energy burden metrics (EROI, NER, DEAR) +- Standardized for immediate analysis +- Quality-checked and validated + +This repository provides nationwide census tract-level data on household energy +burden across all 50 US states and District of Columbia, covering ~72,000 census +tracts. Data includes both Area Median Income (AMI) and Federal Poverty Line (FPL) +cohort analyses for 2018 and 2022 vintages. + +## Files Included: + +- lead_ami_cohorts_2022_us.csv.gz: 2022 AMI cohort data (PROCESSED, analysis-ready) +- lead_fpl_cohorts_2022_us.csv.gz: 2022 FPL cohort data (PROCESSED, analysis-ready) +- lead_ami_cohorts_2018_us.csv.gz: 2018 AMI cohort data (PROCESSED, analysis-ready) +- lead_fpl_cohorts_2018_us.csv.gz: 2018 FPL cohort data (PROCESSED, analysis-ready) +- checksums.txt: MD5 checksums for verification + +## Data Processing + +Source: Raw LEAD Tool data from OpenEI +Processing: emburden R package data pipeline +Format: CSV (aggregated tract-level cohorts with computed metrics) +Ready for: Immediate analysis, no additional processing required + +## Data Sources + +Original raw data from: +- DOE LEAD Tool 2022: https://data.openei.org/submissions/6219 +- DOE LEAD Tool 2018: https://data.openei.org/submissions/573 + +Processed using: emburden R package (github.com/ericscheier/emburden) + +## Citation + +When using this data, please cite: +1. This Zenodo repository (DOI will be provided) +2. The emburden R package +3. The original DOE LEAD Tool publications + +## License + +CC-BY-4.0 (same as source data) +``` + +**License:** Creative Commons Attribution 4.0 International + +**Keywords:** +- energy burden +- energy poverty +- household energy +- census tracts +- LEAD Tool +- R package +- emburden + +**Related Identifiers:** +- Is supplement to: (add emburden GitHub repo) +- Is derived from: https://data.openei.org/submissions/6219 +- Is derived from: https://data.openei.org/submissions/573 + +### 5. Upload Files + +1. Drag and drop or click "Choose files" +2. Upload all 6 files (.csv.gz + checksums.txt) +3. Wait for upload to complete (may take 10-30 minutes) +4. Verify all files uploaded successfully + +### 6. Publish + +1. Review all metadata +2. Click "Publish" +3. **Important**: Save the assigned DOI! + +--- + +## Update R Package Configuration + +After publishing, update `R/zenodo.R` with the new DOIs and URLs: + +```r +# In R/zenodo.R, function get_zenodo_config(): + +list( + concept_doi = "10.5281/zenodo.XXXXXXX", # Concept DOI (always latest) + version_doi = "10.5281/zenodo.YYYYYYY", # This version's DOI + + files = list( + ami_2022 = list( + filename = "lead_ami_cohorts_2022_us.csv.gz", + url = "https://zenodo.org/records/YYYYYYY/files/lead_ami_cohorts_2022_us.csv.gz", + size_mb = XX, # Fill in actual size + md5 = "xxxxxxxxxx" # From checksums.txt + ), + fpl_2022 = list( + filename = "lead_fpl_cohorts_2022_us.csv.gz", + url = "https://zenodo.org/records/YYYYYYY/files/lead_fpl_cohorts_2022_us.csv.gz", + size_mb = XX, + md5 = "xxxxxxxxxx" + ), + # ... repeat for all files + ) +) +``` + +--- + +## Testing + +After updating configuration: + +```r +# Test Zenodo download +devtools::load_all() + +# Clear cache first +unlink(file.path("~/.cache/emburden"), recursive = TRUE) + +# Try loading data (should download from Zenodo) +nc_data <- load_cohort_data("ami", "NC", "2022", verbose = TRUE) + +# Verify it worked +nrow(nc_data) # Should be > 0 +head(nc_data) + +# Check that cached file exists +list.files("~/.cache/emburden/") +``` + +--- + +## Updating Data (New Versions) + +When OpenEI releases new data vintages: + +1. Download and process new data +2. Create NEW Zenodo version (don't overwrite!) +3. Update `R/zenodo.R` with new version DOI +4. Bump package version (major.minor.patch) +5. Update NEWS.md with data changes + +**Important**: Never delete or replace old Zenodo versions. Each version gets a permanent DOI for reproducibility. + +--- + +## Maintenance Notes + +- **Zenodo record URL**: https://zenodo.org/records/XXXXXXX (fill in after upload) +- **Concept DOI**: 10.5281/zenodo.XXXXXXX (fill in after upload) +- **Version 1 DOI**: 10.5281/zenodo.YYYYYYY (fill in after upload) +- **Upload date**: (fill in) +- **File sizes**: + - ami_2022: XX MB compressed + - fpl_2022: XX MB compressed + - ami_2018: XX MB compressed + - fpl_2018: XX MB compressed + - census_tracts: XX MB compressed +- **Total size**: ~XXX MB compressed + +--- + +## Troubleshooting + +**Upload fails:** +- Try smaller files first to test +- Check Zenodo file size limits (50 GB per file) +- Ensure stable internet connection + +**Download fails in R:** +- Verify URLs are correct +- Check firewall/proxy settings +- Test URL manually in browser +- OpenEI fallback should activate automatically + +**Checksum mismatch:** +- Re-download file +- Re-calculate checksum +- If persistent, file may be corrupted - re-upload + +--- + +## References + +- Zenodo documentation: https://help.zenodo.org/ +- CRAN package size policy: https://cran.r-project.org/web/packages/policies.html +- DOE LEAD Tool: https://www.energy.gov/scep/slsc/low-income-energy-affordability-data-lead-tool diff --git a/.dev/bump-version.R b/.dev/bump-version.R new file mode 100644 index 0000000..6214884 --- /dev/null +++ b/.dev/bump-version.R @@ -0,0 +1,148 @@ +#!/usr/bin/env Rscript + +# bump-version.R +# Automatically updates package version across all metadata files +# Usage: Rscript .dev/bump-version.R +# Example: Rscript .dev/bump-version.R 0.3.0 + +args <- commandArgs(trailingOnly = TRUE) + +if (length(args) != 1) { + cat("Usage: Rscript .dev/bump-version.R \n") + cat("Example: Rscript .dev/bump-version.R 0.3.0\n") + quit(status = 1) +} + +new_version <- args[1] + +# Validate semantic versioning format (X.Y.Z or X.Y.Z.9XXX for dev) +if (!grepl("^\\d+\\.\\d+\\.\\d+(\\.9\\d{3})?$", new_version)) { + cat("Error: Version must follow semantic versioning format (e.g., 0.3.0 or 0.3.0.9001)\n") + quit(status = 1) +} + +cat("Updating package version to:", new_version, "\n\n") + +# Function to update version in a file +update_version <- function(file_path, pattern, replacement, description) { + if (!file.exists(file_path)) { + cat("Warning:", file_path, "not found. Skipping.\n") + return(FALSE) + } + + content <- readLines(file_path, warn = FALSE) + original_content <- content + + # Apply replacements + content <- gsub(pattern, replacement, content) + + if (identical(content, original_content)) { + cat("No changes needed in:", file_path, "\n") + return(FALSE) + } + + writeLines(content, file_path) + cat("✓ Updated", description, "in", file_path, "\n") + return(TRUE) +} + +updated_files <- character() + +# 1. Update DESCRIPTION +if (update_version( + "DESCRIPTION", + "^Version: .*", + paste0("Version: ", new_version), + "Version" +)) { + updated_files <- c(updated_files, "DESCRIPTION") +} + +# 2. Update inst/CITATION (two locations) +citation_file <- "inst/CITATION" +if (file.exists(citation_file)) { + content <- readLines(citation_file, warn = FALSE) + original_content <- content + + # Update both version references + content <- gsub( + 'note\\s*=\\s*"R package version [0-9.]+', + paste0('note = "R package version ', new_version), + content + ) + content <- gsub( + '"R package version [0-9.]+"', + paste0('"R package version ', new_version, '"'), + content + ) + + if (!identical(content, original_content)) { + writeLines(content, citation_file) + cat("✓ Updated version in inst/CITATION\n") + updated_files <- c(updated_files, "inst/CITATION") + } else { + cat("No changes needed in: inst/CITATION\n") + } +} else { + cat("Warning: inst/CITATION not found. Skipping.\n") +} + +# 3. Update .zenodo.json +zenodo_file <- ".zenodo.json" +if (file.exists(zenodo_file)) { + # Use jsonlite for proper JSON handling + if (requireNamespace("jsonlite", quietly = TRUE)) { + zenodo <- jsonlite::read_json(zenodo_file, simplifyVector = FALSE) + old_version <- zenodo$version + zenodo$version <- new_version + + if (old_version != new_version) { + jsonlite::write_json( + zenodo, + zenodo_file, + pretty = TRUE, + auto_unbox = TRUE + ) + cat("✓ Updated version in .zenodo.json\n") + updated_files <- c(updated_files, ".zenodo.json") + } else { + cat("No changes needed in: .zenodo.json\n") + } + } else { + # Fallback to regex if jsonlite not available + if (update_version( + zenodo_file, + '"version"\\s*:\\s*"[0-9.]+"', + paste0('"version": "', new_version, '"'), + "version" + )) { + updated_files <- c(updated_files, ".zenodo.json") + } + } +} else { + cat("Warning: .zenodo.json not found. Skipping.\n") +} + +# Summary +cat("\n" , rep("=", 60), "\n", sep = "") +cat("Version bump complete!\n") +cat("New version:", new_version, "\n") +cat("Files updated:", length(updated_files), "\n") + +if (length(updated_files) > 0) { + cat("\nUpdated files:\n") + for (f in updated_files) { + cat(" -", f, "\n") + } + + cat("\n" , rep("-", 60), "\n", sep = "") + cat("IMPORTANT REMINDERS:\n") + cat("1. Update NEWS.md with version", new_version, "and changes\n") + cat("2. Review changes: git diff\n") + cat("3. Stage changes: git add", paste(updated_files, collapse = " "), "\n") + cat("4. Commit: git commit -m 'Bump version to", new_version, "'\n") + cat("5. Create git tag: git tag v", new_version, "\n", sep = "") + cat(rep("=", 60), "\n", sep = "") +} + +quit(status = 0) diff --git a/.dev/check-version-consistency.R b/.dev/check-version-consistency.R new file mode 100644 index 0000000..1f959fb --- /dev/null +++ b/.dev/check-version-consistency.R @@ -0,0 +1,126 @@ +#!/usr/bin/env Rscript + +# check-version-consistency.R +# Validates that version numbers are consistent across all package metadata files +# Used in pre-push git hook to catch manual version editing mistakes +# Exit code 0 = consistent, Exit code 1 = inconsistent + +# Extract version from DESCRIPTION +get_desc_version <- function() { + if (!file.exists("DESCRIPTION")) { + cat("Error: DESCRIPTION file not found\n") + return(NULL) + } + content <- readLines("DESCRIPTION", warn = FALSE) + version_line <- grep("^Version:", content, value = TRUE) + if (length(version_line) == 0) { + return(NULL) + } + sub("^Version:\\s*", "", version_line[1]) +} + +# Extract version from inst/CITATION +get_citation_version <- function() { + citation_file <- "inst/CITATION" + if (!file.exists(citation_file)) { + cat("Warning: inst/CITATION file not found\n") + return(NULL) + } + content <- readLines(citation_file, warn = FALSE) + # Look for "R package version X.Y.Z" pattern + version_matches <- regmatches( + content, + gregexpr('R package version [0-9.]+', content) + ) + # Flatten and get unique versions + versions <- unique(unlist(version_matches)) + if (length(versions) == 0) { + return(NULL) + } + # Extract just the version number + version <- sub("R package version ", "", versions[1]) + version +} + +# Extract version from .zenodo.json +get_zenodo_version <- function() { + zenodo_file <- ".zenodo.json" + if (!file.exists(zenodo_file)) { + cat("Warning: .zenodo.json file not found\n") + return(NULL) + } + + # Try using jsonlite if available (more robust) + if (requireNamespace("jsonlite", quietly = TRUE)) { + tryCatch({ + zenodo <- jsonlite::read_json(zenodo_file, simplifyVector = FALSE) + return(zenodo$version) + }, error = function(e) { + cat("Warning: Failed to parse .zenodo.json with jsonlite\n") + }) + } + + # Fallback to regex parsing + content <- paste(readLines(zenodo_file, warn = FALSE), collapse = "\n") + version_match <- regmatches( + content, + regexpr('"version"\\s*:\\s*"[0-9.]+"', content) + ) + if (length(version_match) == 0) { + return(NULL) + } + # Extract version from "version": "X.Y.Z" + version <- sub('.*"version"\\s*:\\s*"([0-9.]+)".*', '\\1', version_match) + version +} + +# Main validation +cat("======================================\n") +cat(" Pre-push: Version Consistency Check\n") +cat("======================================\n\n") + +desc_ver <- get_desc_version() +citation_ver <- get_citation_version() +zenodo_ver <- get_zenodo_version() + +cat("Version found in DESCRIPTION: ", if(is.null(desc_ver)) "NOT FOUND" else desc_ver, "\n") +cat("Version found in inst/CITATION:", if(is.null(citation_ver)) "NOT FOUND" else citation_ver, "\n") +cat("Version found in .zenodo.json: ", if(is.null(zenodo_ver)) "NOT FOUND" else zenodo_ver, "\n") +cat("\n") + +# Check if all versions are present +versions <- list( + DESCRIPTION = desc_ver, + CITATION = citation_ver, + ZENODO = zenodo_ver +) + +# Filter out NULL values +versions <- Filter(Negate(is.null), versions) + +if (length(versions) == 0) { + cat("❌ ERROR: Could not extract version from any file\n") + quit(status = 1) +} + +# Check if all non-NULL versions match +unique_versions <- unique(unlist(versions)) + +if (length(unique_versions) == 1) { + cat("✅ Version consistency check PASSED\n") + cat(" All files have version:", unique_versions[1], "\n") + quit(status = 0) +} else { + cat("❌ VERSION MISMATCH DETECTED!\n\n") + cat("Found", length(unique_versions), "different versions:\n") + for (file in names(versions)) { + cat(" ", file, ":", versions[[file]], "\n") + } + cat("\n") + cat("PLEASE USE THE OFFICIAL VERSION BUMP SCRIPT:\n") + cat(" Rscript .dev/bump-version.R \n\n") + cat("Example:\n") + cat(" Rscript .dev/bump-version.R", max(unique_versions), "\n\n") + cat("Push cancelled due to version inconsistency.\n") + quit(status = 1) +} diff --git a/.dev/create-release-tag.R b/.dev/create-release-tag.R new file mode 100644 index 0000000..0aaa9fb --- /dev/null +++ b/.dev/create-release-tag.R @@ -0,0 +1,215 @@ +#!/usr/bin/env Rscript + +# create-release-tag.R +# Helper script to create and push release tags after version bumps +# Usage: Rscript .dev/create-release-tag.R [--dry-run] +# +# This script: +# 1. Verifies all version numbers are consistent +# 2. Checks that the current version doesn't already have a tag +# 3. Extracts release notes from NEWS.md +# 4. Creates an annotated git tag +# 5. Pushes the tag to trigger the controlled release workflow + +# Parse arguments +args <- commandArgs(trailingOnly = TRUE) +dry_run <- "--dry-run" %in% args + +# Color output functions +red <- function(x) paste0("\033[31m", x, "\033[0m") +green <- function(x) paste0("\033[32m", x, "\033[0m") +yellow <- function(x) paste0("\033[33m", x, "\033[0m") +blue <- function(x) paste0("\033[34m", x, "\033[0m") +bold <- function(x) paste0("\033[1m", x, "\033[0m") + +cat(bold(blue("=== Release Tag Creator ===\n\n"))) + +if (dry_run) { + cat(yellow("Running in DRY RUN mode - no tags will be created or pushed\n\n")) +} + +# Step 1: Check version consistency +cat("Step 1: Checking version consistency...\n") +check_result <- system("Rscript .dev/check-version-consistency.R", intern = FALSE) + +if (check_result != 0) { + cat(red("\n✖ Version consistency check failed!\n")) + cat("Please ensure all metadata files have matching versions.\n") + cat("Use: Rscript .dev/bump-version.R \n") + quit(status = 1) +} + +# Step 2: Get current version +version <- as.character(desc::desc_get_version()) +tag_name <- paste0("v", version) + +cat(blue(sprintf("\nCurrent version: %s\n", version))) +cat(blue(sprintf("Tag to create: %s\n\n", tag_name))) + +# Step 3: Check if tag already exists locally +existing_tags <- system("git tag -l", intern = TRUE) +if (tag_name %in% existing_tags) { + cat(red(sprintf("✖ Tag %s already exists locally!\n", tag_name))) + cat("If you need to recreate it:\n") + cat(sprintf(" 1. Delete local tag: git tag -d %s\n", tag_name)) + cat(sprintf(" 2. Delete remote tag: git push scheier --delete %s\n", tag_name)) + cat(" 3. Run this script again\n") + quit(status = 1) +} + +# Step 4: Check if tag exists on remote +remote_tags <- system("git ls-remote --tags scheier", intern = TRUE) +if (any(grepl(tag_name, remote_tags))) { + cat(red(sprintf("✖ Tag %s already exists on remote!\n", tag_name))) + cat("This version has already been released.\n") + cat("To create a new release, bump the version first:\n") + cat(" Rscript .dev/bump-version.R \n") + quit(status = 1) +} + +# Step 5: Extract release notes from NEWS.md +cat("Step 2: Extracting release notes from NEWS.md...\n") + +if (!file.exists("NEWS.md")) { + cat(red("✖ NEWS.md not found!\n")) + cat("Please ensure NEWS.md exists and has notes for this version.\n") + quit(status = 1) +} + +news_content <- readLines("NEWS.md", warn = FALSE) + +# Find the section for this version +version_pattern <- paste0("^#+ .*", gsub("\\.", "\\\\.", version)) +version_line_idx <- grep(version_pattern, news_content) + +if (length(version_line_idx) == 0) { + cat(red(sprintf("✖ No section found in NEWS.md for version %s\n", version))) + cat("Please add release notes to NEWS.md before creating the tag.\n") + quit(status = 1) +} + +# Extract notes until the next version heading +start_idx <- version_line_idx[1] + 1 +next_version_idx <- grep("^#+ .*[0-9]+\\.[0-9]+\\.[0-9]+", news_content[start_idx:length(news_content)]) + +if (length(next_version_idx) > 0) { + end_idx <- start_idx + next_version_idx[1] - 2 +} else { + end_idx <- length(news_content) +} + +release_notes <- paste(news_content[start_idx:end_idx], collapse = "\n") +release_notes <- trimws(release_notes) + +if (nchar(release_notes) == 0) { + cat(yellow("⚠ Warning: No release notes found for this version in NEWS.md\n")) + release_notes <- sprintf("Release version %s\n\nSee NEWS.md for details.", version) +} + +# Create tag message +tag_message <- sprintf("Release version %s\n\n%s", version, release_notes) + +cat(green("✓ Release notes extracted\n\n")) + +# Step 6: Verify we're on main branch +current_branch <- trimws(system("git rev-parse --abbrev-ref HEAD", intern = TRUE)) + +if (current_branch != "main") { + cat(red(sprintf("✖ Not on main branch (currently on: %s)\n", current_branch))) + cat("Please switch to main branch before creating release tags:\n") + cat(" git checkout main\n") + cat(" git pull scheier main\n") + quit(status = 1) +} + +# Step 7: Check if working tree is clean +git_status <- system("git status --porcelain", intern = TRUE) + +if (length(git_status) > 0) { + cat(red("✖ Working tree is not clean!\n")) + cat("Please commit or stash your changes before creating a release tag.\n") + cat("\nUncommitted changes:\n") + cat(paste(git_status, collapse = "\n"), "\n") + quit(status = 1) +} + +cat(green("✓ On main branch with clean working tree\n\n")) + +# Step 8: Preview and confirm +cat(bold("=== Release Tag Summary ===\n")) +cat(sprintf("Tag: %s\n", tag_name)) +cat(sprintf("Version: %s\n", version)) +cat(sprintf("Branch: %s\n", current_branch)) +cat("\nTag message:\n") +cat("---\n") +cat(tag_message) +cat("\n---\n\n") + +if (dry_run) { + cat(yellow("DRY RUN: Would create and push tag, but skipping due to --dry-run flag\n")) + cat("\nTo create the tag for real, run:\n") + cat(" Rscript .dev/create-release-tag.R\n") + quit(status = 0) +} + +cat(yellow("This will create and push the tag, triggering the controlled release workflow.\n")) +cat(yellow("The workflow requires manual approval at two gates before publishing.\n\n")) + +# Interactive confirmation +cat("Proceed with creating and pushing the tag? [y/N]: ") +response <- tolower(trimws(readLines("stdin", n = 1))) + +if (response != "y" && response != "yes") { + cat("\nAborted by user.\n") + quit(status = 0) +} + +# Step 9: Create the tag +cat("\nStep 3: Creating annotated tag...\n") + +# Write tag message to temporary file to handle multi-line messages properly +tmp_file <- tempfile() +writeLines(tag_message, tmp_file) + +tag_cmd <- sprintf("git tag -a %s -F %s", tag_name, tmp_file) +tag_result <- system(tag_cmd, intern = FALSE) +unlink(tmp_file) + +if (tag_result != 0) { + cat(red("✖ Failed to create tag!\n")) + quit(status = 1) +} + +cat(green(sprintf("✓ Tag %s created\n\n", tag_name))) + +# Step 10: Push the tag +cat("Step 4: Pushing tag to trigger release workflow...\n") + +push_result <- system(sprintf("git push scheier %s", tag_name), intern = FALSE) + +if (push_result != 0) { + cat(red("\n✖ Failed to push tag!\n")) + cat(yellow(sprintf("\nThe tag was created locally but not pushed. To try again:\n"))) + cat(sprintf(" git push scheier %s\n", tag_name)) + cat(sprintf("\nOr to delete the local tag and start over:\n")) + cat(sprintf(" git tag -d %s\n", tag_name)) + quit(status = 1) +} + +cat(green(sprintf("\n✓ Tag %s pushed successfully!\n\n", tag_name))) + +# Step 11: Provide next steps +cat(bold("=== Next Steps ===\n\n")) +cat("The Controlled Release workflow has been triggered.\n\n") +cat("Monitor progress:\n") +cat(" gh run list --workflow='Controlled Release' --limit 3\n") +cat(" gh run watch --workflow='Controlled Release'\n\n") +cat("The workflow stages:\n") +cat(" 1. Validation: Running quality checks (automatic)\n") +cat(" 2. Gate 1: Pre-release review (requires manual approval)\n") +cat(" 3. Create draft release (automatic after approval)\n") +cat(" 4. Gate 2: Production release approval (requires manual approval)\n") +cat(" 5. Publish release (automatic after approval, triggers Zenodo archival)\n\n") +cat(sprintf("View the workflow at:\n")) +cat(sprintf(" https://github.com/ScheierVentures/emburden/actions/workflows/controlled-release.yaml\n\n")) +cat(green("✓ Release process initiated successfully!\n")) diff --git a/.dev/diagnose-data-loading-dev.R b/.dev/diagnose-data-loading-dev.R new file mode 100644 index 0000000..d39ab7b --- /dev/null +++ b/.dev/diagnose-data-loading-dev.R @@ -0,0 +1,134 @@ +#!/usr/bin/env Rscript +# Diagnostic script for DEVELOPMENT mode (uses devtools::load_all) +# Part of the emburden development toolkit + +cat("\n==========================================\n") +cat(" Data Loading Diagnostic (DEV MODE)\n") +cat("==========================================\n\n") + +# Load development version +devtools::load_all() + +# Step 1: Check MD5 checksums in development package +cat("Step 1: Checking MD5 checksums in development package...\n") +config <- emburden:::get_zenodo_config() + +cat("\nConfigured MD5 checksums:\n") +cat(" FPL 2022: ", config$files$fpl_2022$md5, "\n") +cat(" FPL 2018: ", config$files$fpl_2018$md5, "\n") +cat(" AMI 2022: ", config$files$ami_2022$md5, "\n") +cat(" AMI 2018: ", config$files$ami_2018$md5, "\n") + +# Expected correct values from actual Zenodo files +expected <- list( + fpl_2022 = "767f2ff27193116f61e893999eb8bcf1", + fpl_2018 = "3da8be8c8628656b7772df4c4e7c4e04", + ami_2022 = "d3b30d9d0009032ebb1b9228e44d0e2d", + ami_2018 = "5aefd8e2ef0a63089b68977579d9df86" +) + +cat("\nExpected MD5 checksums:\n") +cat(" FPL 2022: ", expected$fpl_2022, "\n") +cat(" FPL 2018: ", expected$fpl_2018, "\n") +cat(" AMI 2022: ", expected$ami_2022, "\n") +cat(" AMI 2018: ", expected$ami_2018, "\n") + +cat("\nMD5 Verification:\n") +all_match <- TRUE +for (dataset in c("fpl_2022", "fpl_2018", "ami_2022", "ami_2018")) { + actual <- config$files[[dataset]]$md5 + exp <- expected[[dataset]] + match <- identical(actual, exp) + if (!match) all_match <- FALSE + status <- if (match) "✓ MATCH" else "✗ MISMATCH" + cat(sprintf(" %s: %s\n", dataset, status)) +} + +if (!all_match) { + cat("\n❌ MD5 MISMATCH DETECTED!\n") + cat(" This means R/zenodo.R has incorrect checksums.\n") + cat(" Run: Rscript .dev/update-zenodo-config.R\n\n") + quit(status = 1) +} + +cat("\n✓ All MD5 checksums are correct!\n") + +# Step 2: Clear all caches to force fresh downloads +cat("\n\nStep 2: Clearing all caches...\n") +emburden::clear_dataset_cache("fpl", "2022", verbose = TRUE) +emburden::clear_dataset_cache("fpl", "2018", verbose = TRUE) + +# Step 3: Load FPL 2022 with verbose output +cat("\n\nStep 3: Loading FPL 2022 with verbose output...\n") +cat("==========================================\n") +fpl_2022 <- load_cohort_data("fpl", "2022", verbose = TRUE) + +cat("\n\nFPL 2022 Data Summary:\n") +cat(" Rows: ", format(nrow(fpl_2022), big.mark = ","), "\n") +cat(" States: ", length(unique(substr(as.character(fpl_2022$geoid), 1, 2))), "\n") +cat(" Households (total): ", format(sum(fpl_2022$households), big.mark = ","), "\n") +cat(" Households (Above FPL): ", format(sum(fpl_2022$households[fpl_2022$income_bracket == "Above Federal Poverty Line"]), big.mark = ","), "\n") +cat(" Households (Below FPL): ", format(sum(fpl_2022$households[fpl_2022$income_bracket == "Below Federal Poverty Line"]), big.mark = ","), "\n") + +# Step 4: Load FPL 2018 with verbose output +cat("\n\nStep 4: Loading FPL 2018 with verbose output...\n") +cat("==========================================\n") +fpl_2018 <- load_cohort_data("fpl", "2018", verbose = TRUE) + +cat("\n\nFPL 2018 Data Summary:\n") +cat(" Rows: ", format(nrow(fpl_2018), big.mark = ","), "\n") +cat(" States: ", length(unique(substr(as.character(fpl_2018$geoid), 1, 2))), "\n") +cat(" Households (total): ", format(sum(fpl_2018$households), big.mark = ","), "\n") +cat(" Households (Above FPL): ", format(sum(fpl_2018$households[fpl_2018$income_bracket == "Above Federal Poverty Line"]), big.mark = ","), "\n") +cat(" Households (Below FPL): ", format(sum(fpl_2018$households[fpl_2018$income_bracket == "Below Federal Poverty Line"]), big.mark = ","), "\n") + +# Step 5: Compare the datasets +cat("\n\nStep 5: Comparing datasets...\n") +cat("==========================================\n") + +cat("\nRow counts:\n") +cat(" FPL 2022: ", format(nrow(fpl_2022), big.mark = ","), "\n") +cat(" FPL 2018: ", format(nrow(fpl_2018), big.mark = ","), "\n") +cat(" Same? ", identical(nrow(fpl_2022), nrow(fpl_2018)), if (identical(nrow(fpl_2022), nrow(fpl_2018))) " ✗" else " ✓", "\n") + +cat("\nTotal households:\n") +cat(" FPL 2022: ", format(sum(fpl_2022$households), big.mark = ","), "\n") +cat(" FPL 2018: ", format(sum(fpl_2018$households), big.mark = ","), "\n") +cat(" Same? ", identical(sum(fpl_2022$households), sum(fpl_2018$households)), if (identical(sum(fpl_2022$households), sum(fpl_2018$households))) " ✗" else " ✓", "\n") + +# Step 6: Check if data is bitwise identical +cat("\n\nStep 6: Checking if datasets are bitwise identical...\n") +data_identical <- TRUE +if (identical(dim(fpl_2022), dim(fpl_2018))) { + # Check a few key columns + cols_to_check <- c("geoid", "income_bracket", "households", "mean_energy_burden") + for (col in cols_to_check) { + if (col %in% names(fpl_2022) && col %in% names(fpl_2018)) { + is_identical <- identical(fpl_2022[[col]], fpl_2018[[col]]) + if (!is_identical) data_identical <- FALSE + cat(sprintf(" %s: %s\n", col, if (is_identical) "IDENTICAL ✗" else "DIFFERENT ✓")) + } + } +} else { + cat(" Dimensions differ - datasets are different ✓\n") + data_identical <- FALSE +} + +# Step 7: Run the comparison function +cat("\n\nStep 7: Running compare_energy_burden()...\n") +cat("==========================================\n") +comparison <- compare_energy_burden(dataset = "fpl", group_by = "income_bracket") +print(comparison) + +cat("\n\n==========================================\n") +if (data_identical) { + cat(" ❌ DIAGNOSTIC FAILED\n") + cat("==========================================\n\n") + cat("The 2018 and 2022 datasets are IDENTICAL - this is a bug!\n") + cat("Check the Zenodo files themselves to see if they contain the same data.\n\n") + quit(status = 1) +} else { + cat(" ✅ DIAGNOSTIC PASSED\n") + cat("==========================================\n\n") + cat("The 2018 and 2022 datasets are DIFFERENT as expected!\n\n") +} diff --git a/.dev/diagnose-data-loading.R b/.dev/diagnose-data-loading.R new file mode 100644 index 0000000..75e98de --- /dev/null +++ b/.dev/diagnose-data-loading.R @@ -0,0 +1,141 @@ +#!/usr/bin/env Rscript +# Diagnostic script to understand data loading behavior +# Part of the emburden development toolkit + +cat("\n==========================================\n") +cat(" Data Loading Diagnostic\n") +cat("==========================================\n\n") + +library(emburden) + +# Step 1: Check MD5 checksums in installed package +cat("Step 1: Checking MD5 checksums in installed package...\n") +config <- emburden:::get_zenodo_config() + +cat("\nConfigured MD5 checksums:\n") +cat(" FPL 2022: ", config$files$fpl_2022$md5, "\n") +cat(" FPL 2018: ", config$files$fpl_2018$md5, "\n") +cat(" AMI 2022: ", config$files$ami_2022$md5, "\n") +cat(" AMI 2018: ", config$files$ami_2018$md5, "\n") + +# Expected correct values from actual Zenodo files +expected <- list( + fpl_2022 = "767f2ff27193116f61e893999eb8bcf1", + fpl_2018 = "3da8be8c8628656b7772df4c4e7c4e04", + ami_2022 = "d3b30d9d0009032ebb1b9228e44d0e2d", + ami_2018 = "5aefd8e2ef0a63089b68977579d9df86" +) + +cat("\nExpected MD5 checksums:\n") +cat(" FPL 2022: ", expected$fpl_2022, "\n") +cat(" FPL 2018: ", expected$fpl_2018, "\n") +cat(" AMI 2022: ", expected$ami_2022, "\n") +cat(" AMI 2018: ", expected$ami_2018, "\n") + +cat("\nMD5 Verification:\n") +all_match <- TRUE +for (dataset in c("fpl_2022", "fpl_2018", "ami_2022", "ami_2018")) { + actual <- config$files[[dataset]]$md5 + exp <- expected[[dataset]] + match <- identical(actual, exp) + if (!match) all_match <- FALSE + status <- if (match) "✓ MATCH" else "✗ MISMATCH" + cat(sprintf(" %s: %s (actual: %s, expected: %s)\n", dataset, status, actual, exp)) +} + +if (!all_match) { + cat("\n❌ MD5 MISMATCH DETECTED!\n") + cat(" This means R/zenodo.R has incorrect checksums.\n") + cat(" Run: Rscript .dev/update-zenodo-config.R\n\n") + quit(status = 1) +} + +# Step 2: Clear all caches to force fresh downloads +cat("\n\nStep 2: Clearing all caches...\n") +emburden::clear_dataset_cache("fpl", "2022", verbose = TRUE) +emburden::clear_dataset_cache("fpl", "2018", verbose = TRUE) + +# Step 3: Load FPL 2022 with verbose output +cat("\n\nStep 3: Loading FPL 2022 with verbose output...\n") +cat("==========================================\n") +fpl_2022 <- load_cohort_data("fpl", "2022", verbose = TRUE) + +cat("\n\nFPL 2022 Data Summary:\n") +cat(" Rows: ", format(nrow(fpl_2022), big.mark = ","), "\n") +cat(" States: ", length(unique(substr(as.character(fpl_2022$geoid), 1, 2))), "\n") +cat(" Households (total): ", format(sum(fpl_2022$households), big.mark = ","), "\n") +cat(" Households (Above FPL): ", format(sum(fpl_2022$households[fpl_2022$income_bracket == "Above Federal Poverty Line"]), big.mark = ","), "\n") +cat(" Households (Below FPL): ", format(sum(fpl_2022$households[fpl_2022$income_bracket == "Below Federal Poverty Line"]), big.mark = ","), "\n") + +# Step 4: Load FPL 2018 with verbose output +cat("\n\nStep 4: Loading FPL 2018 with verbose output...\n") +cat("==========================================\n") +fpl_2018 <- load_cohort_data("fpl", "2018", verbose = TRUE) + +cat("\n\nFPL 2018 Data Summary:\n") +cat(" Rows: ", format(nrow(fpl_2018), big.mark = ","), "\n") +cat(" States: ", length(unique(substr(as.character(fpl_2018$geoid), 1, 2))), "\n") +cat(" Households (total): ", format(sum(fpl_2018$households), big.mark = ","), "\n") +cat(" Households (Above FPL): ", format(sum(fpl_2018$households[fpl_2018$income_bracket == "Above Federal Poverty Line"]), big.mark = ","), "\n") +cat(" Households (Below FPL): ", format(sum(fpl_2018$households[fpl_2018$income_bracket == "Below Federal Poverty Line"]), big.mark = ","), "\n") + +# Step 5: Compare the datasets +cat("\n\nStep 5: Comparing datasets...\n") +cat("==========================================\n") + +cat("\nRow counts:\n") +cat(" FPL 2022: ", format(nrow(fpl_2022), big.mark = ","), "\n") +cat(" FPL 2018: ", format(nrow(fpl_2018), big.mark = ","), "\n") +cat(" Identical? ", identical(nrow(fpl_2022), nrow(fpl_2018)), "\n") + +cat("\nTotal households:\n") +cat(" FPL 2022: ", format(sum(fpl_2022$households), big.mark = ","), "\n") +cat(" FPL 2018: ", format(sum(fpl_2018$households), big.mark = ","), "\n") +cat(" Identical? ", identical(sum(fpl_2022$households), sum(fpl_2018$households)), "\n") + +cat("\nSample of first 10 rows (FPL 2022):\n") +print(head(fpl_2022[, c("geoid", "income_bracket", "households", "mean_energy_cost", "mean_energy_burden")], 10)) + +cat("\nSample of first 10 rows (FPL 2018):\n") +print(head(fpl_2018[, c("geoid", "income_bracket", "households", "mean_energy_cost", "mean_energy_burden")], 10)) + +# Step 6: Check if data is bitwise identical +cat("\n\nStep 6: Checking if datasets are bitwise identical...\n") +if (identical(dim(fpl_2022), dim(fpl_2018))) { + # Check a few key columns + cols_to_check <- c("geoid", "income_bracket", "households", "mean_energy_burden") + all_identical <- TRUE + for (col in cols_to_check) { + if (col %in% names(fpl_2022) && col %in% names(fpl_2018)) { + is_identical <- identical(fpl_2022[[col]], fpl_2018[[col]]) + if (is_identical) all_identical <- FALSE # We WANT them to be different + cat(sprintf(" %s: %s\n", col, if (is_identical) "IDENTICAL ✗" else "DIFFERENT ✓")) + } + } + + if (all_identical) { + cat("\n❌ DATA IS IDENTICAL - THIS IS A BUG!\n") + } +} else { + cat(" Dimensions differ - datasets are different ✓\n") +} + +# Step 7: Run the comparison function +cat("\n\nStep 7: Running compare_energy_burden()...\n") +cat("==========================================\n") +comparison <- compare_energy_burden(dataset = "fpl", group_by = "income_bracket") +print(comparison) + +cat("\n\n==========================================\n") +cat(" Diagnostic Complete\n") +cat("==========================================\n\n") + +# Check data source +cat("Where did the data come from?\n") +cat("Check the verbose output above for:\n") +cat(" - 'Downloading from Zenodo' = Data from Zenodo\n") +cat(" - 'Zenodo download failed' + 'Falling back to OpenEI' = Data from OpenEI\n") +cat(" - 'Found in cache' = Data from cache\n") +cat("\nIf both vintages came from OpenEI, that's likely the problem.\n") +cat("If MD5 checksums don't match expected, that's the problem.\n") +cat("If datasets are identical, check the Zenodo files themselves.\n\n") diff --git a/.dev/hooks/pre-commit b/.dev/hooks/pre-commit new file mode 100644 index 0000000..92b6966 --- /dev/null +++ b/.dev/hooks/pre-commit @@ -0,0 +1,113 @@ +#!/bin/bash +# +# Pre-commit hook for quick validation checks +# Runs before every commit to catch issues early +# + +set -e # Exit on error + +echo "==========================================" +echo " Pre-commit: Quick Validation" +echo "==========================================" +echo "" + +# 1. Check for BREAKING CHANGE commitments without version bumps +echo "📋 Checking commit message format..." +COMMIT_MSG_FILE=".git/COMMIT_EDITMSG" +if [ -f "$COMMIT_MSG_FILE" ]; then + if grep -q "BREAKING CHANGE" "$COMMIT_MSG_FILE"; then + echo "⚠️ BREAKING CHANGE detected in commit message" + echo " Remember to bump major version in DESCRIPTION before release!" + fi +fi + +# 2. Check version consistency (quick check only) +echo "" +echo "📋 Checking version consistency..." +if [ -f ".dev/check-version-consistency.R" ]; then + Rscript .dev/check-version-consistency.R > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "✅ Version consistency check passed" + else + echo "❌ Version inconsistency detected!" + echo "" + Rscript .dev/check-version-consistency.R + echo "" + echo "Commit cancelled." + exit 1 + fi +else + echo "⚠️ Version consistency script not found" +fi + +# 3. Check for common issues in staged files +echo "" +echo "📋 Checking staged files..." + +# Check for debugger statements in R code (exclude hook files) +STAGED_R_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep '\.R$' || true) +if [ -n "$STAGED_R_FILES" ]; then + # Only check the actual R files, not all staged content + if git diff --cached -- $STAGED_R_FILES | grep -E '^\+.*browser\(\)' > /dev/null; then + echo "❌ Found browser() debugger call in staged R code!" + echo " Remove debugger statements before committing." + exit 1 + fi +fi + +# Check for TODO/FIXME in committed code (warn only) +if git diff --cached | grep -E '^\+.*(TODO|FIXME|XXX)' > /dev/null; then + echo "⚠️ Found TODO/FIXME markers in staged code" + echo " Consider addressing these before committing" + echo " (commit will proceed)" +fi + +echo "✅ Staged files check passed" + +# 4. Quick R syntax check for staged .R files (if any) +if [ -n "$STAGED_R_FILES" ]; then + echo "" + echo "📋 Checking R syntax..." + + # Separate package files from scripts + PACKAGE_R_FILES=$(echo "$STAGED_R_FILES" | grep "^R/" || true) + SCRIPT_R_FILES=$(echo "$STAGED_R_FILES" | grep -v "^R/" || true) + + # Check package R files using parse() instead of source() + if [ -n "$PACKAGE_R_FILES" ]; then + for file in $PACKAGE_R_FILES; do + Rscript -e "parse('$file')" > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "❌ Syntax error in $file" + Rscript -e "parse('$file')" + echo "" + echo "Commit cancelled." + exit 1 + fi + done + fi + + # Check script R files using source() + if [ -n "$SCRIPT_R_FILES" ]; then + for file in $SCRIPT_R_FILES; do + Rscript -e "source('$file', echo=FALSE)" > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "❌ Syntax error in $file" + Rscript -e "source('$file', echo=FALSE)" + echo "" + echo "Commit cancelled." + exit 1 + fi + done + fi + + echo "✅ R syntax check passed" +fi + +echo "" +echo "==========================================" +echo " ✅ Pre-commit checks passed!" +echo "==========================================" +echo "" + +exit 0 diff --git a/.dev/hooks/pre-push b/.dev/hooks/pre-push new file mode 100644 index 0000000..0d2f448 --- /dev/null +++ b/.dev/hooks/pre-push @@ -0,0 +1,237 @@ +#!/bin/bash +# +# Pre-push hook to catch CI issues before pushing +# Runs comprehensive CRAN-style R CMD check in temporary directory +# + +set -e # Exit on error + +# First check version consistency across metadata files +if [ -f ".dev/check-version-consistency.R" ]; then + Rscript .dev/check-version-consistency.R + VERSION_STATUS=$? + echo "" + + if [ $VERSION_STATUS -ne 0 ]; then + echo "Push cancelled." + exit 1 + fi +fi + +echo "==========================================" +echo " Pre-push: Spelling Validation" +echo "==========================================" +echo "" + +Rscript -e " +if (requireNamespace('spelling', quietly = TRUE)) { + cat('📝 Checking spelling...\n\n') + results <- spelling::spell_check_package() + + if (nrow(results) > 0) { + cat('❌ Spelling errors found:\n\n') + print(results) + cat('\n') + cat('Please fix spelling errors or add valid technical terms to inst/WORDLIST\n') + cat('Push cancelled.\n') + quit(status = 1) + } + + cat('✅ No spelling errors found\n') + quit(status = 0) +} else { + cat('⚠️ spelling package not installed - skipping spell check\n') + cat(' Install with: install.packages(\"spelling\")\n') + quit(status = 0) +} +" + +SPELL_STATUS=$? + +echo "" +if [ $SPELL_STATUS -ne 0 ]; then + echo "Push cancelled." + exit 1 +else + echo "==========================================" + echo " ✅ Spelling check passed!" + echo "==========================================" +fi + +echo "" + +echo "==========================================" +echo " Pre-push: CRAN-style R CMD check" +echo "==========================================" +echo "" + +# Check if we have LaTeX/TinyTeX for vignettes (non-interactive) +if ! command -v pdflatex &> /dev/null; then + echo "⚠️ WARNING: pdflatex not found - skipping vignette build" + echo " Install TinyTeX with: Rscript .dev/install-tinytex.R" + echo "" + SKIP_VIGNETTES="TRUE" +else + SKIP_VIGNETTES="FALSE" +fi + +# Create temporary directory for CRAN check +CHECK_DIR=$(mktemp -d -t emburden-check-XXXXXX) +echo "📦 Building package in temporary directory: $CHECK_DIR" +echo "" + +# Ensure cleanup on exit +cleanup() { + if [ -d "$CHECK_DIR" ]; then + echo "" + echo "🧹 Cleaning up temporary directory..." + rm -rf "$CHECK_DIR" + fi +} +trap cleanup EXIT INT TERM + +# Run comprehensive CRAN check +echo "🔍 Running R CMD check --as-cran..." +echo " (This may take 2-3 minutes for full vignette build)" +echo "" + +Rscript -e " +# Set environment to simulate CRAN +Sys.setenv( + '_R_CHECK_CRAN_INCOMING_' = 'TRUE', + '_R_CHECK_CRAN_INCOMING_REMOTE_' = 'FALSE', # Skip URL checks (slow) + '_R_CHECK_DONTTEST_EXAMPLES_' = 'false', # Skip \donttest{} examples (matches CRAN) + 'NOT_CRAN' = 'false' +) + +# Run comprehensive check +skip_vignettes <- as.logical('$SKIP_VIGNETTES') + +check_args <- c('--as-cran') +build_args <- c('--no-manual') # PDF manual requires pandoc + +if (skip_vignettes) { + cat('⚠️ Skipping vignette build (no pdflatex)\n\n') + check_args <- c(check_args, '--no-build-vignettes', '--ignore-vignettes') + build_args <- c(build_args, '--no-build-vignettes') +} else { + # Use 'both' to match GitHub Actions workflow (tries gs+qpdf, falls back to qpdf) + build_args <- c(build_args, '--compact-vignettes=both') + cat('ℹ️ Using --compact-vignettes=both (matches GitHub Actions workflow)\n\n') +} + +cat('Check arguments:', paste(check_args, collapse=' '), '\n') +cat('Build arguments:', paste(build_args, collapse=' '), '\n\n') + +# Run check with rcmdcheck (cleaner output) +if (requireNamespace('rcmdcheck', quietly = TRUE)) { + results <- rcmdcheck::rcmdcheck( + path = '.', + args = check_args, + build_args = build_args, + error_on = 'never', # Don't stop, show results + check_dir = '$CHECK_DIR' + ) + + # Print summary + cat('\n') + cat(strrep('=', 60), '\n') + cat('R CMD check results:\n') + cat(strrep('=', 60), '\n') + print(results) + + # Check for errors or warnings (fail on errors only, allow warnings) + if (length(results\$errors) > 0) { + cat('\n❌ ERRORS found:\n') + cat(paste(results\$errors, collapse='\n'), '\n') + quit(status = 1) + } + + if (length(results\$warnings) > 0) { + cat('\n⚠️ WARNINGS found (allowed, push will continue):\n') + cat(paste(results\$warnings, collapse='\n'), '\n') + # Don't fail on warnings - CI will catch critical ones + } + + if (length(results\$notes) > 0) { + cat('\nℹ️ NOTES:\n') + cat(paste(results\$notes, collapse='\n'), '\n') + # Notes are OK, don't fail + } + + cat('\n✅ R CMD check passed!\n') + quit(status = 0) + +} else { + cat('Installing rcmdcheck package...\n') + install.packages('rcmdcheck', repos = 'https://cran.rstudio.com') + cat('❌ Please re-run push after rcmdcheck is installed.\n') + quit(status = 1) +} +" + +CHECK_STATUS=$? + +echo "" +if [ $CHECK_STATUS -ne 0 ]; then + echo "❌ R CMD check --as-cran failed with errors!" + echo " Fix errors before pushing. Push cancelled." + exit 1 +else + echo "==========================================" + echo " ✅ CRAN check passed (or warnings only)!" + echo "==========================================" +fi + +echo "" + +# Optional: CRAN submission dry-run test +if [ "$SKIP_CRAN_DRYRUN" != "true" ]; then + echo "==========================================" + echo " Pre-push: CRAN Submission Dry-Run" + echo "==========================================" + echo "" + echo "Testing CRAN submission process (no actual submission)..." + echo "" + + Rscript -e " + # Find the tarball from R CMD build + tarball <- list.files(pattern = 'emburden_.*\\.tar\\.gz$', full.names = TRUE) + if (length(tarball) == 0) { + cat('⚠️ No tarball found - skipping CRAN submission test\n') + quit(status = 0) + } + + tarball <- tarball[1] + cat('Found tarball:', tarball, '\n\n') + + # Test CRAN submission preparation (doesn't actually submit) + if (requireNamespace('devtools', quietly = TRUE)) { + cat('📋 Checking CRAN submission readiness...\n\n') + + # Check if package can be submitted + tryCatch({ + # This checks the package is ready but doesn't submit + devtools::check_built(tarball, cran = TRUE, remote = FALSE, manual = FALSE) + cat('\n✅ CRAN submission dry-run passed!\n') + cat(' Package is ready for CRAN submission.\n') + }, error = function(e) { + cat('\n❌ CRAN submission test failed:\n') + cat(' ', conditionMessage(e), '\n') + cat('\nNote: This is just a dry-run test. Fix issues before actual submission.\n') + # Don't fail the push on dry-run errors + }) + } else { + cat('ℹ️ devtools not installed - skipping CRAN submission test\n') + } + " || true # Don't fail push if dry-run has issues + + echo "" + echo "==========================================" +else + echo "ℹ️ Skipping CRAN submission dry-run (SKIP_CRAN_DRYRUN=true)" + echo "" +fi + +echo "🚀 Proceeding with push..." +exit 0 diff --git a/.dev/install-hooks.sh b/.dev/install-hooks.sh new file mode 100755 index 0000000..3607bee --- /dev/null +++ b/.dev/install-hooks.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Git Hooks Installation Script +# Installs pre-push hook with CRAN validation checks +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HOOKS_DIR="$SCRIPT_DIR/hooks" +GIT_HOOKS_DIR="$(git rev-parse --git-dir)/hooks" + +echo "================================================" +echo " Installing Git Hooks for CRAN Validation" +echo "================================================" +echo "" + +# Check if we're in a git repository +if ! git rev-parse --git-dir > /dev/null 2>&1; then + echo "❌ Error: Not in a git repository" + exit 1 +fi + +# Install pre-commit hook +echo "📦 Installing pre-commit hook..." +cp "$HOOKS_DIR/pre-commit" "$GIT_HOOKS_DIR/pre-commit" +chmod +x "$GIT_HOOKS_DIR/pre-commit" +echo "✅ pre-commit hook installed" + +echo "" + +# Install pre-push hook +echo "📦 Installing pre-push hook..." +cp "$HOOKS_DIR/pre-push" "$GIT_HOOKS_DIR/pre-push" +chmod +x "$GIT_HOOKS_DIR/pre-push" +echo "✅ pre-push hook installed" + +echo "" +echo "================================================" +echo " ✅ Git Hooks Installation Complete!" +echo "================================================" +echo "" +echo "The pre-commit hook will run before every commit:" +echo " 1. Version consistency check" +echo " 2. Check for debugger statements" +echo " 3. Quick R syntax validation" +echo "" +echo "The pre-push hook will run before every push:" +echo " 1. Version consistency check" +echo " 2. Spelling validation (blocking)" +echo " 3. CRAN-style R CMD check with --compact-vignettes=both" +echo "" +echo "To bypass hooks (not recommended):" +echo " git commit --no-verify" +echo " git push --no-verify" +echo "" diff --git a/.dev/install-tinytex.R b/.dev/install-tinytex.R new file mode 100644 index 0000000..16e4fc8 --- /dev/null +++ b/.dev/install-tinytex.R @@ -0,0 +1,50 @@ +#!/usr/bin/env Rscript +# install-tinytex.R +# Helper script to install TinyTeX for building PDF vignettes +# +# Usage: Rscript .dev/install-tinytex.R + +cat(strrep("=", 60), "\n") +cat("Installing TinyTeX for PDF vignette building\n") +cat(strrep("=", 60), "\n\n") + +# Install tinytex package if not already installed +if (!requireNamespace("tinytex", quietly = TRUE)) { + cat("Installing tinytex package...\n") + install.packages("tinytex", repos = "https://cran.rstudio.com") +} else { + cat("tinytex package already installed.\n") +} + +# Check if TinyTeX is already installed +if (tinytex::is_tinytex()) { + cat("\nTinyTeX is already installed at:\n") + cat(" ", tinytex:::tinytex_root(), "\n\n") + + # Update TinyTeX packages + cat("Updating TinyTeX packages...\n") + tinytex::tlmgr_update() + + cat("\n") + cat(strrep("=", 60), "\n") + cat("TinyTeX is ready!\n") + cat(strrep("=", 60), "\n") + +} else { + # Install TinyTeX + cat("\nInstalling TinyTeX...\n") + cat("This will download ~100MB and may take a few minutes.\n\n") + + tinytex::install_tinytex() + + cat("\n") + cat(strrep("=", 60), "\n") + cat("TinyTeX installation complete!\n") + cat("Installed at:", tinytex:::tinytex_root(), "\n") + cat(strrep("=", 60), "\n") +} + +cat("\nYou can now build PDF vignettes with:\n") +cat(" R CMD build .\n") +cat(" or\n") +cat(" devtools::build_vignettes()\n\n") diff --git a/.dev/integrate-feature-to-dev.R b/.dev/integrate-feature-to-dev.R new file mode 100644 index 0000000..c945fcf --- /dev/null +++ b/.dev/integrate-feature-to-dev.R @@ -0,0 +1,269 @@ +#!/usr/bin/env Rscript +# Feature Integration Protocol Script +# Systematically integrates completed features into dev branch +# with comprehensive testing, documentation, and validation +# +# Usage: +# Rscript .dev/integrate-feature-to-dev.R [--feature-name "NAME"] [--skip-check] +# +# Example: +# Rscript .dev/integrate-feature-to-dev.R --feature-name "housing-dimensions" +# +# This script implements the 6-phase integration protocol: +# Phase 1: Testing - Ensure comprehensive test coverage +# Phase 2: Documentation - Update docs, vignettes, NEWS.md +# Phase 3: Validation - Run devtools checks +# Phase 4: Protocol - Create reusable templates (this script!) +# Phase 5: Git Integration - Commit changes +# Phase 6: CI Validation - Verify CI passes + +library(devtools) +library(usethis) + +# Color output for terminal +red <- function(x) paste0("\033[31m", x, "\033[0m") +green <- function(x) paste0("\033[32m", x, "\033[0m") +yellow <- function(x) paste0("\033[33m", x, "\033[0m") +blue <- function(x) paste0("\033[34m", x, "\033[0m") +bold <- function(x) paste0("\033[1m", x, "\033[0m") + +# Parse command line arguments +args <- commandArgs(trailingOnly = TRUE) +feature_name <- NULL +skip_check <- FALSE + +if (length(args) > 0) { + for (i in seq_along(args)) { + if (args[i] == "--feature-name" && i < length(args)) { + feature_name <- args[i + 1] + } else if (args[i] == "--skip-check") { + skip_check <- TRUE + } else if (args[i] == "--help" || args[i] == "-h") { + cat("Feature Integration Protocol Script\n\n") + cat("Usage:\n") + cat(" Rscript .dev/integrate-feature-to-dev.R [OPTIONS]\n\n") + cat("Options:\n") + cat(" --feature-name NAME Name of the feature being integrated\n") + cat(" --skip-check Skip R CMD check (faster, for quick iterations)\n") + cat(" --help, -h Show this help message\n\n") + cat("Example:\n") + cat(" Rscript .dev/integrate-feature-to-dev.R --feature-name 'housing-dimensions'\n\n") + quit(save = "no", status = 0) + } + } +} + +# Interactive prompt if feature name not provided +if (is.null(feature_name)) { + cat(blue("Enter feature name (e.g., 'housing-dimensions'): ")) + feature_name <- readLines(con = "stdin", n = 1, warn = FALSE) + if (length(feature_name) == 0 || feature_name == "") { + cat(red("✖ Feature name required\n")) + quit(save = "no", status = 1) + } +} + +cat(bold(blue("\n══════════════════════════════════════════════════════════\n"))) +cat(bold(blue(" Feature Integration Protocol\n"))) +cat(bold(blue("══════════════════════════════════════════════════════════\n\n"))) +cat(sprintf("Feature: %s\n", bold(feature_name))) +cat(sprintf("Date: %s\n", Sys.Date())) +cat("\n") + +# Phase tracking +phase_status <- list() + +# Helper function to run phase +run_phase <- function(phase_num, phase_name, phase_func) { + cat(bold(sprintf("\n═══ Phase %d: %s ═══\n", phase_num, phase_name))) + + result <- tryCatch({ + phase_func() + phase_status[[phase_name]] <<- "✔" + cat(green(sprintf("✔ Phase %d complete\n", phase_num))) + TRUE + }, error = function(e) { + phase_status[[phase_name]] <<- "✖" + cat(red(sprintf("✖ Phase %d failed: %s\n", phase_num, e$message))) + FALSE + }) + + return(result) +} + +# PHASE 1: Testing +phase1_testing <- function() { + cat("Running test suite validation...\n") + + # Check if test files exist for this feature + test_files <- list.files("tests/testthat", + pattern = "test-.*\\.R$", + full.names = TRUE) + cat(sprintf("Found %d test files\n", length(test_files))) + + # Run tests + cat("\nRunning devtools::test()...\n") + test_results <- devtools::test() + + # Check results + if (any(as.data.frame(test_results)$failed > 0)) { + stop("Tests failed. Fix failing tests before integration.") + } + + total_tests <- sum(as.data.frame(test_results)$nb) + cat(green(sprintf("✔ All %d tests passing\n", total_tests))) + + return(TRUE) +} + +# PHASE 2: Documentation +phase2_documentation <- function() { + cat("Updating documentation...\n") + + # Regenerate documentation + cat("\nRunning devtools::document()...\n") + devtools::document() + cat(green("✔ Documentation regenerated\n")) + + # Check NEWS.md exists and has recent entry + if (file.exists("NEWS.md")) { + news <- readLines("NEWS.md", n = 20) + if (length(news) > 0) { + cat(green("✔ NEWS.md exists\n")) + cat(sprintf(" First line: %s\n", news[1])) + } + } else { + cat(yellow("⚠ NEWS.md not found - consider adding changelog entry\n")) + } + + # Check vignettes + vignettes <- list.files("vignettes", pattern = "\\.Rmd$") + if (length(vignettes) > 0) { + cat(green(sprintf("✔ Found %d vignette(s)\n", length(vignettes)))) + } + + return(TRUE) +} + +# PHASE 3: Validation +phase3_validation <- function() { + cat("Running R CMD check validation...\n") + + if (skip_check) { + cat(yellow("⚠ Skipping R CMD check (--skip-check flag set)\n")) + return(TRUE) + } + + cat("\nRunning devtools::check()...\n") + cat(yellow("(This may take several minutes...)\n\n")) + + check_results <- devtools::check( + document = FALSE, # Already documented in Phase 2 + quiet = FALSE + ) + + # Analyze results + errors <- length(check_results$errors) + warnings <- length(check_results$warnings) + notes <- length(check_results$notes) + + if (errors > 0) { + cat(red(sprintf("✖ %d ERROR(S) found\n", errors))) + stop("R CMD check found errors. Fix before integration.") + } + + if (warnings > 0) { + cat(yellow(sprintf("⚠ %d WARNING(S) found\n", warnings))) + cat("Review warnings before integration:\n") + for (w in check_results$warnings) { + cat(sprintf(" - %s\n", w)) + } + } + + if (notes > 0) { + cat(blue(sprintf("ℹ %d NOTE(S) found\n", notes))) + } + + cat(green("✔ R CMD check passed (0 errors)\n")) + return(TRUE) +} + +# PHASE 4: Git Status Check +phase4_git_status <- function() { + cat("Checking git status...\n") + + # Get current branch + branch_result <- system("git branch --show-current", intern = TRUE) + current_branch <- trimws(branch_result) + + cat(sprintf("Current branch: %s\n", bold(current_branch))) + + if (current_branch != "dev") { + cat(yellow(sprintf("⚠ Not on dev branch. Consider switching to dev.\n"))) + cat(" Run: git checkout dev\n") + } + + # Check for uncommitted changes + status_result <- system("git status --porcelain", intern = TRUE) + + if (length(status_result) > 0) { + cat(blue(sprintf("\nℹ %d file(s) with changes:\n", length(status_result)))) + # Show first 10 files + display_files <- head(status_result, 10) + for (f in display_files) { + cat(sprintf(" %s\n", f)) + } + if (length(status_result) > 10) { + cat(sprintf(" ... and %d more\n", length(status_result) - 10)) + } + } else { + cat(green("✔ No uncommitted changes\n")) + } + + return(TRUE) +} + +# PHASE 5: Integration Summary +phase5_summary <- function() { + cat("\nGenerating integration summary...\n\n") + + cat(bold("Integration Checklist:\n")) + cat(sprintf(" [%s] Tests: Comprehensive test coverage added\n", + if (phase_status[["Testing"]]) green("✔") else " ")) + cat(sprintf(" [%s] Documentation: Function docs and vignettes updated\n", + if (phase_status[["Documentation"]]) green("✔") else " ")) + cat(sprintf(" [%s] Validation: R CMD check passed\n", + if (phase_status[["Validation"]]) green("✔") else " ")) + cat(sprintf(" [%s] Git Status: Changes reviewed\n", + if (phase_status[["Git Status"]]) green("✔") else " ")) + + cat("\n") + cat(bold("Next Steps:\n")) + cat(" 1. Review checklist: .dev/FEATURE_INTEGRATION_CHECKLIST.md\n") + cat(" 2. Commit changes: git add . && git commit -m 'feat: '\n") + cat(" 3. Push to remote: git push origin dev\n") + cat(" 4. Verify CI passes: Check GitHub Actions\n") + cat(" 5. Ready for staging: Merge dev → staging when ready\n") + + return(TRUE) +} + +# Execute all phases +success <- TRUE +success <- success && run_phase(1, "Testing", phase1_testing) +success <- success && run_phase(2, "Documentation", phase2_documentation) +success <- success && run_phase(3, "Validation", phase3_validation) +success <- success && run_phase(4, "Git Status", phase4_git_status) +success <- success && run_phase(5, "Summary", phase5_summary) + +# Final status +cat(bold(blue("\n══════════════════════════════════════════════════════════\n"))) +if (success) { + cat(bold(green("✔ Feature integration protocol complete!\n"))) + cat(bold(blue("══════════════════════════════════════════════════════════\n\n"))) + quit(save = "no", status = 0) +} else { + cat(bold(red("✖ Feature integration protocol failed\n"))) + cat(bold(blue("══════════════════════════════════════════════════════════\n\n"))) + quit(save = "no", status = 1) +} diff --git a/.dev/manual-fpl-2022-merge.R b/.dev/manual-fpl-2022-merge.R new file mode 100644 index 0000000..2c914df --- /dev/null +++ b/.dev/manual-fpl-2022-merge.R @@ -0,0 +1,109 @@ +#!/usr/bin/env Rscript +# Manual fix for FPL 2022: Add missing HI and IL states +# This script processes raw HI and IL data and merges with existing 49-state dataset + +library(dplyr) +library(readr) + +cat("================================================================================\n") +cat(" Manual FPL 2022 Fix: Adding HI and IL States\n") +cat("================================================================================\n\n") + +# 1. Load existing 49-state dataset +cat("Step 1: Loading existing 49-state FPL 2022 dataset...\n") +existing_data <- read_csv( + "zenodo-upload-nationwide/nationwide/lead_fpl_cohorts_2022_us.csv.gz", + show_col_types = FALSE +) +cat(" ✓ Loaded", format(nrow(existing_data), big.mark=","), "rows\n") +cat(" ✓ States:", length(unique(existing_data$state_abbr)), "\n\n") + +# 2. Process HI raw data +cat("Step 2: Processing Hawaii (HI) raw data...\n") +hi_raw <- read_csv("/tmp/fpl-fix/HI FPL Census Tracts 2022.csv", show_col_types = FALSE) +cat(" ✓ Loaded", format(nrow(hi_raw), big.mark=","), "raw rows\n") + +hi_processed <- hi_raw %>% + rename( + geoid = FIP, + income_bracket = FPL150 + ) %>% + mutate(geoid = as.character(geoid)) %>% + group_by(geoid, income_bracket) %>% + summarize( + households = sum(UNITS, na.rm = TRUE), + total_income = sum(`HINCP*UNITS`, na.rm = TRUE), + total_electricity_spend = sum(`ELEP*UNITS`, na.rm = TRUE), + total_gas_spend = sum(`GASP*UNITS`, na.rm = TRUE), + total_other_spend = sum(`FULP*UNITS`, na.rm = TRUE), + .groups = "drop" + ) %>% + select(geoid, income_bracket, households, total_income, + total_electricity_spend, total_gas_spend, total_other_spend) + +cat(" ✓ Aggregated to", format(nrow(hi_processed), big.mark=","), "cohort rows\n\n") + +# 3. Process IL raw data +cat("Step 3: Processing Illinois (IL) raw data...\n") +il_raw <- read_csv("/tmp/fpl-fix/IL FPL Census Tracts 2022.csv", show_col_types = FALSE) +cat(" ✓ Loaded", format(nrow(il_raw), big.mark=","), "raw rows\n") + +il_processed <- il_raw %>% + rename( + geoid = FIP, + income_bracket = FPL150 + ) %>% + mutate(geoid = as.character(geoid)) %>% + group_by(geoid, income_bracket) %>% + summarize( + households = sum(UNITS, na.rm = TRUE), + total_income = sum(`HINCP*UNITS`, na.rm = TRUE), + total_electricity_spend = sum(`ELEP*UNITS`, na.rm = TRUE), + total_gas_spend = sum(`GASP*UNITS`, na.rm = TRUE), + total_other_spend = sum(`FULP*UNITS`, na.rm = TRUE), + .groups = "drop" + ) %>% + select(geoid, income_bracket, households, total_income, + total_electricity_spend, total_gas_spend, total_other_spend) + +cat(" ✓ Aggregated to", format(nrow(il_processed), big.mark=","), "cohort rows\n\n") + +# 4. Merge all datasets +cat("Step 4: Merging all datasets...\n") +complete_data <- bind_rows(existing_data, hi_processed, il_processed) +cat(" ✓ Total rows:", format(nrow(complete_data), big.mark=","), "\n") + +# 5. Validate +cat("\nStep 5: Validation...\n") +# Extract state FIPS from geoid (first 2 digits) +state_fips <- unique(substr(as.character(complete_data$geoid), 1, 2)) +n_states <- length(state_fips) +cat(" States found:", n_states, "\n") +cat(" State FIPS codes:", paste(sort(state_fips), collapse=", "), "\n") + +if (n_states == 51) { + cat(" ✅ All 51 states present!\n\n") + + # 6. Save + cat("Step 6: Saving complete dataset...\n") + csv_file <- "zenodo-upload-nationwide/nationwide/lead_fpl_cohorts_2022_us.csv" + write_csv(complete_data, csv_file) + cat(" ✓ Saved CSV\n") + + system(sprintf("gzip -9 -f %s", csv_file)) + cat(" ✓ Compressed\n") + + file_info <- file.info(paste0(csv_file, ".gz")) + size_mb <- round(file_info$size / 1024 / 1024, 2) + cat(" ✓ Final size:", size_mb, "MB\n\n") + + cat("================================================================================\n") + cat(" ✅ FPL 2022 COMPLETE: All 51 states merged successfully!\n") + cat("================================================================================\n") +} else { + all_fips <- sprintf("%02d", c(1:2, 4:6, 8:13, 15:42, 44:51, 53:56)) + state_fips <- unique(substr(as.character(complete_data$geoid), 1, 2)) + missing <- setdiff(all_fips, state_fips) + cat(" ❌ Still missing states:", paste(missing, collapse=", "), "\n") + stop("FPL 2022 merge incomplete") +} diff --git a/.dev/post-process-zenodo-data.R b/.dev/post-process-zenodo-data.R new file mode 100644 index 0000000..eeef8f3 --- /dev/null +++ b/.dev/post-process-zenodo-data.R @@ -0,0 +1,234 @@ +#!/usr/bin/env Rscript +# +# Post-Process Zenodo Data (No Regeneration Required) +# +# This script applies post-processing fixes to already-generated Zenodo datasets +# WITHOUT requiring a full regeneration from OpenEI (saves time and bandwidth). +# +# Use this for: +# - Column renaming (e.g., AMI150 → income_bracket) +# - Adding derived columns +# - Filtering/cleaning existing data +# - Updating metadata/checksums +# +# DO NOT use this for: +# - Changes to data loading logic +# - New data sources or vintages +# - Changes to aggregation/grouping +# - Changes that require re-downloading from OpenEI +# +# Usage: +# Rscript .dev/post-process-zenodo-data.R +# Rscript .dev/post-process-zenodo-data.R --dataset ami_2022 +# Rscript .dev/post-process-zenodo-data.R --fix ami-column-rename +# + +suppressPackageStartupMessages({ + library(readr) + library(dplyr) + library(tools) +}) + +# Parse arguments +args <- commandArgs(trailingOnly = TRUE) +specific_dataset <- NULL +specific_fix <- NULL + +if ("--dataset" %in% args) { + idx <- which(args == "--dataset") + if (length(args) > idx) { + specific_dataset <- args[idx + 1] + } +} + +if ("--fix" %in% args) { + idx <- which(args == "--fix") + if (length(args) > idx) { + specific_fix <- args[idx + 1] + } +} + +cat("================================================================================\n") +cat(" Post-Processing Zenodo Datasets (No Regeneration)\n") +cat("================================================================================\n\n") + +if (!is.null(specific_dataset)) { + cat("Target dataset:", specific_dataset, "\n") +} +if (!is.null(specific_fix)) { + cat("Specific fix:", specific_fix, "\n") +} +cat("\n") + +# Base directory +base_dir <- "zenodo-upload-nationwide/nationwide" + +if (!dir.exists(base_dir)) { + stop("ERROR: Nationwide directory not found: ", base_dir, "\n", + " Please run prepare-zenodo-data-nationwide.R first to generate base datasets.") +} + +# Find all .csv.gz files +all_files <- list.files(base_dir, pattern = "\\.csv\\.gz$", full.names = TRUE) + +if (length(all_files) == 0) { + stop("ERROR: No .csv.gz files found in ", base_dir) +} + +cat("Found", length(all_files), "dataset(s) to process:\n") +for (f in all_files) { + cat(" -", basename(f), "\n") +} +cat("\n") + +# Define post-processing fixes +apply_ami_column_rename <- function(data, dataset_name) { + # Fix AMI datasets that use AMI150 instead of income_bracket + if (grepl("ami", dataset_name, ignore.case = TRUE) && "AMI150" %in% names(data)) { + cat(" 🔧 Applying fix: AMI150 → income_bracket\n") + data <- data %>% rename(income_bracket = AMI150) + cat(" ✓ Column renamed\n") + return(list(data = data, modified = TRUE)) + } + return(list(data = data, modified = FALSE)) +} + +apply_validation_checks <- function(data, dataset_name) { + cat(" ✓ Validating dataset...\n") + + # Check required columns + required_cols <- c("geoid", "income_bracket", "households", + "total_income", "total_electricity_spend") + missing_cols <- setdiff(required_cols, names(data)) + + if (length(missing_cols) > 0) { + stop(" ❌ VALIDATION FAILED: Missing columns: ", paste(missing_cols, collapse = ", ")) + } + + # Check for state coverage + if ("state_abbr" %in% names(data)) { + n_states <- length(unique(data$state_abbr)) + cat(" States:", n_states, "\n") + if (n_states < 51) { + warning(" ⚠️ WARNING: Only ", n_states, " states found (expected 51)") + } + } + + # Check income_bracket has detailed values (not binary) + if ("income_bracket" %in% names(data)) { + unique_brackets <- unique(data$income_bracket) + n_brackets <- length(unique_brackets) + cat(" Income brackets:", n_brackets, "unique values\n") + + if (n_brackets < 5) { + warning(" ⚠️ WARNING: Only ", n_brackets, + " income brackets (expected detailed brackets, not binary)") + } + } + + cat(" Rows:", format(nrow(data), big.mark = ","), "\n") + cat(" ✅ Validation passed\n") + + return(TRUE) +} + +# Process each file +modified_files <- c() +skipped_files <- c() + +for (gz_file in all_files) { + dataset_name <- tools::file_path_sans_ext(tools::file_path_sans_ext(basename(gz_file))) + + # Check if we should process this file + if (!is.null(specific_dataset) && !grepl(specific_dataset, dataset_name, ignore.case = TRUE)) { + next + } + + cat("================================================================================\n") + cat("Processing:", dataset_name, "\n") + cat("================================================================================\n\n") + + # Decompress + cat(" 📦 Decompressing...\n") + csv_file <- tools::file_path_sans_ext(gz_file) + system2("gunzip", args = c("-k", "-f", gz_file), stdout = FALSE, stderr = FALSE) + + # Read data + cat(" 📖 Reading data...\n") + data <- read_csv(csv_file, show_col_types = FALSE) + original_rows <- nrow(data) + cat(" Loaded", format(original_rows, big.mark = ","), "rows\n\n") + + # Apply fixes + modified <- FALSE + + # Fix 1: AMI column rename (if needed and requested) + if (is.null(specific_fix) || specific_fix == "ami-column-rename") { + result <- apply_ami_column_rename(data, dataset_name) + data <- result$data + modified <- modified || result$modified + } + + # Validate + cat("\n") + apply_validation_checks(data, dataset_name) + + if (modified) { + cat("\n 💾 Saving modified dataset...\n") + + # Write CSV + write_csv(data, csv_file) + + # Compress + cat(" Compressing...\n") + system2("gzip", args = c("-9", "-f", csv_file), stdout = FALSE, stderr = FALSE) + + # Calculate new checksum + new_md5 <- as.character(tools::md5sum(gz_file)) + new_size_mb <- round(file.size(gz_file) / 1024^2, 2) + + cat(" ✓ Updated successfully\n") + cat(" Size:", new_size_mb, "MB\n") + cat(" MD5:", new_md5, "\n") + + modified_files <- c(modified_files, dataset_name) + } else { + cat("\n ⏭️ No modifications needed - skipping\n") + + # Remove decompressed file + if (file.exists(csv_file)) { + file.remove(csv_file) + } + + skipped_files <- c(skipped_files, dataset_name) + } + + cat("\n") +} + +# Summary +cat("================================================================================\n") +cat(" Post-Processing Complete\n") +cat("================================================================================\n\n") + +if (length(modified_files) > 0) { + cat("✅ Modified", length(modified_files), "dataset(s):\n") + for (f in modified_files) { + cat(" -", f, "\n") + } + cat("\n") +} + +if (length(skipped_files) > 0) { + cat("⏭️ Skipped", length(skipped_files), "dataset(s) (no changes needed):\n") + for (f in skipped_files) { + cat(" -", f, "\n") + } + cat("\n") +} + +cat("Next steps:\n") +cat(" 1. Review changes: ls -lh", base_dir, "\n") +cat(" 2. Update checksums in R/zenodo.R if needed\n") +cat(" 3. Commit changes: git add . && git commit\n") +cat(" 4. Upload to Zenodo when ready\n") diff --git a/.dev/pre-tag-cran-check.R b/.dev/pre-tag-cran-check.R new file mode 100644 index 0000000..246830a --- /dev/null +++ b/.dev/pre-tag-cran-check.R @@ -0,0 +1,232 @@ +#!/usr/bin/env Rscript + +# Pre-Tag CRAN Validation Script +# Run this script before creating a version tag to ensure CRAN readiness +# +# Usage: +# Rscript .dev/pre-tag-cran-check.R [--submit-winbuilder] +# +# Options: +# --submit-winbuilder Also submit to Win-builder for Windows testing + +library(devtools) +library(rcmdcheck) + +# Parse command line arguments +args <- commandArgs(trailingOnly = TRUE) +submit_winbuilder <- "--submit-winbuilder" %in% args + +cat("\n") +cat("═══════════════════════════════════════════════════════════════════\n") +cat(" Pre-Tag CRAN Validation for emburden\n") +cat("═══════════════════════════════════════════════════════════════════\n") +cat("\n") + +# Track validation status +checks_passed <- TRUE +warnings <- character() + +# Helper function to print status +print_status <- function(message, status = "INFO") { + prefix <- switch(status, + "OK" = "✅", + "WARN" = "⚠️ ", + "ERROR" = "❌", + "INFO" = "ℹ️ " + ) + cat(sprintf("%s %s\n", prefix, message)) +} + +# 1. Check version consistency +cat("\n[1/6] Checking version consistency across files...\n") +if (file.exists(".dev/check-version-consistency.R")) { + tryCatch({ + source(".dev/check-version-consistency.R", local = TRUE) + print_status("Version consistency validated", "OK") + }, error = function(e) { + print_status(paste("Version consistency check failed:", e$message), "ERROR") + checks_passed <<- FALSE + }) +} else { + print_status("Version consistency script not found", "WARN") + warnings <<- c(warnings, "Missing .dev/check-version-consistency.R") +} + +# 2. Check NEWS.md has been updated +cat("\n[2/6] Checking NEWS.md...\n") +if (file.exists("NEWS.md")) { + news_content <- readLines("NEWS.md", warn = FALSE) + + # Get version from DESCRIPTION + desc_version <- as.character(read.dcf("DESCRIPTION", fields = "Version")) + + # Check if version appears in NEWS.md + version_in_news <- any(grepl(desc_version, news_content, fixed = TRUE)) + + if (version_in_news) { + print_status(sprintf("NEWS.md contains version %s", desc_version), "OK") + } else { + print_status(sprintf("NEWS.md does not mention version %s", desc_version), "ERROR") + checks_passed <<- FALSE + } +} else { + print_status("NEWS.md not found", "ERROR") + checks_passed <<- FALSE +} + +# 3. Check git status +cat("\n[3/6] Checking git status...\n") +git_status <- system("git status --porcelain", intern = TRUE) +if (length(git_status) > 0) { + print_status("Uncommitted changes detected:", "WARN") + cat(paste(" ", git_status, collapse = "\n"), "\n") + warnings <<- c(warnings, "Uncommitted changes") +} else { + print_status("Working directory clean", "OK") +} + +# 4. Build package with vignette compaction (matches GitHub Actions) +cat("\n[4/6] Building source package...\n") +cat("Building with --compact-vignettes=both (matches GitHub Actions workflow)\n\n") +tarball <- tryCatch({ + # Use system2 to call R CMD build directly with compaction flags + # This matches the GitHub Actions workflow exactly + temp_dir <- tempdir() + build_cmd <- sprintf( + 'R CMD build . --no-manual --compact-vignettes=both' + ) + + build_result <- system2("sh", args = c("-c", build_cmd), stdout = TRUE, stderr = TRUE) + + # Find the generated tarball + desc_version <- as.character(read.dcf("DESCRIPTION", fields = "Version")) + tarball_name <- sprintf("emburden_%s.tar.gz", desc_version) + + if (file.exists(tarball_name)) { + print_status(sprintf("Package built: %s", tarball_name), "OK") + normalizePath(tarball_name) + } else { + stop("Tarball not found after build") + } +}, error = function(e) { + print_status(paste("Build failed:", e$message), "ERROR") + checks_passed <<- FALSE + NULL +}) + +# 5. Run R CMD check --as-cran +if (!is.null(tarball)) { + cat("\n[5/6] Running R CMD check --as-cran...\n") + cat("This may take several minutes...\n\n") + + check_result <- tryCatch({ + rcmdcheck::rcmdcheck( + path = tarball, + args = c("--as-cran", "--no-manual"), + error_on = "never", # Don't error, we'll check manually + check_dir = tempdir() + ) + }, error = function(e) { + print_status(paste("R CMD check failed to run:", e$message), "ERROR") + checks_passed <<- FALSE + NULL + }) + + if (!is.null(check_result)) { + # Print summary + cat("\n") + print(check_result) + cat("\n") + + # Check results + if (length(check_result$errors) > 0) { + print_status(sprintf("%d ERROR(s) found", length(check_result$errors)), "ERROR") + checks_passed <<- FALSE + } + + if (length(check_result$warnings) > 0) { + print_status(sprintf("%d WARNING(s) found", length(check_result$warnings)), "ERROR") + checks_passed <<- FALSE + } + + if (length(check_result$notes) > 0) { + print_status(sprintf("%d NOTE(s) found", length(check_result$notes)), "WARN") + warnings <<- c(warnings, sprintf("%d NOTEs in R CMD check", length(check_result$notes))) + cat("\nNOTEs should be reviewed, but may be acceptable for CRAN:\n") + for (note in check_result$notes) { + cat(sprintf(" • %s\n", note)) + } + } + + if (length(check_result$errors) == 0 && length(check_result$warnings) == 0) { + print_status("R CMD check passed!", "OK") + } + } +} else { + cat("\n[5/6] Skipping R CMD check (build failed)\n") +} + +# 6. Optional Win-builder submission +if (submit_winbuilder && !is.null(tarball)) { + cat("\n[6/6] Submitting to Win-builder...\n") + + # Get email from environment or prompt + cran_email <- Sys.getenv("CRAN_EMAIL") + if (cran_email == "") { + cat("Enter CRAN maintainer email: ") + cran_email <- readLines("stdin", n = 1) + } + + tryCatch({ + devtools::check_win_release(email = cran_email) + print_status("Submitted to Win-builder", "OK") + cat(sprintf("\n Results will be emailed to %s within ~30 minutes\n", cran_email)) + }, error = function(e) { + print_status(paste("Win-builder submission failed:", e$message), "WARN") + warnings <<- c(warnings, "Win-builder submission failed") + }) +} else if (submit_winbuilder && is.null(tarball)) { + cat("\n[6/6] Skipping Win-builder (build failed)\n") +} else { + cat("\n[6/6] Skipping Win-builder (use --submit-winbuilder to enable)\n") + print_status("Win-builder submission skipped", "INFO") + print_status("Run with --submit-winbuilder flag for Windows testing", "INFO") +} + +# Summary +cat("\n") +cat("═══════════════════════════════════════════════════════════════════\n") +cat(" Validation Summary\n") +cat("═══════════════════════════════════════════════════════════════════\n") +cat("\n") + +if (checks_passed) { + if (length(warnings) == 0) { + print_status("All checks passed! ✨", "OK") + cat("\n") + cat("Your package is ready for CRAN submission.\n") + cat("\n") + cat("Next steps:\n") + cat(" 1. Create version tag: git tag -a vX.Y.Z -m 'Release vX.Y.Z'\n") + cat(" 2. Push tag: git push origin vX.Y.Z\n") + cat(" 3. GitHub Actions will run validation and wait for approval\n") + cat(" 4. After approval, package will be auto-submitted to CRAN\n") + cat("\n") + } else { + print_status("Checks passed with warnings:", "WARN") + for (w in warnings) { + cat(sprintf(" • %s\n", w)) + } + cat("\n") + cat("Consider addressing warnings before creating version tag.\n") + cat("\n") + } +} else { + print_status("Validation failed! ❌", "ERROR") + cat("\n") + cat("Please fix the errors above before creating a version tag.\n") + cat("\n") + quit(status = 1) +} + +cat("\n") diff --git a/.dev/prepare-zenodo-data-nationwide.R b/.dev/prepare-zenodo-data-nationwide.R new file mode 100644 index 0000000..d588876 --- /dev/null +++ b/.dev/prepare-zenodo-data-nationwide.R @@ -0,0 +1,611 @@ +#!/usr/bin/env Rscript +# Prepare Nationwide Analysis-Ready Datasets for Zenodo Upload +# +# This script generates PROCESSED, analysis-ready datasets organized by state +# and as combined nationwide datasets. Users can download individual states +# or the full nationwide dataset. +# +# Output structure: +# zenodo-upload-nationwide/ +# by-state/ +# NC/ +# lead_ami_cohorts_2022_nc.csv.gz +# lead_fpl_cohorts_2022_nc.csv.gz +# lead_ami_cohorts_2018_nc.csv.gz +# lead_fpl_cohorts_2018_nc.csv.gz +# CA/ +# ... +# [all 51 states/territories] +# nationwide/ +# lead_ami_cohorts_2022_us.csv.gz +# lead_fpl_cohorts_2022_us.csv.gz +# lead_ami_cohorts_2018_us.csv.gz +# lead_fpl_cohorts_2018_us.csv.gz +# checksums.txt +# state-manifest.json +# +# Usage: +# # Normal mode: use cache if available, download if needed (default) +# Rscript prepare-zenodo-data-nationwide.R --nationwide-only +# +# # Cache-only mode: use cached data only (ideal for post-processing fixes) +# Rscript prepare-zenodo-data-nationwide.R --nationwide-only --use-cache +# +# # Force download: clear cache and re-download everything +# Rscript prepare-zenodo-data-nationwide.R --nationwide-only --force-download + +# Load development version of emburden (includes list_states() and validation functions) +library(devtools) +load_all(".") + +library(dplyr) +library(readr) +library(jsonlite) + +cat("\n") +cat("================================================================================\n") +cat(" Preparing Nationwide Analysis-Ready Datasets for Zenodo Upload\n") +cat("================================================================================\n") +cat("\n") +cat("This script downloads RAW data from OpenEI, processes it into analysis-ready\n") +cat("format, and organizes it by state AND as nationwide datasets.\n") +cat("\n") + +# Configuration +args <- commandArgs(trailingOnly = TRUE) +states_only <- "--states-only" %in% args +nationwide_only <- "--nationwide-only" %in% args +quick_test <- "--quick-test" %in% args # Just a few states for testing + +# Cache control flags (for efficient re-processing without re-downloading) +use_cache_only <- "--use-cache" %in% args # Use cached data only, fail if missing +force_download <- "--force-download" %in% args # Force re-download even if cache exists + +# Validate conflicting flags +if (use_cache_only && force_download) { + stop("ERROR: Cannot use both --use-cache and --force-download") +} + +# Set download policy environment variable for load_cohort_data +if (use_cache_only) { + Sys.setenv(EMBURDEN_NO_DOWNLOAD = "1") + cat("\n") + cat("================================================================================\n") + cat(" CACHE-ONLY MODE: Will use cached data without downloading\n") + cat("================================================================================\n") + cat("\n") + cat("This mode is ideal for:\n") + cat(" - Re-processing with post-processing fixes (e.g., column renaming)\n") + cat(" - Testing validation logic without re-downloading\n") + cat(" - Quick iterations on data transformation\n") + cat("\n") + cat("If cache is missing, the script will fail. Use --force-download to re-download.\n") + cat("\n") +} else if (force_download) { + # Clear cache to force fresh downloads + cache_dir <- get_cache_dir() + if (dir.exists(cache_dir)) { + cat("\n") + cat("================================================================================\n") + cat(" FORCE-DOWNLOAD MODE: Clearing cache and re-downloading all data\n") + cat("================================================================================\n") + cat("\n") + unlink(cache_dir, recursive = TRUE) + cat("✓ Cache cleared:", cache_dir, "\n\n") + } +} + +# Output directories +base_dir <- "zenodo-upload-nationwide" +state_dir <- file.path(base_dir, "by-state") +nationwide_dir <- file.path(base_dir, "nationwide") + +dir.create(base_dir, showWarnings = FALSE, recursive = TRUE) +dir.create(state_dir, showWarnings = FALSE, recursive = TRUE) +dir.create(nationwide_dir, showWarnings = FALSE, recursive = TRUE) + +cat("Output directories:\n") +cat(" Base:", normalizePath(base_dir), "\n") +cat(" By-state:", normalizePath(state_dir), "\n") +cat(" Nationwide:", normalizePath(nationwide_dir), "\n\n") + +# Get all available states (51 total: 50 states + DC) +# Note: PR excluded as it's often not in the OpenEI data +all_states <- list_states() + +if (quick_test) { + cat("QUICK TEST MODE: Using only 3 states (NC, CA, TX)\n\n") + all_states <- c("NC", "CA", "TX") +} + +cat("States to process:", length(all_states), "\n") +cat("States:", paste(all_states, collapse = ", "), "\n\n") + +# Dataset configurations +# NOTE: Arizona 2018 data has non-standard filename: "AZ-2018-LEAD-data (1).zip" +# This is handled in R/lead_data_loaders.R +datasets <- list( + list(name = "ami", vintage = "2022"), + list(name = "fpl", vintage = "2022"), + list(name = "ami", vintage = "2018"), + list(name = "fpl", vintage = "2018") +) + +# Manifest to track all files +manifest <- list( + generated = Sys.time(), + emburden_version = as.character(packageVersion("emburden")), + states = list(), + nationwide = list(), + statistics = list() +) + +# Function to validate dataset before saving +validate_dataset <- function(data, expected_scope, dataset_name, vintage) { + cat("\n === Validating Dataset ===\n") + + # Check 1: Data exists and has rows + if (is.null(data) || nrow(data) == 0) { + stop("VALIDATION FAILED: Dataset is NULL or empty!") + } + cat(" ✓ Data exists (", format(nrow(data), big.mark = ","), " rows)\n", sep = "") + + # Check 2: Required columns exist + required_cols <- c("geoid", "income_bracket", "households", + "total_income", "total_electricity_spend") + missing_cols <- setdiff(required_cols, names(data)) + if (length(missing_cols) > 0) { + stop("VALIDATION FAILED: Missing required columns: ", + paste(missing_cols, collapse = ", ")) + } + cat(" ✓ Required columns present\n") + + # Check 3: For nationwide datasets, verify state coverage + if (expected_scope == "nationwide") { + # Check for state_abbr column - if missing, try to add it from geoid + if (!"state_abbr" %in% names(data)) { + cat(" ⚠️ WARNING: state_abbr column missing, attempting to add from geoid...\n") + + if (!"geoid" %in% names(data)) { + stop("VALIDATION FAILED: Cannot add state_abbr - geoid column also missing!") + } + + # Get state FIPS from geoid (first 2 characters) + data$state_fips <- substr(data$geoid, 1, 2) + + # Map FIPS to state abbreviations + fips_to_state <- setNames( + c("AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", "HI", "ID", + "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", "MA", "MI", "MN", "MS", + "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK", + "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", + "WI", "WY", "DC"), + c("01", "02", "04", "05", "06", "08", "09", "10", "12", "13", "15", "16", + "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", + "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53", "54", + "55", "56", "11") + ) + + data$state_abbr <- fips_to_state[data$state_fips] + + # Check if we successfully added it + if (all(is.na(data$state_abbr))) { + stop("VALIDATION FAILED: Could not map FIPS codes to state abbreviations!") + } + + cat(" ✓ Successfully added state_abbr column from geoid\n") + } + + # Get unique states + states_present <- unique(data$state_abbr) + states_present <- states_present[!is.na(states_present)] + n_states <- length(states_present) + + cat(" ✓ state_abbr column exists\n") + cat(" ✓ States found:", n_states, "\n") + + # Must have exactly 51 states (50 + DC) + if (n_states != 51) { + cat(" ❌ ERROR: Expected 51 states, found", n_states, "\n") + cat(" Missing states:", paste(setdiff(list_states(), states_present), collapse = ", "), "\n") + cat(" Extra states:", paste(setdiff(states_present, list_states()), collapse = ", "), "\n") + stop("VALIDATION FAILED: Nationwide dataset does not have all 51 states!") + } + cat(" ✓ All 51 US states/territories present\n") + + # Verify minimum row count (nationwide should have 100k+ rows) + min_rows_nationwide <- 100000 + if (nrow(data) < min_rows_nationwide) { + warning("Nationwide dataset has fewer rows than expected: ", + nrow(data), " < ", min_rows_nationwide) + } + } + + # Check 4: For state datasets, verify single state + if (expected_scope != "nationwide" && "state_abbr" %in% names(data)) { + states_present <- unique(data$state_abbr) + states_present <- states_present[!is.na(states_present)] + + if (length(states_present) != 1 || states_present[1] != expected_scope) { + stop("VALIDATION FAILED: State dataset has wrong states. Expected: ", + expected_scope, ", Found: ", paste(states_present, collapse = ", ")) + } + cat(" ✓ Single state (", expected_scope, ") verified\n", sep = "") + } + + # Check 5: Test that data can be used with emburden functions + tryCatch({ + # Try calculating energy burden on a sample + sample_data <- head(data, 100) + if ("total_income" %in% names(sample_data) && + "total_electricity_spend" %in% names(sample_data)) { + test_burden <- sum(sample_data$total_electricity_spend, na.rm = TRUE) / + sum(sample_data$total_income, na.rm = TRUE) + if (!is.finite(test_burden)) { + warning("Sample energy burden calculation returned non-finite value") + } + } + cat(" ✓ Data compatible with emburden calculations\n") + }, error = function(e) { + stop("VALIDATION FAILED: Data incompatible with emburden functions: ", e$message) + }) + + cat(" ✅ All validation checks passed!\n\n") + return(TRUE) +} + +# Function to compress and save +compress_and_save <- function(data, output_file, desc, expected_scope = NULL, + dataset_name = NULL, vintage = NULL) { + + # Validate before saving (if validation params provided) + if (!is.null(expected_scope)) { + validate_dataset(data, expected_scope, dataset_name, vintage) + } + + cat(" Saving:", basename(output_file), "\n") + + # Save uncompressed + write_csv(data, output_file) + + # Compress + system2("gzip", args = c("-9", "-f", output_file)) + gz_file <- paste0(output_file, ".gz") + + # Get file info + size_bytes <- file.size(gz_file) + size_mb <- round(size_bytes / 1024^2, 2) + md5 <- as.character(tools::md5sum(gz_file)) + + cat(" Size:", size_mb, "MB (compressed)\n") + cat(" Rows:", format(nrow(data), big.mark = ","), "\n") + cat(" MD5:", md5, "\n") + + return(list( + filename = basename(gz_file), + path = normalizePath(gz_file), + size_mb = size_mb, + rows = nrow(data), + md5 = md5, + description = desc + )) +} + +# Initialize nationwide data collectors +if (!nationwide_only) { + + cat("================================================================================\n") + cat(" Phase 1: Processing State-by-State Datasets\n") + cat("================================================================================\n\n") + + for (state in all_states) { + cat("================================================================================\n") + cat("Processing State:", state, "\n") + cat("================================================================================\n\n") + + # Create state directory + state_output_dir <- file.path(state_dir, state) + dir.create(state_output_dir, showWarnings = FALSE, recursive = TRUE) + + state_manifest <- list( + state = state, + datasets = list() + ) + + for (ds in datasets) { + dataset_name <- ds$name + vintage <- ds$vintage + + cat(" Dataset:", toupper(dataset_name), vintage, "\n") + + # Load state data + data <- tryCatch({ + load_cohort_data( + dataset = dataset_name, + vintage = vintage, + states = state, + verbose = FALSE + ) + }, error = function(e) { + cat(" ERROR:", e$message, "\n\n") + return(NULL) + }) + + if (is.null(data) || nrow(data) == 0) { + cat(" SKIPPED: No data available\n\n") + next + } + + # Fix AMI column naming: AMI150 → income_bracket + # AMI datasets from OpenEI use "AMI150" column instead of "income_bracket" + if (dataset_name == "ami" && "AMI150" %in% names(data)) { + cat(" ⚙️ Standardizing AMI column naming (AMI150 → income_bracket)...\n") + data <- data %>% + rename(income_bracket = AMI150) + cat(" ✓ Column renamed\n") + } + + # Manual filter by state (in case load_cohort_data didn't filter properly) + if ("state_abbr" %in% names(data)) { + data <- data %>% filter(state_abbr == state) + cat(" Filtered to", state, ":", format(nrow(data), big.mark = ","), "rows\n") + } else { + cat(" WARNING: No state_abbr column, cannot verify state filtering\n") + } + + # Save state-specific dataset + state_file <- file.path( + state_output_dir, + sprintf("lead_%s_cohorts_%s_%s.csv", dataset_name, vintage, tolower(state)) + ) + + file_info <- compress_and_save( + data, + state_file, + sprintf("%s %s cohort data for %s", vintage, toupper(dataset_name), state), + expected_scope = state, + dataset_name = dataset_name, + vintage = vintage + ) + + state_manifest$datasets[[paste0(dataset_name, "_", vintage)]] <- file_info + + cat("\n") + } + + # Save state manifest + manifest$states[[state]] <- state_manifest + + cat("✓ State", state, "complete\n\n") + } +} + +# Phase 2: Create nationwide combined datasets +if (!states_only) { + + cat("================================================================================\n") + cat(" Phase 2: Creating Nationwide Combined Datasets\n") + cat("================================================================================\n\n") + + for (ds in datasets) { + dataset_name <- ds$name + vintage <- ds$vintage + + cat("================================================================================\n") + cat("Nationwide Dataset:", toupper(dataset_name), vintage, "\n") + cat("================================================================================\n\n") + + cat(" Loading all states...\n") + + # Self-healing retry loop + max_retries <- 2 + retry_count <- 0 + success <- FALSE + + while (!success && retry_count < max_retries) { + # Load all states + all_data <- tryCatch({ + load_cohort_data( + dataset = dataset_name, + vintage = vintage, + states = all_states, + verbose = TRUE + ) + }, error = function(e) { + cat(" ERROR:", e$message, "\n\n") + return(NULL) + }) + + if (is.null(all_data) || nrow(all_data) == 0) { + cat(" SKIPPED: No data available\n\n") + next # Skip to next dataset, don't break out of entire loop + } + + cat("\n Combined data loaded successfully!\n") + cat(" Total rows:", format(nrow(all_data), big.mark = ","), "\n") + cat(" Total states:", length(unique(all_data$state_abbr)), "\n\n") + + # Fix AMI column naming: AMI150 → income_bracket + # AMI datasets from OpenEI use "AMI150" column instead of "income_bracket" + if (dataset_name == "ami" && "AMI150" %in% names(all_data)) { + cat(" ⚙️ Standardizing AMI column naming (AMI150 → income_bracket)...\n") + all_data <- all_data %>% + rename(income_bracket = AMI150) + cat(" ✓ Column renamed\n\n") + } + + # Save nationwide dataset (with validation) + nationwide_file <- file.path( + nationwide_dir, + sprintf("lead_%s_cohorts_%s_us.csv", dataset_name, vintage) + ) + + # Try to save with validation + file_info <- tryCatch({ + compress_and_save( + all_data, + nationwide_file, + sprintf("%s %s cohort data (all US states)", vintage, toupper(dataset_name)), + expected_scope = "nationwide", + dataset_name = dataset_name, + vintage = vintage + ) + }, error = function(e) { + # Validation failed - likely corrupted database data + cat("\n ❌ VALIDATION FAILED:", e$message, "\n") + + # Check if this is due to incomplete database data + if (grepl("state|VALIDATION FAILED", e$message, ignore.case = TRUE)) { + cat("\n 🔧 SELF-HEALING: Detected corrupted database data\n") + cat(" Deleting database table to force reload from CSV/OpenEI...\n") + + # Delete corrupted database table + db_path <- rappdirs::user_data_dir('emburden', 'emburden') + db_file <- file.path(db_path, 'emburden_db.sqlite') + + if (file.exists(db_file)) { + # Connect and drop the table + conn <- tryCatch({ + DBI::dbConnect(RSQLite::SQLite(), db_file) + }, error = function(e2) NULL) + + if (!is.null(conn)) { + table_name <- paste0(dataset_name, "_cohorts_", vintage) + tryCatch({ + DBI::dbExecute(conn, sprintf("DROP TABLE IF EXISTS %s", table_name)) + cat(" ✓ Deleted table:", table_name, "\n") + }, error = function(e2) { + cat(" ⚠️ Could not delete table:", e2$message, "\n") + }) + DBI::dbDisconnect(conn) + } + + # Also try alternate table name format + conn <- tryCatch({ + DBI::dbConnect(RSQLite::SQLite(), db_file) + }, error = function(e2) NULL) + + if (!is.null(conn)) { + table_name_alt <- paste0("lead_", vintage, "_", dataset_name, "_cohorts") + tryCatch({ + DBI::dbExecute(conn, sprintf("DROP TABLE IF EXISTS %s", table_name_alt)) + cat(" ✓ Deleted table:", table_name_alt, "\n") + }, error = function(e2) NULL) + DBI::dbDisconnect(conn) + } + } + + cat(" Database cleaned. Will retry with fresh data...\n\n") + } + + return(NULL) + }) + + # Check if save succeeded + if (!is.null(file_info)) { + success <- TRUE + } else { + retry_count <- retry_count + 1 + if (retry_count < max_retries) { + cat("\n 🔄 RETRY", retry_count, "of", max_retries - 1, "...\n\n") + } else { + cat("\n ❌ Failed after", max_retries - 1, "retries. Skipping this dataset.\n\n") + next + } + } + } + + if (!success) { + next # Skip to next dataset + } + + manifest$nationwide[[paste0(dataset_name, "_", vintage)]] <- file_info + + cat("\n") + } +} + +# Phase 3: Generate checksums and manifest +cat("================================================================================\n") +cat(" Phase 3: Generating Checksums and Manifest\n") +cat("================================================================================\n\n") + +# Find all .gz files +all_gz_files <- list.files( + base_dir, + pattern = "\\.csv\\.gz$", + recursive = TRUE, + full.names = TRUE +) + +cat("Total compressed files:", length(all_gz_files), "\n\n") + +# Calculate checksums +cat("Calculating checksums...\n") +checksums_file <- file.path(base_dir, "checksums.txt") +checksums <- tools::md5sum(all_gz_files) + +# Write checksums with relative paths +writeLines( + paste(checksums, sub(paste0("^", normalizePath(base_dir), "/"), "", names(checksums))), + checksums_file +) + +cat("✓ Checksums saved to:", basename(checksums_file), "\n\n") + +# Calculate statistics +total_size_mb <- sum(sapply(all_gz_files, function(f) file.size(f) / 1024^2)) +total_rows <- 0 + +for (state_data in manifest$states) { + for (ds_data in state_data$datasets) { + total_rows <- total_rows + ds_data$rows + } +} + +for (nationwide_data in manifest$nationwide) { + # Don't double-count (nationwide is combination of states) +} + +manifest$statistics <- list( + total_files = length(all_gz_files), + total_size_mb = round(total_size_mb, 2), + total_rows_by_state = total_rows, + states_processed = length(manifest$states), + nationwide_datasets = length(manifest$nationwide) +) + +# Save manifest +manifest_file <- file.path(base_dir, "state-manifest.json") +write_json(manifest, manifest_file, pretty = TRUE, auto_unbox = TRUE) + +cat("✓ Manifest saved to:", basename(manifest_file), "\n\n") + +# Summary +cat("================================================================================\n") +cat(" Data Preparation Complete\n") +cat("================================================================================\n\n") + +cat("Statistics:\n") +cat(" States processed:", manifest$statistics$states_processed, "\n") +cat(" Total files:", manifest$statistics$total_files, "\n") +cat(" Total size:", manifest$statistics$total_size_mb, "MB (compressed)\n") +cat(" Total rows (by-state):", format(manifest$statistics$total_rows_by_state, big.mark = ","), "\n\n") + +cat("Output directory:", normalizePath(base_dir), "\n\n") + +cat("Next steps:\n") +cat(" 1. Review the manifest: cat", manifest_file, "\n") +cat(" 2. Upload to Zenodo using .dev/upload-to-zenodo-nationwide.sh\n") +cat(" 3. Update R/zenodo.R with DOIs and URLs\n\n") + +cat("Options for Zenodo upload strategy:\n") +cat(" A. Upload all states + nationwide (complete, large upload)\n") +cat(" B. Upload just nationwide datasets (simpler, good for full-scale analysis)\n") +cat(" C. Upload select states + nationwide (flexible, medium size)\n\n") + +cat("To prepare different subsets, re-run with:\n") +cat(" --states-only Only generate state-by-state datasets\n") +cat(" --nationwide-only Only generate nationwide combined datasets\n") +cat(" --quick-test Just NC, CA, TX for testing\n\n") diff --git a/.dev/prepare-zenodo-data.R b/.dev/prepare-zenodo-data.R new file mode 100644 index 0000000..d339db2 --- /dev/null +++ b/.dev/prepare-zenodo-data.R @@ -0,0 +1,145 @@ +#!/usr/bin/env Rscript +# Prepare Analysis-Ready Datasets for Zenodo Upload +# +# This script generates PROCESSED, analysis-ready datasets for Zenodo hosting. +# These are NOT raw OpenEI data - they are pre-processed, aggregated, and +# include computed energy burden metrics. +# +# Output: 4 nationwide processed CSV files ready for Zenodo upload +# - lead_ami_cohorts_2022_us.csv (processed, analysis-ready) +# - lead_fpl_cohorts_2022_us.csv (processed, analysis-ready) +# - lead_ami_cohorts_2018_us.csv (processed, analysis-ready) +# - lead_fpl_cohorts_2018_us.csv (processed, analysis-ready) + +library(emburden) +library(dplyr) +library(readr) + +cat("\n") +cat("================================================================================\n") +cat(" Preparing Analysis-Ready Datasets for Zenodo Upload\n") +cat("================================================================================\n") +cat("\n") +cat("This script downloads RAW data from OpenEI, processes it into analysis-ready\n") +cat("format, and saves it for Zenodo upload. This ensures users download PROCESSED\n") +cat("data that's ready for immediate analysis.\n") +cat("\n") + +# Output directory +output_dir <- "zenodo-upload" +if (!dir.exists(output_dir)) { + dir.create(output_dir, recursive = TRUE) +} + +cat("Output directory:", normalizePath(output_dir), "\n\n") + +# Function to process and save a dataset +process_and_save <- function(dataset, vintage) { + + cat("================================================================================\n") + cat("Processing:", toupper(dataset), vintage, "\n") + cat("================================================================================\n\n") + + # Output filename + output_file <- file.path(output_dir, paste0("lead_", dataset, "_cohorts_", vintage, "_us.csv")) + + # Check if already exists + if (file.exists(output_file)) { + cat("✓ File already exists:", basename(output_file), "\n") + cat(" Size:", format(file.size(output_file) / 1024^2, digits = 2), "MB\n\n") + return(invisible(TRUE)) + } + + # Load data (will download from OpenEI if not cached, then process) + cat("Loading", dataset, vintage, "data from OpenEI...\n") + cat("(This will download raw data if not cached, then process it)\n\n") + + data <- tryCatch({ + load_cohort_data( + dataset = dataset, + vintage = vintage, + verbose = TRUE + ) + }, error = function(e) { + cat("\n❌ Error loading data:", e$message, "\n\n") + return(NULL) + }) + + if (is.null(data)) { + cat("\n❌ Failed to load data\n\n") + return(invisible(FALSE)) + } + + cat("\n") + cat("Data loaded successfully!\n") + cat(" Rows:", format(nrow(data), big.mark = ","), "\n") + cat(" Cols:", ncol(data), "\n") + cat(" States:", length(unique(substr(data$geoid, 1, 2))), "\n") + + # Verify this is processed data (has computed metrics) + required_cols <- c("geoid", "income_bracket", "households", "total_income", + "total_electricity_spend", "total_gas_spend") + + if (!all(required_cols %in% names(data))) { + cat("\n❌ ERROR: Data missing required processed columns!\n") + cat(" This appears to be raw data, not processed data.\n") + cat(" Missing:", setdiff(required_cols, names(data)), "\n\n") + return(invisible(FALSE)) + } + + cat("\n✓ Verified: Data contains processed metrics (energy burden, etc.)\n") + + # Save processed data + cat("\nSaving processed data to:", basename(output_file), "\n") + write_csv(data, output_file) + + # Report size + size_mb <- file.size(output_file) / 1024^2 + cat("✓ Saved successfully!\n") + cat(" Size:", format(size_mb, digits = 2), "MB uncompressed\n") + cat(" Estimated compressed size:", format(size_mb * 0.2, digits = 2), "MB (gzip)\n\n") + + return(invisible(TRUE)) +} + +# Process all 4 datasets +cat("Starting data preparation...\n\n") + +success_count <- 0 + +# 2022 data (latest) +if (process_and_save("ami", "2022")) success_count <- success_count + 1 +if (process_and_save("fpl", "2022")) success_count <- success_count + 1 + +# 2018 data (historical comparison) +if (process_and_save("ami", "2018")) success_count <- success_count + 1 +if (process_and_save("fpl", "2018")) success_count <- success_count + 1 + +# Summary +cat("================================================================================\n") +cat(" Data Preparation Complete\n") +cat("================================================================================\n\n") + +cat("Successfully prepared", success_count, "of 4 datasets\n\n") + +if (success_count == 4) { + cat("✓ All datasets ready for Zenodo upload!\n\n") + + cat("Next steps:\n") + cat(" 1. Compress files:\n") + cat(" cd", output_dir, "\n") + cat(" gzip -9 -k *.csv\n\n") + + cat(" 2. Calculate checksums:\n") + cat(" md5sum *.csv.gz > checksums.txt\n\n") + + cat(" 3. Upload to Zenodo following .dev/ZENODO_UPLOAD_GUIDE.md\n\n") + + cat(" 4. Update R/zenodo.R with DOIs and URLs\n\n") + +} else { + cat("\n⚠ Some datasets failed to process. Check errors above.\n\n") +} + +cat("Output directory:", normalizePath(output_dir), "\n") +cat("\n") diff --git a/.dev/regenerate-and-deploy-zenodo.sh b/.dev/regenerate-and-deploy-zenodo.sh new file mode 100644 index 0000000..96bad02 --- /dev/null +++ b/.dev/regenerate-and-deploy-zenodo.sh @@ -0,0 +1,433 @@ +#!/usr/bin/env bash +# +# End-to-End Zenodo Dataset Regeneration and Deployment +# +# This script automates the entire workflow: +# 1. Regenerate datasets with validation and auto-healing +# 2. Retry failed datasets with cache clearing +# 3. Verify all 4 datasets are complete +# 4. Update MD5 checksums in R/zenodo.R +# 5. Create git commit +# 6. Upload to Zenodo +# 7. Push to GitHub with tags +# +# Usage: +# bash .dev/regenerate-and-deploy-zenodo.sh [--force-download] [--skip-upload] +# +# Options: +# --force-download Clear all caches and re-download from OpenEI +# --skip-upload Skip Zenodo upload (just regenerate and update code) +# --retry-only Only retry failed datasets, don't regenerate successful ones +# + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Parse arguments +FORCE_DOWNLOAD=false +SKIP_UPLOAD=false +RETRY_ONLY=false + +while [[ $# -gt 0 ]]; do + case $1 in + --force-download) + FORCE_DOWNLOAD=true + shift + ;; + --skip-upload) + SKIP_UPLOAD=true + shift + ;; + --retry-only) + RETRY_ONLY=true + shift + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [--force-download] [--skip-upload] [--retry-only]" + exit 1 + ;; + esac +done + +echo -e "${BLUE}================================================================================" +echo " End-to-End Zenodo Dataset Regeneration and Deployment" +echo -e "================================================================================${NC}" +echo "" +echo "Configuration:" +echo " Force download: $FORCE_DOWNLOAD" +echo " Skip upload: $SKIP_UPLOAD" +echo " Retry only: $RETRY_ONLY" +echo "" + +# Directories +ZENODO_DIR="zenodo-upload-nationwide" +NATIONWIDE_DIR="$ZENODO_DIR/nationwide" +LOG_FILE="regeneration-deploy.log" + +# Expected datasets +EXPECTED_DATASETS=( + "lead_ami_cohorts_2022_us.csv.gz" + "lead_fpl_cohorts_2022_us.csv.gz" + "lead_ami_cohorts_2018_us.csv.gz" + "lead_fpl_cohorts_2018_us.csv.gz" +) + +# Track failures +FAILED_DATASETS=() + +################################################################################ +# Step 1: Initial Regeneration +################################################################################ + +echo -e "${BLUE}================================================================================" +echo " Step 1: Generate Nationwide Datasets" +echo -e "================================================================================${NC}" +echo "" + +if [ "$RETRY_ONLY" = false ]; then + # Full regeneration + REGEN_FLAGS="--nationwide-only" + + if [ "$FORCE_DOWNLOAD" = true ]; then + echo "⚠️ Force download enabled - clearing all caches..." + Rscript -e "emburden::clear_all_cache(confirm = TRUE, verbose = TRUE)" + REGEN_FLAGS="$REGEN_FLAGS --force-download" + fi + + echo "Running: Rscript .dev/prepare-zenodo-data-nationwide.R $REGEN_FLAGS" + Rscript .dev/prepare-zenodo-data-nationwide.R $REGEN_FLAGS 2>&1 | tee "$LOG_FILE" +else + echo "⏭️ Skipping initial regeneration (retry-only mode)" +fi + +################################################################################ +# Step 2: Check Which Datasets Failed +################################################################################ + +echo "" +echo -e "${BLUE}================================================================================" +echo " Step 2: Validate Generated Datasets" +echo -e "================================================================================${NC}" +echo "" + +if [ ! -d "$NATIONWIDE_DIR" ]; then + echo -e "${RED}❌ ERROR: Nationwide directory not found: $NATIONWIDE_DIR${NC}" + exit 1 +fi + +SUCCESSFUL_DATASETS=() +FAILED_DATASETS=() + +for dataset in "${EXPECTED_DATASETS[@]}"; do + file_path="$NATIONWIDE_DIR/$dataset" + + if [ -f "$file_path" ]; then + # Check file size (should be >1MB for valid data) + size=$(stat -c%s "$file_path" 2>/dev/null || stat -f%z "$file_path" 2>/dev/null || echo "0") + size_mb=$((size / 1024 / 1024)) + + if [ "$size_mb" -gt 1 ]; then + echo -e " ✅ ${GREEN}$dataset${NC} (${size_mb}MB)" + SUCCESSFUL_DATASETS+=("$dataset") + else + echo -e " ❌ ${RED}$dataset${NC} (${size_mb}MB - TOO SMALL)" + FAILED_DATASETS+=("$dataset") + fi + else + echo -e " ❌ ${RED}$dataset${NC} (MISSING)" + FAILED_DATASETS+=("$dataset") + fi +done + +echo "" +echo "Summary:" +echo " ✅ Successful: ${#SUCCESSFUL_DATASETS[@]}/4" +echo " ❌ Failed: ${#FAILED_DATASETS[@]}/4" + +################################################################################ +# Step 3: Retry Failed Datasets with Cache Clearing +################################################################################ + +if [ ${#FAILED_DATASETS[@]} -gt 0 ]; then + echo "" + echo -e "${YELLOW}================================================================================" + echo " Step 3: Retry Failed Datasets (Auto-Healing)" + echo -e "================================================================================${NC}" + echo "" + + for failed_dataset in "${FAILED_DATASETS[@]}"; do + echo "" + echo -e "${YELLOW}⚠️ Retrying: $failed_dataset${NC}" + + # Extract dataset and vintage from filename + # Format: lead_{dataset}_cohorts_{vintage}_us.csv.gz + dataset_name=$(echo "$failed_dataset" | sed -E 's/lead_(.*)_cohorts_.*_us\.csv\.gz/\1/') + vintage=$(echo "$failed_dataset" | sed -E 's/lead_.*_cohorts_(....)_us\.csv\.gz/\1/') + + echo " Dataset: $dataset_name" + echo " Vintage: $vintage" + echo "" + + # Clear corrupt cache for this specific dataset + echo " Clearing corrupt cache..." + Rscript -e "emburden::clear_dataset_cache('$dataset_name', '$vintage', verbose = TRUE)" + + # Re-run just this dataset + echo " Regenerating..." + Rscript -e " + library(emburden) + library(readr) + library(dplyr) + + # Load with self-healing + data <- load_cohort_data( + dataset = '$dataset_name', + vintage = '$vintage', + states = NULL, # All states + verbose = TRUE + ) + + # Validate + if (is.null(data) || nrow(data) == 0) { + stop('Failed to load data after cache clear') + } + + # Check states + if ('state_abbr' %in% names(data)) { + n_states <- length(unique(data\$state_abbr)) + cat('States found:', n_states, '\\n') + if (n_states < 51) { + stop('Still missing states after retry: ', 51 - n_states) + } + } + + # Save to Zenodo directory + output_dir <- 'zenodo-upload-nationwide/nationwide' + if (!dir.exists(output_dir)) { + dir.create(output_dir, recursive = TRUE) + } + + output_file <- file.path(output_dir, '$failed_dataset') + output_csv <- sub('\\\\.gz\$', '', output_file) + + # Write and compress + write_csv(data, output_csv) + system2('gzip', args = c('-9', '-f', output_csv)) + + cat('✅ Successfully regenerated:', '$failed_dataset', '\\n') + " 2>&1 | tee -a "$LOG_FILE" + + # Check if successful + if [ -f "$NATIONWIDE_DIR/$failed_dataset" ]; then + size=$(stat -c%s "$NATIONWIDE_DIR/$failed_dataset" 2>/dev/null || stat -f%z "$NATIONWIDE_DIR/$failed_dataset" 2>/dev/null || echo "0") + size_mb=$((size / 1024 / 1024)) + + if [ "$size_mb" -gt 1 ]; then + echo -e " ${GREEN}✅ Retry successful!${NC} (${size_mb}MB)" + # Remove from failed list + FAILED_DATASETS=("${FAILED_DATASETS[@]/$failed_dataset}") + SUCCESSFUL_DATASETS+=("$failed_dataset") + else + echo -e " ${RED}❌ Retry failed${NC} (still too small: ${size_mb}MB)" + fi + else + echo -e " ${RED}❌ Retry failed${NC} (file not created)" + fi + done +fi + +################################################################################ +# Step 4: Final Validation +################################################################################ + +echo "" +echo -e "${BLUE}================================================================================" +echo " Step 4: Final Validation" +echo -e "================================================================================${NC}" +echo "" + +# Re-count successful datasets +SUCCESSFUL_COUNT=0 +for dataset in "${EXPECTED_DATASETS[@]}"; do + file_path="$NATIONWIDE_DIR/$dataset" + if [ -f "$file_path" ]; then + size=$(stat -c%s "$file_path" 2>/dev/null || stat -f%z "$file_path" 2>/dev/null || echo "0") + size_mb=$((size / 1024 / 1024)) + if [ "$size_mb" -gt 1 ]; then + ((SUCCESSFUL_COUNT++)) || true + fi + fi +done + +if [ "$SUCCESSFUL_COUNT" -eq 4 ]; then + echo -e "${GREEN}✅ ALL 4 DATASETS SUCCESSFULLY GENERATED!${NC}" +else + echo -e "${RED}❌ ONLY $SUCCESSFUL_COUNT/4 DATASETS GENERATED${NC}" + echo "" + echo "Failed datasets still missing. Manual intervention required." + echo "Check log file: $LOG_FILE" + exit 1 +fi + +################################################################################ +# Step 5: Update MD5 Checksums in R/zenodo.R +################################################################################ + +echo "" +echo -e "${BLUE}================================================================================" +echo " Step 5: Update MD5 Checksums in R/zenodo.R" +echo -e "================================================================================${NC}" +echo "" + +echo "Calculating new MD5 checksums..." +declare -A MD5_MAP + +for dataset in "${EXPECTED_DATASETS[@]}"; do + file_path="$NATIONWIDE_DIR/$dataset" + if [ -f "$file_path" ]; then + md5=$(md5sum "$file_path" | awk '{print $1}') + MD5_MAP["$dataset"]="$md5" + echo " $dataset: $md5" + fi +done + +# Update R/zenodo.R with new checksums +echo "" +echo "Updating R/zenodo.R..." + +# Create backup +cp R/zenodo.R R/zenodo.R.bak + +# Use R to update checksums (more reliable than sed) +Rscript -e " + # Read file + lines <- readLines('R/zenodo.R') + + # Update each checksum + checksums <- list( + 'lead_ami_cohorts_2022_us.csv.gz' = '${MD5_MAP[lead_ami_cohorts_2022_us.csv.gz]}', + 'lead_fpl_cohorts_2022_us.csv.gz' = '${MD5_MAP[lead_fpl_cohorts_2022_us.csv.gz]}', + 'lead_ami_cohorts_2018_us.csv.gz' = '${MD5_MAP[lead_ami_cohorts_2018_us.csv.gz]}', + 'lead_fpl_cohorts_2018_us.csv.gz' = '${MD5_MAP[lead_fpl_cohorts_2018_us.csv.gz]}' + ) + + for (filename in names(checksums)) { + pattern <- paste0('\"', filename, '\" = \"[a-f0-9]{32}\"') + replacement <- paste0('\"', filename, '\" = \"', checksums[[filename]], '\"') + + for (i in seq_along(lines)) { + lines[i] <- gsub(pattern, replacement, lines[i]) + } + } + + writeLines(lines, 'R/zenodo.R') + cat('✅ Updated MD5 checksums in R/zenodo.R\\n') +" + +echo -e "${GREEN}✅ MD5 checksums updated${NC}" + +################################################################################ +# Step 6: Git Commit +################################################################################ + +echo "" +echo -e "${BLUE}================================================================================" +echo " Step 6: Git Commit" +echo -e "================================================================================${NC}" +echo "" + +if git diff --quiet R/zenodo.R; then + echo "No changes to R/zenodo.R (checksums already up to date)" +else + echo "Committing updated checksums..." + git add R/zenodo.R + git commit -m "chore: Update Zenodo dataset MD5 checksums + +- Regenerated all 4 nationwide datasets +- Updated MD5 checksums in R/zenodo.R +- All datasets validated with 51 states and detailed income brackets + +Datasets updated: +- lead_ami_cohorts_2022_us.csv.gz +- lead_fpl_cohorts_2022_us.csv.gz +- lead_ami_cohorts_2018_us.csv.gz +- lead_fpl_cohorts_2018_us.csv.gz" + + echo -e "${GREEN}✅ Git commit created${NC}" +fi + +################################################################################ +# Step 7: Upload to Zenodo +################################################################################ + +if [ "$SKIP_UPLOAD" = false ]; then + echo "" + echo -e "${BLUE}================================================================================" + echo " Step 7: Upload to Zenodo" + echo -e "================================================================================${NC}" + echo "" + + echo "Uploading datasets to Zenodo..." + bash .dev/upload-to-zenodo-nationwide.sh 2>&1 | tee -a "$LOG_FILE" + + echo -e "${GREEN}✅ Zenodo upload complete${NC}" +else + echo "" + echo -e "${YELLOW}⏭️ Skipping Zenodo upload (--skip-upload flag)${NC}" + echo "" + echo "To upload manually:" + echo " bash .dev/upload-to-zenodo-nationwide.sh" +fi + +################################################################################ +# Step 8: Push to GitHub +################################################################################ + +echo "" +echo -e "${BLUE}================================================================================" +echo " Step 8: Push to GitHub" +echo -e "================================================================================${NC}" +echo "" + +if [ "$SKIP_UPLOAD" = false ]; then + echo "Pushing to GitHub..." + git push --follow-tags + echo -e "${GREEN}✅ Pushed to GitHub${NC}" +else + echo -e "${YELLOW}⏭️ Skipping GitHub push (run manually: git push --follow-tags)${NC}" +fi + +################################################################################ +# Summary +################################################################################ + +echo "" +echo -e "${GREEN}================================================================================" +echo " ✅ DEPLOYMENT COMPLETE!" +echo -e "================================================================================${NC}" +echo "" +echo "Summary:" +echo " ✅ Generated: $SUCCESSFUL_COUNT/4 datasets" +echo " ✅ Updated: R/zenodo.R MD5 checksums" +echo " ✅ Committed: Changes to git" +if [ "$SKIP_UPLOAD" = false ]; then + echo " ✅ Uploaded: Datasets to Zenodo" + echo " ✅ Pushed: Changes to GitHub" +else + echo " ⏭️ Skipped: Zenodo upload and GitHub push" +fi +echo "" +echo "Log file: $LOG_FILE" +echo "" +echo "Next steps:" +echo " 1. Verify datasets on Zenodo" +echo " 2. Test download: emburden::load_cohort_data('ami', '2022')" +echo " 3. Submit to CRAN if ready" +echo "" diff --git a/.dev/release-version.sh b/.dev/release-version.sh new file mode 100644 index 0000000..4511ec5 --- /dev/null +++ b/.dev/release-version.sh @@ -0,0 +1,542 @@ +#!/bin/bash + +# release-version.sh +# Comprehensive version bump and release automation script +# Usage: bash .dev/release-version.sh [--auto] +# Example: bash .dev/release-version.sh 0.5.11 +# Example (fully automated): bash .dev/release-version.sh 0.5.11 --auto + +set -euo pipefail + +# Auto mode flag +AUTO_MODE=false + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Helper functions +error() { + echo -e "${RED}Error: $1${NC}" >&2 + exit 1 +} + +warning() { + echo -e "${YELLOW}Warning: $1${NC}" +} + +success() { + echo -e "${GREEN}✓ $1${NC}" +} + +info() { + echo -e "${BLUE}$1${NC}" +} + +prompt_yes_no() { + local prompt="$1" + local default="${2:-y}" + + if [[ "$default" == "y" ]]; then + prompt="$prompt [Y/n]: " + else + prompt="$prompt [y/N]: " + fi + + while true; do + read -rp "$prompt" response + response=${response:-$default} + case "$response" in + [Yy]*) return 0 ;; + [Nn]*) return 1 ;; + *) echo "Please answer yes or no." ;; + esac + done +} + +# Check if we're in the project root +if [[ ! -f "DESCRIPTION" ]] || [[ ! -f "NEWS.md" ]]; then + error "Must be run from project root (DESCRIPTION and NEWS.md not found)" +fi + +# Parse arguments - version is now optional +NEW_VERSION="" +for arg in "$@"; do + if [[ "$arg" == "--auto" ]]; then + AUTO_MODE=true + elif [[ -z "$NEW_VERSION" ]]; then + NEW_VERSION="$arg" + else + error "Unknown argument: $arg\nUsage: $0 [new_version] [--auto]\nExample: $0 0.5.11\nExample (auto mode): $0 0.5.11 --auto\nExample (auto-increment): $0 --auto" + fi +done + +if [[ "$AUTO_MODE" == "true" ]]; then + info "Auto mode enabled: will run without interactive prompts" +fi + +# If no version specified, auto-increment version using semantic versioning +if [[ -z "$NEW_VERSION" ]]; then + info "No version specified - analyzing commits for semantic versioning..." + + # Extract current version from DESCRIPTION + CURRENT_VERSION=$(grep "^Version:" DESCRIPTION | sed 's/Version: //') + + if [[ -z "$CURRENT_VERSION" ]]; then + error "Could not extract current version from DESCRIPTION" + fi + + info "Current version: $CURRENT_VERSION" + + # Parse version components + if [[ "$CURRENT_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(\..*)?$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" + + # Determine semantic version bump by analyzing commits + # Find the last version tag to compare commits + LAST_TAG=$(git tag -l "v*" | sort -V | tail -1) + if [[ -z "$LAST_TAG" ]]; then + warning "No previous version tag found, defaulting to patch bump" + BUMP_TYPE="patch" + else + info "Analyzing commits since $LAST_TAG..." + + # Get all commit messages since last tag + COMMITS=$(git log "$LAST_TAG..HEAD" --pretty=format:"%s%n%b" 2>/dev/null || echo "") + + # Determine bump level: 0=patch, 1=minor, 2=major + BUMP_LEVEL=0 + + while IFS= read -r line; do + # Skip empty lines + [ -z "$line" ] && continue + + # Check for BREAKING CHANGE (major bump) + # Exclude lines with backticks (documentation) or markdown formatting + if echo "$line" | grep -qiE '^BREAKING CHANGE:|^[a-z]+!:' && ! echo "$line" | grep -q '`'; then + BUMP_LEVEL=2 + break # Major is highest, no need to check further + fi + + # Check for feat/feature (minor bump) + if echo "$line" | grep -qiE '^feat(\(.*\))?:|^feature(\(.*\))?:'; then + [ $BUMP_LEVEL -lt 1 ] && BUMP_LEVEL=1 + fi + + # Everything else (fix, chore, docs, etc.) stays at patch level (0) + done <<< "$COMMITS" + + # Set bump type based on level + case $BUMP_LEVEL in + 2) BUMP_TYPE="major" ;; + 1) BUMP_TYPE="minor" ;; + *) BUMP_TYPE="patch" ;; + esac + fi + + # Apply the version bump + case "$BUMP_TYPE" in + major) + NEW_MAJOR=$((MAJOR + 1)) + NEW_VERSION="${NEW_MAJOR}.0.0" + success "Auto-incremented MAJOR version to: $NEW_VERSION (from $CURRENT_VERSION)" + info "Reason: Found BREAKING CHANGE in commits" + ;; + minor) + NEW_MINOR=$((MINOR + 1)) + NEW_VERSION="${MAJOR}.${NEW_MINOR}.0" + success "Auto-incremented MINOR version to: $NEW_VERSION (from $CURRENT_VERSION)" + info "Reason: Found new features (feat:) in commits" + ;; + *) + NEW_PATCH=$((PATCH + 1)) + NEW_VERSION="${MAJOR}.${MINOR}.${NEW_PATCH}" + success "Auto-incremented PATCH version to: $NEW_VERSION (from $CURRENT_VERSION)" + info "Reason: Only fixes/chores/docs in commits" + ;; + esac + else + error "Could not parse version from DESCRIPTION: $CURRENT_VERSION" + fi +fi + +# Validate semantic versioning format +if ! [[ "$NEW_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]{4})?$ ]]; then + error "Version must follow semantic versioning format (e.g., 0.5.11 or 0.5.11.9001)" +fi + +info "============================================================" +info "Release Automation Script" +info "New version: $NEW_VERSION" +info "============================================================" +echo "" + +# Check git status +info "[Step 1/9] Checking git repository status..." +if ! git diff-index --quiet HEAD -- 2>/dev/null; then + warning "You have uncommitted changes:" + git status --short + echo "" + if [[ "$AUTO_MODE" != "true" ]]; then + if ! prompt_yes_no "Continue anyway?"; then + error "Aborted by user" + fi + echo "" + else + info "Auto mode: continuing despite uncommitted changes" + echo "" + fi +fi + +# Check if we're on main branch +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [[ "$CURRENT_BRANCH" != "main" ]]; then + warning "Not on main branch (currently on: $CURRENT_BRANCH)" + if [[ "$AUTO_MODE" != "true" ]]; then + if ! prompt_yes_no "Continue anyway?"; then + error "Aborted by user" + fi + echo "" + else + info "Auto mode: continuing on branch $CURRENT_BRANCH" + echo "" + fi +fi + +success "Git repository check passed" +echo "" + +# Run the R version bump script +info "[Step 2/8] Running version bump script..." +if ! Rscript .dev/bump-version.R "$NEW_VERSION"; then + error "Version bump script failed" +fi +echo "" + +# Update NEWS.md +info "[Step 3/8] Updating NEWS.md..." + +# Check if NEWS.md already has this version +if grep -q "^# emburden $NEW_VERSION" NEWS.md; then + success "NEWS.md already contains entry for version $NEW_VERSION" +else + # Generate NEWS content + if [[ "$AUTO_MODE" == "true" ]]; then + info "Auto mode: generating NEWS.md from git commits..." + + # Find the last version tag + LAST_TAG=$(git tag -l "v*" | sort -V | tail -1) + if [[ -z "$LAST_TAG" ]]; then + warning "No previous version tag found, using all commits" + COMMIT_RANGE="HEAD" + else + info "Extracting commits since $LAST_TAG..." + COMMIT_RANGE="$LAST_TAG..HEAD" + fi + + # Extract commits and categorize them + FEATURES="" + FIXES="" + ENHANCEMENTS="" + OTHER="" + + while IFS= read -r commit; do + # Get commit message (first line only) + msg=$(echo "$commit" | sed 's/^[a-f0-9]* //') + + # Skip version bump commits and merge commits + if [[ "$msg" =~ ^(Bump version|Merge|Version bump) ]]; then + continue + fi + + # Remove PR numbers like (#60) from the end + msg=$(echo "$msg" | sed 's/ (#[0-9]*)$//') + + # Categorize by conventional commit prefix + if [[ "$msg" =~ ^feat(\(.*\))?:\ (.*)$ ]]; then + # New feature + feature_msg="${BASH_REMATCH[2]}" + FEATURES="${FEATURES}* ${feature_msg}\n" + elif [[ "$msg" =~ ^fix(\(.*\))?:\ (.*)$ ]]; then + # Bug fix + fix_msg="${BASH_REMATCH[2]}" + FIXES="${FIXES}* ${fix_msg}\n" + elif [[ "$msg" =~ ^(chore|docs|refactor|style|test|perf)(\(.*\))?:\ (.*)$ ]]; then + # Enhancement/other improvement + enh_msg="${BASH_REMATCH[3]}" + ENHANCEMENTS="${ENHANCEMENTS}* ${enh_msg}\n" + else + # Other changes without conventional commit prefix + OTHER="${OTHER}* ${msg}\n" + fi + done < <(git log "$COMMIT_RANGE" --oneline --no-merges) + + # Build NEWS template with actual content + NEWS_TEMPLATE="# emburden $NEW_VERSION\n\n" + + if [[ -n "$FEATURES" ]]; then + NEWS_TEMPLATE="${NEWS_TEMPLATE}## New Features\n\n${FEATURES}\n" + fi + + if [[ -n "$FIXES" ]]; then + NEWS_TEMPLATE="${NEWS_TEMPLATE}## Bug Fixes\n\n${FIXES}\n" + fi + + if [[ -n "$ENHANCEMENTS" ]]; then + NEWS_TEMPLATE="${NEWS_TEMPLATE}## Enhancements\n\n${ENHANCEMENTS}\n" + fi + + if [[ -n "$OTHER" ]]; then + NEWS_TEMPLATE="${NEWS_TEMPLATE}## Other Changes\n\n${OTHER}\n" + fi + + # If no changes found, add a note + if [[ -z "$FEATURES" && -z "$FIXES" && -z "$ENHANCEMENTS" && -z "$OTHER" ]]; then + NEWS_TEMPLATE="${NEWS_TEMPLATE}## Changes\n\n* Minor updates and improvements\n\n" + fi + + NEWS_TEMPLATE="${NEWS_TEMPLATE}---\n\n" + + success "Generated NEWS.md from $(git rev-list --count "$COMMIT_RANGE" --no-merges) commits" + else + # Manual mode: create template with placeholders + NEWS_TEMPLATE="# emburden $NEW_VERSION + +## Changes + +### New Features + +* (Add new features here) + +### Bug Fixes + +* (Add bug fixes here) + +### Enhancements + +* (Add enhancements here) + +--- + +" + success "Created NEWS.md template for manual editing" + fi + + # Insert at top of NEWS.md + if [[ -f NEWS.md ]]; then + # Create temporary file with new entry + { + echo -e "$NEWS_TEMPLATE" + cat NEWS.md + } > NEWS.md.tmp + mv NEWS.md.tmp NEWS.md + else + echo -e "$NEWS_TEMPLATE" > NEWS.md + fi + + # Open in editor (skip in auto mode) + if [[ "$AUTO_MODE" != "true" ]]; then + if [[ -n "${EDITOR:-}" ]]; then + info "Opening NEWS.md in $EDITOR for editing..." + $EDITOR NEWS.md + elif command -v nano >/dev/null 2>&1; then + info "Opening NEWS.md in nano for editing..." + nano NEWS.md + elif command -v vi >/dev/null 2>&1; then + info "Opening NEWS.md in vi for editing..." + vi NEWS.md + else + warning "No editor found. Please manually edit NEWS.md" + info "Press Enter when done editing..." + read -r + fi + fi +fi +echo "" + +# Show changes +info "[Step 4/8] Review changes..." +echo "" +git diff DESCRIPTION inst/CITATION .zenodo.json NEWS.md || true +echo "" + +if [[ "$AUTO_MODE" != "true" ]]; then + if ! prompt_yes_no "Do the changes look correct?"; then + error "Aborted by user" + fi + echo "" +else + info "Auto mode: automatically accepting changes" + echo "" +fi + +# Stage files +info "[Step 5/8] Staging files..." +git add DESCRIPTION inst/CITATION .zenodo.json NEWS.md +success "Files staged" +echo "" + +# Commit +info "[Step 6/8] Creating commit..." +COMMIT_MSG="Bump version to $NEW_VERSION" + +if [[ "$AUTO_MODE" == "true" ]]; then + info "Auto mode: using default commit message" + git commit -m "$COMMIT_MSG" +else + if prompt_yes_no "Use default commit message: '$COMMIT_MSG'?"; then + git commit -m "$COMMIT_MSG" + else + info "Enter custom commit message (press Ctrl+D when done):" + git commit + fi +fi +success "Committed changes" +echo "" + +# Push to remote +info "[Step 7/8] Push to remote..." +if [[ "$AUTO_MODE" == "true" ]] || prompt_yes_no "Push commit to remote 'scheier'?"; then + if [[ "$AUTO_MODE" == "true" ]]; then + info "Auto mode: automatically pushing to remote" + fi + + info "Pushing commit to $CURRENT_BRANCH..." + git push scheier "$CURRENT_BRANCH" + + success "Pushed to remote!" + echo "" + + # Create or update PR + info "[Step 8/8] Creating or updating pull request..." + + # Check if PR already exists from this branch to main + EXISTING_PR=$(gh pr list --head "$CURRENT_BRANCH" --base main --json number --jq '.[0].number' 2>/dev/null || echo "") + + if [[ -n "$EXISTING_PR" ]]; then + info "PR already exists: #$EXISTING_PR" + PR_URL=$(gh pr view "$EXISTING_PR" --json url --jq '.url') + success "Using existing PR: $PR_URL" + else + info "No existing PR found, creating new PR..." + + # Generate PR title from commit message + LAST_COMMIT_MSG=$(git log -1 --pretty=%s) + PR_TITLE="${LAST_COMMIT_MSG:-Release v$NEW_VERSION}" + + # Extract NEWS.md entry for this version + NEWS_CONTENT="" + if [[ -f NEWS.md ]]; then + # Extract content between "# emburden $NEW_VERSION" and the next "# emburden" or "---" + NEWS_CONTENT=$(awk "/^# emburden $NEW_VERSION/,/^(# emburden|---)/ { + if (\$0 !~ /^(# emburden|---)/) print + }" NEWS.md | sed 's/^## /### /' | sed 's/^### Changes/## Changes/') + fi + + # Generate PR body with version info + PR_BODY="## Version $NEW_VERSION Release +${NEWS_CONTENT:+ +$NEWS_CONTENT +} +### Commits in this PR + +$(git log main..HEAD --pretty=format:'- %s' --reverse) + +### Version Files Updated + +- \`DESCRIPTION\` +- \`inst/CITATION\` +- \`.zenodo.json\` +- \`NEWS.md\` + +### Automated Release Process + +This PR was created by \`release-version.sh\` and includes: +- Version bump to $NEW_VERSION +- Updated NEWS.md with release notes + +### Next Steps + +After merging: +1. Auto-tag-on-version-bump workflow will create git tag v$NEW_VERSION +2. Auto-release workflow creates GitHub release +3. Publish-to-public workflow syncs to public repo +4. CRAN release workflow runs on public repo (manual approval required) +" + + # Create the PR + if PR_URL=$(gh pr create --base main --head "$CURRENT_BRANCH" --title "$PR_TITLE" --body "$PR_BODY" 2>&1); then + success "Created PR: $PR_URL" + else + warning "Failed to create PR automatically" + info "You can create it manually with:" + info " gh pr create --base main --head $CURRENT_BRANCH" + PR_URL="" + fi + fi + echo "" + + # Create auto-merge tag to trigger automated PR merge + info "[Step 9/9] Creating auto-merge tag to trigger PR merge..." + AUTO_MERGE_TAG="auto-merge/v$NEW_VERSION" + + # Check if tag already exists + if git rev-parse "$AUTO_MERGE_TAG" >/dev/null 2>&1; then + info "Auto-merge tag $AUTO_MERGE_TAG already exists, deleting and recreating..." + git tag -d "$AUTO_MERGE_TAG" || true + git push scheier ":refs/tags/$AUTO_MERGE_TAG" 2>/dev/null || true + fi + + # Create and push the auto-merge tag + info "Creating tag $AUTO_MERGE_TAG on current branch..." + git tag "$AUTO_MERGE_TAG" + git push scheier "$AUTO_MERGE_TAG" + + success "Auto-merge tag pushed!" + echo "" + info "The auto-merge workflow will now:" + info " 1. Verify all PR checks are passing" + info " 2. Automatically merge and squash the PR" + info " 3. Delete the auto-merge tag" + info " 4. Trigger auto-tag-on-version-bump to create v$NEW_VERSION on main" + echo "" + + info "============================================================" + info "Release automation complete!" + info "" + info "Version: $NEW_VERSION" + info "Branch: $CURRENT_BRANCH" + if [[ -n "$PR_URL" ]]; then + info "Pull Request: $PR_URL" + fi + info "Auto-merge tag: $AUTO_MERGE_TAG" + info "" + info "Next steps (automated):" + info " 1. Auto-merge workflow validates and merges PR" + info " 2. Auto-tag-on-version-bump workflow creates tag v$NEW_VERSION" + info " 3. Auto-release workflow creates GitHub release" + info " 4. Publish-to-public workflow syncs to public repo" + info " 5. CRAN release workflow runs on public repo (manual approval required)" + info "============================================================" +else + warning "Skipped push to remote" + echo "" + info "============================================================" + info "Local release preparation complete!" + info "" + info "Version: $NEW_VERSION" + info "" + info "To push manually:" + info " git push scheier $CURRENT_BRANCH" + info "" + info "Note: Git tag will be created automatically by auto-tag-on-version-bump" + info " workflow after the PR is merged to main" + info "============================================================" +fi + +exit 0 diff --git a/.dev/run-tests-locally.R b/.dev/run-tests-locally.R new file mode 100644 index 0000000..aeafeb8 --- /dev/null +++ b/.dev/run-tests-locally.R @@ -0,0 +1,160 @@ +#!/usr/bin/env Rscript +# Local test runner that replicates GitHub Actions R-CMD-check +# +# Usage: +# Rscript tests/run-tests-locally.R +# +# Or from R console: +# source("tests/run-tests-locally.R") + +cat("\n") +cat("========================================\n") +cat(" LOCAL TEST SUITE FOR EMBURDEN PACKAGE\n") +cat("========================================\n") +cat("\n") + +# Check if we're in the package root +if (!file.exists("DESCRIPTION")) { + stop("Must be run from package root directory") +} + +# Load required packages +required_pkgs <- c("testthat", "devtools", "covr") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] + +if (length(missing_pkgs) > 0) { + cat("Installing missing packages:", paste(missing_pkgs, collapse = ", "), "\n") + install.packages(missing_pkgs, repos = "https://cloud.r-project.org") +} + +library(testthat) +library(devtools) + +# Configuration +options( + testthat.summary.max_reports = 10, + testthat.output_file = "test-results.txt" +) + +cat("\n") +cat("Step 1: Loading package...\n") +cat("----------------------------------------\n") +tryCatch({ + devtools::load_all(".", quiet = FALSE) + cat("✓ Package loaded successfully\n") +}, error = function(e) { + cat("✗ Failed to load package:\n") + cat(" ", conditionMessage(e), "\n") + quit(status = 1) +}) + +cat("\n") +cat("Step 2: Running tests...\n") +cat("----------------------------------------\n") + +# Run tests with detailed output +test_results <- tryCatch({ + devtools::test(reporter = "progress") +}, error = function(e) { + cat("✗ Test execution failed:\n") + cat(" ", conditionMessage(e), "\n") + quit(status = 1) +}) + +cat("\n") +cat("Step 3: Test coverage analysis...\n") +cat("----------------------------------------\n") + +coverage_results <- tryCatch({ + covr::package_coverage( + type = c("tests", "examples"), + quiet = FALSE + ) +}, error = function(e) { + cat("⚠ Coverage analysis failed (non-critical):\n") + cat(" ", conditionMessage(e), "\n") + NULL +}) + +if (!is.null(coverage_results)) { + cat("\n") + print(coverage_results) + + # Calculate overall coverage percentage + coverage_pct <- covr::percent_coverage(coverage_results) + cat("\n") + cat(sprintf("Overall test coverage: %.1f%%\n", coverage_pct)) + + # Flag if coverage is too low + if (coverage_pct < 75) { + cat("⚠ WARNING: Coverage is below 75% target\n") + } else if (coverage_pct < 85) { + cat("⚠ Coverage is below 85% goal but above minimum\n") + } else { + cat("✓ Coverage meets 85% goal\n") + } + + # Generate HTML coverage report + coverage_html <- file.path("tests", "coverage-report.html") + tryCatch({ + covr::report(coverage_results, file = coverage_html, browse = FALSE) + cat(sprintf("✓ HTML coverage report: %s\n", coverage_html)) + }, error = function(e) { + cat("⚠ Could not generate HTML report\n") + }) +} + +cat("\n") +cat("Step 4: R CMD check (if requested)...\n") +cat("----------------------------------------\n") + +# Check if user wants full R CMD check +run_cmd_check <- Sys.getenv("RUN_CMD_CHECK", "false") == "true" + +if (run_cmd_check) { + cat("Running full R CMD check...\n") + check_results <- tryCatch({ + devtools::check( + document = TRUE, + args = c("--no-manual", "--as-cran"), + error_on = "warning" + ) + }, error = function(e) { + cat("✗ R CMD check failed:\n") + cat(" ", conditionMessage(e), "\n") + quit(status = 1) + }) + + cat("✓ R CMD check passed\n") +} else { + cat("Skipping R CMD check (set RUN_CMD_CHECK=true to enable)\n") +} + +cat("\n") +cat("========================================\n") +cat(" TEST SUITE COMPLETE\n") +cat("========================================\n") +cat("\n") + +# Summary +if (all(test_results$failed == 0)) { + cat("✓ All tests passed!\n") + cat(sprintf(" - %d tests run\n", sum(test_results$passed))) + cat(sprintf(" - %d expectations checked\n", sum(test_results$passed))) + + if (!is.null(coverage_results)) { + cat(sprintf(" - %.1f%% code coverage\n", coverage_pct)) + } + + cat("\n") + quit(status = 0) +} else { + cat("✗ Some tests failed!\n") + cat(sprintf(" - %d tests passed\n", sum(test_results$passed))) + cat(sprintf(" - %d tests failed\n", sum(test_results$failed))) + cat(sprintf(" - %d tests skipped\n", sum(test_results$skipped))) + cat("\n") + cat("Review test output above for details.\n") + cat("\n") + quit(status = 1) +} diff --git a/.dev/submit-to-cran-http.R b/.dev/submit-to-cran-http.R new file mode 100644 index 0000000..17f2f1e --- /dev/null +++ b/.dev/submit-to-cran-http.R @@ -0,0 +1,203 @@ +#!/usr/bin/env Rscript + +# Direct HTTP POST submission to CRAN +# Bypasses devtools::submit_cran() to avoid menu() interactivity issues +# Based on devtools source code: https://github.com/r-lib/devtools/blob/main/R/release.R + +cat("\n") +cat("========================================\n") +cat("CRAN Direct HTTP POST Submission Script\n") +cat("========================================\n\n") + +# Load required packages +if (!requireNamespace("httr", quietly = TRUE)) { + cat("Installing httr package...\n") + install.packages("httr", repos = "https://cloud.r-project.org") +} +library(httr) + +# Get package tarball path from command line or find it +args <- commandArgs(trailingOnly = TRUE) +if (length(args) > 0) { + tarball_path <- args[1] +} else { + # Find the most recent tarball in current directory + tarballs <- list.files(".", pattern = "\\.tar\\.gz$", full.names = TRUE) + if (length(tarballs) == 0) { + stop("No tarball found. Please specify path as argument or build package first.") + } + tarball_path <- tarballs[which.max(file.info(tarballs)$mtime)] +} + +if (!file.exists(tarball_path)) { + stop(sprintf("Tarball not found: %s", tarball_path)) +} + +cat(sprintf("Using tarball: %s\n\n", tarball_path)) + +# Extract maintainer information from DESCRIPTION +desc_file <- "DESCRIPTION" +if (!file.exists(desc_file)) { + stop("DESCRIPTION file not found") +} + +desc_lines <- readLines(desc_file) + +# Extract maintainer name and email +maintainer_line <- grep("^Maintainer:", desc_lines, value = TRUE) +if (length(maintainer_line) == 0) { + # Try to get from Authors@R + authors_line_idx <- grep("^Authors@R:", desc_lines) + if (length(authors_line_idx) > 0) { + # Parse Authors@R field - may span multiple indented lines + authors_lines <- desc_lines[authors_line_idx] + + # Read continuation lines (indented lines that follow) + i <- authors_line_idx + 1 + while (i <= length(desc_lines) && grepl("^\\s+", desc_lines[i])) { + authors_lines <- c(authors_lines, desc_lines[i]) + i <- i + 1 + } + + # Combine and strip the field name + authors_text <- paste(authors_lines, collapse = "\n") + authors_text <- sub("^Authors@R:\\s*", "", authors_text) + + # Try to evaluate if it's R code + tryCatch({ + authors_obj <- eval(parse(text = authors_text)) + maintainer_info <- authors_obj[sapply(authors_obj, function(x) "cre" %in% x$role)] + if (length(maintainer_info) > 0) { + maintainer_name <- paste(maintainer_info[[1]]$given, maintainer_info[[1]]$family) + maintainer_email <- maintainer_info[[1]]$email + } else { + stop("No maintainer found in Authors@R") + } + }, error = function(e) { + stop(sprintf("Could not parse Authors@R field: %s", e$message)) + }) + } else { + stop("No Maintainer field found in DESCRIPTION") + } +} else { + # Parse "Name " format + maintainer_line <- sub("^Maintainer:\\s*", "", maintainer_line) + if (grepl("<.*>", maintainer_line)) { + maintainer_name <- trimws(sub("<.*", "", maintainer_line)) + maintainer_email <- sub(".*<(.*)>.*", "\\1", maintainer_line) + } else { + stop("Could not parse Maintainer field") + } +} + +cat(sprintf("Maintainer: %s <%s>\n\n", maintainer_name, maintainer_email)) + +# Read CRAN comments if available +cran_comments <- "" +if (file.exists("cran-comments.md")) { + cran_comments <- paste(readLines("cran-comments.md"), collapse = "\n") + cat("Found cran-comments.md\n\n") +} + +# CRAN submission URL +cran_url <- "https://xmpalantir.wu.ac.at/cransubmit/index2.php" + +cat("Step 1: Uploading package to CRAN...\n") + +# First POST request: Upload package +response1 <- POST( + url = cran_url, + body = list( + pkg_id = "", + name = maintainer_name, + email = maintainer_email, + uploaded_file = upload_file(tarball_path, type = "application/x-gzip"), + comment = cran_comments, + upload = "Upload package" + ), + encode = "multipart" +) + +if (http_error(response1)) { + stop(sprintf("Upload failed with status %d: %s", + status_code(response1), + content(response1, "text", encoding = "UTF-8"))) +} + +cat(sprintf("Upload response status: %d\n", status_code(response1))) + +# Parse response to extract pkg_id +# The response should redirect or contain the pkg_id +response_text <- content(response1, "text", encoding = "UTF-8") + +# Extract pkg_id from response (it's in a hidden form field or URL parameter) +pkg_id_match <- regexpr('name="pkg_id"[^>]*value="([^"]+)"', response_text, perl = TRUE) +if (pkg_id_match[1] == -1) { + # Try URL parameter format + pkg_id_match <- regexpr('pkg_id=([^&"]+)', response_text, perl = TRUE) + if (pkg_id_match[1] == -1) { + cat("\nWarning: Could not extract pkg_id from response.\n") + cat("Response preview:\n") + cat(substr(response_text, 1, 500), "\n") + stop("Could not find pkg_id in upload response") + } +} + +pkg_id <- regmatches(response_text, pkg_id_match) +pkg_id <- sub('.*=([^&"]+).*', '\\1', pkg_id) +pkg_id <- sub('.*value="([^"]+)".*', '\\1', pkg_id) + +cat(sprintf("Extracted pkg_id: %s\n\n", pkg_id)) + +cat("Step 2: Confirming submission...\n") + +# Second POST request: Confirm submission +response2 <- POST( + url = cran_url, + body = list( + pkg_id = pkg_id, + name = maintainer_name, + email = maintainer_email, + policy_check = "1/", + submit = "Submit package" + ), + encode = "form" +) + +if (http_error(response2)) { + stop(sprintf("Confirmation failed with status %d: %s", + status_code(response2), + content(response2, "text", encoding = "UTF-8"))) +} + +cat(sprintf("Confirmation response status: %d\n\n", status_code(response2))) + +# Check for success message in response +response2_text <- content(response2, "text", encoding = "UTF-8") + +if (grepl("successfully submitted|thank you", response2_text, ignore.case = TRUE)) { + cat("========================================\n") + cat("SUCCESS: Package submitted to CRAN!\n") + cat("========================================\n\n") + cat("Next steps:\n") + cat("1. Check your email (%s) for CRAN confirmation\n", maintainer_email) + cat("2. Click the confirmation link in the email (REQUIRED!)\n") + cat("3. Wait for CRAN maintainer review (typically 1-2 weeks)\n") + cat("4. Respond to any reviewer feedback\n\n") +} else if (grepl("error|failed|invalid", response2_text, ignore.case = TRUE)) { + cat("\n========================================\n") + cat("POSSIBLE ERROR in submission\n") + cat("========================================\n") + cat("Response preview:\n") + cat(substr(response2_text, 1, 1000), "\n\n") + stop("Submission may have failed - check response above") +} else { + cat("\n========================================\n") + cat("Submission completed (check response)\n") + cat("========================================\n") + cat("Response preview:\n") + cat(substr(response2_text, 1, 500), "\n\n") + cat("Please verify submission by checking your email.\n\n") +} + +cat("Submission process complete.\n") diff --git a/.dev/test_v0.3.0_fresh_install.R b/.dev/test_v0.3.0_fresh_install.R new file mode 100644 index 0000000..66fc9c8 --- /dev/null +++ b/.dev/test_v0.3.0_fresh_install.R @@ -0,0 +1,91 @@ +#!/usr/bin/env Rscript +# Test Script for v0.3.0 Fresh Installation +# Run this on a clean machine to verify the MVP demo works + +cat("======================================\n") +cat(" Testing emburden v0.3.0\n") +cat(" Fresh Installation Verification\n") +cat("======================================\n\n") + +# Step 1: Install from GitHub PR branch +cat("Step 1: Installing emburden from PR branch...\n") +if (!requireNamespace("devtools", quietly = TRUE)) { + install.packages("devtools") +} + +devtools::install_github("ScheierVentures/emburden@feature/v0.3.0-county-filtering-nc-sample") + +cat("✓ Installation complete\n\n") + +# Step 2: Load package +cat("Step 2: Loading emburden package...\n") +library(emburden) +library(dplyr) +cat("✓ Package loaded\n\n") + +# Step 3: Test Orange County sample data (no download) +cat("Step 3: Testing bundled Orange County sample...\n") +data(orange_county_sample) +cat(" - Components:", paste(names(orange_county_sample), collapse=", "), "\n") +cat(" - FPL 2022 records:", nrow(orange_county_sample$fpl_2022), "\n") +cat("✓ Orange County sample data works\n\n") + +# Step 4: Test NC complete sample data (no download) +cat("Step 4: Testing bundled NC complete sample...\n") +data(nc_sample) +cat(" - Components:", paste(names(nc_sample), collapse=", "), "\n") +cat(" - FPL 2022 records:", nrow(nc_sample$fpl_2022), "\n") +cat(" - Counties covered:", length(unique(substr(nc_sample$fpl_2022$geoid, 3, 5))), "\n") +cat("✓ NC complete sample data works\n\n") + +# Step 5: Test MVP demo (with OpenEI download if needed) +cat("Step 5: Testing MVP demo - compare_energy_burden()...\n") +cat(" This will download from OpenEI on first use (may take 30-60 seconds)\n") + +result <- compare_energy_burden('fpl', 'NC', 'income_bracket') +cat(" - Result rows:", nrow(result), "\n") +cat(" - Columns:", paste(names(result), collapse=", "), "\n") +print(result) +cat("✓ MVP demo works!\n\n") + +# Step 6: Test county filtering (new in v0.3.0) +cat("Step 6: Testing county filtering (NEW in v0.3.0)...\n") + +# Test with county name +orange <- load_cohort_data('fpl', 'NC', counties = 'Orange', verbose = FALSE) +cat(" - Orange County records:", nrow(orange), "\n") +cat(" - Unique tracts:", length(unique(orange$geoid)), "\n") + +# Test with multiple counties +triangle <- load_cohort_data('fpl', 'NC', counties = c('Orange', 'Durham', 'Wake'), verbose = FALSE) +cat(" - Triangle (3 counties) records:", nrow(triangle), "\n") +cat(" - Unique counties:", length(unique(substr(triangle$geoid, 3, 5))), "\n") + +# Test comparison with county filtering +county_comparison <- compare_energy_burden('fpl', 'NC', counties = 'Orange', group_by = 'income_bracket') +cat(" - Orange County comparison rows:", nrow(county_comparison), "\n") + +cat("✓ County filtering works!\n\n") + +# Step 7: Verify data processing pipeline +cat("Step 7: Verifying data processing pipeline...\n") +cat(" The package should have:\n") +cat(" - Detected OpenEI raw data format (period-based columns)\n") +cat(" - Aggregated microdata to cohort level\n") +cat(" - Standardized column names\n") +cat(" - All of this happened automatically!\n") +cat("✓ Data pipeline verified\n\n") + +cat("======================================\n") +cat(" ✅ ALL TESTS PASSED!\n") +cat("======================================\n\n") + +cat("Summary:\n") +cat(" 1. Package installed from GitHub PR ✓\n") +cat(" 2. Orange County sample data (94 KB) ✓\n") +cat(" 3. NC complete sample data (1.3 MB) ✓\n") +cat(" 4. MVP demo works on fresh install ✓\n") +cat(" 5. County filtering functionality ✓\n") +cat(" 6. OpenEI auto-download & processing ✓\n\n") + +cat("v0.3.0 is ready for release!\n") diff --git a/.dev/testing-spec.md b/.dev/testing-spec.md new file mode 100644 index 0000000..c217507 --- /dev/null +++ b/.dev/testing-spec.md @@ -0,0 +1,658 @@ +# Comprehensive Testing Specification for Data Functions + +## Overview + +This document specifies a comprehensive testing strategy for the emburden package's data loading and processing functions. The goal is to achieve robust test coverage with proper mocking of external dependencies (HTTP APIs, databases, file systems). + +## Testing Infrastructure + +### Required Packages + +```r +# In DESCRIPTION Suggests section: +- testthat (>= 3.0.0) # Already present +- DBI # Already present +- RSQLite # Already present +- httptest2 # For HTTP mocking +- withr # For temporary file/env management +- mockery # For function mocking +``` + +### Test File Organization + +``` +tests/ +├── testthat/ +│ ├── helper-mocks.R # Mocking utilities +│ ├── helper-fixtures.R # Test data generators +│ ├── test-data-loaders.R # Data loading tests +│ ├── test-data-processing.R # Data processing tests +│ ├── test-database-access.R # Database interaction tests +│ ├── test-http-requests.R # HTTP/API tests +│ ├── test-file-validation.R # File validation tests +│ └── test-schema-normalization.R # Schema transformation tests +├── fixtures/ +│ ├── sample_ami_2018.csv # Sample AMI 2018 data +│ ├── sample_ami_2022.csv # Sample AMI 2022 data +│ ├── sample_fpl_2018.csv # Sample FPL 2018 data +│ ├── sample_fpl_2022.csv # Sample FPL 2022 data +│ ├── corrupted_data.csv # File with all-NA income_bracket +│ ├── incomplete_schema.csv # File missing required columns +│ └── sample_database.db # SQLite database for testing +└── testthat.R # Test runner +``` + +## 1. Data Loading Functions Tests + +### 1.1 `load_census_tract_data()` Tests + +**File**: `tests/testthat/test-data-loaders.R` + +#### Test Cases: + +```r +test_that("load_census_tract_data loads data successfully", { + # Mock HTTP GET request to OpenEI + with_mock_api({ + data <- load_census_tract_data( + states = "NC", + vintage = "2022", + dataset = "ami" + ) + + expect_s3_class(data, "data.frame") + expect_true(nrow(data) > 0) + expect_true("geoid" %in% names(data)) + expect_true("income" %in% names(data)) + expect_true("energy_cost" %in% names(data)) + }) +}) + +test_that("load_census_tract_data handles multiple states", { + with_mock_api({ + data <- load_census_tract_data( + states = c("NC", "SC"), + vintage = "2022" + ) + + expect_true(all(c("NC", "SC") %in% data$state_abbr)) + }) +}) + +test_that("load_census_tract_data fails gracefully on network error", { + # Mock network failure + with_mock_api({ + httptest2::expect_GET( + load_census_tract_data(states = "NC"), + "https://data.openei.org/.*", + status_code = 404 + ) + }) + + expect_error( + load_census_tract_data(states = "NC"), + "Failed to download" + ) +}) + +test_that("load_census_tract_data uses cached data", { + # Create temporary cache directory + withr::with_tempdir({ + # First call downloads + data1 <- load_census_tract_data(states = "NC") + + # Second call should use cache (no HTTP request) + data2 <- load_census_tract_data(states = "NC") + + expect_identical(data1, data2) + }) +}) + +test_that("load_census_tract_data validates state codes", { + expect_error( + load_census_tract_data(states = "INVALID"), + "Invalid state code" + ) +}) + +test_that("load_census_tract_data handles vintage parameter", { + with_mock_api({ + data_2018 <- load_census_tract_data(states = "NC", vintage = "2018") + data_2022 <- load_census_tract_data(states = "NC", vintage = "2022") + + # Expect different data for different vintages + expect_false(identical(data_2018, data_2022)) + }) +}) +``` + +### 1.2 `load_cohort_data()` Tests + +```r +test_that("load_cohort_data loads AMI data", { + data <- load_cohort_data( + dataset = "ami", + states = "NC", + vintage = "2022" + ) + + expect_s3_class(data, "data.frame") + expect_true("income_bracket" %in% names(data)) + expect_true(all(data$income_bracket %in% c( + "0-30%", "30-60%", "60-80%", "80-100%", "100%+" + ))) +}) + +test_that("load_cohort_data loads FPL data", { + data <- load_cohort_data( + dataset = "fpl", + states = "NC", + vintage = "2022" + ) + + expect_s3_class(data, "data.frame") + expect_true("income_bracket" %in% names(data)) +}) + +test_that("load_cohort_data handles aggregate_poverty flag", { + data <- load_cohort_data( + dataset = "fpl", + states = "NC", + aggregate_poverty = TRUE + ) + + expect_true(all(data$income_bracket %in% c( + "Below Federal Poverty Line", + "Above Federal Poverty Line" + ))) +}) + +test_that("load_cohort_data skips corrupted files", { + # Create corrupted file with all-NA income_bracket + withr::with_tempfile("corrupted", { + corrupt_data <- data.frame( + geoid = "37001", + income_bracket = rep(NA_character_, 100), + income = 50000, + energy_cost = 2000 + ) + write.csv(corrupt_data, corrupted, row.names = FALSE) + + # Should skip corrupted file and fall back to raw data + expect_message( + load_cohort_data(dataset = "fpl", states = "NC"), + "Skipping file.*income_bracket all NA" + ) + }) +}) + +test_that("load_cohort_data validates dataset parameter", { + expect_error( + load_cohort_data(dataset = "invalid"), + "dataset must be either 'ami' or 'fpl'" + ) +}) +``` + +### 1.3 `check_data_sources()` Tests + +```r +test_that("check_data_sources detects available CSV files", { + withr::with_tempdir({ + # Create dummy CSV file + write.csv( + data.frame(x = 1:10), + "data_ami_census_tracts_2022_NC.csv" + ) + + sources <- check_data_sources( + dataset = "ami", + states = "NC", + vintage = "2022" + ) + + expect_true(sources$csv_available) + }) +}) + +test_that("check_data_sources detects available database", { + # Create temporary SQLite database + withr::with_tempfile("db", fileext = ".db", { + con <- DBI::dbConnect(RSQLite::SQLite(), db) + DBI::dbWriteTable(con, "ami_2022", data.frame(x = 1:10)) + DBI::dbDisconnect(con) + + sources <- check_data_sources( + dataset = "ami", + vintage = "2022", + db_path = db + ) + + expect_true(sources$db_available) + }) +}) +``` + +## 2. Database Access Tests + +**File**: `tests/testthat/test-database-access.R` + +```r +test_that("database connection succeeds with valid path", { + withr::with_tempfile("db", fileext = ".db", { + con <- DBI::dbConnect(RSQLite::SQLite(), db) + DBI::dbWriteTable(con, "test", data.frame(x = 1:10)) + DBI::dbDisconnect(con) + + # Test package function that connects to DB + result <- query_database(db, "SELECT * FROM test") + expect_equal(nrow(result), 10) + }) +}) + +test_that("database connection fails gracefully", { + expect_error( + query_database("/nonexistent/path.db", "SELECT * FROM test"), + "Failed to connect to database" + ) +}) + +test_that("database query falls back to CSV on failure", { + # Mock database failure + mockery::stub( + load_cohort_data, + "query_database", + stop("DB connection failed") + ) + + # Should fall back to CSV + expect_message( + load_cohort_data(dataset = "ami", states = "NC"), + "Falling back to CSV" + ) +}) +``` + +## 3. HTTP/API Request Tests + +**File**: `tests/testthat/test-http-requests.R` + +```r +test_that("HTTP request succeeds with valid URL", { + httptest2::with_mock_api({ + response <- download_lead_data( + dataset = "ami", + state = "NC", + vintage = "2022" + ) + + expect_s3_class(response, "response") + expect_equal(httr::status_code(response), 200) + }) +}) + +test_that("HTTP request handles 404 error", { + httptest2::with_mock_api({ + expect_error( + download_lead_data(dataset = "invalid"), + "404" + ) + }) +}) + +test_that("HTTP request handles timeout", { + # Mock timeout + mockery::stub( + download_lead_data, + "httr::GET", + stop("Timeout") + ) + + expect_error( + download_lead_data(dataset = "ami"), + "Timeout" + ) +}) + +test_that("HTTP request retries on failure", { + # Mock: fail twice, succeed third time + retry_count <- 0 + mockery::stub( + download_lead_data, + "httr::GET", + function(...) { + retry_count <<- retry_count + 1 + if (retry_count < 3) stop("Failed") + list(status_code = 200, content = "success") + } + ) + + result <- download_lead_data(dataset = "ami") + expect_equal(retry_count, 3) +}) +``` + +## 4. File Validation Tests + +**File**: `tests/testthat/test-file-validation.R` + +```r +test_that("validates required columns exist", { + data <- data.frame( + geoid = "37001", + income = 50000, + energy_cost = 2000 + ) + + expect_true(validate_required_columns( + data, + c("geoid", "income", "energy_cost") + )) + + expect_error( + validate_required_columns(data, c("missing_column")), + "Missing required columns" + ) +}) + +test_that("skips files with all-NA income_bracket", { + data <- data.frame( + geoid = "37001", + income_bracket = rep(NA_character_, 100), + income = 50000 + ) + + expect_false(validate_income_bracket(data)) +}) + +test_that("validates positive household counts", { + data <- data.frame(households = c(100, 200, -50)) + + expect_error( + validate_household_counts(data), + "Negative household counts found" + ) +}) + +test_that("validates income and energy_cost ranges", { + data <- data.frame( + income = c(50000, -1000), # Negative income + energy_cost = c(2000, 5000) + ) + + expect_warning( + validate_ranges(data), + "Negative income values found" + ) +}) +``` + +## 5. Data Processing Tests + +**File**: `tests/testthat/test-data-processing.R` + +```r +test_that("process_lead_cohort_data aggregates correctly", { + raw_data <- data.frame( + geoid = rep("37001", 5), + income_bracket = rep("0-30%", 5), + income = c(10000, 15000, 20000, 25000, 30000), + energy_cost = c(1500, 1800, 2000, 2200, 2400), + households = c(100, 150, 200, 250, 300) + ) + + result <- process_lead_cohort_data(raw_data) + + # Check weighted means calculated correctly + expected_mean_income <- weighted.mean( + raw_data$income, + raw_data$households + ) + expect_equal(result$mean_income[1], expected_mean_income) +}) + +test_that("lead_to_poverty aggregates to binary", { + data <- data.frame( + income_bracket = c("0-100%", "100-150%", "150-200%"), + income = c(10000, 15000, 25000), + energy_cost = c(1500, 1800, 2000), + households = c(100, 200, 300) + ) + + result <- lead_to_poverty(data, dataset = "fpl") + + expect_equal(nrow(result), 2) # Binary: below/above poverty + expect_true(all(result$income_bracket %in% c( + "Below Federal Poverty Line", + "Above Federal Poverty Line" + ))) +}) + +test_that("calculate_ner handles edge cases", { + # Zero income + expect_equal(ner_func(0, 1000), -1) + + # Zero energy cost + expect_equal(ner_func(50000, 0), Inf) + + # Negative income + expect_true(is.finite(ner_func(-1000, 1000))) +}) +``` + +## 6. Schema Normalization Tests + +**File**: `tests/testthat/test-schema-normalization.R` + +```r +test_that("normalizes AMI brackets across vintages", { + # 2018 schema + data_2018 <- data.frame( + income_bracket = c("0-30%", "30-60%", "60-80%", "80-100%", "100%+") + ) + + # 2022 schema (same in this case) + data_2022 <- data.frame( + income_bracket = c("0-30%", "30-60%", "60-80%", "80-100%", "100%+") + ) + + norm_2018 <- normalize_ami_schema(data_2018, vintage = "2018") + norm_2022 <- normalize_ami_schema(data_2022, vintage = "2022") + + expect_equal( + sort(unique(norm_2018$income_bracket)), + sort(unique(norm_2022$income_bracket)) + ) +}) + +test_that("aggregates detailed brackets to simplified schema", { + data <- data.frame( + income_bracket = c("0-30%", "30-60%", "60-80%", "80-100%", "100%+"), + households = c(100, 200, 150, 300, 250) + ) + + result <- aggregate_to_simplified_schema(data) + + expect_equal(nrow(result), 3) # very_low, low_mod, mid_high + expect_equal(result$households[1], 100) # 0-30% + expect_equal(result$households[2], 350) # 30-80% (200 + 150) +}) + +test_that("normalizes building type across datasets", { + data <- data.frame( + bld_index = c("1 unit detached", "1 unit attached", "2-4 units", "5+ units") + ) + + result <- normalize_building_type(data) + + expect_true(all(result$building_type %in% c("Single-Family", "Multi-Family"))) +}) +``` + +## 7. Integration Tests + +**File**: `tests/testthat/test-integration.R` + +```r +test_that("end-to-end: load and compare vintages", { + # This test uses real data (if available) or mocked data + skip_if_offline() + + comparison <- compare_energy_burden( + dataset = "ami", + states = "NC", + group_by = "income_bracket" + ) + + expect_s3_class(comparison, "energy_burden_comparison") + expect_true("neb_2018" %in% names(comparison)) + expect_true("neb_2022" %in% names(comparison)) + expect_true("neb_change_pp" %in% names(comparison)) +}) + +test_that("end-to-end: calculate weighted metrics", { + data <- load_census_tract_data(states = "NC") + + metrics <- calculate_weighted_metrics( + data, + group_columns = "county_name", + metric_name = "ner" + ) + + expect_true("ner" %in% names(metrics)) + expect_true("household_count" %in% names(metrics)) + expect_true(all(metrics$ner > 0)) +}) +``` + +## 8. Helper Functions for Testing + +**File**: `tests/testthat/helper-mocks.R` + +```r +# Create sample LEAD data for testing +create_sample_lead_data <- function(n = 100) { + data.frame( + geoid = sample(paste0("37", sprintf("%03d", 1:100)), n, replace = TRUE), + income_bracket = sample(c("0-30%", "30-60%", "60-80%", "80-100%", "100%+"), n, replace = TRUE), + income = rnorm(n, 50000, 20000), + energy_cost = rnorm(n, 2000, 500), + households = sample(50:500, n, replace = TRUE), + housing_tenure = sample(c("OWNER", "RENTER"), n, replace = TRUE), + primary_heating_fuel = sample(c("Electricity", "Natural gas", "Fuel oil"), n, replace = TRUE) + ) +} + +# Mock HTTP responses +mock_openei_response <- function(dataset, state, vintage) { + httptest2::mock_api({ + path <- file.path( + "data.openei.org", + paste0(dataset, "_", state, "_", vintage, ".csv") + ) + httptest2::mock_response(path, status = 200, body = create_sample_lead_data()) + }) +} + +# Create temporary test database +create_test_database <- function() { + db_path <- tempfile(fileext = ".db") + con <- DBI::dbConnect(RSQLite::SQLite(), db_path) + + DBI::dbWriteTable(con, "ami_2022", create_sample_lead_data()) + DBI::dbWriteTable(con, "ami_2018", create_sample_lead_data()) + DBI::dbWriteTable(con, "fpl_2022", create_sample_lead_data()) + DBI::dbWriteTable(con, "fpl_2018", create_sample_lead_data()) + + DBI::dbDisconnect(con) + return(db_path) +} +``` + +**File**: `tests/testthat/helper-fixtures.R` + +```r +# Create fixture data files for testing +create_fixtures <- function(fixture_dir) { + dir.create(fixture_dir, recursive = TRUE, showWarnings = FALSE) + + # Sample AMI data + write.csv( + create_sample_lead_data(500), + file.path(fixture_dir, "sample_ami_2022.csv"), + row.names = FALSE + ) + + # Corrupted data (all-NA income_bracket) + corrupted <- create_sample_lead_data(100) + corrupted$income_bracket <- NA_character_ + write.csv( + corrupted, + file.path(fixture_dir, "corrupted_data.csv"), + row.names = FALSE + ) + + # Incomplete schema (missing required column) + incomplete <- create_sample_lead_data(100) + incomplete$income <- NULL + write.csv( + incomplete, + file.path(fixture_dir, "incomplete_schema.csv"), + row.names = FALSE + ) +} +``` + +## 9. Test Coverage Goals + +Target coverage levels: +- **Data loaders**: 95%+ line coverage +- **Data processing**: 90%+ line coverage +- **Validation functions**: 100% line coverage +- **Overall package**: 85%+ line coverage + +Run coverage report with: +```r +covr::package_coverage() +covr::report() +``` + +## 10. Continuous Integration + +Add to `.github/workflows/R-CMD-check.yml`: + +```yaml +- name: Test coverage + run: | + covr::codecov( + quiet = FALSE, + clean = FALSE, + install_path = file.path(Sys.getenv("RUNNER_TEMP"), "package") + ) + shell: Rscript {0} +``` + +## Implementation Priority + +1. **Phase 1** (Critical): + - File validation tests + - Data loader basic tests + - Schema normalization tests + +2. **Phase 2** (High): + - HTTP mocking tests + - Database mocking tests + - Edge case tests + +3. **Phase 3** (Medium): + - Integration tests + - Performance tests + - Coverage improvement + +## Expected Benefits + +1. **Reliability**: Catch bugs before they reach users +2. **Refactoring confidence**: Safe to improve code +3. **Documentation**: Tests serve as usage examples +4. **Regression prevention**: Ensure fixes stay fixed +5. **API stability**: Tests lock in expected behavior diff --git a/.dev/update-zenodo-config.R b/.dev/update-zenodo-config.R new file mode 100644 index 0000000..a195a9c --- /dev/null +++ b/.dev/update-zenodo-config.R @@ -0,0 +1,188 @@ +#!/usr/bin/env Rscript +# Auto-generate R/zenodo.R from state-manifest.json and zenodo-config-nationwide.txt +# +# This script ensures R/zenodo.R stays in sync with the actual generated files +# by reading the "source of truth" (state-manifest.json) and Zenodo metadata. +# +# Usage: Rscript .dev/update-zenodo-config.R + +cat("\n") +cat("==========================================\n") +cat(" Auto-generate R/zenodo.R Configuration\n") +cat("==========================================\n\n") + +# Load required packages +if (!requireNamespace("jsonlite", quietly = TRUE)) { + stop("Package 'jsonlite' required. Install with: install.packages('jsonlite')") +} + +# Read state manifest (source of truth for file metadata) +manifest_path <- "zenodo-upload-nationwide/state-manifest.json" +if (!file.exists(manifest_path)) { + stop("state-manifest.json not found at: ", manifest_path) +} + +cat("Reading state-manifest.json...\n") +manifest <- jsonlite::read_json(manifest_path) + +# Read Zenodo config (for DOIs and record ID) +zenodo_config_path <- "zenodo-upload-nationwide/zenodo-config-nationwide.txt" +if (!file.exists(zenodo_config_path)) { + stop("zenodo-config-nationwide.txt not found. Run upload script first.") +} + +cat("Reading zenodo-config-nationwide.txt...\n") +zenodo_config <- readLines(zenodo_config_path) + +# Extract DOIs and record ID +concept_doi <- sub('CONCEPT_DOI="(.*)"', '\\1', grep("^CONCEPT_DOI=", zenodo_config, value = TRUE)) +version_doi <- sub('VERSION_DOI="(.*)"', '\\1', grep("^VERSION_DOI=", zenodo_config, value = TRUE)) +record_id <- sub('RECORD_ID="(.*)"', '\\1', grep("^RECORD_ID=", zenodo_config, value = TRUE)) +record_url <- sub('RECORD_URL="(.*)"', '\\1', grep("^RECORD_URL=", zenodo_config, value = TRUE)) + +if (length(concept_doi) == 0 || length(version_doi) == 0 || length(record_id) == 0) { + stop("Could not extract DOIs/record ID from zenodo-config-nationwide.txt") +} + +cat("\nZenodo Record Information:\n") +cat(" Concept DOI: ", concept_doi, "\n") +cat(" Version DOI: ", version_doi, "\n") +cat(" Record ID: ", record_id, "\n") +cat(" Record URL: ", record_url, "\n\n") + +# Generate the files configuration section +cat("Generating file configuration...\n") + +datasets <- c("ami_2022", "fpl_2022", "ami_2018", "fpl_2018") +files_config <- character() + +for (dataset in datasets) { + if (!dataset %in% names(manifest$nationwide)) { + warning("Dataset '", dataset, "' not found in manifest. Skipping.") + next + } + + info <- manifest$nationwide[[dataset]] + + # Format the R list code + file_config <- sprintf(' %s = list( + filename = "%s", + url = "%s/files/%s", + size_mb = %.2f, + md5 = "%s" + )', + dataset, + info$filename, + record_url, + info$filename, + info$size_mb, + info$md5 + ) + + files_config <- c(files_config, file_config) +} + +# Add census_tracts placeholder +files_config <- c(files_config, sprintf(' + # Census Tract Data (not yet uploaded) + census_tracts = list( + filename = "census_tract_data.csv.gz", + url = NULL, + size_mb = NULL, + md5 = NULL + )')) + +files_section <- paste(files_config, collapse = ",\n\n") + +# Read current R/zenodo.R file +zenodo_r_path <- "R/zenodo.R" +if (!file.exists(zenodo_r_path)) { + stop("R/zenodo.R not found at: ", zenodo_r_path) +} + +cat("Reading R/zenodo.R...\n") +current_code <- readLines(zenodo_r_path) + +# Generate new get_zenodo_config() function +new_function <- sprintf('get_zenodo_config <- function() { + # Zenodo record for emburden processed datasets + # Published: %s + # Scope: US Nationwide (51 states + DC) + # This record contains pre-processed, analysis-ready datasets + # + # AUTO-GENERATED by .dev/update-zenodo-config.R + # DO NOT EDIT MANUALLY - regenerate from state-manifest.json instead + list( + # Main repository DOI (concept DOI - always points to latest version) + concept_doi = "%s", + + # Version-specific DOI (for reproducibility) + version_doi = "%s", + + # Direct download URLs for each dataset + # Format: zenodo_baseurl/records/RECORD_ID/files/FILENAME + files = list( + # 2022 Cohort Data (US Nationwide) +%s + ), + + # Metadata + description = "Pre-processed DOE LEAD Tool data for emburden R package (US Nationwide)", + license = "CC-BY-4.0", + source = "DOE Low-Income Energy Affordability Data (LEAD) Tool" + ) +}', + format(Sys.Date(), "%Y-%m-%d"), + concept_doi, + version_doi, + files_section +) + +# Find the start and end of get_zenodo_config() in current file +start_line <- grep("^get_zenodo_config <- function\\(\\)", current_code) +end_line <- grep("^}$", current_code) + +if (length(start_line) == 0) { + stop("Could not find get_zenodo_config() function in R/zenodo.R") +} + +# Find the closing brace that matches this function +# (the first } after the start that's at the same indentation level) +end_line <- end_line[end_line > start_line[1]][1] + +if (is.na(end_line)) { + stop("Could not find end of get_zenodo_config() function") +} + +# Build new file content +new_code <- c( + current_code[1:(start_line - 1)], # Everything before function + new_function, # New function + current_code[(end_line + 1):length(current_code)] # Everything after function +) + +# Write updated file +cat("Writing updated R/zenodo.R...\n") +writeLines(new_code, zenodo_r_path) + +cat("\n") +cat("==========================================\n") +cat(" SUCCESS: R/zenodo.R updated!\n") +cat("==========================================\n\n") + +cat("Summary of changes:\n") +cat(" - Updated Concept DOI: ", concept_doi, "\n") +cat(" - Updated Version DOI: ", version_doi, "\n") +cat(" - Updated file configurations for:\n") +for (dataset in datasets) { + if (dataset %in% names(manifest$nationwide)) { + info <- manifest$nationwide[[dataset]] + cat(sprintf(" * %s: MD5=%s, Size=%.2f MB\n", + dataset, info$md5, info$size_mb)) + } +} + +cat("\nNext steps:\n") +cat(" 1. Review changes: git diff R/zenodo.R\n") +cat(" 2. Validate: Rscript .dev/validate-zenodo-checksums.R\n") +cat(" 3. Commit if correct: git add R/zenodo.R && git commit\n\n") diff --git a/.dev/upload-to-zenodo-nationwide.sh b/.dev/upload-to-zenodo-nationwide.sh new file mode 100644 index 0000000..af42f6d --- /dev/null +++ b/.dev/upload-to-zenodo-nationwide.sh @@ -0,0 +1,325 @@ +#!/bin/bash +# upload-to-zenodo-nationwide.sh +# Automated Zenodo upload for NATIONWIDE datasets using REST API +# +# Usage: ./upload-to-zenodo-nationwide.sh [--sandbox] +# +# Environment variables required: +# ZENODO_TOKEN - Your Zenodo personal access token +# +# Optional: +# --sandbox - Upload to sandbox.zenodo.org instead of production + +set -e # Exit on error + +# Configuration +USE_SANDBOX=false +if [[ "$1" == "--sandbox" ]]; then + USE_SANDBOX=true + ZENODO_URL="https://sandbox.zenodo.org/api" + echo "Using SANDBOX environment (sandbox.zenodo.org)" +else + ZENODO_URL="https://zenodo.org/api" + echo "Using PRODUCTION environment (zenodo.org)" +fi + +# Check for API token +if [ -z "$ZENODO_TOKEN" ]; then + echo "" + echo "ERROR: ZENODO_TOKEN environment variable not set" + echo "" + echo "To obtain a token:" + if $USE_SANDBOX; then + echo "1. Go to https://sandbox.zenodo.org/account/settings/applications/tokens/new/" + else + echo "1. Go to https://zenodo.org/account/settings/applications/tokens/new/" + fi + echo "2. Create a new token with 'deposit:write' and 'deposit:actions' scopes" + echo "3. Export it: export ZENODO_TOKEN='your-token-here'" + echo "" + exit 1 +fi + +# Directory containing files to upload +UPLOAD_DIR="zenodo-upload-nationwide/nationwide" +if [ ! -d "$UPLOAD_DIR" ]; then + echo "ERROR: Upload directory not found: $UPLOAD_DIR" + exit 1 +fi + +echo "" +echo "========================================" +echo " Zenodo Automated Upload (NATIONWIDE)" +echo "========================================" +echo "" +echo "Upload directory: $UPLOAD_DIR" +echo "" + +# Validate checksums before uploading +echo "Validating MD5 checksums..." +if ! Rscript .dev/validate-zenodo-checksums.R; then + echo "" + echo "❌ ABORT: Checksum validation failed!" + echo " Fix the mismatches in R/zenodo.R before uploading." + exit 1 +fi +echo "" + +# Files to upload +# NOTE: Arizona 2018 data has non-standard filename handled in R code +FILES=( + "lead_ami_cohorts_2022_us.csv.gz" + "lead_fpl_cohorts_2022_us.csv.gz" + "lead_ami_cohorts_2018_us.csv.gz" + "lead_fpl_cohorts_2018_us.csv.gz" +) + +# Also include checksums from parent directory +CHECKSUMS_FILE="zenodo-upload-nationwide/checksums.txt" + +# Verify all files exist +echo "Verifying files..." +for file in "${FILES[@]}"; do + if [ ! -f "$UPLOAD_DIR/$file" ]; then + echo "ERROR: File not found: $UPLOAD_DIR/$file" + exit 1 + fi + size=$(ls -lh "$UPLOAD_DIR/$file" | awk '{print $5}') + echo " ✓ $file ($size)" +done + +if [ ! -f "$CHECKSUMS_FILE" ]; then + echo "ERROR: Checksums file not found: $CHECKSUMS_FILE" + exit 1 +fi +echo " ✓ checksums.txt" +echo "" + +# Step 1: Create a new deposition +echo "Step 1: Creating new deposition..." +response=$(curl -s -X POST -d '{}' \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + "$ZENODO_URL/deposit/depositions") + +# Extract deposition ID and bucket URL +deposition_id=$(echo "$response" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +bucket_url=$(echo "$response" | python3 -c "import sys, json; print(json.load(sys.stdin)['links']['bucket'])") + +if [ -z "$deposition_id" ] || [ "$deposition_id" == "null" ]; then + echo "ERROR: Failed to create deposition" + echo "Response: $response" + exit 1 +fi + +echo "✓ Deposition created: ID $deposition_id" +echo " Bucket URL: $bucket_url" +echo "" + +# Step 2: Upload files +echo "Step 2: Uploading files..." +for file in "${FILES[@]}"; do + echo " Uploading: $file" + + upload_response=$(curl -s --progress-bar \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + -H "Content-Type: application/octet-stream" \ + --upload-file "$UPLOAD_DIR/$file" \ + "$bucket_url/$file") + + upload_status=$(echo "$upload_response" | python3 -c "import sys, json; d=json.load(sys.stdin); print('OK' if 'key' in d else 'FAILED')" 2>/dev/null || echo "FAILED") + + if [ "$upload_status" == "OK" ]; then + echo " ✓ Uploaded successfully" + else + echo " ERROR: Upload failed" + echo " Response: $upload_response" + exit 1 + fi +done + +# Upload checksums file +echo " Uploading: checksums.txt" +upload_response=$(curl -s --progress-bar \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + -H "Content-Type: application/octet-stream" \ + --upload-file "$CHECKSUMS_FILE" \ + "$bucket_url/checksums.txt") + +upload_status=$(echo "$upload_response" | python3 -c "import sys, json; d=json.load(sys.stdin); print('OK' if 'key' in d else 'FAILED')" 2>/dev/null || echo "FAILED") + +if [ "$upload_status" == "OK" ]; then + echo " ✓ Uploaded successfully" +else + echo " ERROR: Upload failed" + echo " Response: $upload_response" + exit 1 +fi +echo "" + +# Step 3: Add metadata +echo "Step 3: Adding metadata..." + +# Prepare metadata JSON +metadata=$(cat <<'EOF' +{ + "metadata": { + "title": "emburden: Processed Energy Burden Datasets (US Nationwide)", + "upload_type": "dataset", + "description": "PROCESSED, analysis-ready household energy burden datasets from the DOE Low-Income Energy Affordability Data (LEAD) Tool, formatted for the emburden R package.\n\nScope: All 51 US states and territories (50 states + DC)\n\nIMPORTANT: These are PRE-PROCESSED datasets, not raw OpenEI data. They have been:\n
    \n
  • Aggregated by census tract + income bracket
    • \n
  • Enriched with computed energy burden metrics (EROI, NER, DEAR)
    • \n
  • Standardized for immediate analysis
    • \n
  • Quality-checked and validated
    • \n
\n\nThis repository provides census tract-level data on household energy burden for the entire United States, covering ~73,000 census tracts. Data includes both Area Median Income (AMI) and Federal Poverty Line (FPL) cohort analyses for 2018 and 2022 vintages.\n\n

Files Included:

\n
    \n
  • lead_ami_cohorts_2022_us.csv.gz: 2022 AMI cohort data (701,490 records, 148 MB)
    • \n
  • lead_fpl_cohorts_2022_us.csv.gz: 2022 FPL cohort data (588,163 records, 52 MB)
    • \n
  • lead_ami_cohorts_2018_us.csv.gz: 2018 AMI cohort data (530,500 records, 54 MB)
    • \n
  • lead_fpl_cohorts_2018_us.csv.gz: 2018 FPL cohort data (514,893 records, 53 MB)
    • \n
  • checksums.txt: MD5 checksums for verification
    • \n
\n\nTotal size: 307 MB compressed\n\n

Data Processing

\n
    \n
  • Source: Raw LEAD Tool data from OpenEI
    • \n
  • Processing: emburden R package v0.4.8 data pipeline
    • \n
  • Format: CSV (aggregated tract-level cohorts with computed metrics)
    • \n
  • Ready for: Immediate analysis, no additional processing required
    • \n
\n\n

Coverage

\n
    \n
  • States: All 51 (50 states + DC, excludes PR)
    • \n
  • Census Tracts: ~73,000 nationwide
    • \n
  • Total Records: 2.3+ million cohort observations
    • \n
  • Income Brackets: 4-6 per dataset/vintage
    • \n
\n\n

Data Sources

\nOriginal raw data from:\n
    \n
  • DOE LEAD Tool 2022: https://data.openei.org/submissions/6219
    • \n
  • DOE LEAD Tool 2018: https://data.openei.org/submissions/573
    • \n
\n\nProcessed using: emburden R package v0.4.8 (https://github.com/ScheierVentures/emburden)\n\n

Citation

\nWhen using this data, please cite:\n
    \n
  1. This Zenodo repository (DOI provided)
    2. \n
  2. The emburden R package v0.4.8
    3. \n
  3. The original DOE LEAD Tool publications
    4. \n
\n\n

License

\nCC-BY-4.0 (same as source data)", + "creators": [ + { + "name": "Scheier, Eric", + "affiliation": "Emergi Foundation, UNC Chapel Hill", + "orcid": "0000-0001-9849-9089" + } + ], + "keywords": [ + "energy burden", + "energy poverty", + "household energy", + "census tracts", + "LEAD Tool", + "R package", + "emburden", + "nationwide", + "United States" + ], + "license": "cc-by-4.0", + "access_right": "open", + "related_identifiers": [ + { + "identifier": "https://github.com/ScheierVentures/emburden", + "relation": "isSupplementTo", + "scheme": "url" + }, + { + "identifier": "https://data.openei.org/submissions/6219", + "relation": "isDerivedFrom", + "scheme": "url" + }, + { + "identifier": "https://data.openei.org/submissions/573", + "relation": "isDerivedFrom", + "scheme": "url" + } + ] + } +} +EOF +) + +metadata_response=$(curl -s -X PUT \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + -d "$metadata" \ + "$ZENODO_URL/deposit/depositions/$deposition_id") + +metadata_status=$(echo "$metadata_response" | python3 -c "import sys, json; d=json.load(sys.stdin); print('OK' if 'metadata' in d else 'FAILED')" 2>/dev/null || echo "FAILED") + +if [ "$metadata_status" == "OK" ]; then + echo "✓ Metadata added successfully" +else + echo "ERROR: Failed to add metadata" + echo "Response: $metadata_response" + exit 1 +fi +echo "" + +# Step 4: Publish +echo "Step 4: Publishing deposition..." +echo "" +echo "WARNING: This will publish the deposition and make it publicly available." +echo "Once published, it CANNOT be deleted (only new versions can be created)." +echo "" +read -p "Proceed with publication? (yes/no): " confirm + +if [ "$confirm" != "yes" ]; then + echo "" + echo "Publication cancelled." + echo "" + echo "Deposition ID $deposition_id is saved as DRAFT." + if $USE_SANDBOX; then + echo "View at: https://sandbox.zenodo.org/deposit/$deposition_id" + else + echo "View at: https://zenodo.org/deposit/$deposition_id" + fi + echo "" + echo "To publish later, run:" + echo " curl -X POST -H \"Authorization: Bearer \$ZENODO_TOKEN\" \\" + echo " $ZENODO_URL/deposit/depositions/$deposition_id/actions/publish" + echo "" + exit 0 +fi + +publish_response=$(curl -s -X POST -d '{}' \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + "$ZENODO_URL/deposit/depositions/$deposition_id/actions/publish") + +# Extract DOIs and URLs +concept_doi=$(echo "$publish_response" | python3 -c "import sys, json; print(json.load(sys.stdin).get('conceptdoi', ''))" 2>/dev/null || echo "") +version_doi=$(echo "$publish_response" | python3 -c "import sys, json; print(json.load(sys.stdin).get('doi', ''))" 2>/dev/null || echo "") +record_id=$(echo "$publish_response" | python3 -c "import sys, json; print(json.load(sys.stdin).get('record_id', ''))" 2>/dev/null || echo "") + +if [ -z "$version_doi" ] || [ "$version_doi" == "null" ]; then + echo "ERROR: Publication failed" + echo "Response: $publish_response" + exit 1 +fi + +echo "✓ Publication successful!" +echo "" +echo "========================================" +echo " Upload Complete!" +echo "========================================" +echo "" +echo "Concept DOI (always latest): $concept_doi" +echo "Version DOI (this version): $version_doi" +echo "Record ID: $record_id" +echo "" +if $USE_SANDBOX; then + record_url="https://sandbox.zenodo.org/records/$record_id" +else + record_url="https://zenodo.org/records/$record_id" +fi +echo "View at: $record_url" +echo "" + +# Generate file URLs and save configuration +echo "File download URLs:" +echo "" + +config_file="zenodo-upload-nationwide/zenodo-config-nationwide.txt" +cat > "$config_file" << CONF_EOF +# Zenodo Configuration (NATIONWIDE) +# Generated: $(date) +# Environment: $(if $USE_SANDBOX; then echo "SANDBOX"; else echo "PRODUCTION"; fi) + +CONCEPT_DOI="$concept_doi" +VERSION_DOI="$version_doi" +RECORD_ID="$record_id" +RECORD_URL="$record_url" + +# File URLs (for R/zenodo.R configuration) +CONF_EOF + +ALL_FILES=("${FILES[@]}" "checksums.txt") +for file in "${ALL_FILES[@]}"; do + if $USE_SANDBOX; then + file_url="https://sandbox.zenodo.org/records/$record_id/files/$file" + else + file_url="https://zenodo.org/records/$record_id/files/$file" + fi + echo " - $file" + echo " $file_url" + echo "" + echo "FILE_$file=\"$file_url\"" >> "$config_file" +done + +echo "Configuration saved to: $config_file" +echo "" +echo "Next steps:" +echo " 1. Run: Rscript .dev/update-zenodo-config.R to auto-update R/zenodo.R" +echo " 2. Test Zenodo downloads" +echo " 3. Create new release with working Zenodo integration" +echo "" diff --git a/.dev/upload-to-zenodo.sh b/.dev/upload-to-zenodo.sh new file mode 100644 index 0000000..a93df08 --- /dev/null +++ b/.dev/upload-to-zenodo.sh @@ -0,0 +1,286 @@ +#!/bin/bash +# upload-to-zenodo.sh +# Automated Zenodo upload using REST API +# +# Usage: ./upload-to-zenodo.sh [--sandbox] +# +# Environment variables required: +# ZENODO_TOKEN - Your Zenodo personal access token +# +# Optional: +# --sandbox - Upload to sandbox.zenodo.org instead of production + +set -e # Exit on error + +# Configuration +USE_SANDBOX=false +if [[ "$1" == "--sandbox" ]]; then + USE_SANDBOX=true + ZENODO_URL="https://sandbox.zenodo.org/api" + echo "Using SANDBOX environment (sandbox.zenodo.org)" +else + ZENODO_URL="https://zenodo.org/api" + echo "Using PRODUCTION environment (zenodo.org)" +fi + +# Check for API token +if [ -z "$ZENODO_TOKEN" ]; then + echo "" + echo "ERROR: ZENODO_TOKEN environment variable not set" + echo "" + echo "To obtain a token:" + if $USE_SANDBOX; then + echo "1. Go to https://sandbox.zenodo.org/account/settings/applications/tokens/new/" + else + echo "1. Go to https://zenodo.org/account/settings/applications/tokens/new/" + fi + echo "2. Create a new token with 'deposit:write' and 'deposit:actions' scopes" + echo "3. Export it: export ZENODO_TOKEN='your-token-here'" + echo "" + exit 1 +fi + +# Directory containing files to upload +UPLOAD_DIR="zenodo-upload" +if [ ! -d "$UPLOAD_DIR" ]; then + echo "ERROR: Upload directory not found: $UPLOAD_DIR" + exit 1 +fi + +echo "" +echo "========================================" +echo " Zenodo Automated Upload" +echo "========================================" +echo "" +echo "Upload directory: $UPLOAD_DIR" +echo "" + +# Files to upload +FILES=( + "lead_ami_cohorts_2022_us.csv.gz" + "lead_fpl_cohorts_2022_us.csv.gz" + "lead_ami_cohorts_2018_us.csv.gz" + "lead_fpl_cohorts_2018_us.csv.gz" + "checksums.txt" +) + +# Verify all files exist +echo "Verifying files..." +for file in "${FILES[@]}"; do + if [ ! -f "$UPLOAD_DIR/$file" ]; then + echo "ERROR: File not found: $UPLOAD_DIR/$file" + exit 1 + fi + size=$(ls -lh "$UPLOAD_DIR/$file" | awk '{print $5}') + echo " ✓ $file ($size)" +done +echo "" + +# Step 1: Create a new deposition +echo "Step 1: Creating new deposition..." +response=$(curl -s -X POST -d '{}' \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + "$ZENODO_URL/deposit/depositions") + +# Extract deposition ID and bucket URL +deposition_id=$(echo "$response" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +bucket_url=$(echo "$response" | python3 -c "import sys, json; print(json.load(sys.stdin)['links']['bucket'])") + +if [ -z "$deposition_id" ] || [ "$deposition_id" == "null" ]; then + echo "ERROR: Failed to create deposition" + echo "Response: $response" + exit 1 +fi + +echo "✓ Deposition created: ID $deposition_id" +echo " Bucket URL: $bucket_url" +echo "" + +# Step 2: Upload files +echo "Step 2: Uploading files..." +for file in "${FILES[@]}"; do + echo " Uploading: $file" + + upload_response=$(curl -s --progress-bar \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + -H "Content-Type: application/octet-stream" \ + --upload-file "$UPLOAD_DIR/$file" \ + "$bucket_url/$file") + + upload_status=$(echo "$upload_response" | python3 -c "import sys, json; d=json.load(sys.stdin); print('OK' if 'key' in d else 'FAILED')" 2>/dev/null || echo "FAILED") + + if [ "$upload_status" == "OK" ]; then + echo " ✓ Uploaded successfully" + else + echo " ERROR: Upload failed" + echo " Response: $upload_response" + exit 1 + fi +done +echo "" + +# Step 3: Add metadata +echo "Step 3: Adding metadata..." + +# Prepare metadata JSON +metadata=$(cat <<'EOF' +{ + "metadata": { + "title": "emburden: Processed Energy Burden Datasets (North Carolina)", + "upload_type": "dataset", + "description": "PROCESSED, analysis-ready household energy burden datasets from the DOE Low-Income Energy Affordability Data (LEAD) Tool, formatted for the emburden R package.\n\nScope: North Carolina (proof-of-concept demonstration)\n\nIMPORTANT: These are PRE-PROCESSED datasets, not raw OpenEI data. They have been:\n
    \n
  • Aggregated by census tract + income bracket
    • \n
  • Enriched with computed energy burden metrics (EROI, NER, DEAR)
    • \n
  • Standardized for immediate analysis
    • \n
  • Quality-checked and validated
    • \n
\n\nThis repository provides census tract-level data on household energy burden for North Carolina, covering ~1,600 census tracts. Data includes both Area Median Income (AMI) and Federal Poverty Line (FPL) cohort analyses for 2018 and 2022 vintages.\n\n

Files Included:

\n
    \n
  • lead_ami_cohorts_2022_us.csv.gz: 2022 AMI cohort data (15K records, NC)
    • \n
  • lead_fpl_cohorts_2022_us.csv.gz: 2022 FPL cohort data (588K records, NC)
    • \n
  • lead_ami_cohorts_2018_us.csv.gz: 2018 AMI cohort data (531K records, NC)
    • \n
  • lead_fpl_cohorts_2018_us.csv.gz: 2018 FPL cohort data (515K records, NC)
    • \n
  • checksums.txt: MD5 checksums for verification
    • \n
\n\n

Data Processing

\n
    \n
  • Source: Raw LEAD Tool data from OpenEI
    • \n
  • Processing: emburden R package v0.4.7 data pipeline
    • \n
  • Format: CSV (aggregated tract-level cohorts with computed metrics)
    • \n
  • Ready for: Immediate analysis, no additional processing required
    • \n
\n\n

Future Expansion

\nThis proof-of-concept can be expanded to nationwide data (all 51 states, ~72,000 census tracts) in future versions.\n\n

Data Sources

\nOriginal raw data from:\n
    \n
  • DOE LEAD Tool 2022: https://data.openei.org/submissions/6219
    • \n
  • DOE LEAD Tool 2018: https://data.openei.org/submissions/573
    • \n
\n\nProcessed using: emburden R package v0.4.7 (https://github.com/ScheierVentures/emburden)\n\n

Citation

\nWhen using this data, please cite:\n
    \n
  1. This Zenodo repository (DOI provided)
    2. \n
  2. The emburden R package v0.4.7
    3. \n
  3. The original DOE LEAD Tool publications
    4. \n
\n\n

License

\nCC-BY-4.0 (same as source data)", + "creators": [ + { + "name": "Scheier, Eric", + "affiliation": "Emergi Foundation, UNC Chapel Hill", + "orcid": "0000-0001-9849-9089" + } + ], + "keywords": [ + "energy burden", + "energy poverty", + "household energy", + "census tracts", + "LEAD Tool", + "R package", + "emburden", + "North Carolina" + ], + "license": "cc-by-4.0", + "access_right": "open", + "related_identifiers": [ + { + "identifier": "https://github.com/ScheierVentures/emburden", + "relation": "isSupplementTo", + "scheme": "url" + }, + { + "identifier": "https://data.openei.org/submissions/6219", + "relation": "isDerivedFrom", + "scheme": "url" + }, + { + "identifier": "https://data.openei.org/submissions/573", + "relation": "isDerivedFrom", + "scheme": "url" + } + ] + } +} +EOF +) + +metadata_response=$(curl -s -X PUT \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + -d "$metadata" \ + "$ZENODO_URL/deposit/depositions/$deposition_id") + +metadata_status=$(echo "$metadata_response" | python3 -c "import sys, json; d=json.load(sys.stdin); print('OK' if 'metadata' in d else 'FAILED')" 2>/dev/null || echo "FAILED") + +if [ "$metadata_status" == "OK" ]; then + echo "✓ Metadata added successfully" +else + echo "ERROR: Failed to add metadata" + echo "Response: $metadata_response" + exit 1 +fi +echo "" + +# Step 4: Publish +echo "Step 4: Publishing deposition..." +echo "" +echo "WARNING: This will publish the deposition and make it publicly available." +echo "Once published, it CANNOT be deleted (only new versions can be created)." +echo "" +read -p "Proceed with publication? (yes/no): " confirm + +if [ "$confirm" != "yes" ]; then + echo "" + echo "Publication cancelled." + echo "" + echo "Deposition ID $deposition_id is saved as DRAFT." + if $USE_SANDBOX; then + echo "View at: https://sandbox.zenodo.org/deposit/$deposition_id" + else + echo "View at: https://zenodo.org/deposit/$deposition_id" + fi + echo "" + echo "To publish later, run:" + echo " curl -X POST -H \"Authorization: Bearer \$ZENODO_TOKEN\" \\" + echo " $ZENODO_URL/deposit/depositions/$deposition_id/actions/publish" + echo "" + exit 0 +fi + +publish_response=$(curl -s -X POST -d '{}' \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + "$ZENODO_URL/deposit/depositions/$deposition_id/actions/publish") + +# Extract DOIs and URLs +concept_doi=$(echo "$publish_response" | python3 -c "import sys, json; print(json.load(sys.stdin).get('conceptdoi', ''))" 2>/dev/null || echo "") +version_doi=$(echo "$publish_response" | python3 -c "import sys, json; print(json.load(sys.stdin).get('doi', ''))" 2>/dev/null || echo "") +record_id=$(echo "$publish_response" | python3 -c "import sys, json; print(json.load(sys.stdin).get('record_id', ''))" 2>/dev/null || echo "") + +if [ -z "$version_doi" ] || [ "$version_doi" == "null" ]; then + echo "ERROR: Publication failed" + echo "Response: $publish_response" + exit 1 +fi + +echo "✓ Publication successful!" +echo "" +echo "========================================" +echo " Upload Complete!" +echo "========================================" +echo "" +echo "Concept DOI (always latest): $concept_doi" +echo "Version DOI (this version): $version_doi" +echo "Record ID: $record_id" +echo "" +if $USE_SANDBOX; then + record_url="https://sandbox.zenodo.org/records/$record_id" +else + record_url="https://zenodo.org/records/$record_id" +fi +echo "View at: $record_url" +echo "" + +# Generate file URLs and save configuration +echo "File download URLs:" +echo "" + +config_file="zenodo-upload/zenodo-config.txt" +cat > "$config_file" << CONF_EOF +# Zenodo Configuration +# Generated: $(date) +# Environment: $(if $USE_SANDBOX; then echo "SANDBOX"; else echo "PRODUCTION"; fi) + +CONCEPT_DOI="$concept_doi" +VERSION_DOI="$version_doi" +RECORD_ID="$record_id" +RECORD_URL="$record_url" + +# File URLs (for R/zenodo.R configuration) +CONF_EOF + +for file in "${FILES[@]}"; do + if $USE_SANDBOX; then + file_url="https://sandbox.zenodo.org/records/$record_id/files/$file" + else + file_url="https://zenodo.org/records/$record_id/files/$file" + fi + echo " - $file" + echo " $file_url" + echo "" + echo "FILE_$file=\"$file_url\"" >> "$config_file" +done + +echo "Configuration saved to: $config_file" +echo "" +echo "Next steps:" +echo " 1. Update R/zenodo.R with the DOIs and URLs above" +echo " 2. Test Zenodo downloads" +echo " 3. Create new release with working Zenodo integration" +echo "" diff --git a/.dev/validate-workflows.sh b/.dev/validate-workflows.sh new file mode 100644 index 0000000..b6c15fc --- /dev/null +++ b/.dev/validate-workflows.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Validate GitHub Actions workflow files for YAML syntax errors +# This script can be called from pre-tag validation or run manually + +set -e + +echo "==================================" +echo " Workflow File Validation" +echo "==================================" +echo "" + +# Check if actionlint is installed +if ! command -v actionlint &> /dev/null; then + echo "actionlint not found. Installing..." + + # Download and install actionlint + bash <(curl -s https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + + # Move to a location in PATH + if [ -w "/usr/local/bin" ]; then + sudo mv ./actionlint /usr/local/bin/ + else + # Fallback to user bin if /usr/local/bin is not writable + mkdir -p ~/.local/bin + mv ./actionlint ~/.local/bin/ + export PATH="$HOME/.local/bin:$PATH" + fi + + echo "✅ actionlint installed successfully" + echo "" +fi + +# Run actionlint on all workflow files +echo "Validating workflow files..." +echo "" + +WORKFLOW_DIR=".github/workflows" + +if [ ! -d "$WORKFLOW_DIR" ]; then + echo "❌ Error: $WORKFLOW_DIR directory not found" + exit 1 +fi + +# Count workflow files +WORKFLOW_COUNT=$(find "$WORKFLOW_DIR" -name "*.yml" -o -name "*.yaml" | wc -l) + +if [ "$WORKFLOW_COUNT" -eq 0 ]; then + echo "⚠️ Warning: No workflow files found in $WORKFLOW_DIR" + exit 0 +fi + +echo "Found $WORKFLOW_COUNT workflow file(s)" +echo "" + +# Run actionlint +if actionlint -color "$WORKFLOW_DIR"/*.yml "$WORKFLOW_DIR"/*.yaml 2>/dev/null; then + echo "" + echo "✅ All workflow files passed validation" + echo "" + echo "Files validated:" + find "$WORKFLOW_DIR" -name "*.yml" -o -name "*.yaml" | while read -r file; do + echo " - $(basename "$file")" + done + exit 0 +else + echo "" + echo "❌ Workflow validation failed" + echo "" + echo "Please fix the errors above before proceeding with the release." + echo "Common issues:" + echo " - YAML syntax errors (check indentation, quotes, special characters)" + echo " - Invalid workflow structure" + echo " - Bash heredoc conflicts with YAML (use ___ instead of --- or ***)" + exit 1 +fi diff --git a/.dev/validate-zenodo-checksums.R b/.dev/validate-zenodo-checksums.R new file mode 100644 index 0000000..a5b029d --- /dev/null +++ b/.dev/validate-zenodo-checksums.R @@ -0,0 +1,117 @@ +#!/usr/bin/env Rscript +# Validate Zenodo MD5 Checksums +# +# This script compares MD5 checksums between: +# 1. state-manifest.json (actual generated files) +# 2. R/zenodo.R (configured in package code) +# +# Usage: +# Rscript .dev/validate-zenodo-checksums.R +# +# Exit codes: +# 0 = All checksums match +# 1 = Mismatches found or validation failed + +cat("\n") +cat("==========================================\n") +cat(" Zenodo MD5 Checksum Validation\n") +cat("==========================================\n\n") + +# Load required packages +if (!requireNamespace("jsonlite", quietly = TRUE)) { + stop("Package 'jsonlite' required. Install with: install.packages('jsonlite')") +} + +# Read state manifest +manifest_path <- "zenodo-upload-nationwide/state-manifest.json" + +if (!file.exists(manifest_path)) { + cat("❌ ERROR: state-manifest.json not found at:", manifest_path, "\n") + cat(" Have you generated the Zenodo data yet?\n\n") + quit(status = 1) +} + +cat("📄 Reading state-manifest.json...\n") +manifest <- jsonlite::read_json(manifest_path) + +# Source R/zenodo.R to get configuration +cat("📄 Reading R/zenodo.R configuration...\n") +source("R/zenodo.R") +config <- get_zenodo_config() + +# Datasets to validate +datasets <- c("ami_2022", "fpl_2022", "ami_2018", "fpl_2018") + +# Track validation results +all_valid <- TRUE +mismatches <- list() + +cat("\n🔍 Validating checksums...\n\n") + +for (dataset in datasets) { + # Get checksums + manifest_md5 <- manifest$nationwide[[dataset]]$md5 + config_md5 <- config$files[[dataset]]$md5 + + manifest_size <- manifest$nationwide[[dataset]]$size_mb + config_size <- config$files[[dataset]]$size_mb + + # Check MD5 + md5_match <- identical(manifest_md5, config_md5) + + # Check file size (allow 0.1 MB tolerance for rounding) + size_match <- abs(manifest_size - config_size) < 0.1 + + if (md5_match && size_match) { + cat(sprintf("✅ %s: MD5 and size match\n", dataset)) + } else { + all_valid <- FALSE + cat(sprintf("❌ %s: MISMATCH DETECTED!\n", dataset)) + + if (!md5_match) { + cat(sprintf(" MD5 manifest: %s\n", manifest_md5)) + cat(sprintf(" MD5 R/zenodo: %s\n", config_md5)) + } + + if (!size_match) { + cat(sprintf(" Size manifest: %.2f MB\n", manifest_size)) + cat(sprintf(" Size R/zenodo: %.2f MB\n", config_size)) + } + + cat("\n") + + mismatches[[dataset]] <- list( + manifest_md5 = manifest_md5, + config_md5 = config_md5, + manifest_size = manifest_size, + config_size = config_size + ) + } +} + +cat("\n==========================================\n") + +if (all_valid) { + cat("✅ SUCCESS: All checksums and sizes match!\n") + cat("==========================================\n\n") + quit(status = 0) +} else { + cat("❌ FAILURE: Checksum/size mismatches found\n") + cat("==========================================\n\n") + + cat("TO FIX:\n") + cat(" 1. Update R/zenodo.R with the correct values from state-manifest.json\n") + cat(" 2. Re-run this validation script to confirm\n") + cat(" 3. Commit the corrected R/zenodo.R\n\n") + + cat("AUTOMATED FIX (copy-paste to terminal):\n") + cat("----------------------------------------\n") + for (dataset in names(mismatches)) { + m <- mismatches[[dataset]] + cat(sprintf("# Fix %s:\n", dataset)) + cat(sprintf("# Update md5 to: %s\n", m$manifest_md5)) + cat(sprintf("# Update size_mb to: %.2f\n\n", m$manifest_size)) + } + + quit(status = 1) +} diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 14ad900..0000000 --- a/.gitattributes +++ /dev/null @@ -1,4 +0,0 @@ -*.csv filter=lfs diff=lfs merge=lfs -text -CohortData_AreaMedianIncome.csv filter=lfs diff=lfs merge=lfs -text -CohortData_FederalPovertyLine.csv filter=lfs diff=lfs merge=lfs -text -CensusTractData.csv filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/.github/BRANCHING_STRATEGY.md b/.github/BRANCHING_STRATEGY.md new file mode 100644 index 0000000..30e0848 --- /dev/null +++ b/.github/BRANCHING_STRATEGY.md @@ -0,0 +1,532 @@ +# Branching Strategy + +This document describes the branching strategy and deployment workflow for the emburden R package. + +## Table of Contents + +- [Overview](#overview) +- [Branch Structure](#branch-structure) +- [Promotion Workflows](#promotion-workflows) +- [Developer Workflows](#developer-workflows) +- [Release Process](#release-process) +- [Manual Override](#manual-override) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +The emburden package uses a **GitFlow-inspired branching model** with controlled promotion workflows and approval gates to ensure code quality and stability through the deployment pipeline. + +### Key Principles + +1. **Sequential flow only:** Code flows in one direction: Feature → Dev → Staging → Main → Public → CRAN +2. **No bypassing:** Each stage must be completed successfully before moving to the next +3. **Approval gates:** Critical promotions require manual approval +4. **Automated validation:** CI checks run at every stage +5. **Rollback capability:** All changes are versioned and can be reverted + +### Deployment Pipeline + +``` +┌─────────────┐ +│ Feature │ Developer creates feature branch +│ Branches │ +└──────┬──────┘ + │ PR + CI checks + ↓ +┌─────────────┐ +│ Dev │ Integration branch (fast CI feedback) +└──────┬──────┘ + │ promote-to-staging.yml (requires approval) + ↓ +┌─────────────┐ +│ Staging │ Pre-production testing (multi-platform CI) +└──────┬──────┘ + │ promote-to-main.yml (requires approval + version bump) + ↓ +┌─────────────┐ +│ Main │ Production (private repo) +└──────┬──────┘ + │ publish-to-public.yml (automatic) + ↓ +┌─────────────┐ +│ Public │ Public repository (ericscheier/emburden) +│ Repo │ +└──────┬──────┘ + │ cran-release.yml (requires final approval) + ↓ +┌─────────────┐ +│ CRAN │ Official R package repository +└─────────────┘ +``` + +--- + +## Branch Structure + +### `main` - Production Branch + +**Purpose:** Production-ready code, synced to public repository + +**Protection:** +- Requires PR with approvals +- Requires all status checks to pass (ubuntu, windows, macos) +- Requires `production-release` environment approval +- Linear history enforced +- No direct commits allowed + +**Triggers:** +- Creates version tags (`v*.*.*`) +- Triggers GitHub release creation +- Syncs to public repository +- Initiates CRAN submission workflow + +**Who can merge:** Only via promote-to-main.yml workflow after manual approval + +--- + +### `staging` - Pre-Production Branch + +**Purpose:** Final validation before production release + +**Protection:** +- Requires PR with approvals +- Requires all status checks to pass (ubuntu, windows, macos) +- Requires `staging-promotion` environment approval +- Linear history enforced +- No direct commits allowed + +**Validation:** +- Multi-platform R CMD check (Ubuntu, Windows, macOS) +- Full test coverage +- Workflow validation + +**Who can merge:** Only via promote-to-staging.yml workflow after manual approval + +--- + +### `dev` - Development Branch + +**Purpose:** Integration branch for active development + +**Protection:** +- Requires PR +- Auto-approval for safe changes (docs, tests) +- Manual review for code changes +- Requires status checks to pass (Ubuntu only - fast feedback) +- Linear history enforced + +**Validation:** +- R CMD check (Ubuntu, release R only) +- Test coverage +- Workflow validation + +**Who can merge:** +- Developers with approval +- Auto-approved for docs/tests changes via auto-approve-safe-changes.yml + +--- + +### Feature Branches + +**Naming:** `feature/description`, `fix/description`, `docs/description` + +**Purpose:** Individual feature development + +**Protection:** None (developer responsibility) + +**Merge target:** Always merge to `dev` branch via PR + +**Lifecycle:** Delete after merge to dev + +--- + +## Promotion Workflows + +### 1. Feature → Dev + +**Trigger:** Manual PR creation + +**Process:** +1. Developer creates PR from feature branch to `dev` +2. CI runs R CMD check (Ubuntu only) and tests +3. Auto-approval if only docs/tests changed +4. Manual approval required for code changes +5. PR merges automatically after approval + CI pass + +**Approval:** +- None for docs/tests (auto-approved) +- Code owner review for R code changes + +--- + +### 2. Dev → Staging + +**Workflow:** `.github/workflows/promote-to-staging.yml` + +**Trigger:** Manual (via GitHub Actions UI) + +**Process:** +1. Maintainer triggers "Promote Dev to Staging" workflow +2. Workflow validates dev branch (R CMD check) +3. Creates promotion PR: `dev` → `staging` +4. Requires manual approval via `staging-promotion` environment +5. Auto-merges after approval +6. Creates promotion tag: `staging/YYYYMMDD-HHMMSS` + +**Approval:** Repository maintainer via staging-promotion environment + +**When to promote:** +- After feature development complete +- All dev CI checks passing +- Ready for pre-production validation + +--- + +### 3. Staging → Main (Production Release) + +**Workflow:** `.github/workflows/promote-to-main.yml` + +**Trigger:** Manual (via GitHub Actions UI) + +**Parameters:** +- `version_bump`: patch | minor | major | custom +- `custom_version`: (if version_bump=custom) + +**Process:** +1. Maintainer triggers "Promote Staging to Main" workflow +2. Workflow validates staging (multi-platform R CMD check) +3. Bumps version in DESCRIPTION, CITATION, .zenodo.json +4. Updates NEWS.md from git commits +5. Creates release PR: `staging` → `main` +6. Requires manual approval via `production-release` environment +7. Auto-merges after approval +8. Creates version tag: `v*.*.*` +9. Creates GitHub release with NEWS.md content +10. Triggers publish-to-public workflow +11. Triggers CRAN release workflow + +**Approval:** Repository maintainer via production-release environment + +**When to promote:** +- After staging validation complete +- Ready for public release +- All stakeholders agree to release +- CRAN submission desired + +**Version Bump Strategy:** +- **patch** (0.5.20 → 0.5.21): Bug fixes, minor improvements +- **minor** (0.5.20 → 0.6.0): New features, enhancements +- **major** (0.5.20 → 1.0.0): Breaking changes, major releases +- **custom**: Specify exact version (e.g., for release candidates) + +--- + +### 4. Main → Public Repo + +**Workflow:** `.github/workflows/publish-to-public.yml` + +**Trigger:** Automatic when GitHub release is published on main + +**Process:** +1. Triggered automatically by GitHub release event +2. Full CRAN validation (with vignettes) +3. Requires `cran-production` environment approval +4. Cleans repository (removes private files, AI attributions) +5. Force-pushes to public repository: `ericscheier/emburden` +6. Pushes version tag to public repo +7. Creates GitHub release on public repo + +**Approval:** Repository maintainer via cran-production environment + +**Automated cleaning:** +- Removes `.private/` directory +- Removes `*.log` files +- Removes `*_cache/` and `*_files/` directories +- Removes private workflow files +- Strips AI attributions from commit messages + +--- + +### 5. Public Repo → CRAN + +**Workflow:** `.github/workflows/cran-release.yml` (on public repo) + +**Trigger:** Automatic when version tag pushed to public repo + +**Process:** +1. Builds source package with vignettes +2. Runs full CRAN checks +3. Provides instructions for manual CRAN submission +4. Uploads tarball to GitHub release + +**Note:** CRAN submission requires manual interaction and cannot be fully automated + +--- + +## Developer Workflows + +### Starting New Work + +```bash +# Ensure you're on dev and up to date +git checkout dev +git pull scheier dev + +# Create feature branch +git checkout -b feature/my-new-feature + +# Make your changes... +# Commit frequently +git add . +git commit -m "feat: Add new feature" + +# Push to remote +git push scheier feature/my-new-feature +``` + +### Creating a Pull Request + +```bash +# Via GitHub CLI +gh pr create --base dev --head feature/my-new-feature \ + --title "Add new feature" \ + --body "Description of changes" + +# Or via GitHub web UI +# Navigate to repository and click "New Pull Request" +``` + +### PR Guidelines + +- **Title:** Use conventional commits format (`feat:`, `fix:`, `docs:`, etc.) +- **Description:** Explain what and why, not how +- **Tests:** Include tests for new features +- **Documentation:** Update docs and vignettes as needed +- **Changelog:** Notable changes should be mentioned (will auto-generate from commits) + +### Auto-Approval for Safe Changes + +PRs that **only** modify these files are auto-approved: +- Documentation: `*.md`, `man/`, `vignettes/` +- Tests: `tests/` +- Dev tools: `.github/`, `.dev/`, `.lintr`, etc. +- Data docs: `data-raw/` + +All other changes require manual code review. + +--- + +## Release Process + +### Preparing a Release + +**1. Complete development on dev branch** +```bash +# All features merged to dev +# All CI checks passing on dev +# Ready for staging validation +``` + +**2. Promote dev → staging** +```bash +# Via GitHub Actions UI: +# Actions → "Promote Dev to Staging" → Run workflow +# +# Then approve the staging-promotion environment when PR is created +``` + +**3. Validate on staging** +```bash +# Wait for multi-platform CI to complete +# Test the package manually if needed +# Verify all changes are ready for production +``` + +**4. Promote staging → main (production release)** +```bash +# Via GitHub Actions UI: +# Actions → "Promote Staging to Main (Production Release)" → Run workflow +# version_bump: patch | minor | major +# +# Workflow will: +# - Validate staging +# - Bump version automatically +# - Create release PR +# +# Then approve the production-release environment +``` + +**5. Automatic publishing** +```bash +# After merge to main, workflows automatically: +# - Create version tag (v*.*.*) +# - Create GitHub release +# - Publish to public repo +# - Trigger CRAN submission workflow +``` + +**6. CRAN submission** +```bash +# After publish-to-public completes: +# - Review CRAN workflow output +# - Download tarball from GitHub release +# - Submit via CRAN web form or email +``` + +### Hotfix Process + +For critical bugs in production: + +```bash +# 1. Create hotfix branch from main +git checkout main +git pull scheier main +git checkout -b hotfix/critical-bug + +# 2. Fix the bug +# ... make changes ... +git commit -m "fix: Critical bug in production" + +# 3. Create PR to staging (not dev) +gh pr create --base staging --head hotfix/critical-bug + +# 4. After merge to staging, promote to main immediately +# Use promote-to-main workflow + +# 5. Backport to dev if needed +git checkout dev +git cherry-pick +git push scheier dev +``` + +--- + +## Manual Override + +### Emergency Procedures + +**When to use:** Critical production issues only + +**Process:** +1. Identify the issue and severity +2. Get approval from repository owner +3. Create hotfix branch from affected branch +4. Apply minimal fix +5. Follow expedited promotion process +6. Document in post-mortem + +### Bypassing Workflows + +**NOT RECOMMENDED** but possible in emergencies: + +```bash +# Emergency push to main (requires admin) +# This bypasses all protections - use with extreme caution +git push --force scheier main + +# Better: Use workflow_dispatch with skip_checks parameter +# Actions → promote-to-main → Run workflow → skip_checks: true +``` + +--- + +## Troubleshooting + +### CI Checks Failing + +**Problem:** R CMD check fails on PR + +**Solutions:** +1. Check error message in GitHub Actions log +2. Run `R CMD check` locally: `devtools::check()` +3. Fix issues and push new commit +4. CI will re-run automatically + +### Merge Conflicts + +**Problem:** PR has merge conflicts + +**Solutions:** +```bash +# Update your branch with latest dev +git checkout feature/my-feature +git fetch scheier dev +git merge scheier/dev + +# Resolve conflicts manually +# ... edit files ... + +git add . +git commit -m "chore: Resolve merge conflicts" +git push scheier feature/my-feature +``` + +### Promotion Workflow Stuck + +**Problem:** Promotion workflow waiting for approval + +**Solutions:** +1. Check GitHub Actions → Environments +2. Review and approve the pending deployment +3. Workflow will resume automatically + +### Version Bump Issues + +**Problem:** Automated version bump created wrong version + +**Solutions:** +1. Do NOT merge the release PR yet +2. Close the PR +3. Manually edit DESCRIPTION, CITATION, .zenodo.json +4. Commit to staging: `git commit -am "fix: Correct version number"` +5. Re-run promote-to-main workflow with custom version + +--- + +## Configuration Reference + +### GitHub Environments + +| Environment | Purpose | Approvers | Required For | +|------------|---------|-----------|--------------| +| `staging-promotion` | Dev → Staging | Maintainer | Promotion to staging | +| `production-release` | Staging → Main | Maintainer | Production releases | +| `cran-production` | Final CRAN gate | Maintainer | CRAN submission | + +### Branch Protection Rules + +| Branch | PR Required | Approvals | Status Checks | Linear History | +|--------|-------------|-----------|---------------|----------------| +| main | ✅ | 1 | All (3 platforms) | ✅ | +| staging | ✅ | 1 | All (3 platforms) | ✅ | +| dev | ✅ | 0* | Ubuntu only | ✅ | + +*Auto-approved for docs/tests, manual review for code + +### Status Checks + +| Check | Branches | Platforms | Required | +|-------|----------|-----------|----------| +| R-CMD-check | dev, staging, main | Ubuntu (dev), All (staging/main) | ✅ | +| test-coverage | dev, staging, main | Ubuntu only | ✅ | +| validate-workflows | dev, staging, main | Ubuntu only | ✅ | + +--- + +## Additional Resources + +- [CODEOWNERS](.github/CODEOWNERS) - Code review requirements +- [BRANCH_PROTECTION_SETUP.md](.github/BRANCH_PROTECTION_SETUP.md) - GitHub configuration +- [DEPRECATED_WORKFLOWS.md](.github/workflows/DEPRECATED_WORKFLOWS.md) - Migration notes +- [CONTRIBUTING.md](../CONTRIBUTING.md) - Contribution guidelines + +--- + +## Questions or Issues? + +- **Workflow failures:** Check GitHub Actions logs and adjust code +- **Approval questions:** Contact repository maintainer +- **Feature requests:** Open an issue on GitHub +- **Emergency support:** Email eric@scheier.org diff --git a/.github/BRANCH_PROTECTION_SETUP.md b/.github/BRANCH_PROTECTION_SETUP.md new file mode 100644 index 0000000..ed370ad --- /dev/null +++ b/.github/BRANCH_PROTECTION_SETUP.md @@ -0,0 +1,247 @@ +# Branch Protection & Environment Setup + +This document describes the manual GitHub configuration required for the branching strategy. + +## Branch Protection Rules + +These must be configured via GitHub UI: `Settings` → `Branches` → `Branch protection rules` + +### 1. `dev` Branch Protection + +**Pattern:** `dev` + +**Settings:** +- ✅ Require a pull request before merging + - ✅ Require approvals: **0** (auto-approve safe changes via workflow) + - ✅ Dismiss stale pull request approvals when new commits are pushed + - ❌ Require review from Code Owners (optional for feature → dev) +- ✅ Require status checks to pass before merging + - ✅ Require branches to be up to date before merging + - **Required status checks:** + - `ubuntu-latest (release)` (from R-CMD-check) + - `test-coverage` (from test-coverage) +- ✅ Require conversation resolution before merging +- ✅ Require linear history (no merge commits) +- ✅ Do not allow bypassing the above settings +- ❌ Allow force pushes (disabled) +- ❌ Allow deletions (disabled) + +**Rationale:** Dev is the integration branch. All feature work merges here first. We want basic CI checks but allow auto-approval for safe changes (docs, tests). + +--- + +### 2. `staging` Branch Protection + +**Pattern:** `staging` + +**Settings:** +- ✅ Require a pull request before merging + - ✅ Require approvals: **1** + - ✅ Dismiss stale pull request approvals when new commits are pushed + - ✅ Require review from Code Owners + - ✅ Restrict who can dismiss pull request reviews (Admins only) +- ✅ Require status checks to pass before merging + - ✅ Require branches to be up to date before merging + - **Required status checks:** + - `ubuntu-latest (release)` (from R-CMD-check) + - `windows-latest (release)` (from R-CMD-check) + - `macos-latest (release)` (from R-CMD-check) + - `test-coverage` (from test-coverage) + - `validate-workflows` (from validate-workflows) +- ✅ Require deployments to succeed before merging + - **Required environment:** `staging-promotion` +- ✅ Require conversation resolution before merging +- ✅ Require linear history +- ✅ Do not allow bypassing the above settings +- ❌ Allow force pushes (disabled) +- ❌ Allow deletions (disabled) + +**Rationale:** Staging is pre-production. Only promoted from dev after full multi-platform validation and manual approval. This is the final testing ground before main. + +--- + +### 3. `main` Branch Protection + +**Pattern:** `main` + +**Settings:** +- ✅ Require a pull request before merging + - ✅ Require approvals: **1** + - ✅ Dismiss stale pull request approvals when new commits are pushed + - ✅ Require review from Code Owners + - ✅ Restrict who can dismiss pull request reviews (Admins only) +- ✅ Require status checks to pass before merging + - ✅ Require branches to be up to date before merging + - **Required status checks:** + - `ubuntu-latest (release)` (from R-CMD-check) + - `windows-latest (release)` (from R-CMD-check) + - `macos-latest (release)` (from R-CMD-check) + - `test-coverage` (from test-coverage) + - `validate-workflows` (from validate-workflows) +- ✅ Require deployments to succeed before merging + - **Required environment:** `production-release` +- ✅ Require conversation resolution before merging +- ✅ Require signed commits (optional but recommended) +- ✅ Require linear history +- ✅ Do not allow bypassing the above settings +- ❌ Allow force pushes (disabled) +- ❌ Allow deletions (disabled) +- ✅ Lock branch (optional: prevent all pushes) + +**Rationale:** Main is production. Only accepts promotions from staging. Triggers version bumps, CRAN submission, and public repo publishing. Maximum protection. + +**Note:** Main already has protection configured. Verify it matches the above settings. + +--- + +## GitHub Environments + +These must be configured via GitHub UI: `Settings` → `Environments` + +### 1. `dev-deployment` + +**Purpose:** Auto-deploy to development environment (if applicable) + +**Protection rules:** +- ❌ Required reviewers: None (auto-deploy) +- ✅ Deployment branches: Only `dev` branch + +**Environment secrets:** (if needed) +- None currently required + +--- + +### 2. `staging-promotion` + +**Purpose:** Manual approval gate for dev → staging promotions + +**Protection rules:** +- ✅ Required reviewers: **You** (repository owner/maintainer) +- ⏱️ Wait timer: **0 minutes** (immediate after approval) +- ✅ Deployment branches: Only `dev` branch (via promote-to-staging workflow) + +**Environment secrets:** +- None required + +**Approval strategy:** +- Reviewer verifies all dev CI passes +- Reviewer confirms feature completion +- Reviewer checks changelogs/release notes +- Manual approval triggers promotion merge + +--- + +### 3. `production-release` + +**Purpose:** Manual approval gate for staging → main promotions (production releases) + +**Protection rules:** +- ✅ Required reviewers: **You** (repository owner/maintainer) +- ⏱️ Wait timer: **0 minutes** (immediate after approval) +- ✅ Deployment branches: Only `staging` branch (via promote-to-main workflow) + +**Environment secrets:** +- None required + +**Approval strategy:** +- Reviewer verifies all staging CI passes (including multi-platform) +- Reviewer confirms staging testing complete +- Reviewer reviews version bump strategy +- Manual approval triggers: + - Version bump (DESCRIPTION, CITATION, .zenodo.json) + - Merge to main + - Git tag creation + - Public repo publishing + - CRAN release workflow trigger + +--- + +### 4. `cran-production` (Already exists) + +**Purpose:** Final approval for CRAN submission + +**Protection rules:** +- ✅ Required reviewers: **You** +- ⏱️ Wait timer: **0 minutes** +- ✅ Deployment branches: Only `main` branch + +**Environment secrets:** +- `CRAN_EMAIL` - Email for CRAN submission + +**Note:** Keep as-is. Used by existing `cran-release.yml` workflow. + +--- + +## CODEOWNERS File + +Create `.github/CODEOWNERS` to enforce code review requirements. + +**Required for:** +- Dev → Staging promotions (require staging-promotion environment approval) +- Staging → Main promotions (require production-release environment approval) +- Critical files (DESCRIPTION, workflows, etc.) + +See `.github/CODEOWNERS` (to be created in Phase 2). + +--- + +## Workflow Triggers + +After configuration, workflows will trigger as follows: + +| Workflow | Trigger | Branches | Purpose | +|----------|---------|----------|---------| +| R-CMD-check | push, PR | dev, staging, main | Multi-platform validation | +| test-coverage | push, PR | dev, staging, main | Code coverage reporting | +| validate-workflows | push, PR | dev, staging, main | Workflow syntax validation | +| promote-to-staging | manual | dev → staging | Promote dev to staging | +| promote-to-main | manual | staging → main | Release to production | +| auto-approve-safe-changes | PR | dev | Auto-approve docs/tests | +| publish-to-public | tag push | main only | Sync to public repo | +| cran-release | tag push | main only | Submit to CRAN | + +--- + +## Setup Checklist + +Use this checklist to configure GitHub settings: + +### Branch Protection +- [ ] Configure `dev` branch protection rules +- [ ] Configure `staging` branch protection rules +- [ ] Verify `main` branch protection rules (already exists) + +### Environments +- [ ] Create `dev-deployment` environment +- [ ] Create `staging-promotion` environment (add yourself as required reviewer) +- [ ] Create `production-release` environment (add yourself as required reviewer) +- [ ] Verify `cran-production` environment exists with `CRAN_EMAIL` secret + +### Code Owners +- [ ] Create `.github/CODEOWNERS` file +- [ ] Verify CODEOWNERS enforcement is enabled in branch protection + +### Workflow Files +- [ ] Deploy `promote-to-staging.yml` +- [ ] Deploy `promote-to-main.yml` +- [ ] Deploy `auto-approve-safe-changes.yml` +- [ ] Update `R-CMD-check.yml` for multi-branch +- [ ] Update `test-coverage.yml` for multi-branch +- [ ] Update `publish-to-public.yml` to only trigger from main +- [ ] Remove `auto-merge-version-bumps.yml` +- [ ] Update `auto-tag-on-version-bump.yml` (or remove if redundant) + +### Documentation +- [ ] Create `BRANCHING_STRATEGY.md` +- [ ] Update `CONTRIBUTING.md` with new workflow +- [ ] Create `.dev/promote.sh` helper script +- [ ] Update `.dev/release-version.sh` script + +--- + +## Notes + +- **Main branch already protected:** The main branch currently has protection requiring PRs and status checks. Verify these match the requirements above. +- **Environment approvals are critical:** Without proper environment configuration, promotions will auto-merge without human review. +- **Linear history:** All branches require linear history (rebase, no merge commits) for cleaner git history. +- **Status checks:** Workflows must complete successfully before merges are allowed. Failing CI blocks all promotions. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..283110a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,53 @@ +# Code Owners for emburden +# This file defines who must review changes to specific files/directories +# Patterns follow .gitignore syntax + +# Default owner for everything in the repo +# Replace @ericscheier with the actual GitHub username(s) of the maintainer(s) +* @ericscheier + +# Critical package configuration files +# These require careful review as they affect package metadata and behavior +DESCRIPTION @ericscheier +NAMESPACE @ericscheier +inst/CITATION @ericscheier +.zenodo.json @ericscheier + +# Workflow files +# Changes to CI/CD workflows should be reviewed carefully +/.github/workflows/ @ericscheier +/.github/CODEOWNERS @ericscheier +/.github/BRANCH_PROTECTION_SETUP.md @ericscheier + +# Version management and release scripts +# Critical for maintaining versioning consistency +/.dev/release-version.sh @ericscheier +/.dev/bump-version.R @ericscheier + +# Build and dependency configuration +.Rbuildignore @ericscheier +_pkgdown.yml @ericscheier + +# Security-sensitive files +/.github/workflows/*release*.yml @ericscheier +/.github/workflows/*publish*.yml @ericscheier +/.github/workflows/*cran*.yml @ericscheier + +# Core package code +# You may want to relax this for collaborative development +# Uncomment to require review of all R source code +# /R/ @ericscheier + +# Documentation can be auto-approved (see auto-approve-safe-changes.yml) +# So we don't add owners for: +# - README.md +# - NEWS.md +# - vignettes/ +# - man/ +# - tests/ + +# Data and analysis scripts don't need strict ownership +# These are often exploratory and not part of the package +# - analysis/ +# - research/ +# - data-raw/ diff --git a/.github/RELEASE_WORKFLOW.md b/.github/RELEASE_WORKFLOW.md index a8be792..920ee0d 100644 --- a/.github/RELEASE_WORKFLOW.md +++ b/.github/RELEASE_WORKFLOW.md @@ -433,7 +433,7 @@ If you encounter issues with the release workflow: 1. Check this documentation first 2. Review GitHub Actions logs for error details 3. Open an issue in the repository -4. Contact the package maintainer: eric.scheier@gmail.com +4. Contact the package maintainer: eric@scheier.org --- diff --git a/.github/workflows/.disabled/auto-approve-safe-changes.yml b/.github/workflows/.disabled/auto-approve-safe-changes.yml new file mode 100644 index 0000000..6dd14fd --- /dev/null +++ b/.github/workflows/.disabled/auto-approve-safe-changes.yml @@ -0,0 +1,285 @@ +name: Auto-Approve Safe Changes + +# Automatically approve PRs that only modify documentation, tests, or other safe files +# Still requires CI checks to pass before merge + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - dev # Only auto-approve for dev branch + +permissions: + pull-requests: write + contents: read + +jobs: + check-safe-files: + name: "Check if PR contains only safe changes" + runs-on: ubuntu-latest + if: github.repository == 'ScheierVentures/emburden' + + outputs: + is_safe: ${{ steps.check_files.outputs.is_safe }} + unsafe_files: ${{ steps.check_files.outputs.unsafe_files }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed_files + run: | + # Get base branch + BASE_REF="${{ github.event.pull_request.base.ref }}" + HEAD_REF="${{ github.event.pull_request.head.ref }}" + + echo "Comparing $BASE_REF...$HEAD_REF" + + # Fetch base and head + git fetch origin "$BASE_REF:$BASE_REF" || true + git fetch origin "$HEAD_REF:$HEAD_REF" || true + + # Get list of changed files + CHANGED_FILES=$(git diff --name-only "origin/$BASE_REF...origin/$HEAD_REF" | sort | uniq) + + echo "Changed files:" + echo "$CHANGED_FILES" + echo "" + + # Save to file for next step + echo "$CHANGED_FILES" > /tmp/changed_files.txt + + - name: Check if files are safe + id: check_files + run: | + # Define safe file patterns (grep -E regex) + # These are files that can be auto-approved without human review + SAFE_PATTERNS=( + # Documentation + '^README\.md$' + '^NEWS\.md$' + '^CHANGELOG\.md$' + '^CONTRIBUTING\.md$' + '^CODE_OF_CONDUCT\.md$' + '^LICENSE\.md$' + '^.*\.md$' # All markdown files in general + '^docs/' + '^\.github/.*\.md$' + + # Tests + '^tests/' + '^testthat\.R$' + + # Generated documentation + '^man/' + '^vignettes/' + + # Comments and roxygen + '^R/.*#.*$' # Comment-only changes (detected by git diff later) + + # Development tools + '^\.github/workflows/' + '^\.dev/' + '^\.lintr$' + '^\.Rbuildignore$' + '^\.gitignore$' + + # Configuration (some safe configs) + '^codecov\.yml$' + '^_pkgdown\.yml$' + + # Data documentation + '^data-raw/' + + # Presentation/analysis (not in package) + '^analysis/' + '^research/' + '^deprecated/' + + # R Markdown / Quarto + '^.*\.Rmd$' + '^.*\.qmd$' + + # Other documentation formats + '^.*\.bib$' + '^.*\.csl$' + ) + + # Build grep pattern + PATTERN=$(printf "|%s" "${SAFE_PATTERNS[@]}") + PATTERN="${PATTERN:1}" # Remove leading | + + echo "Safe file pattern: $PATTERN" + echo "" + + # Check each file + IS_SAFE=true + UNSAFE_FILES="" + + while IFS= read -r file; do + if [[ -z "$file" ]]; then + continue + fi + + # Check if file matches safe patterns + if echo "$file" | grep -E "$PATTERN" >/dev/null; then + echo "✅ SAFE: $file" + else + echo "❌ UNSAFE: $file" + IS_SAFE=false + UNSAFE_FILES="${UNSAFE_FILES}${file}\n" + fi + done < /tmp/changed_files.txt + + echo "" + echo "====================" + if [[ "$IS_SAFE" == "true" ]]; then + echo "✅ All files are safe for auto-approval" + echo "is_safe=true" >> $GITHUB_OUTPUT + else + echo "❌ Some files require manual review:" + echo -e "$UNSAFE_FILES" + echo "is_safe=false" >> $GITHUB_OUTPUT + echo "unsafe_files<> $GITHUB_OUTPUT + echo -e "$UNSAFE_FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + fi + echo "====================" + + auto-approve: + name: "Auto-Approve Safe Changes" + needs: [check-safe-files] + runs-on: ubuntu-latest + if: | + needs.check-safe-files.outputs.is_safe == 'true' && + github.event.pull_request.user.login != 'dependabot[bot]' && + github.event.pull_request.draft == false + + steps: + - name: Approve PR + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + + console.log(`Auto-approving PR #${pr_number}...`); + + try { + await github.rest.pulls.createReview({ + owner: repo.owner, + repo: repo.repo, + pull_request_number: pr_number, + event: 'APPROVE', + body: `## ✅ Auto-Approved: Safe Changes Only + +This PR has been automatically approved because it only modifies safe files: +- Documentation (README, NEWS, vignettes, etc.) +- Tests +- Development tools +- Configuration files + +> **Note:** CI checks must still pass before this PR can be merged. + +--- + +### Safe file categories +- 📝 Documentation: \`*.md\`, \`man/\`, \`vignettes/\` +- 🧪 Tests: \`tests/\`, \`testthat.R\` +- 🔧 Dev tools: \`.github/\`, \`.dev/\` +- ⚙️ Config: \`.Rbuildignore\`, \`_pkgdown.yml\`, etc. + +### Still required +- ✅ All CI checks must pass (R CMD check, test coverage, etc.) +- ✅ Branch protection rules still apply +- ✅ Manual approval can override if needed + +If you believe this auto-approval was incorrect, please leave a review.` + }); + + console.log(`✅ PR #${pr_number} auto-approved successfully`); + } catch (error) { + console.error(`Failed to approve PR #${pr_number}:`, error.message); + throw error; + } + + - name: Add auto-merge label + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + + await github.rest.issues.addLabels({ + owner: repo.owner, + repo: repo.repo, + issue_number: pr_number, + labels: ['auto-approved', 'safe-changes'] + }); + + notify-unsafe: + name: "Notify: Manual Review Required" + needs: [check-safe-files] + runs-on: ubuntu-latest + if: needs.check-safe-files.outputs.is_safe == 'false' + + steps: + - name: Comment on PR + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + const unsafe_files = `${{ needs.check-safe-files.outputs.unsafe_files }}`; + + const comment_body = `## 🔍 Manual Review Required + +This PR modifies files that require manual code review and cannot be auto-approved. + +**Files requiring review:** +\`\`\` +${unsafe_files} +\`\`\` + +**What this means:** +- A code review approval is required before merge +- All CI checks must still pass +- Reviewers should examine code quality, logic, and potential side effects + +**Safe files** (would be auto-approved): +- 📝 Documentation: \`*.md\`, \`man/\`, \`vignettes/\` +- 🧪 Tests: \`tests/\` +- 🔧 Dev tools: \`.github/\`, \`.dev/\` +- ⚙️ Config: \`.Rbuildignore\`, \`_pkgdown.yml\`, etc. + +--- + +**For reviewers:** Please review the changes and approve if they meet quality standards.`; + + await github.rest.issues.createComment({ + owner: repo.owner, + repo: repo.repo, + issue_number: pr_number, + body: comment_body + }); + + - name: Add review required label + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + + await github.rest.issues.addLabels({ + owner: repo.owner, + repo: repo.repo, + issue_number: pr_number, + labels: ['review-required'] + }); diff --git a/.github/workflows/.disabled/promote-to-main.yml b/.github/workflows/.disabled/promote-to-main.yml new file mode 100644 index 0000000..3594b16 --- /dev/null +++ b/.github/workflows/.disabled/promote-to-main.yml @@ -0,0 +1,580 @@ +name: Promote Staging to Main (Production Release) + +# Production release workflow: staging → main +# Validates staging, bumps version, creates PR, requires manual approval +# After merge, triggers publish-to-public and cran-release workflows + +on: + workflow_dispatch: # Manual trigger only + inputs: + version_bump: + description: 'Version bump type' + required: true + default: 'patch' + type: choice + options: + - 'patch' + - 'minor' + - 'major' + - 'custom' + custom_version: + description: 'Custom version (only if version_bump=custom)' + required: false + type: string + skip_checks: + description: 'Skip CI checks (emergency only)' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + # Validate staging branch before promotion + validate-staging: + name: "Validate Staging Branch" + runs-on: ubuntu-latest + if: github.repository == 'ScheierVentures/emburden' && inputs.skip_checks != 'true' + + outputs: + staging_sha: ${{ steps.get_sha.outputs.staging_sha }} + staging_short_sha: ${{ steps.get_sha.outputs.staging_short_sha }} + current_version: ${{ steps.get_version.outputs.current_version }} + new_version: ${{ steps.calculate_version.outputs.new_version }} + + steps: + - name: Checkout staging branch + uses: actions/checkout@v4 + with: + ref: staging + fetch-depth: 0 + + - name: Get staging branch SHA + id: get_sha + run: | + STAGING_SHA=$(git rev-parse HEAD) + STAGING_SHORT_SHA=$(git rev-parse --short HEAD) + echo "staging_sha=$STAGING_SHA" >> $GITHUB_OUTPUT + echo "staging_short_sha=$STAGING_SHORT_SHA" >> $GITHUB_OUTPUT + echo "Staging branch SHA: $STAGING_SHA" + + - name: Get current version + id: get_version + run: | + CURRENT_VERSION=$(grep "^Version:" DESCRIPTION | sed 's/Version: //') + echo "current_version=$CURRENT_VERSION" >> $GITHUB_OUTPUT + echo "Current version: $CURRENT_VERSION" + + - name: Calculate new version + id: calculate_version + run: | + CURRENT_VERSION="${{ steps.get_version.outputs.current_version }}" + BUMP_TYPE="${{ inputs.version_bump }}" + CUSTOM_VERSION="${{ inputs.custom_version }}" + + if [[ "$BUMP_TYPE" == "custom" ]]; then + if [[ -z "$CUSTOM_VERSION" ]]; then + echo "❌ Error: custom version specified but no custom_version provided" + exit 1 + fi + NEW_VERSION="$CUSTOM_VERSION" + else + # Parse current version + if [[ "$CURRENT_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(\..*)?$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" + + case "$BUMP_TYPE" in + major) + NEW_VERSION="$((MAJOR + 1)).0.0" + ;; + minor) + NEW_VERSION="${MAJOR}.$((MINOR + 1)).0" + ;; + patch) + NEW_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))" + ;; + *) + echo "❌ Error: Invalid bump type: $BUMP_TYPE" + exit 1 + ;; + esac + else + echo "❌ Error: Could not parse version: $CURRENT_VERSION" + exit 1 + fi + fi + + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "📦 Version bump: $CURRENT_VERSION → $NEW_VERSION (type: $BUMP_TYPE)" + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Setup R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - name: Setup Pandoc + uses: r-lib/actions/setup-pandoc@v2 + + - name: Run R CMD check on staging + run: | + echo "🔍 Running R CMD check on staging branch..." + Rscript -e "rcmdcheck::rcmdcheck( + args = c('--no-manual', '--as-cran'), + error_on = 'warning', + check_dir = 'check' + )" + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: staging-check-results + path: check/ + + # Multi-platform validation (runs in parallel with ubuntu check above) + validate-multiplatform: + name: "Multi-Platform Validation" + runs-on: ${{ matrix.os }} + if: github.repository == 'ScheierVentures/emburden' && inputs.skip_checks != 'true' + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + + steps: + - name: Checkout staging branch + uses: actions/checkout@v4 + with: + ref: staging + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Setup R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - name: Run R CMD check + run: | + Rscript -e "rcmdcheck::rcmdcheck( + args = c('--no-manual'), + error_on = 'warning', + check_dir = 'check' + )" + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-check-results + path: check/ + + # Create promotion PR with version bump + create-release-pr: + name: "Create Production Release PR" + needs: [validate-staging, validate-multiplatform] + if: | + always() && + (needs.validate-staging.result == 'success' || inputs.skip_checks == 'true') && + (needs.validate-multiplatform.result == 'success' || inputs.skip_checks == 'true') + runs-on: ubuntu-latest + + outputs: + pr_number: ${{ steps.create_pr.outputs.pr_number }} + + steps: + - name: Checkout main branch + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Create release branch + id: create_branch + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") + NEW_VERSION="${{ needs.validate-staging.outputs.new_version || inputs.custom_version }}" + BRANCH_NAME="release/v${NEW_VERSION}" + echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + + # Create release branch from main + git checkout -b "$BRANCH_NAME" + + # Merge staging into release branch + git fetch origin staging:staging + git merge --no-ff staging -m "chore: Promote staging to main - Release v${NEW_VERSION}" + + echo "Created release branch: $BRANCH_NAME" + + - name: Bump version + id: bump_version + run: | + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + + echo "🔧 Bumping version to $NEW_VERSION using release-version.sh..." + + # Run version bump script in auto mode (no interactive prompts) + # This updates DESCRIPTION, CITATION, .zenodo.json, and NEWS.md + bash .dev/release-version.sh "$NEW_VERSION" --auto + + echo "✅ Version bumped successfully" + + - name: Amend merge commit with version bump + run: | + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + + # Stage version files + git add DESCRIPTION inst/CITATION .zenodo.json NEWS.md + + # Amend the merge commit to include version bump + git commit --amend --no-edit + + echo "✅ Merge commit amended with version bump" + + - name: Push release branch + run: | + BRANCH_NAME="${{ steps.create_branch.outputs.branch_name }}" + git push origin "$BRANCH_NAME" + echo "✅ Pushed release branch: $BRANCH_NAME" + + - name: Create pull request + id: create_pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BRANCH_NAME="${{ steps.create_branch.outputs.branch_name }}" + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + CURRENT_VERSION="${{ needs.validate-staging.outputs.current_version || 'N/A' }}" + STAGING_SHA="${{ needs.validate-staging.outputs.staging_sha || 'N/A' }}" + STAGING_SHORT_SHA="${{ needs.validate-staging.outputs.staging_short_sha || 'N/A' }}" + TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + + # Get commits between main and staging + COMMITS=$(git log main..staging --oneline --no-merges | head -20) + COMMIT_COUNT=$(git rev-list --count main..staging) + + # Extract NEWS.md entry for this version + NEWS_CONTENT="" + if [[ -f NEWS.md ]]; then + NEWS_CONTENT=$(awk "/^# emburden $NEW_VERSION/,/^(# emburden|---)/ { + if (\$0 !~ /^(# emburden|---)/) print + }" NEWS.md | sed 's/^## /### /') + fi + + # Create PR body (without single quotes to allow variable expansion) + cat > pr_body.md << EOFBODY +## 🚀 Production Release v$NEW_VERSION + +- **Release timestamp:** $TIMESTAMP +- **Version bump:** $CURRENT_VERSION → **$NEW_VERSION** +- **Staging SHA:** \`$STAGING_SHORT_SHA\` (\`$STAGING_SHA\`) +- **Commits included:** $COMMIT_COUNT + +--- + +### 📋 Release Notes + +$NEWS_CONTENT + +--- + +### 📦 Changes in this release + +\`\`\` +$COMMITS +\`\`\` + +--- + +### ✅ Pre-release validation + +- ✅ Staging R CMD check: **PASSED** +- ✅ Multi-platform checks (ubuntu, windows, macos): **PASSED** +- ✅ All staging CI checks: **PASSED** +- ✅ Version bumped: DESCRIPTION, CITATION, .zenodo.json, NEWS.md + +--- + +### 🔐 Required approvals + +This production release requires: +1. ✅ Manual code review approval +2. ✅ Manual approval via \`production-release\` environment +3. ✅ All main branch CI checks must pass + +--- + +### 🎯 What happens after approval and merge? + +1. **Version tag created:** \`v$NEW_VERSION\` on main branch +2. **GitHub release:** Auto-created with changelog from NEWS.md +3. **Publish to public repo:** Workflow syncs main branch to \`ericscheier/emburden\` +4. **CRAN release:** Workflow triggers on public repo (requires final manual approval) + +--- + +### ⚠️ IMPORTANT + +**This is a production release.** Review all changes carefully before approving. + +Once merged to main: +- Public repository will be updated +- CRAN submission workflow will be triggered +- Version tag will be permanent + +**Do not approve unless:** +- All changes have been thoroughly tested in staging +- Documentation is complete and accurate +- Breaking changes are clearly documented +- You are ready for public release and CRAN submission +EOFBODY + + # Substitute variables + sed -i "s/\$TIMESTAMP/$TIMESTAMP/g" pr_body.md + sed -i "s/\$NEW_VERSION/$NEW_VERSION/g" pr_body.md + sed -i "s/\$CURRENT_VERSION/$CURRENT_VERSION/g" pr_body.md + sed -i "s/\$STAGING_SHA/$STAGING_SHA/g" pr_body.md + sed -i "s/\$STAGING_SHORT_SHA/$STAGING_SHORT_SHA/g" pr_body.md + sed -i "s/\$COMMIT_COUNT/$COMMIT_COUNT/g" pr_body.md + sed -i "s/\$COMMITS/$COMMITS/g" pr_body.md + sed -i "s/\$NEWS_CONTENT/$NEWS_CONTENT/g" pr_body.md + + # Create PR + PR_URL=$(gh pr create \ + --base main \ + --head "$BRANCH_NAME" \ + --title "🚀 Release v${NEW_VERSION}" \ + --body-file pr_body.md \ + --label "release" \ + --label "production") + + # Extract PR number + PR_NUMBER=$(echo "$PR_URL" | grep -o '[0-9]*$') + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT + + echo "Created production release PR: $PR_URL" + echo "PR Number: $PR_NUMBER" + + - name: Add release checklist + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ steps.create_pr.outputs.pr_number }}" + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + + gh pr comment "$PR_NUMBER" --body "$(cat <<'EOF' +### 📋 Production Release Checklist + +Before approving this release, verify: + +#### Code Quality +- [ ] All changes have been reviewed and tested in staging +- [ ] All CI checks pass (ubuntu, windows, macos) +- [ ] Code coverage is maintained or improved +- [ ] No security vulnerabilities introduced + +#### Documentation +- [ ] NEWS.md accurately describes all changes +- [ ] README.md is up to date +- [ ] Vignettes reflect current functionality +- [ ] Function documentation is complete and accurate + +#### Version Management +- [ ] Version number follows semantic versioning +- [ ] DESCRIPTION, CITATION, and .zenodo.json are synchronized +- [ ] Version bump is appropriate (patch/minor/major) + +#### Release Readiness +- [ ] Breaking changes are clearly documented (if any) +- [ ] Deprecation warnings are in place (if applicable) +- [ ] All known bugs are resolved or documented +- [ ] CRAN submission is desired at this time + +--- + +### 🎯 Approval Process + +1. **Review:** Check all changes in this PR +2. **Verify:** All CI status checks are green +3. **Approve:** GitHub PR review approval +4. **Environment:** Approve via \`production-release\` environment gate +5. **Auto-merge:** PR merges automatically after approval + +--- + +### 📦 Post-Merge Automation + +After merge, the following will happen automatically: + +1. ✅ Version tag \`v$NEW_VERSION\` created on main +2. ✅ GitHub release created with NEWS.md content +3. ✅ Publish-to-public workflow syncs to public repo +4. ⏳ CRAN release workflow runs (requires your final approval) + +--- + +**Questions or concerns?** Leave a comment before approving. +EOF + )" | sed "s/\$NEW_VERSION/$NEW_VERSION/g" + + # Wait for approval and auto-merge + auto-merge-release: + name: "Auto-Merge Production Release" + needs: [create-release-pr, validate-staging] + runs-on: ubuntu-latest + environment: production-release # Requires manual approval + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Enable auto-merge + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ needs.create-release-pr.outputs.pr_number }}" + + echo "🔄 Enabling auto-merge for production release PR #$PR_NUMBER..." + + # Enable auto-merge with merge commit strategy (preserve full history) + gh pr merge "$PR_NUMBER" --auto --merge --delete-branch + + echo "✅ Auto-merge enabled. PR will merge automatically after:" + echo " - All status checks pass" + echo " - Required reviews are approved" + echo " - production-release environment approval granted" + + - name: Wait for merge + run: | + PR_NUMBER="${{ needs.create-release-pr.outputs.pr_number }}" + + echo "⏳ Waiting for PR #$PR_NUMBER to merge..." + + # Poll PR status for up to 5 minutes + for i in {1..30}; do + PR_STATE=$(gh pr view "$PR_NUMBER" --json state --jq '.state') + + if [[ "$PR_STATE" == "MERGED" ]]; then + echo "✅ PR merged successfully!" + break + elif [[ "$PR_STATE" == "CLOSED" ]]; then + echo "❌ PR was closed without merging" + exit 1 + fi + + echo " Still waiting... (attempt $i/30)" + sleep 10 + done + + - name: Create version tag + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + NEW_VERSION="${{ needs.validate-staging.outputs.new_version }}" + TAG_NAME="v${NEW_VERSION}" + + echo "📌 Creating version tag: $TAG_NAME" + + # Fetch latest main + git fetch origin main:main + git checkout main + + # Create annotated tag + git tag -a "$TAG_NAME" -m "Release version $NEW_VERSION" + git push origin "$TAG_NAME" + + echo "✅ Created and pushed tag: $TAG_NAME" + + # Create GitHub release + gh release create "$TAG_NAME" \ + --title "emburden v${NEW_VERSION}" \ + --notes-file <(awk "/^# emburden $NEW_VERSION/,/^(# emburden|---)/ { if (\$0 !~ /^(# emburden|---)/) print }" NEWS.md) \ + --verify-tag + + echo "✅ Created GitHub release for $TAG_NAME" + + - name: Release summary + if: always() + run: | + NEW_VERSION="${{ needs.validate-staging.outputs.new_version }}" + PR_NUMBER="${{ needs.create-release-pr.outputs.pr_number }}" + + cat >> $GITHUB_STEP_SUMMARY << 'EOF' +## ✅ Production Release Complete + +**Version:** v$NEW_VERSION +**PR Number:** #$PR_NUMBER +**Status:** Merged and tagged + +--- + +### 🎯 Next Steps (Automated) + +The following workflows will now run automatically: + +1. ✅ **publish-to-public.yml** - Syncing to public repo (ericscheier/emburden) +2. ⏳ **cran-release.yml** - CRAN submission (requires your final approval) + +--- + +### 📊 Release Pipeline Status + +\`\`\` +Feature → Dev → Staging → [Main] → Public Repo → CRAN + ^^^^^^^^ + YOU ARE HERE + ↓ + Tag: v$NEW_VERSION + ↓ + GitHub Release + ↓ + Public Repo Sync + ↓ + CRAN Submission +\`\`\` + +--- + +**Monitor:** +- Public repo: https://github.com/ericscheier/emburden +- CRAN workflow: Approve when ready for CRAN submission +EOF + + sed -i "s/\$NEW_VERSION/$NEW_VERSION/g" $GITHUB_STEP_SUMMARY + sed -i "s/\$PR_NUMBER/$PR_NUMBER/g" $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/.disabled/promote-to-staging.yml b/.github/workflows/.disabled/promote-to-staging.yml new file mode 100644 index 0000000..fd39b0d --- /dev/null +++ b/.github/workflows/.disabled/promote-to-staging.yml @@ -0,0 +1,305 @@ +name: Promote Dev to Staging + +# Manual promotion workflow: dev → staging +# Validates dev branch and creates PR requiring manual approval + +on: + workflow_dispatch: # Manual trigger only + inputs: + skip_checks: + description: 'Skip CI checks (emergency only)' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + # Validate dev branch before promotion + validate-dev: + name: "Validate Dev Branch" + runs-on: ubuntu-latest + if: github.repository == 'ScheierVentures/emburden' && inputs.skip_checks != 'true' + + outputs: + dev_sha: ${{ steps.get_sha.outputs.dev_sha }} + dev_short_sha: ${{ steps.get_sha.outputs.dev_short_sha }} + + steps: + - name: Checkout dev branch + uses: actions/checkout@v4 + with: + ref: dev + fetch-depth: 0 + + - name: Get dev branch SHA + id: get_sha + run: | + DEV_SHA=$(git rev-parse HEAD) + DEV_SHORT_SHA=$(git rev-parse --short HEAD) + echo "dev_sha=$DEV_SHA" >> $GITHUB_OUTPUT + echo "dev_short_sha=$DEV_SHORT_SHA" >> $GITHUB_OUTPUT + echo "Dev branch SHA: $DEV_SHA" + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Setup R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - name: Setup Pandoc + uses: r-lib/actions/setup-pandoc@v2 + + - name: Run R CMD check on dev + run: | + echo "🔍 Running R CMD check on dev branch..." + Rscript -e "rcmdcheck::rcmdcheck( + args = c('--no-manual', '--as-cran'), + error_on = 'warning', + check_dir = 'check' + )" + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: dev-check-results + path: check/ + + # Create promotion PR + create-promotion-pr: + name: "Create Promotion PR" + needs: [validate-dev] + if: | + always() && + (needs.validate-dev.result == 'success' || inputs.skip_checks == 'true') + runs-on: ubuntu-latest + + outputs: + pr_number: ${{ steps.create_pr.outputs.pr_number }} + + steps: + - name: Checkout staging branch + uses: actions/checkout@v4 + with: + ref: staging + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create promotion branch + id: create_branch + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") + BRANCH_NAME="promote/dev-to-staging-${TIMESTAMP}" + echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT + + # Create promotion branch from staging + git checkout -b "$BRANCH_NAME" + + # Merge dev into promotion branch + git fetch origin dev:dev + git merge --no-ff dev -m "chore: Promote dev to staging (${TIMESTAMP})" + + # Push promotion branch + git push origin "$BRANCH_NAME" + + echo "Created promotion branch: $BRANCH_NAME" + + - name: Create pull request + id: create_pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BRANCH_NAME="${{ steps.create_branch.outputs.branch_name }}" + DEV_SHA="${{ needs.validate-dev.outputs.dev_sha || 'N/A' }}" + DEV_SHORT_SHA="${{ needs.validate-dev.outputs.dev_short_sha || 'N/A' }}" + TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + + # Get commits between staging and dev + COMMITS=$(git log staging..dev --oneline --no-merges | head -20) + COMMIT_COUNT=$(git rev-list --count staging..dev) + + # Create PR body (without single quotes to allow variable expansion) + cat > pr_body.md << EOF +## 🚀 Dev → Staging Promotion + +- **Promotion timestamp:** $TIMESTAMP +- **Dev branch SHA:** \`$DEV_SHORT_SHA\` (\`$DEV_SHA\`) +- **Commits included:** $COMMIT_COUNT + +### Changes in this promotion + +\`\`\` +$COMMITS +\`\`\` + +### Pre-promotion validation + +- ✅ Dev branch R CMD check: **PASSED** +- ✅ All dev CI checks: **PASSED** + +### Required approvals + +This promotion requires: +1. ✅ Manual approval via \`staging-promotion\` environment +2. ✅ All staging CI checks must pass +3. ✅ Code review approval + +### What happens after approval? + +1. PR will auto-merge to staging +2. Staging CI will run (multi-platform) +3. Promotion tag \`staging/YYYYMMDD-HHMMSS\` will be created +4. Staging is then ready for production release + +--- + +**⚠️ IMPORTANT:** Review all changes carefully before approving. Once merged, staging becomes the source of truth for the next production release. +EOF + + # Substitute variables + sed -i "s/\$TIMESTAMP/$TIMESTAMP/g" pr_body.md + sed -i "s/\$DEV_SHA/$DEV_SHA/g" pr_body.md + sed -i "s/\$DEV_SHORT_SHA/$DEV_SHORT_SHA/g" pr_body.md + sed -i "s/\$COMMIT_COUNT/$COMMIT_COUNT/g" pr_body.md + sed -i "s/\$COMMITS/$COMMITS/g" pr_body.md + + # Create PR + PR_URL=$(gh pr create \ + --base staging \ + --head "$BRANCH_NAME" \ + --title "🚀 Promote dev to staging ($(date -u +"%Y-%m-%d"))" \ + --body-file pr_body.md \ + --label "promotion" \ + --label "staging") + + # Extract PR number + PR_NUMBER=$(echo "$PR_URL" | grep -o '[0-9]*$') + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT + + echo "Created promotion PR: $PR_URL" + echo "PR Number: $PR_NUMBER" + + - name: Add promotion comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ steps.create_pr.outputs.pr_number }}" + + gh pr comment "$PR_NUMBER" --body "$(cat <<'EOF' +### 📋 Promotion Checklist + +Before approving this promotion, verify: + +- [ ] All changes in dev have been reviewed and tested +- [ ] No breaking changes unless documented +- [ ] All CI checks pass (ubuntu, windows, macos) +- [ ] Code coverage is maintained or improved +- [ ] Documentation is up to date +- [ ] CHANGELOG/NEWS is updated (if applicable) + +**Approval process:** +1. Review all changes in this PR +2. Verify CI status checks are green +3. Approve via GitHub PR review +4. Approve via \`staging-promotion\` environment gate +5. PR will auto-merge after approval + +**Questions or issues?** Leave a comment and mark the checklist items that need attention. +EOF + )" + + # Wait for approval and auto-merge + auto-merge: + name: "Auto-Merge After Approval" + needs: [create-promotion-pr] + runs-on: ubuntu-latest + environment: staging-promotion # Requires manual approval + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Enable auto-merge + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ needs.create-promotion-pr.outputs.pr_number }}" + + echo "🔄 Enabling auto-merge for PR #$PR_NUMBER..." + + # Enable auto-merge with squash strategy + gh pr merge "$PR_NUMBER" --auto --squash --delete-branch + + echo "✅ Auto-merge enabled. PR will merge automatically after:" + echo " - All status checks pass" + echo " - Required reviews are approved" + echo " - staging-promotion environment approval granted" + + - name: Tag promotion + if: success() + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") + TAG_NAME="staging/${TIMESTAMP}" + + echo "📌 Creating promotion tag: $TAG_NAME" + + # Wait a moment for merge to complete + sleep 10 + + # Fetch latest staging + git fetch origin staging:staging + git checkout staging + + # Create and push tag + git tag -a "$TAG_NAME" -m "Promotion from dev to staging at $TIMESTAMP" + git push origin "$TAG_NAME" + + echo "✅ Created tag: $TAG_NAME" + + - name: Promotion summary + if: always() + run: | + PR_NUMBER="${{ needs.create-promotion-pr.outputs.pr_number }}" + + cat >> $GITHUB_STEP_SUMMARY << 'EOF' +## ✅ Dev → Staging Promotion Complete + +**PR Number:** #$PR_NUMBER +**Status:** Merged and tagged +**Next steps:** +- Staging CI is running +- Review staging deployment +- When ready, promote staging → main for production release + +--- + +**Branching flow:** +``` +Feature branches → dev → [staging] → main → public repo → CRAN + ^^^^^^^^ + YOU ARE HERE +``` +EOF + + sed -i "s/\$PR_NUMBER/$PR_NUMBER/g" $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/.temp-disabled/promote-to-main.yml b/.github/workflows/.temp-disabled/promote-to-main.yml new file mode 100644 index 0000000..c1b1d90 --- /dev/null +++ b/.github/workflows/.temp-disabled/promote-to-main.yml @@ -0,0 +1,570 @@ +name: Promote Staging to Main (Production Release) + +# Production release workflow: staging → main +# Validates staging, bumps version, creates PR, requires manual approval +# After merge, triggers publish-to-public and cran-release workflows + +on: + workflow_dispatch: # Manual trigger only + inputs: + version_bump: + description: 'Version bump type' + required: true + default: 'patch' + type: choice + options: + - 'patch' + - 'minor' + - 'major' + - 'custom' + custom_version: + description: 'Custom version (only if version_bump=custom)' + required: false + type: string + skip_checks: + description: 'Skip CI checks (emergency only)' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + # Validate staging branch before promotion + validate-staging: + name: "Validate Staging Branch" + runs-on: ubuntu-latest + if: github.repository == 'ScheierVentures/emburden' && inputs.skip_checks != 'true' + + outputs: + staging_sha: ${{ steps.get_sha.outputs.staging_sha }} + staging_short_sha: ${{ steps.get_sha.outputs.staging_short_sha }} + current_version: ${{ steps.get_version.outputs.current_version }} + new_version: ${{ steps.calculate_version.outputs.new_version }} + + steps: + - name: Checkout staging branch + uses: actions/checkout@v4 + with: + ref: staging + fetch-depth: 0 + + - name: Get staging branch SHA + id: get_sha + run: | + STAGING_SHA=$(git rev-parse HEAD) + STAGING_SHORT_SHA=$(git rev-parse --short HEAD) + echo "staging_sha=$STAGING_SHA" >> $GITHUB_OUTPUT + echo "staging_short_sha=$STAGING_SHORT_SHA" >> $GITHUB_OUTPUT + echo "Staging branch SHA: $STAGING_SHA" + + - name: Get current version + id: get_version + run: | + CURRENT_VERSION=$(grep "^Version:" DESCRIPTION | sed 's/Version: //') + echo "current_version=$CURRENT_VERSION" >> $GITHUB_OUTPUT + echo "Current version: $CURRENT_VERSION" + + - name: Calculate new version + id: calculate_version + run: | + CURRENT_VERSION="${{ steps.get_version.outputs.current_version }}" + BUMP_TYPE="${{ inputs.version_bump }}" + CUSTOM_VERSION="${{ inputs.custom_version }}" + + if [[ "$BUMP_TYPE" == "custom" ]]; then + if [[ -z "$CUSTOM_VERSION" ]]; then + echo "❌ Error: custom version specified but no custom_version provided" + exit 1 + fi + NEW_VERSION="$CUSTOM_VERSION" + else + # Parse current version + if [[ "$CURRENT_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(\..*)?$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" + + case "$BUMP_TYPE" in + major) + NEW_VERSION="$((MAJOR + 1)).0.0" + ;; + minor) + NEW_VERSION="${MAJOR}.$((MINOR + 1)).0" + ;; + patch) + NEW_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))" + ;; + *) + echo "❌ Error: Invalid bump type: $BUMP_TYPE" + exit 1 + ;; + esac + else + echo "❌ Error: Could not parse version: $CURRENT_VERSION" + exit 1 + fi + fi + + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "📦 Version bump: $CURRENT_VERSION → $NEW_VERSION (type: $BUMP_TYPE)" + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Setup R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - name: Setup Pandoc + uses: r-lib/actions/setup-pandoc@v2 + + - name: Run R CMD check on staging + run: | + echo "🔍 Running R CMD check on staging branch..." + Rscript -e "rcmdcheck::rcmdcheck( + args = c('--no-manual', '--as-cran'), + error_on = 'warning', + check_dir = 'check' + )" + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: staging-check-results + path: check/ + + # Multi-platform validation (runs in parallel with ubuntu check above) + validate-multiplatform: + name: "Multi-Platform Validation" + runs-on: ${{ matrix.os }} + if: github.repository == 'ScheierVentures/emburden' && inputs.skip_checks != 'true' + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + + steps: + - name: Checkout staging branch + uses: actions/checkout@v4 + with: + ref: staging + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Setup R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - name: Run R CMD check + run: | + Rscript -e "rcmdcheck::rcmdcheck( + args = c('--no-manual'), + error_on = 'warning', + check_dir = 'check' + )" + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-check-results + path: check/ + + # Create promotion PR with version bump + create-release-pr: + name: "Create Production Release PR" + needs: [validate-staging, validate-multiplatform] + if: | + always() && + (needs.validate-staging.result == 'success' || inputs.skip_checks == 'true') && + (needs.validate-multiplatform.result == 'success' || inputs.skip_checks == 'true') + runs-on: ubuntu-latest + + outputs: + pr_number: ${{ steps.create_pr.outputs.pr_number }} + + steps: + - name: Checkout main branch + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Create release branch + id: create_branch + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") + NEW_VERSION="${{ needs.validate-staging.outputs.new_version || inputs.custom_version }}" + BRANCH_NAME="release/v${NEW_VERSION}" + echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + + # Create release branch from main + git checkout -b "$BRANCH_NAME" + + # Merge staging into release branch + git fetch origin staging:staging + git merge --no-ff staging -m "chore: Promote staging to main - Release v${NEW_VERSION}" + + echo "Created release branch: $BRANCH_NAME" + + - name: Bump version + id: bump_version + run: | + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + + echo "🔧 Bumping version to $NEW_VERSION using release-version.sh..." + + # Run version bump script in auto mode (no interactive prompts) + # This updates DESCRIPTION, CITATION, .zenodo.json, and NEWS.md + bash .dev/release-version.sh "$NEW_VERSION" --auto + + echo "✅ Version bumped successfully" + + - name: Amend merge commit with version bump + run: | + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + + # Stage version files + git add DESCRIPTION inst/CITATION .zenodo.json NEWS.md + + # Amend the merge commit to include version bump + git commit --amend --no-edit + + echo "✅ Merge commit amended with version bump" + + - name: Push release branch + run: | + BRANCH_NAME="${{ steps.create_branch.outputs.branch_name }}" + git push origin "$BRANCH_NAME" + echo "✅ Pushed release branch: $BRANCH_NAME" + + - name: Create pull request + id: create_pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BRANCH_NAME="${{ steps.create_branch.outputs.branch_name }}" + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + CURRENT_VERSION="${{ needs.validate-staging.outputs.current_version || 'N/A' }}" + STAGING_SHA="${{ needs.validate-staging.outputs.staging_sha || 'N/A' }}" + STAGING_SHORT_SHA="${{ needs.validate-staging.outputs.staging_short_sha || 'N/A' }}" + TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + + # Get commits between main and staging + COMMITS=$(git log main..staging --oneline --no-merges | head -20) + COMMIT_COUNT=$(git rev-list --count main..staging) + + # Extract NEWS.md entry for this version + NEWS_CONTENT="" + if [[ -f NEWS.md ]]; then + NEWS_CONTENT=$(awk "/^# emburden $NEW_VERSION/,/^(# emburden|---)/ { + if (\$0 !~ /^(# emburden|---)/) print + }" NEWS.md | sed 's/^## /### /') + fi + + # Create PR body using echo to avoid heredoc YAML conflicts + { + echo "## 🚀 Production Release v$NEW_VERSION" + echo "" + echo "- **Release timestamp:** $TIMESTAMP" + echo "- **Version bump:** $CURRENT_VERSION → **$NEW_VERSION**" + echo "- **Staging SHA:** \`$STAGING_SHORT_SHA\` (\`$STAGING_SHA\`)" + echo "- **Commits included:** $COMMIT_COUNT" + echo "" + echo "___" + echo "" + echo "### 📋 Release Notes" + echo "" + echo "$NEWS_CONTENT" + echo "" + echo "___" + echo "" + echo "### 📦 Changes in this release" + echo "" + echo "\`\`\`" + echo "$COMMITS" + echo "\`\`\`" + echo "" + echo "___" + echo "" + echo "### ✅ Pre-release validation" + echo "" + echo "- ✅ Staging R CMD check: **PASSED**" + echo "- ✅ Multi-platform checks (ubuntu, windows, macos): **PASSED**" + echo "- ✅ All staging CI checks: **PASSED**" + echo "- ✅ Version bumped: DESCRIPTION, CITATION, .zenodo.json, NEWS.md" + echo "" + echo "___" + echo "" + echo "### 🔐 Required approvals" + echo "" + echo "This production release requires:" + echo "1. ✅ Manual code review approval" + echo "2. ✅ Manual approval via \`production-release\` environment" + echo "3. ✅ All main branch CI checks must pass" + echo "" + echo "___" + echo "" + echo "### 🎯 What happens after approval and merge?" + echo "" + echo "1. **Version tag created:** \`v$NEW_VERSION\` on main branch" + echo "2. **GitHub release:** Auto-created with changelog from NEWS.md" + echo "3. **Publish to public repo:** Workflow syncs main branch to \`ericscheier/emburden\`" + echo "4. **CRAN release:** Workflow triggers on public repo (requires final manual approval)" + echo "" + echo "___" + echo "" + echo "### ⚠️ IMPORTANT" + echo "" + echo "**This is a production release.** Review all changes carefully before approving." + echo "" + echo "Once merged to main:" + echo "- Public repository will be updated" + echo "- CRAN submission workflow will be triggered" + echo "- Version tag will be permanent" + echo "" + echo "**Do not approve unless:**" + echo "- All changes have been thoroughly tested in staging" + echo "- Documentation is complete and accurate" + echo "- Breaking changes are clearly documented" + echo "- You are ready for public release and CRAN submission" + } > pr_body.md + + # Create PR + PR_URL=$(gh pr create \ + --base main \ + --head "$BRANCH_NAME" \ + --title "🚀 Release v${NEW_VERSION}" \ + --body-file pr_body.md \ + --label "release" \ + --label "production") + + # Extract PR number + PR_NUMBER=$(echo "$PR_URL" | grep -o '[0-9]*$') + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT + + echo "Created production release PR: $PR_URL" + echo "PR Number: $PR_NUMBER" + + - name: Add release checklist + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ steps.create_pr.outputs.pr_number }}" + NEW_VERSION="${{ steps.create_branch.outputs.new_version }}" + + gh pr comment "$PR_NUMBER" --body "$(cat <<'EOF' +### 📋 Production Release Checklist + +Before approving this release, verify: + +#### Code Quality +- [ ] All changes have been reviewed and tested in staging +- [ ] All CI checks pass (ubuntu, windows, macos) +- [ ] Code coverage is maintained or improved +- [ ] No security vulnerabilities introduced + +#### Documentation +- [ ] NEWS.md accurately describes all changes +- [ ] README.md is up to date +- [ ] Vignettes reflect current functionality +- [ ] Function documentation is complete and accurate + +#### Version Management +- [ ] Version number follows semantic versioning +- [ ] DESCRIPTION, CITATION, and .zenodo.json are synchronized +- [ ] Version bump is appropriate (patch/minor/major) + +#### Release Readiness +- [ ] Breaking changes are clearly documented (if any) +- [ ] Deprecation warnings are in place (if applicable) +- [ ] All known bugs are resolved or documented +- [ ] CRAN submission is desired at this time + +--- + +### 🎯 Approval Process + +1. **Review:** Check all changes in this PR +2. **Verify:** All CI status checks are green +3. **Approve:** GitHub PR review approval +4. **Environment:** Approve via \`production-release\` environment gate +5. **Auto-merge:** PR merges automatically after approval + +--- + +### 📦 Post-Merge Automation + +After merge, the following will happen automatically: + +1. ✅ Version tag \`v$NEW_VERSION\` created on main +2. ✅ GitHub release created with NEWS.md content +3. ✅ Publish-to-public workflow syncs to public repo +4. ⏳ CRAN release workflow runs (requires your final approval) + +--- + +**Questions or concerns?** Leave a comment before approving. +EOF + )" | sed "s/\$NEW_VERSION/$NEW_VERSION/g" + + # Wait for approval and auto-merge + auto-merge-release: + name: "Auto-Merge Production Release" + needs: [create-release-pr, validate-staging] + runs-on: ubuntu-latest + environment: production-release # Requires manual approval + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Enable auto-merge + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ needs.create-release-pr.outputs.pr_number }}" + + echo "🔄 Enabling auto-merge for production release PR #$PR_NUMBER..." + + # Enable auto-merge with merge commit strategy (preserve full history) + gh pr merge "$PR_NUMBER" --auto --merge --delete-branch + + echo "✅ Auto-merge enabled. PR will merge automatically after:" + echo " - All status checks pass" + echo " - Required reviews are approved" + echo " - production-release environment approval granted" + + - name: Wait for merge + run: | + PR_NUMBER="${{ needs.create-release-pr.outputs.pr_number }}" + + echo "⏳ Waiting for PR #$PR_NUMBER to merge..." + + # Poll PR status for up to 5 minutes + for i in {1..30}; do + PR_STATE=$(gh pr view "$PR_NUMBER" --json state --jq '.state') + + if [[ "$PR_STATE" == "MERGED" ]]; then + echo "✅ PR merged successfully!" + break + elif [[ "$PR_STATE" == "CLOSED" ]]; then + echo "❌ PR was closed without merging" + exit 1 + fi + + echo " Still waiting... (attempt $i/30)" + sleep 10 + done + + - name: Create version tag + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + NEW_VERSION="${{ needs.validate-staging.outputs.new_version }}" + TAG_NAME="v${NEW_VERSION}" + + echo "📌 Creating version tag: $TAG_NAME" + + # Fetch latest main + git fetch origin main:main + git checkout main + + # Create annotated tag + git tag -a "$TAG_NAME" -m "Release version $NEW_VERSION" + git push origin "$TAG_NAME" + + echo "✅ Created and pushed tag: $TAG_NAME" + + # Create GitHub release + gh release create "$TAG_NAME" \ + --title "emburden v${NEW_VERSION}" \ + --notes-file <(awk "/^# emburden $NEW_VERSION/,/^(# emburden|---)/ { if (\$0 !~ /^(# emburden|---)/) print }" NEWS.md) \ + --verify-tag + + echo "✅ Created GitHub release for $TAG_NAME" + + - name: Release summary + if: always() + run: | + NEW_VERSION="${{ needs.validate-staging.outputs.new_version }}" + PR_NUMBER="${{ needs.create-release-pr.outputs.pr_number }}" + + cat >> $GITHUB_STEP_SUMMARY << 'EOF' +## ✅ Production Release Complete + +**Version:** v$NEW_VERSION +**PR Number:** #$PR_NUMBER +**Status:** Merged and tagged + +--- + +### 🎯 Next Steps (Automated) + +The following workflows will now run automatically: + +1. ✅ **publish-to-public.yml** - Syncing to public repo (ericscheier/emburden) +2. ⏳ **cran-release.yml** - CRAN submission (requires your final approval) + +--- + +### 📊 Release Pipeline Status + +\`\`\` +Feature → Dev → Staging → [Main] → Public Repo → CRAN + ^^^^^^^^ + YOU ARE HERE + ↓ + Tag: v$NEW_VERSION + ↓ + GitHub Release + ↓ + Public Repo Sync + ↓ + CRAN Submission +\`\`\` + +--- + +**Monitor:** +- Public repo: https://github.com/ericscheier/emburden +- CRAN workflow: Approve when ready for CRAN submission +EOF + + sed -i "s/\$NEW_VERSION/$NEW_VERSION/g" $GITHUB_STEP_SUMMARY + sed -i "s/\$PR_NUMBER/$PR_NUMBER/g" $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/.temp-disabled/promote-to-staging.yml b/.github/workflows/.temp-disabled/promote-to-staging.yml new file mode 100644 index 0000000..4dbdba2 --- /dev/null +++ b/.github/workflows/.temp-disabled/promote-to-staging.yml @@ -0,0 +1,298 @@ +name: Promote Dev to Staging + +# Manual promotion workflow: dev → staging +# Validates dev branch and creates PR requiring manual approval + +on: + workflow_dispatch: # Manual trigger only + inputs: + skip_checks: + description: 'Skip CI checks (emergency only)' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + # Validate dev branch before promotion + validate-dev: + name: "Validate Dev Branch" + runs-on: ubuntu-latest + if: github.repository == 'ScheierVentures/emburden' && inputs.skip_checks != 'true' + + outputs: + dev_sha: ${{ steps.get_sha.outputs.dev_sha }} + dev_short_sha: ${{ steps.get_sha.outputs.dev_short_sha }} + + steps: + - name: Checkout dev branch + uses: actions/checkout@v4 + with: + ref: dev + fetch-depth: 0 + + - name: Get dev branch SHA + id: get_sha + run: | + DEV_SHA=$(git rev-parse HEAD) + DEV_SHORT_SHA=$(git rev-parse --short HEAD) + echo "dev_sha=$DEV_SHA" >> $GITHUB_OUTPUT + echo "dev_short_sha=$DEV_SHORT_SHA" >> $GITHUB_OUTPUT + echo "Dev branch SHA: $DEV_SHA" + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + use-public-rspm: true + + - name: Setup R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - name: Setup Pandoc + uses: r-lib/actions/setup-pandoc@v2 + + - name: Run R CMD check on dev + run: | + echo "🔍 Running R CMD check on dev branch..." + Rscript -e "rcmdcheck::rcmdcheck( + args = c('--no-manual', '--as-cran'), + error_on = 'warning', + check_dir = 'check' + )" + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: dev-check-results + path: check/ + + # Create promotion PR + create-promotion-pr: + name: "Create Promotion PR" + needs: [validate-dev] + if: | + always() && + (needs.validate-dev.result == 'success' || inputs.skip_checks == 'true') + runs-on: ubuntu-latest + + outputs: + pr_number: ${{ steps.create_pr.outputs.pr_number }} + + steps: + - name: Checkout staging branch + uses: actions/checkout@v4 + with: + ref: staging + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create promotion branch + id: create_branch + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") + BRANCH_NAME="promote/dev-to-staging-${TIMESTAMP}" + echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT + + # Create promotion branch from staging + git checkout -b "$BRANCH_NAME" + + # Merge dev into promotion branch + git fetch origin dev:dev + git merge --no-ff dev -m "chore: Promote dev to staging (${TIMESTAMP})" + + # Push promotion branch + git push origin "$BRANCH_NAME" + + echo "Created promotion branch: $BRANCH_NAME" + + - name: Create pull request + id: create_pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BRANCH_NAME="${{ steps.create_branch.outputs.branch_name }}" + DEV_SHA="${{ needs.validate-dev.outputs.dev_sha || 'N/A' }}" + DEV_SHORT_SHA="${{ needs.validate-dev.outputs.dev_short_sha || 'N/A' }}" + TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + + # Get commits between staging and dev + COMMITS=$(git log staging..dev --oneline --no-merges | head -20) + COMMIT_COUNT=$(git rev-list --count staging..dev) + + # Create PR body using echo to avoid heredoc YAML conflicts + { + echo "## 🚀 Dev → Staging Promotion" + echo "" + echo "- **Promotion timestamp:** $TIMESTAMP" + echo "- **Dev branch SHA:** \`$DEV_SHORT_SHA\` (\`$DEV_SHA\`)" + echo "- **Commits included:** $COMMIT_COUNT" + echo "" + echo "### Changes in this promotion" + echo "" + echo "\`\`\`" + echo "$COMMITS" + echo "\`\`\`" + echo "" + echo "### Pre-promotion validation" + echo "" + echo "- ✅ Dev branch R CMD check: **PASSED**" + echo "- ✅ All dev CI checks: **PASSED**" + echo "" + echo "### Required approvals" + echo "" + echo "This promotion requires:" + echo "1. ✅ Manual approval via \`staging-promotion\` environment" + echo "2. ✅ All staging CI checks must pass" + echo "3. ✅ Code review approval" + echo "" + echo "### What happens after approval?" + echo "" + echo "1. PR will auto-merge to staging" + echo "2. Staging CI will run (multi-platform)" + echo "3. Promotion tag \`staging/YYYYMMDD-HHMMSS\` will be created" + echo "4. Staging is then ready for production release" + echo "" + echo "___" + echo "" + echo "**⚠️ IMPORTANT:** Review all changes carefully before approving. Once merged, staging becomes the source of truth for the next production release." + } > pr_body.md + + # Create PR + PR_URL=$(gh pr create \ + --base staging \ + --head "$BRANCH_NAME" \ + --title "🚀 Promote dev to staging ($(date -u +"%Y-%m-%d"))" \ + --body-file pr_body.md \ + --label "promotion" \ + --label "staging") + + # Extract PR number + PR_NUMBER=$(echo "$PR_URL" | grep -o '[0-9]*$') + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT + + echo "Created promotion PR: $PR_URL" + echo "PR Number: $PR_NUMBER" + + - name: Add promotion comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ steps.create_pr.outputs.pr_number }}" + + gh pr comment "$PR_NUMBER" --body "$(cat <<'EOF' +### 📋 Promotion Checklist + +Before approving this promotion, verify: + +- [ ] All changes in dev have been reviewed and tested +- [ ] No breaking changes unless documented +- [ ] All CI checks pass (ubuntu, windows, macos) +- [ ] Code coverage is maintained or improved +- [ ] Documentation is up to date +- [ ] CHANGELOG/NEWS is updated (if applicable) + +**Approval process:** +1. Review all changes in this PR +2. Verify CI status checks are green +3. Approve via GitHub PR review +4. Approve via \`staging-promotion\` environment gate +5. PR will auto-merge after approval + +**Questions or issues?** Leave a comment and mark the checklist items that need attention. +EOF + )" + + # Wait for approval and auto-merge + auto-merge: + name: "Auto-Merge After Approval" + needs: [create-promotion-pr] + runs-on: ubuntu-latest + environment: staging-promotion # Requires manual approval + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Enable auto-merge + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ needs.create-promotion-pr.outputs.pr_number }}" + + echo "🔄 Enabling auto-merge for PR #$PR_NUMBER..." + + # Enable auto-merge with squash strategy + gh pr merge "$PR_NUMBER" --auto --squash --delete-branch + + echo "✅ Auto-merge enabled. PR will merge automatically after:" + echo " - All status checks pass" + echo " - Required reviews are approved" + echo " - staging-promotion environment approval granted" + + - name: Tag promotion + if: success() + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") + TAG_NAME="staging/${TIMESTAMP}" + + echo "📌 Creating promotion tag: $TAG_NAME" + + # Wait a moment for merge to complete + sleep 10 + + # Fetch latest staging + git fetch origin staging:staging + git checkout staging + + # Create and push tag + git tag -a "$TAG_NAME" -m "Promotion from dev to staging at $TIMESTAMP" + git push origin "$TAG_NAME" + + echo "✅ Created tag: $TAG_NAME" + + - name: Promotion summary + if: always() + run: | + PR_NUMBER="${{ needs.create-promotion-pr.outputs.pr_number }}" + + cat >> $GITHUB_STEP_SUMMARY << 'EOF' +## ✅ Dev → Staging Promotion Complete + +**PR Number:** #$PR_NUMBER +**Status:** Merged and tagged +**Next steps:** +- Staging CI is running +- Review staging deployment +- When ready, promote staging → main for production release + +--- + +**Branching flow:** +``` +Feature branches → dev → [staging] → main → public repo → CRAN + ^^^^^^^^ + YOU ARE HERE +``` +EOF + + sed -i "s/\$PR_NUMBER/$PR_NUMBER/g" $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/CRAN-RELEASE.md b/.github/workflows/CRAN-RELEASE.md new file mode 100644 index 0000000..cff9676 --- /dev/null +++ b/.github/workflows/CRAN-RELEASE.md @@ -0,0 +1,184 @@ +# CRAN Release Workflow + +This GitHub Actions workflow automates the CRAN package validation and submission process for the `emburden` package. + +## Overview + +The workflow is triggered automatically when you push a version tag (e.g., `v0.5.7`, `v1.0.0`) and performs: + +1. **CRAN Validation** - Runs `R CMD check --as-cran` on multiple platforms +2. **Manual Approval Gate** - Requires approval via GitHub environment `cran-production` +3. **Package Submission** - Prepares package for CRAN submission +4. **GitHub Release** - Creates a release with the source tarball + +## Setup + +### 1. Create GitHub Environment + +You need to create a GitHub environment called `cran-production` with manual approval: + +1. Go to your repository **Settings** → **Environments** +2. Click **New environment** +3. Name it: `cran-production` +4. Under **Deployment protection rules**, check **Required reviewers** +5. Add yourself (or maintainers) as required reviewers +6. Click **Save protection rules** + +### 2. Configure Secrets (Optional) + +If you want automatic CRAN submission (currently commented out in workflow): + +1. Go to **Settings** → **Secrets and variables** → **Actions** +2. Add a secret: `CRAN_EMAIL` with your CRAN maintainer email + +## Usage + +### Creating a CRAN Release + +The workflow triggers automatically when you bump the version and push a tag: + +```bash +# 1. Update version in DESCRIPTION file +vim DESCRIPTION # Change Version: 0.5.7 to Version: 0.5.8 + +# 2. Commit the version bump +git add DESCRIPTION +git commit -m "Bump version to 0.5.8 for CRAN" +git push + +# 3. The auto-tag workflow will create v0.5.8 tag automatically +# 4. The tag triggers this CRAN release workflow +``` + +**Or create the tag manually:** + +```bash +git tag -a v0.5.8 -m "Release v0.5.8" +git push origin v0.5.8 +``` + +### Workflow Steps + +1. **Automatic Validation** (5-10 minutes) + - Checks out code + - Builds source package (`.tar.gz`) + - Runs `R CMD check --as-cran` + - Uploads check results and tarball as artifacts + +2. **Manual Approval** (human intervention required) + - GitHub sends you a notification + - Go to **Actions** tab → Select the workflow run + - Review the CRAN check results + - Click **Review deployments** → **Approve** (or Reject) + +3. **CRAN Submission** (after approval) + - Downloads the validated tarball + - Creates CRAN submission comments + - Prepares for submission (currently manual) + - Creates GitHub Release with tarball attached + +## Manual CRAN Submission + +The workflow currently **does not** auto-submit to CRAN (safety measure). Instead: + +### Option 1: Via GitHub Release + +1. Go to the [Releases page](https://github.com/ericscheier/emburden/releases) +2. Download the `.tar.gz` file from the release +3. Upload it manually at: https://cran.r-project.org/submit.html + +### Option 2: Via R Console + +```r +# Download the tarball from the GitHub release +tarball <- "emburden_0.5.8.tar.gz" + +# Submit to CRAN +devtools::submit_cran(tarball) +``` + +### Option 3: Enable Auto-Submit (Advanced) + +Uncomment this line in `.github/workflows/cran-release.yml`: + +```yaml +# Rscript -e "devtools::submit_cran('$TARBALL', email = Sys.getenv('CRAN_EMAIL'))" +``` + +**Warning**: This will automatically submit to CRAN after approval. Make sure you're ready! + +## Monitoring + +### During Validation + +Check the workflow run in the **Actions** tab: + +```bash +# Via GitHub CLI +gh run list --workflow=cran-release.yml +gh run view +``` + +### After Submission + +1. **CRAN Incoming**: https://cran.r-project.org/incoming/ +2. **Your Package Page**: https://cran.r-project.org/web/packages/emburden/ +3. **CRAN Checks**: https://cran.r-project.org/web/checks/check_results_emburden.html + +## Troubleshooting + +### Check Failed with Errors + +1. View the workflow run in **Actions** tab +2. Download the **cran-check-results** artifact +3. Review the `00check.log` and `00install.out` files +4. Fix issues and create a new version tag + +### Approval Timed Out + +GitHub environments have a timeout (default: 30 days). If you don't approve in time: + +1. The workflow will be marked as "cancelled" or "timed out" +2. Simply push a new tag to re-trigger: `git push origin v0.5.8 --force` (if same version) +3. Or bump to a new version: `v0.5.9` + +### No Reviewers Configured + +If you see "This environment has no protection rules": + +1. Go to **Settings** → **Environments** → **cran-production** +2. Add required reviewers +3. Re-run the workflow + +## Best Practices + +1. **Test locally first**: Run `R CMD check --as-cran` on your machine before pushing +2. **Review artifacts**: Download and inspect the check results before approving +3. **Version bumps**: Always update `DESCRIPTION`, `NEWS.md`, and `inst/CITATION` +4. **CRAN policy**: Review [CRAN Repository Policy](https://cran.r-project.org/web/packages/policies.html) +5. **Submission frequency**: CRAN prefers infrequent submissions (not more than once every 1-2 months) + +## Example Timeline + +``` +10:00 - Push v0.5.8 tag +10:01 - Workflow starts validation +10:08 - Validation complete, tarball uploaded +10:08 - Waiting for approval (manual step) +10:30 - Maintainer reviews and approves +10:31 - CRAN submission prepared +10:32 - GitHub release created +10:35 - Manual upload to CRAN (or auto-submit if enabled) +``` + +## Related Workflows + +- **auto-tag-on-version-bump.yml** - Automatically creates version tags when `DESCRIPTION` changes +- **auto-release.yml** - Creates GitHub releases from version tags +- **publish-to-public.yml** - Syncs this repository to the public mirror + +## References + +- [CRAN Submission Policy](https://cran.r-project.org/web/packages/policies.html) +- [GitHub Environments](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment) +- [devtools::submit_cran()](https://devtools.r-lib.org/reference/submit_cran.html) diff --git a/.github/workflows/DEPRECATED_WORKFLOWS.md b/.github/workflows/DEPRECATED_WORKFLOWS.md new file mode 100644 index 0000000..06a0846 --- /dev/null +++ b/.github/workflows/DEPRECATED_WORKFLOWS.md @@ -0,0 +1,107 @@ +# Deprecated Workflows + +This document tracks workflows that have been deprecated and replaced by the new branching strategy. + +## Branching Strategy Migration + +**Date:** 2025-01-XX +**Migration:** Feature → Dev → Staging → Main branching model + +### Deprecated Workflows + +The following workflows have been **disabled** and replaced by the new promotion workflows: + +#### 1. `auto-tag-on-version-bump.yml` + +**Original Purpose:** +Automatically created git tags when DESCRIPTION version changed + +**Replaced By:** +`promote-to-main.yml` - Version tagging is now part of the production release process + +**Why Deprecated:** +- Version bumping is now controlled through staging → main promotions +- Tags are created explicitly during the promote-to-main workflow +- More control over when tags are created (only after full approval) + +--- + +#### 2. `auto-release.yml` + +**Original Purpose:** +Automatically created GitHub releases when version tags were pushed + +**Replaced By:** +`promote-to-main.yml` - GitHub release creation is integrated into the production release workflow + +**Why Deprecated:** +- Release creation is now part of the atomic promote-to-main workflow +- Ensures releases are only created after all validations pass +- Better integration with the promotion approval process + +--- + +#### 3. `auto-merge-version-bumps.yml` + +**Original Purpose:** +Automatically merged version bump PRs based on special tags + +**Replaced By:** +`promote-to-staging.yml` and `promote-to-main.yml` - Controlled promotion workflows with approval gates + +**Why Deprecated:** +- New branching model uses explicit promotion workflows with environment approvals +- Better control over what gets merged and when +- No special tagging mechanism needed - approvals are explicit +- Supports dev → staging and staging → main promotions + +--- + +## New Workflow Architecture + +### Promotion Workflows + +| Workflow | Purpose | Approval Required | +|----------|---------|-------------------| +| `promote-to-staging.yml` | Dev → Staging | staging-promotion environment | +| `promote-to-main.yml` | Staging → Main (production release) | production-release environment | + +### Supporting Workflows + +| Workflow | Purpose | +|----------|---------| +| `auto-approve-safe-changes.yml` | Auto-approve PRs with only docs/tests changes | +| `R-CMD-check.yml` | Multi-platform CI (dev, staging, main) | +| `test-coverage.yml` | Code coverage tracking (all branches) | +| `publish-to-public.yml` | Sync main to public repo | +| `cran-release.yml` | CRAN submission (public repo) | + +--- + +## Migration Notes + +### For Developers + +- **Version bumps:** Use `promote-to-main.yml` manual trigger, which automatically bumps version +- **Creating releases:** Releases are created automatically when staging → main promotion completes +- **Merging changes:** Use promotion workflows instead of direct PRs to main + +### Rollback Plan + +If the new branching strategy needs to be temporarily reverted: + +1. Re-enable deprecated workflows by removing `if: false` condition +2. Update trigger branches back to `main` only +3. Disable promotion workflows + +--- + +## Cleanup Schedule + +**Phase 1 (Current):** Workflows disabled via `if: false` condition +**Phase 2 (After 30 days):** Move to `.github/workflows/deprecated/` directory +**Phase 3 (After 90 days):** Remove entirely from repository + +--- + +**Questions?** See `.github/BRANCHING_STRATEGY.md` for full documentation of the new workflow. diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml deleted file mode 100644 index d39bf0b..0000000 --- a/.github/workflows/R-CMD-check.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples -# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help -on: - push: - branches: [main, master, package-transformation] - pull_request: - branches: [main, master, package-transformation] - -name: R-CMD-check - -jobs: - R-CMD-check: - runs-on: ${{ matrix.config.os }} - - name: ${{ matrix.config.os }} (${{ matrix.config.r }}) - - strategy: - fail-fast: false - matrix: - config: - - {os: macos-latest, r: 'release'} - - {os: windows-latest, r: 'release'} - - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - - {os: ubuntu-latest, r: 'release'} - - {os: ubuntu-latest, r: 'oldrel-1'} - - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - R_KEEP_PKG_SOURCE: yes - - steps: - - uses: actions/checkout@v4 - - - uses: r-lib/actions/setup-pandoc@v2 - - - uses: r-lib/actions/setup-r@v2 - with: - r-version: ${{ matrix.config.r }} - http-user-agent: ${{ matrix.config.http-user-agent }} - use-public-rspm: true - - - uses: r-lib/actions/setup-r-dependencies@v2 - with: - extra-packages: any::rcmdcheck - needs: check - - - uses: r-lib/actions/check-r-package@v2 - with: - upload-snapshots: true - build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' diff --git a/.github/workflows/R-CMD-check.yml b/.github/workflows/R-CMD-check.yml new file mode 100644 index 0000000..9e1249e --- /dev/null +++ b/.github/workflows/R-CMD-check.yml @@ -0,0 +1,85 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +# +# Multi-branch CI strategy: +# - dev: Fast feedback with ubuntu-latest only +# - staging: Multi-platform (ubuntu, windows, macos) +# - main: Full validation (all platforms + R devel + oldrel) + +name: R-CMD-check + +on: + pull_request: + branches: [main] + paths: + - 'DESCRIPTION' + - 'NAMESPACE' + - 'inst/CITATION' + - '.zenodo.json' + - 'NEWS.md' + - 'R/**' + - 'src/**' + - 'data/**' + - 'tests/**' + - 'vignettes/**' + - '.github/workflows/**' + push: + branches: [main] + workflow_dispatch: + +# Cancel in-progress runs when a new run is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: macos-latest, r: 'release'} + - {os: windows-latest, r: 'release'} + - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} + - {os: ubuntu-latest, r: 'release'} + - {os: ubuntu-latest, r: 'oldrel-1'} + + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + R_KEEP_PKG_SOURCE: yes + _R_CHECK_DONTTEST_EXAMPLES_: false + + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.r }} + http-user-agent: ${{ matrix.config.http-user-agent }} + use-public-rspm: true + + - uses: r-lib/actions/setup-tinytex@v2 + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + cache: false + + - name: Install LaTeX packages for vignettes + run: | + tinytex::tlmgr_install(c("orcidlink", "jss")) + shell: Rscript {0} + + - uses: r-lib/actions/check-r-package@v2 + with: + upload-snapshots: true + build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' + args: 'c("--as-cran")' + error-on: '"error"' diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..9a389d0 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,55 @@ +# Workflow Organization + +This document explains how release workflows are organized between the private and public repositories. + +## Private Repository (ScheierVentures/emburden) + +**Workflows:** +- `auto-release.yml` - Automatic GitHub release creation + - Triggers: Push tags matching `v*` (e.g., v0.5.9) + - Creates initial GitHub release with release notes + - Triggers `publish-to-public.yml` to sync to public repo + +- `publish-to-public.yml` - Syncs releases to public repository + - Triggered by auto-release workflow + - Pushes code and tags to ericscheier/emburden + +**Purpose:** Quick, automatic releases that publish to the public repository + +## Public Repository (ericscheier/emburden) + +**Workflows:** +- `controlled-release.yaml` - CRAN submission workflow (**lives in public repo**) + - Triggers: Automatically when release is published (synced from private repo) + - Also supports manual workflow_dispatch as fallback + - Updates existing GitHub release (created by auto-release) + - Comprehensive CRAN validation (R CMD check --as-cran, Win-builder) + - Dual approval gates (pre-release-review, public-release) + - CRAN submission guidance + +**Purpose:** CRAN releases with automatic triggering, approval gates, and comprehensive validation + +## Release Process + +### Regular Release (GitHub only) +1. Push version tag from private repo: `git tag v0.5.9 && git push origin v0.5.9` +2. auto-release.yml creates GitHub release +3. publish-to-public.yml syncs to public repo +4. Done! + +### CRAN Release (Automatic) +1. Complete regular release process above +2. CRAN workflow triggers automatically in public repo +3. Workflow runs comprehensive CRAN validation +4. Approve at pre-release-review gate (after reviewing validation results) +5. Approve at public-release gate (final approval) +6. Follow CRAN submission instructions from workflow output + +**Note:** The workflow can also be triggered manually via Actions tab if needed + +## Why This Organization? + +- **Separation of concerns**: Auto-release handles fast GitHub releases, controlled-release handles CRAN +- **No conflicts**: Workflows don't compete for same release +- **CRAN from public repo**: CRAN submissions should come from the public repository +- **Flexible**: Can do GitHub releases without CRAN, or add CRAN validation later diff --git a/.github/workflows/auto-approve-safe-changes.yml b/.github/workflows/auto-approve-safe-changes.yml new file mode 100644 index 0000000..e347a57 --- /dev/null +++ b/.github/workflows/auto-approve-safe-changes.yml @@ -0,0 +1,292 @@ +name: Auto-Approve Safe Changes + +# Automatically approve PRs that only modify documentation, tests, or other safe files +# Still requires CI checks to pass before merge + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - dev # Only auto-approve for dev branch + +permissions: + pull-requests: write + contents: read + +jobs: + check-safe-files: + name: "Check if PR contains only safe changes" + runs-on: ubuntu-latest + if: github.repository == 'ScheierVentures/emburden' + + outputs: + is_safe: ${{ steps.check_files.outputs.is_safe }} + unsafe_files: ${{ steps.check_files.outputs.unsafe_files }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed_files + run: | + # Get base branch + BASE_REF="${{ github.event.pull_request.base.ref }}" + HEAD_REF="${{ github.event.pull_request.head.ref }}" + + echo "Comparing $BASE_REF...$HEAD_REF" + + # Fetch base and head + git fetch origin "$BASE_REF:$BASE_REF" || true + git fetch origin "$HEAD_REF:$HEAD_REF" || true + + # Get list of changed files + CHANGED_FILES=$(git diff --name-only "origin/$BASE_REF...origin/$HEAD_REF" | sort | uniq) + + echo "Changed files:" + echo "$CHANGED_FILES" + echo "" + + # Save to file for next step + echo "$CHANGED_FILES" > /tmp/changed_files.txt + + - name: Check if files are safe + id: check_files + run: | + # Define safe file patterns (grep -E regex) + # These are files that can be auto-approved without human review + SAFE_PATTERNS=( + # Documentation + '^README\.md$' + '^NEWS\.md$' + '^CHANGELOG\.md$' + '^CONTRIBUTING\.md$' + '^CODE_OF_CONDUCT\.md$' + '^LICENSE\.md$' + '^.*\.md$' # All markdown files in general + '^docs/' + '^\.github/.*\.md$' + + # Tests + '^tests/' + '^testthat\.R$' + + # Generated documentation + '^man/' + '^vignettes/' + + # Comments and roxygen + '^R/.*#.*$' # Comment-only changes (detected by git diff later) + + # Development tools + '^\.github/workflows/' + '^\.dev/' + '^\.lintr$' + '^\.Rbuildignore$' + '^\.gitignore$' + + # Configuration (some safe configs) + '^codecov\.yml$' + '^_pkgdown\.yml$' + + # Data documentation + '^data-raw/' + + # Presentation/analysis (not in package) + '^analysis/' + '^research/' + '^deprecated/' + + # R Markdown / Quarto + '^.*\.Rmd$' + '^.*\.qmd$' + + # Other documentation formats + '^.*\.bib$' + '^.*\.csl$' + ) + + # Build grep pattern + PATTERN=$(printf "|%s" "${SAFE_PATTERNS[@]}") + PATTERN="${PATTERN:1}" # Remove leading | + + echo "Safe file pattern: $PATTERN" + echo "" + + # Check each file + IS_SAFE=true + UNSAFE_FILES="" + + while IFS= read -r file; do + if [[ -z "$file" ]]; then + continue + fi + + # Check if file matches safe patterns + if echo "$file" | grep -E "$PATTERN" >/dev/null; then + echo "✅ SAFE: $file" + else + echo "❌ UNSAFE: $file" + IS_SAFE=false + UNSAFE_FILES="${UNSAFE_FILES}${file}\n" + fi + done < /tmp/changed_files.txt + + echo "" + echo "====================" + if [[ "$IS_SAFE" == "true" ]]; then + echo "✅ All files are safe for auto-approval" + echo "is_safe=true" >> $GITHUB_OUTPUT + else + echo "❌ Some files require manual review:" + echo -e "$UNSAFE_FILES" + echo "is_safe=false" >> $GITHUB_OUTPUT + echo "unsafe_files<> $GITHUB_OUTPUT + echo -e "$UNSAFE_FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + fi + echo "====================" + + auto-approve: + name: "Auto-Approve Safe Changes" + needs: [check-safe-files] + runs-on: ubuntu-latest + if: | + needs.check-safe-files.outputs.is_safe == 'true' && + github.event.pull_request.user.login != 'dependabot[bot]' && + github.event.pull_request.draft == false + + steps: + - name: Approve PR + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + + console.log(`Auto-approving PR #${pr_number}...`); + + try { + const approvalBody = [ + '## ✅ Auto-Approved' + ': Safe Changes Only', + '', + 'This PR has been automatically approved because it only modifies safe files' + ':', + '- Documentation (README, NEWS, vignettes, etc.)', + '- Tests', + '- Development tools', + '- Configuration files', + '', + '**Note' + ':** CI checks must still pass before this PR can be merged.', + '', + '___', + '', + '### Safe file categories', + '- 📝 Documentation' + ': `*.md`, `man/`, `vignettes/`', + '- 🧪 Tests' + ': `tests/`, `testthat.R`', + '- 🔧 Dev tools' + ': `.github/`, `.dev/`', + '- ⚙️ Config' + ': `.Rbuildignore`, `_pkgdown.yml`, etc.', + '', + '### Still required', + '- ✅ All CI checks must pass (R CMD check, test coverage, etc.)', + '- ✅ Branch protection rules still apply', + '- ✅ Manual approval can override if needed', + '', + 'If you believe this auto-approval was incorrect, please leave a review.' + ].join('\n'); + + await github.rest.pulls.createReview({ + owner: repo.owner, + repo: repo.repo, + pull_request_number: pr_number, + event: 'APPROVE', + body: approvalBody + }); + + console.log(`✅ PR #${pr_number} auto-approved successfully`); + } catch (error) { + console.error(`Failed to approve PR #${pr_number}:`, error.message); + throw error; + } + + - name: Add auto-merge label + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + + await github.rest.issues.addLabels({ + owner: repo.owner, + repo: repo.repo, + issue_number: pr_number, + labels: ['auto-approved', 'safe-changes'] + }); + + notify-unsafe: + name: "Notify: Manual Review Required" + needs: [check-safe-files] + runs-on: ubuntu-latest + if: needs.check-safe-files.outputs.is_safe == 'false' + + steps: + - name: Comment on PR + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + const unsafe_files = `${{ needs.check-safe-files.outputs.unsafe_files }}`; + + const commentParts = [ + '## 🔍 Manual Review Required', + '', + 'This PR modifies files that require manual code review and cannot be auto-approved.', + '', + '**Files requiring review' + ':**', + '```', + unsafe_files, + '```', + '', + '**What this means' + ':**', + '- A code review approval is required before merge', + '- All CI checks must still pass', + '- Reviewers should examine code quality, logic, and potential side effects', + '', + '**Safe files** (would be auto-approved)' + ':', + '- 📝 Documentation' + ': `*.md`, `man/`, `vignettes/`', + '- 🧪 Tests' + ': `tests/`', + '- 🔧 Dev tools' + ': `.github/`, `.dev/`', + '- ⚙️ Config' + ': `.Rbuildignore`, `_pkgdown.yml`, etc.', + '', + '___', + '', + '**For reviewers' + ':** Please review the changes and approve if they meet quality standards.' + ]; + const comment_body = commentParts.join('\n'); + + await github.rest.issues.createComment({ + owner: repo.owner, + repo: repo.repo, + issue_number: pr_number, + body: comment_body + }); + + - name: Add review required label + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr_number = context.payload.pull_request.number; + const repo = context.repo; + + await github.rest.issues.addLabels({ + owner: repo.owner, + repo: repo.repo, + issue_number: pr_number, + labels: ['review-required'] + }); diff --git a/.github/workflows/auto-merge-version-bumps.yml b/.github/workflows/auto-merge-version-bumps.yml new file mode 100644 index 0000000..fe6ac04 --- /dev/null +++ b/.github/workflows/auto-merge-version-bumps.yml @@ -0,0 +1,262 @@ +name: Auto-Merge Version Bump PRs + +# ⚠️ DEPRECATED: This workflow has been replaced by promote-to-staging.yml and promote-to-main.yml +# See .github/workflows/DEPRECATED_WORKFLOWS.md for details +# +# ORIGINAL PURPOSE: +# Automatically merged and squashed PRs when triggered by auto-merge tag +# The release-version.sh script created an auto-merge/vX.X.X tag +# which triggered this workflow to: +# 1. Verify all PR checks were passing +# 2. Auto-merge the PR with squash +# 3. Delete the auto-merge tag + +on: + push: + tags: + - 'auto-merge/**' # Triggers when release-version.sh creates auto-merge tag + +permissions: + contents: write + pull-requests: write + checks: read + +jobs: + auto-merge: + name: Auto-merge version bump PR + runs-on: ubuntu-latest + if: false # DEPRECATED - Disabled in favor of promote-to-staging.yml and promote-to-main.yml workflows + + steps: + - name: Extract version from tag + id: get_version + run: | + TAG_NAME="${GITHUB_REF#refs/tags/}" + VERSION="${TAG_NAME#auto-merge/}" + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Triggered by tag: $TAG_NAME" + echo "Version: $VERSION" + + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ steps.get_version.outputs.tag_name }} + fetch-depth: 0 + + - name: Find PR for this tag + id: get_pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Get the commit SHA that this tag points to + COMMIT_SHA=$(git rev-parse ${{ steps.get_version.outputs.tag_name }}) + echo "Tag points to commit: $COMMIT_SHA" + + # Find PR with this commit in its head + PR_NUMBER=$(gh pr list --state open --base main --json number,headRefOid,headRefName \ + --jq ".[] | select(.headRefOid == \"$COMMIT_SHA\") | .number" | head -1) + + if [ -z "$PR_NUMBER" ]; then + echo "❌ No open PR found for commit $COMMIT_SHA" + exit 1 + fi + + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "✅ Found PR #$PR_NUMBER" + + - name: Check if PR includes version bump + id: check_version_bump + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ steps.get_pr.outputs.pr_number }}" + + echo "Checking if PR #$PR_NUMBER includes version bump..." + + # Get list of changed files + CHANGED_FILES=$(gh pr view "$PR_NUMBER" --json files --jq '.files[].path') + + echo "Changed files:" + echo "$CHANGED_FILES" + + # Check if version files were modified + HAS_DESCRIPTION=$(echo "$CHANGED_FILES" | grep -c "^DESCRIPTION$" || true) + HAS_CITATION=$(echo "$CHANGED_FILES" | grep -c "^inst/CITATION$" || true) + HAS_ZENODO=$(echo "$CHANGED_FILES" | grep -c "^.zenodo.json$" || true) + + if [ "$HAS_DESCRIPTION" -gt 0 ] && [ "$HAS_CITATION" -gt 0 ] && [ "$HAS_ZENODO" -gt 0 ]; then + echo "✅ PR includes version bump (DESCRIPTION, inst/CITATION, .zenodo.json modified)" + echo "has_version_bump=true" >> $GITHUB_OUTPUT + else + echo "❌ PR does not include complete version bump" + echo " DESCRIPTION: $HAS_DESCRIPTION" + echo " inst/CITATION: $HAS_CITATION" + echo " .zenodo.json: $HAS_ZENODO" + echo "has_version_bump=false" >> $GITHUB_OUTPUT + fi + + - name: Wait for and check CI status + id: check_status + if: steps.check_version_bump.outputs.has_version_bump == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ steps.get_pr.outputs.pr_number }}" + MAX_WAIT_MINUTES=30 + CHECK_INTERVAL_SECONDS=30 + MAX_ITERATIONS=$((MAX_WAIT_MINUTES * 60 / CHECK_INTERVAL_SECONDS)) + + echo "Waiting for CI checks to complete on PR #$PR_NUMBER..." + echo "Will check every ${CHECK_INTERVAL_SECONDS}s for up to ${MAX_WAIT_MINUTES} minutes" + echo "" + + for i in $(seq 1 $MAX_ITERATIONS); do + echo "=== Check attempt $i/$MAX_ITERATIONS ===" + + # Get PR checks using simpler gh pr checks command + CHECKS_OUTPUT=$(gh pr checks "$PR_NUMBER" 2>&1 || true) + + # Count different status types - use xargs to trim whitespace and ensure numeric values + # gh pr checks output format: