From 2f139eb39d7b63cdcd818d1ef75574073665744a Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Mon, 20 Apr 2026 20:08:41 -0600 Subject: [PATCH 01/66] Initial work on removing old deployment files and setting up new config - Remove old deployment files and dependencies. Some of these were even from older setups no longer used. - Try to begin work on cleaning up configuration approach to make things easier to configure via environment variables. The previous approach had several layers of legacy approaches that had built up over time. There was also a lot of duplicate config where there were subtle drifts in configuration that this attempts to de-duplicate and unify. None of this is actually hooked up yet, this is just the beginnings of trying to work on new deployments. --- .github/scripts/decrypt.sh | 6 - .github/scripts/make_keys.py.sh | 6 - .github/workflows/pull_request_tests.yml | 10 +- .github/workflows/push_tests.yml | 10 +- .gitignore | 1 - .helm/namespaces/app.yaml | 7 - .helm/namespaces/shared-resources.yaml | 7 - .helm/secret-values.development.yaml | 2 - .helm/secret-values.production.yaml | 2 - .helm/secret-values.staging.yaml | 2 - .helm/secret-values.yaml | 2 - .helm/secret/development-keys.py | 1 - .helm/secret/production-keys.py | 1 - .helm/secret/staging-keys.py | 1 - .helm/templates/base-config-map.yaml | 25 - .helm/templates/celery-deployment.yaml | 103 -- .helm/templates/db-migrate-job.yaml | 27 - .helm/templates/django-deployment.yaml | 91 -- .helm/templates/django-ingress.yaml | 40 - .helm/templates/django-service.yaml | 10 - .helm/templates/ecr-image-pull-secret.yaml | 108 --- .helm/templates/julia-deployment.yaml | 59 -- .helm/templates/julia-service.yaml | 10 - .helm/templates/redis-service.yaml | 12 - .helm/templates/redis-stateful-set.yaml | 41 - .helm/templates/secrets.yaml | 10 - .helm/values.deploy.yaml | 1 - .helm/values.development.yaml | 0 .helm/values.production.yaml | 16 - .helm/values.staging.yaml | 2 - .helm/values.yaml | 25 - Capfile | 21 - Gemfile | 17 - Gemfile.lock | 79 -- Jenkinsfile | 213 ----- Jenkinsfile-deploy | 77 -- Jenkinsfile-deploy-c110p | 60 -- Jenkinsfile-image-cleanup | 59 -- Jenkinsfile-restart-celery-julia | 114 --- Jenkinsfile-undeploy-c110p | 48 - Jenkinsfile-undeploy-develop | 76 -- Jenkinsfile-undeploy-rhel | 57 -- Jenkinsfile-undeploy-staging | 76 -- checkdb.py | 39 +- config/deploy.rb | 55 -- config/deploy/development.rb | 9 - config/deploy/internal_c110p.rb | 5 - config/deploy/production.rb | 7 - config/deploy/staging.rb | 6 - config/deploy/templates/foreman/env.erb | 12 - config/gunicorn.conf.py | 12 +- docker-compose.nginx.yml | 16 +- docker-compose.nojulia.yml | 16 +- docker-compose.yml | 16 +- keys.py | 18 + manage.py | 2 +- reopt_api/celery.py | 46 +- reopt_api/dev_settings.py | 165 ---- reopt_api/internal_c110p_settings.py | 135 --- .../{production_settings.py => settings.py} | 41 +- reopt_api/staging_settings.py | 159 ---- transcrypt/.editorconfig | 16 - transcrypt/.gitattributes | 1 - transcrypt/INSTALL.md | 61 -- transcrypt/LICENSE | 20 - transcrypt/README.md | 296 ------ transcrypt/contrib/bash/transcrypt | 55 -- .../contrib/packaging/pacman/.gitignore | 6 - transcrypt/contrib/packaging/pacman/PKGBUILD | 23 - transcrypt/contrib/zsh/_transcrypt | 37 - transcrypt/man/transcrypt.1 | 118 --- transcrypt/man/transcrypt.1.ronn | 107 --- transcrypt/transcrypt | 887 ------------------ werf-giterminism.yaml | 12 - werf.yaml | 19 - 75 files changed, 108 insertions(+), 3844 deletions(-) delete mode 100755 .github/scripts/decrypt.sh delete mode 100755 .github/scripts/make_keys.py.sh delete mode 100644 .helm/namespaces/app.yaml delete mode 100644 .helm/namespaces/shared-resources.yaml delete mode 100644 .helm/secret-values.development.yaml delete mode 100644 .helm/secret-values.production.yaml delete mode 100644 .helm/secret-values.staging.yaml delete mode 100644 .helm/secret-values.yaml delete mode 100644 .helm/secret/development-keys.py delete mode 100644 .helm/secret/production-keys.py delete mode 100644 .helm/secret/staging-keys.py delete mode 100644 .helm/templates/base-config-map.yaml delete mode 100644 .helm/templates/celery-deployment.yaml delete mode 100644 .helm/templates/db-migrate-job.yaml delete mode 100644 .helm/templates/django-deployment.yaml delete mode 100644 .helm/templates/django-ingress.yaml delete mode 100644 .helm/templates/django-service.yaml delete mode 100644 .helm/templates/ecr-image-pull-secret.yaml delete mode 100644 .helm/templates/julia-deployment.yaml delete mode 100644 .helm/templates/julia-service.yaml delete mode 100644 .helm/templates/redis-service.yaml delete mode 100644 .helm/templates/redis-stateful-set.yaml delete mode 100644 .helm/templates/secrets.yaml delete mode 100644 .helm/values.deploy.yaml delete mode 100644 .helm/values.development.yaml delete mode 100644 .helm/values.production.yaml delete mode 100644 .helm/values.staging.yaml delete mode 100644 .helm/values.yaml delete mode 100644 Capfile delete mode 100644 Gemfile delete mode 100644 Gemfile.lock delete mode 100644 Jenkinsfile delete mode 100644 Jenkinsfile-deploy delete mode 100644 Jenkinsfile-deploy-c110p delete mode 100644 Jenkinsfile-image-cleanup delete mode 100644 Jenkinsfile-restart-celery-julia delete mode 100644 Jenkinsfile-undeploy-c110p delete mode 100644 Jenkinsfile-undeploy-develop delete mode 100644 Jenkinsfile-undeploy-rhel delete mode 100644 Jenkinsfile-undeploy-staging delete mode 100644 config/deploy.rb delete mode 100644 config/deploy/development.rb delete mode 100644 config/deploy/internal_c110p.rb delete mode 100644 config/deploy/production.rb delete mode 100644 config/deploy/staging.rb delete mode 100644 config/deploy/templates/foreman/env.erb create mode 100644 keys.py delete mode 100644 reopt_api/dev_settings.py delete mode 100755 reopt_api/internal_c110p_settings.py rename reopt_api/{production_settings.py => settings.py} (79%) delete mode 100755 reopt_api/staging_settings.py delete mode 100644 transcrypt/.editorconfig delete mode 100644 transcrypt/.gitattributes delete mode 100644 transcrypt/INSTALL.md delete mode 100644 transcrypt/LICENSE delete mode 100644 transcrypt/README.md delete mode 100644 transcrypt/contrib/bash/transcrypt delete mode 100644 transcrypt/contrib/packaging/pacman/.gitignore delete mode 100644 transcrypt/contrib/packaging/pacman/PKGBUILD delete mode 100644 transcrypt/contrib/zsh/_transcrypt delete mode 100644 transcrypt/man/transcrypt.1 delete mode 100644 transcrypt/man/transcrypt.1.ronn delete mode 100755 transcrypt/transcrypt delete mode 100644 werf-giterminism.yaml delete mode 100644 werf.yaml diff --git a/.github/scripts/decrypt.sh b/.github/scripts/decrypt.sh deleted file mode 100755 index 32d3ae913..000000000 --- a/.github/scripts/decrypt.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# transcrypt/transcrypt --flush-credentials -y || true -# transcrypt/transcrypt -c aes-256-cbc -p "$TRANSCRYPT_PASSWORD" -y - - diff --git a/.github/scripts/make_keys.py.sh b/.github/scripts/make_keys.py.sh deleted file mode 100755 index fde27a36e..000000000 --- a/.github/scripts/make_keys.py.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -cp keys.py.template keys.py -echo "pvwatts_api_key = '${NREL_DEV_API_KEY}'" >> keys.py -echo "developer_nrel_gov_key = '${NREL_DEV_API_KEY}'" >> keys.py -echo "Successfully created keys.py" diff --git a/.github/workflows/pull_request_tests.yml b/.github/workflows/pull_request_tests.yml index 92a96768b..308e72fe0 100644 --- a/.github/workflows/pull_request_tests.yml +++ b/.github/workflows/pull_request_tests.yml @@ -18,16 +18,10 @@ jobs: steps: - uses: actions/checkout@v2 - # - name: Decrypt - # env: - # TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }} - # run: ./.github/scripts/decrypt.sh - - name: Make keys.py - env: - NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - run: ./.github/scripts/make_keys.py.sh - name: Build containers run: docker compose up -d + env: + NLR_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - name: Check running containers run: docker ps -a - name: Wait for julia_api diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index a37ea849a..f2c174ba9 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -14,16 +14,10 @@ jobs: steps: - uses: actions/checkout@v2 - # - name: Decrypt - # env: - # TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }} - # run: ./.github/scripts/decrypt.sh - - name: Make keys.py - env: - NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - run: ./.github/scripts/make_keys.py.sh - name: Build containers run: docker compose up -d + env: + NLR_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - name: Check running containers run: docker ps -a - name: Wait for julia_api diff --git a/.gitignore b/.gitignore index aba3fad8d..e9a3e7c50 100755 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ __pycache__/ *.py[cod] *$py.class -keys.py .idea input_files/* !input_files/**/ diff --git a/.helm/namespaces/app.yaml b/.helm/namespaces/app.yaml deleted file mode 100644 index 7e88af61b..000000000 --- a/.helm/namespaces/app.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Env.DEPLOY_APP_NAMESPACE_NAME }} - annotations: - field.cattle.io/projectId: {{ .Env.RANCHER_PROJECT_ID }} - field.cattle.io/resourceQuota: '{"limit":{"pods":"{{ .Env.DEPLOY_APP_NAMESPACE_POD_LIMIT }}"}}' diff --git a/.helm/namespaces/shared-resources.yaml b/.helm/namespaces/shared-resources.yaml deleted file mode 100644 index 7e1be5241..000000000 --- a/.helm/namespaces/shared-resources.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Env.DEPLOY_SHARED_RESOURCES_NAMESPACE_NAME }} - annotations: - field.cattle.io/projectId: {{ .Env.RANCHER_PROJECT_ID }} - field.cattle.io/resourceQuota: '{"limit":{"pods":"{{ .Env.DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT }}"}}' diff --git a/.helm/secret-values.development.yaml b/.helm/secret-values.development.yaml deleted file mode 100644 index e8bf8a455..000000000 --- a/.helm/secret-values.development.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 10002e0d33ad1785006b074cda33268e9021e5a649773438a3556c0679aad0162ff4d80dec917ca4582168b2a633c5f905ed diff --git a/.helm/secret-values.production.yaml b/.helm/secret-values.production.yaml deleted file mode 100644 index 414a71a19..000000000 --- a/.helm/secret-values.production.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 10009752210666abab8694970f80e8248197733947c60c9ba27f5287e5839b74db8b293b8ebcb18a173132ee928133b62f04 diff --git a/.helm/secret-values.staging.yaml b/.helm/secret-values.staging.yaml deleted file mode 100644 index 1c053f4ae..000000000 --- a/.helm/secret-values.staging.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 10006ed43b5d95373fd966af13014d65342e6fc31658633262ca150fe1ec338ee3f2cde10def52af5336b19f0268ff445e74 diff --git a/.helm/secret-values.yaml b/.helm/secret-values.yaml deleted file mode 100644 index e3a719463..000000000 --- a/.helm/secret-values.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 1000fb94d50114b50c0457a01b0fc1b9245b79c3780d3f3b21ea587d89311204933e diff --git a/.helm/secret/development-keys.py b/.helm/secret/development-keys.py deleted file mode 100644 index 66224418d..000000000 --- a/.helm/secret/development-keys.py +++ /dev/null @@ -1 +0,0 @@ -1000a287d4991b6127153631eee92ee306ab1a68dc7a9bdb8524a3de50ca2a666e9efb35319928d708a9da1f06efb7b15b0601339cb7995ddd6b3a0ffa00aa5597abdb6f553608acdc8e21fd3c6146f17d7f98ca01295a58b45a05e2a35e019fc23f7457409ec8d0a76cc4abb3e973348ac7b24e6d965769ebdd6104394b473c83b0725339f9346e0055d303178e5d855572027bb6b7ea4b871af5cc9367186d4baad982656a36805a122d9be9276f4bc91f57c3a373e0d9ef19084b8871ec8a4dea5f4e930f0097c7f1bb454f20bd8809cec4a96825304ca6061140551e792a983538fccde5137a069e0db683fba346b82b064bcf19abbba922a4a868425d1c47983a43f55046c0ad6e4cb56278aa59a2153141bf97e61a3b307ce8a325e816856604808d0b70d129366bb0ee7d333ae777164afa80fbe9c0101637f05094fb157da0935f5a580a3e5dc0ddf08c3d17e8d3256acb231af9ac4c6f8f013fb2f6497b640c3bf3ee2d2e4cc5ba2b396584faf6b5957948956799481ab6888fcc036ecb70425117ab15931e66184448f5e294e6c54d1b9d2142ec552d18b899f5c3e24bf2870e49a709e46355b62eb9a436baee47aad53a6504ee47d8c2e54724ed5b1120a222361e18bf7162eabae1d87c613bd87ef4b9d578bae3ffca241eb2a89cb90d7d7648767819a89bfaf96eb6ec53dac17271dd030706cec9ddac2454459a25d40384f7f5026fd085df6e16f2eaf1c49bd307e2fe62bc95f76d79fe4c71b8776fe60bd92e4a370b1cd2707bdbee5576fa0139c8277a47160fddc92bb167efc41726acebc32e63fa6fb6896b0102b179da146be946193996e22e2271574e70bf9d4191c9410536416611cf358e23acc324db381228aa03a9119d2dad8fad493ee3570302eb2d6df8dc78d1fec6fac406ce55ee6ba2b36ade3a68e47a5a2b76d563b47b583cc2d8e258bded75ea8edba123a95fd88eac3f4f5a45520f1fb417311c6f469e580150e0f4bb96e35f1cbec5ccd0d2d330a1ec799504bec44cfa256f9977ea55e1877f760f05e5457ef10cd6b2a34fd42ab5c02ad7fd6658f3d38fcb6a144cf27ee0581a90803e136155e9e891a60fcf2eba3a9d60e23a346e251cfc8a60a35c636f01b273179c3d514b4610f541237661b134a052159597f8e312856f22567831afbd9893dad5664a65474146abb8c3028b9520a8d7df4ae1424a976903aa14abe433b41d283b5e00e9b2d0b5e5eabe301bf16e8f22896b3455e53b914d90709b1241245c6606ae3b981a1c diff --git a/.helm/secret/production-keys.py b/.helm/secret/production-keys.py deleted file mode 100644 index 21f436ab9..000000000 --- a/.helm/secret/production-keys.py +++ /dev/null @@ -1 +0,0 @@ -100025a03b71714752567c2de575616c8b7579c7dadc9b11159121ab5ac4540b5066127f7806996f427e595a4b0704b5704c3a188a7a4dc7f38ec0276018e45e5476168601ff37467ed008284468c45601eac86636cfc0deb1df467202a9049c9130e963b39eaad74e6e3165c7d8adb1ff1ffb78ff61ece5ca7db4f9ed4eb81a5692987eff93ad3a7b45b6c0f9440ab0841fee83840a59f2c3c4423bdf45bd675d202156e7a216a6b5effcdfa0db1c05407ce36686b115498e823e47117707dd298ecf0e6b7594ea56a4e3720038ccd77ffdc69f917929e99c2e168056e46c318c269a0ca39fa75e97cb78b168f4fd8c55a4fea8f49086c3ef2081d27b310f899c10a7ae7ce7a9c9f44af09e7de46ee934ac849c0779cfb576f02124f40889f075b8e33427fd16e31b2a1171b89145993c1a958ca71fde968818cf18aebcad411dcf2bc749b4ec78d4e2a1ab777629851da1a3e91fe79b8049b5f6cae106667bd30321225d7c94489f1d59512472477e8a8127e6591992afd15cf379a70c5110790ab27429fcaa54eeaed5935854451bf88dea6ba2d0b6b7e77797e2d5053309d0c610dba06d51ba94afe35115a88e0664809c38f7906846f8e12dfc79c430de26039011c57cef6351a23b0751e91f1b33ea86e02d9e2779a7a79ec4045b8bc505d4076ab5814a9eba2c6155c54701a42810101c3c7b2579f5501976af502e88f6eca17db31bb0cd5baa846552fcd0c3e1b1d706919479122d543f4c0fa6b7bd5dba77d3794f30c989bd84b8f64cb692df5ca3e4e3f365bfefddaa2a37dad56f9aabebaaafa12ccc285a0cefe4d711f2b990141b759d8303c2288f59a8fe19d0f9988a012f161581723f8d0cea748f215889f9ba4cad109d042f488547a8fb16b2d742c5172a53654d609e0da25d163619349a6734cfa6407cbd928e579637be614753311b3c6eb65bc453eb666ad628b0279f4f1ada9d708e11c5d104cda2ce8dde01e40462ac89f3183997e54f28010acd8c6bebf9ea34d7882b4f021add6f1534426b85d06058664a63b331170ee54413ac047b2d18a233e0e5fbd2199058d594366c142e1d07c41a4efa98593847f2bde01c652b975a0be21be155a1d2f1628cae8ef3fb69a4340f1ec4f87299f127b065a8b19fe11b102adbad45e337ec1a8bd8e6bb30a5521466e86c603537f7549c4f4caa26160f25984e5065ca44c8a85d diff --git a/.helm/secret/staging-keys.py b/.helm/secret/staging-keys.py deleted file mode 100644 index 8e527bd04..000000000 --- a/.helm/secret/staging-keys.py +++ /dev/null @@ -1 +0,0 @@ -1000c06d9305e04dc9f413f0087fc91909d40984d91ddfbfba505c33325695725cbce0d378841dda920fb9a88a9782b89090103ff193ce519db8912bf14b85bc4750deb4a6a81b84022b71c87b6d7a769625b1a8d7086cea3674db349856efe37a3b5b14e212cd5f817762055fe9fb2b34965bdd16fdba30ece9f1953e0fac0aa920e1d8835bc3b55f7a3a17a233260b44b51683a742a301bc4e775f8543c507d73f28ff4cf51f4a359322bd3dfcdcf17aa6cc56faca4d9aef3690b3b48e08dc5af9cd9846a6f9f28e91447ddc0951cf31bd07565314f4823e10c7f89476f32f9327e81b539118ff4c19d394c366b6ec248d9b3270b85d0dfe0e604b8b88863d303265a3b91490a47d517cadd08670d3a6744dc5c21b2b509ac65d2bd359a64aaa25e9c9744b6da1baab3d3413c510f1cfdb424bb00b810477c74cb9a094c0a59c773f8b13c8b2b27e0e052a66817d874bb995491568a1cdd5c64760345c9db3f8bc53553944eebd26b1ce0a001148980a765c8f38ed05388bf8eb40a20f15fd3508f14525986ea2e0877427de86cb7683747372559a11a3c546e7273afcb08871119df76689f22ce688a0a8fe145a4d0cf7397e80ce14486ebc646428f6789a2e94308adbe780c39f711734e00e1b9ea93f0dfdb5da0d9f8e4afe389e68dea05bbdf14b9a4a8986419da2ad0759845487fce519358a7b0009375fcf585f1e077ef177ab25544097ce6e739c9db8a6b49b6bc1989041264047dec69d70d3e854796f070fc1696f810105700270eb847c7346b51574fde2bf87e71039a440a9c235abb5ccb0f6ea5eec63cb3323b9080eec85a84574f531b316a800ba56b735d3ed678690a2cbde5f6448d022fa4c41a75f7d5ebcb20e3e0f2e4a10a97645c3e61c0ca7ff5519174919495ba8d69b9c8b510cceb152b32b8af714903101139224f66fc695dc2464f51b31545540b36b0296ba106444e121007bbda0d1e30a4fc7c26fb926c665d098c71ba686e19f9844b7ea8ad9ddafb40776dd2f68e965b8607f253349262b20799dc4d041d35f3730e494673fe896a334944101758435c9cd934a36d724f47729ac132ca1717ff657b5a579e0f8eeea0549a22eea69dd4c1cedf29a59998bd16667164a458623676c06b05f5565997868cbfec3a8fe727d56d7d5f2e1882e5e146baa07d41f0551755f96844118b121eab3927086702c523a1d77ac6fdf6ca11e4e7b0c67a404331a80db3cfc136e8b5b7646d5cdf477652f8012fdd0a2fa7716744c223ea423a1ce00252fe170368590d90adc459398347dcf03f25e05acbbbe605d7ce2f1045625c2c89db1fd2e3df89228b7960237546ac1063441e5d28871576fb5770c05bd508fb704f541b77e933cb243cd46f83b81d44401dfd267e299d0ff223ebc000feee0713dfba95d1fef13de4bf4662cc89b05559ecd08cdaff2c602ff93159d38f1867275c89c6a4073a23bb8c305c87cc008c0086d1b7d3dbe2b413835a3552af21edce243005e357073d72d469e1b0a302a60 diff --git a/.helm/templates/base-config-map.yaml b/.helm/templates/base-config-map.yaml deleted file mode 100644 index 13d7a4dae..000000000 --- a/.helm/templates/base-config-map.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Chart.Name }}-base-config-map -data: - APP_ENV: {{ .Values.appEnv | quote }} - {{ if .Values.branchName }} - BRANCH_NAME: {{ .Values.branchName | quote }} - {{ end }} - {{ if .Values.dbName }} - DB_NAME: {{ .Values.dbName | quote }} - # Use the database name as the queue name so that separate branches or - # databases on staging remain in separate queues. - APP_QUEUE_NAME: {{ .Values.dbName | quote }} - {{ else }} - # In production (or where the database name isn't explicitly overridden), - # just use the app env as the queue name. - APP_QUEUE_NAME: {{ .Values.appEnv | quote }} - {{ end }} - DJANGO_SETTINGS_MODULE: {{ .Values.djangoSettingsModule | quote }} - SOLVER: xpress - JULIA_HOST: {{ tpl .Values.juliaHost . | quote }} - REDIS_HOST: {{ tpl .Values.redisHost . | quote }} - REDIS_PORT: {{ .Values.redisPort | quote }} - K8S_DEPLOY: "true" diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml deleted file mode 100644 index 14daae8ad..000000000 --- a/.helm/templates/celery-deployment.yaml +++ /dev/null @@ -1,103 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Chart.Name }}-celery-deployment - labels: - app: {{ .Chart.Name }}-celery -spec: - replicas: {{ .Values.celeryReplicas }} - selector: - matchLabels: - app: {{ .Chart.Name }}-celery - template: - metadata: - labels: - app: {{ .Chart.Name }}-celery - appImageTagChecksum: {{ index .Values.werf.image "reopt-api" | sha1sum }} - spec: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ .Chart.Name }}-celery - - key: appImageTagChecksum - operator: In - values: - - {{ index .Values.werf.image "reopt-api" | sha1sum }} - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - volumes: - - name: {{ .Chart.Name }}-secrets-volume - secret: - secretName: {{ .Chart.Name }}-secrets - initContainers: - - name: {{ .Chart.Name }}-ready-wait - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/ready-wait"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py - containers: - - name: {{ .Chart.Name }}-celery - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/worker"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - env: - - name: JULIA_HOST - value: "localhost" - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py -# readinessProbe: -# exec: -# command: ["pgrep", "-f", "bin/celery"] -# periodSeconds: 5 -# timeoutSeconds: 3 -# failureThreshold: 3 -# livenessProbe: -# exec: -# command: ["pgrep", "-f", "bin/celery"] -# initialDelaySeconds: 30 -# periodSeconds: 60 -# timeoutSeconds: 30 -# failureThreshold: 10 - resources: - requests: - cpu: {{ .Values.celeryCpuRequest | quote }} - memory: {{ .Values.celeryMemoryRequest | quote }} - limits: - cpu: {{ .Values.celeryCpuLimit | quote }} - memory: {{ .Values.celeryMemoryLimit | quote }} - - - name: {{ .Chart.Name }}-julia - image: {{ index .Values.werf.image "julia-api" }} - args: ["julia", "--project=/opt/julia_src", "http.jl"] - ports: - - containerPort: 8081 - env: - - name: JULIA_NUM_THREADS - value: "4" - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - resources: - requests: - cpu: {{ .Values.juliaCpuRequest | quote }} - memory: {{ .Values.juliaMemoryRequest | quote }} - limits: - cpu: {{ .Values.juliaCpuLimit | quote }} - memory: {{ .Values.juliaMemoryLimit | quote }} diff --git a/.helm/templates/db-migrate-job.yaml b/.helm/templates/db-migrate-job.yaml deleted file mode 100644 index 55966b26b..000000000 --- a/.helm/templates/db-migrate-job.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Chart.Name }}-db-migrate-job-{{ .Release.Revision }} -spec: - backoffLimit: 0 - template: - spec: - restartPolicy: Never - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - volumes: - - name: {{ .Chart.Name }}-secrets-volume - secret: - secretName: {{ .Chart.Name }}-secrets - containers: - - name: {{ .Chart.Name }}-job - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/migrate"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py diff --git a/.helm/templates/django-deployment.yaml b/.helm/templates/django-deployment.yaml deleted file mode 100644 index 26a0d89a3..000000000 --- a/.helm/templates/django-deployment.yaml +++ /dev/null @@ -1,91 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Chart.Name }}-django-deployment - labels: - app: {{ .Chart.Name }}-django -spec: - replicas: {{ .Values.djangoReplicas }} - selector: - matchLabels: - app: {{ .Chart.Name }}-django - template: - metadata: - labels: - app: {{ .Chart.Name }}-django - appImageTagChecksum: {{ index .Values.werf.image "reopt-api" | sha1sum }} - spec: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ .Chart.Name }}-django - - key: appImageTagChecksum - operator: In - values: - - {{ index .Values.werf.image "reopt-api" | sha1sum }} - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - volumes: - - name: {{ .Chart.Name }}-secrets-volume - secret: - secretName: {{ .Chart.Name }}-secrets - initContainers: - - name: {{ .Chart.Name }}-ready-wait - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/ready-wait"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py - containers: - - name: {{ .Chart.Name }}-django - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/server"] - ports: - - containerPort: 8000 - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py -# readinessProbe: -# httpGet: -# path: /_health -# port: 8000 -# httpHeaders: -# - name: X-Forwarded-Proto -# value: https -# periodSeconds: 5 -# timeoutSeconds: 60 -# failureThreshold: 10 -# livenessProbe: -# httpGet: -# path: /_health -# port: 8000 -# httpHeaders: -# - name: X-Forwarded-Proto -# value: https -# initialDelaySeconds: 30 -# periodSeconds: 60 -# timeoutSeconds: 60 -# failureThreshold: 10 - resources: - requests: - cpu: {{ .Values.djangoCpuRequest | quote }} - memory: {{ .Values.djangoMemoryRequest | quote }} - limits: - cpu: {{ .Values.djangoCpuLimit | quote }} - memory: {{ .Values.djangoMemoryLimit | quote }} diff --git a/.helm/templates/django-ingress.yaml b/.helm/templates/django-ingress.yaml deleted file mode 100644 index 8c3d5428c..000000000 --- a/.helm/templates/django-ingress.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ .Chart.Name }}-django-ingress - annotations: - # Size should be kept in sync with "client_max_body_size" in - # config/templates/nginx/site.conf.tmpl. - nginx.ingress.kubernetes.io/proxy-body-size: "2048m" - nginx.ingress.kubernetes.io/proxy-read-timeout: "800" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "800" -spec: - tls: - - hosts: - - {{ .Values.ingressHost }} - {{ if .Values.tempIngressHost }} - - {{ .Values.tempIngressHost }} - {{ end }} - rules: - - host: {{ .Values.ingressHost }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ .Chart.Name }}-django-service - port: - number: 8000 - {{ if .Values.tempIngressHost }} - - host: {{ .Values.tempIngressHost }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ .Chart.Name }}-django-service - port: - number: 8000 - {{ end }} diff --git a/.helm/templates/django-service.yaml b/.helm/templates/django-service.yaml deleted file mode 100644 index d6c649772..000000000 --- a/.helm/templates/django-service.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Chart.Name }}-django-service -spec: - selector: - app: {{ .Chart.Name }}-django - ports: - - protocol: TCP - port: 8000 diff --git a/.helm/templates/ecr-image-pull-secret.yaml b/.helm/templates/ecr-image-pull-secret.yaml deleted file mode 100644 index 6f4f759a1..000000000 --- a/.helm/templates/ecr-image-pull-secret.yaml +++ /dev/null @@ -1,108 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-service-account - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-role - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "update", "get", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-role-binding - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Chart.Name }}-ecr-login-renew-role -subjects: - - kind: ServiceAccount - name: {{ .Chart.Name }}-ecr-login-renew-service-account ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-config-map - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -data: - DOCKER_SECRET_NAME: {{ .Chart.Name }}-ecr-image-pull-secret - TARGET_NAMESPACE: {{ .Release.Namespace | quote }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-secrets - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -type: Opaque -data: - AWS_REGION: {{ .Values.ecrAwsRegion | b64enc | quote }} - AWS_ACCESS_KEY_ID: {{ .Values.ecrAwsAccessKeyId | b64enc | quote }} - AWS_SECRET_ACCESS_KEY: {{ .Values.ecrAwsSecretAccessKey | b64enc | quote }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-initial-setup - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "2" -spec: - template: - spec: - restartPolicy: OnFailure - dnsConfig: - options: - - name: ndots - value: "1" - serviceAccountName: {{ .Chart.Name }}-ecr-login-renew-service-account - containers: - - name: {{ .Chart.Name }}-ecr-login-renew - image: ghcr.io/nabsul/k8s-ecr-login-renew:v1.7.1 - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-ecr-login-renew-config-map - - secretRef: - name: {{ .Chart.Name }}-ecr-login-renew-secrets ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-cron-job -spec: - schedule: "37 */6 * * *" - jobTemplate: - spec: - template: - spec: - restartPolicy: OnFailure - dnsConfig: - options: - - name: ndots - value: "1" - serviceAccountName: {{ .Chart.Name }}-ecr-login-renew-service-account - containers: - - name: {{ .Chart.Name }}-ecr-login-renew - image: ghcr.io/nabsul/k8s-ecr-login-renew:v1.7.1 - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-ecr-login-renew-config-map - - secretRef: - name: {{ .Chart.Name }}-ecr-login-renew-secrets diff --git a/.helm/templates/julia-deployment.yaml b/.helm/templates/julia-deployment.yaml deleted file mode 100644 index dea2880a6..000000000 --- a/.helm/templates/julia-deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Chart.Name }}-julia-deployment - labels: - app: {{ .Chart.Name }}-julia -spec: - replicas: {{ .Values.juliaReplicas }} - selector: - matchLabels: - app: {{ .Chart.Name }}-julia - template: - metadata: - labels: - app: {{ .Chart.Name }}-julia - appImageTagChecksum: {{ index .Values.werf.image "julia-api" | sha1sum }} - spec: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ .Chart.Name }}-julia - - key: appImageTagChecksum - operator: In - values: - - {{ index .Values.werf.image "julia-api" | sha1sum }} - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - containers: - - name: {{ .Chart.Name }}-julia - image: {{ index .Values.werf.image "julia-api" }} - args: ["julia", "--project=/opt/julia_src", "http.jl"] - ports: - - containerPort: 8081 - env: - - name: JULIA_NUM_THREADS - value: "4" - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - readinessProbe: - httpGet: - path: /health - port: 8081 - periodSeconds: 5 - timeoutSeconds: 60 - failureThreshold: 10 - resources: - requests: - cpu: {{ .Values.juliaCpuRequest | quote }} - memory: "2000Mi" - limits: - cpu: {{ .Values.juliaCpuLimit | quote }} - memory: "2000Mi" diff --git a/.helm/templates/julia-service.yaml b/.helm/templates/julia-service.yaml deleted file mode 100644 index 8bd9be4ef..000000000 --- a/.helm/templates/julia-service.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Chart.Name }}-julia-service -spec: - selector: - app: {{ .Chart.Name }}-julia - ports: - - protocol: TCP - port: 8081 diff --git a/.helm/templates/redis-service.yaml b/.helm/templates/redis-service.yaml deleted file mode 100644 index ce7665369..000000000 --- a/.helm/templates/redis-service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Chart.Name }}-redis-service - annotations: - "helm.sh/resource-policy": keep -spec: - selector: - app: {{ .Chart.Name }}-redis - ports: - - protocol: TCP - port: 6379 diff --git a/.helm/templates/redis-stateful-set.yaml b/.helm/templates/redis-stateful-set.yaml deleted file mode 100644 index 339705613..000000000 --- a/.helm/templates/redis-stateful-set.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ .Chart.Name }}-redis-stateful-set - labels: - app: {{ .Chart.Name }}-redis - annotations: - "helm.sh/resource-policy": keep -spec: - replicas: 1 - selector: - matchLabels: - app: {{ .Chart.Name }}-redis - serviceName: {{ .Chart.Name }}-redis - template: - metadata: - labels: - app: {{ .Chart.Name }}-redis - spec: - containers: - - name: {{ .Chart.Name }}-redis - image: redis:6.0.8-alpine - args: ["sh", "-c", "redis-server --requirepass $$REDIS_PASSWORD --save 900 1 --save 300 10 --save 60 10000 --appendonly yes --maxmemory 2048mb --maxmemory-policy allkeys-lru"] - # volumeMounts: - # - name: {{ .Chart.Name }}-redis-data-volume - # mountPath: /data - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-secrets - key: redis_password - # volumeClaimTemplates: - # - metadata: - # name: {{ .Chart.Name }}-redis-data-volume - # spec: - # storageClassName: {{ .Values.redisDataVolumeStorageClassName | quote }} - # accessModes: ["ReadWriteOnce"] - # resources: - # requests: - # storage: 1.5Gi diff --git a/.helm/templates/secrets.yaml b/.helm/templates/secrets.yaml deleted file mode 100644 index 707e59311..000000000 --- a/.helm/templates/secrets.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Chart.Name }}-secrets -type: Opaque -data: - development-keys.py: {{ werf_secret_file "development-keys.py" | b64enc | quote }} - staging-keys.py: {{ werf_secret_file "staging-keys.py" | b64enc | quote }} - production-keys.py: {{ werf_secret_file "production-keys.py" | b64enc | quote }} - redis_password: {{ .Values.secrets.redis_password | b64enc | quote }} diff --git a/.helm/values.deploy.yaml b/.helm/values.deploy.yaml deleted file mode 100644 index e8bd27bfa..000000000 --- a/.helm/values.deploy.yaml +++ /dev/null @@ -1 +0,0 @@ -redisDataVolumeStorageClassName: netappnfs diff --git a/.helm/values.development.yaml b/.helm/values.development.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/.helm/values.production.yaml b/.helm/values.production.yaml deleted file mode 100644 index 46abcafb6..000000000 --- a/.helm/values.production.yaml +++ /dev/null @@ -1,16 +0,0 @@ -appEnv: production -djangoSettingsModule: reopt_api.production_settings -djangoReplicas: 10 -djangoCpuRequest: "1000m" -djangoCpuLimit: "2000m" -djangoMemoryRequest: "2000Mi" -djangoMemoryLimit: "2000Mi" -celeryReplicas: 10 -juliaReplicas: 5 -juliaCpuRequest: "2000m" -juliaCpuLimit: "4000m" -juliaMemoryRequest: "12000Mi" -juliaMemoryLimit: "12000Mi" -juliaDeploymentStrategy: - rollingUpdate: - maxSurge: "0%" diff --git a/.helm/values.staging.yaml b/.helm/values.staging.yaml deleted file mode 100644 index d7cfc7107..000000000 --- a/.helm/values.staging.yaml +++ /dev/null @@ -1,2 +0,0 @@ -appEnv: staging -djangoSettingsModule: reopt_api.staging_settings \ No newline at end of file diff --git a/.helm/values.yaml b/.helm/values.yaml deleted file mode 100644 index b6dae9237..000000000 --- a/.helm/values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -ecrAwsRegion: us-east-2 -ecrAwsAccessKeyId: -ecrAwsSecretAccessKey: -ingressHost: localhost -appEnv: development -djangoSettingsModule: reopt_api.dev_settings -redisDataVolumeStorageClassName: -juliaHost: "{{ .Chart.Name }}-julia-service" -redisHost: "{{ .Chart.Name }}-redis-service" -redisPort: 6379 -djangoReplicas: 1 -djangoCpuRequest: "500m" -djangoCpuLimit: "2000m" -djangoMemoryRequest: "700Mi" -djangoMemoryLimit: "700Mi" -celeryReplicas: 1 -celeryCpuRequest: "100m" -celeryCpuLimit: "1000m" -celeryMemoryRequest: "1600Mi" -celeryMemoryLimit: "1600Mi" -juliaReplicas: 1 -juliaCpuRequest: "500m" -juliaCpuLimit: "2000m" -juliaMemoryRequest: "6000Mi" -juliaMemoryLimit: "6000Mi" diff --git a/Capfile b/Capfile deleted file mode 100644 index 50fae920c..000000000 --- a/Capfile +++ /dev/null @@ -1,21 +0,0 @@ -# Load DSL and set up stages -require "capistrano/setup" - -# Include default deployment tasks -require "capistrano/deploy" - -# Load the SCM plugin -require "capistrano/scm/git" -install_plugin Capistrano::SCM::Git - -# Includes additional plugins -require "capistrano/rbenv" -require "capistrano/bundler" -require "capistrano/file-permissions" -require "captastic/subdomains" -require "captastic/foreman" -require "captastic/nginx" -require "capistrano/tada-defaults" - -# Load custom tasks from `lib/capistrano/tasks` if you have any defined -Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } \ No newline at end of file diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 4745ba95e..000000000 --- a/Gemfile +++ /dev/null @@ -1,17 +0,0 @@ -source "https://rubygems.org" - -# Process management -gem "foreman", "~> 0.87.0" - -group :development do - # Deployment - gem "capistrano", "~> 3.12.0" - gem "capistrano-rbenv", "~> 2.1.4" - gem "capistrano-bundler", "~> 1.6.0" - gem "capistrano-file-permissions", "~> 1.0.0" - gem "capistrano-tada-defaults", :git => "https://github.nrel.gov/TADA/capistrano-tada-defaults.git" - gem "captastic", :git => "https://github.nrel.gov/TADA/captastic.git" - gem "captastic-nginx", :git => "https://github.nrel.gov/TADA/captastic-nginx.git" - gem "captastic-foreman", :git => "https://github.nrel.gov/TADA/captastic-foreman.git" - gem "captastic-subdomains", :git => "https://github.nrel.gov/TADA/captastic-subdomains.git" -end \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index f55d163fa..000000000 --- a/Gemfile.lock +++ /dev/null @@ -1,79 +0,0 @@ -GIT - remote: https://github.nrel.gov/TADA/capistrano-tada-defaults.git - revision: b9921ac074f5b5f13b6f34fd3c70ab0e6e13cc37 - specs: - capistrano-tada-defaults (0.2.0) - -GIT - remote: https://github.nrel.gov/TADA/captastic-foreman.git - revision: fc5ca792a9673e5f401717ee1c4c92a35d2d9702 - specs: - captastic-foreman (0.2.1) - captastic - inifile - -GIT - remote: https://github.nrel.gov/TADA/captastic-nginx.git - revision: 8198edc9e1b9378f5912c4c1a025adbf46182200 - specs: - captastic-nginx (0.1.0) - -GIT - remote: https://github.nrel.gov/TADA/captastic-subdomains.git - revision: 77e0965208cefdb95640130858f9dd1fbc80373b - specs: - captastic-subdomains (0.1.2) - -GIT - remote: https://github.nrel.gov/TADA/captastic.git - revision: f51c58441b98c9d969ec2df46d16141d1fff728f - specs: - captastic (0.1.2) - -GEM - remote: https://rubygems.org/ - specs: - airbrussh (1.4.0) - sshkit (>= 1.6.1, != 1.7.0) - capistrano (3.12.0) - airbrussh (>= 1.0.0) - i18n - rake (>= 10.0.0) - sshkit (>= 1.9.0) - capistrano-bundler (1.6.0) - capistrano (~> 3.1) - capistrano-file-permissions (1.0.0) - capistrano (~> 3.0) - capistrano-rbenv (2.1.6) - capistrano (~> 3.1) - sshkit (~> 1.3) - concurrent-ruby (1.1.6) - foreman (0.87.0) - i18n (1.8.2) - concurrent-ruby (~> 1.0) - inifile (3.0.0) - net-scp (2.0.0) - net-ssh (>= 2.6.5, < 6.0.0) - net-ssh (5.2.0) - rake (13.0.1) - sshkit (1.20.0) - net-scp (>= 1.1.2) - net-ssh (>= 2.8.0) - -PLATFORMS - ruby - -DEPENDENCIES - capistrano (~> 3.12.0) - capistrano-bundler (~> 1.6.0) - capistrano-file-permissions (~> 1.0.0) - capistrano-rbenv (~> 2.1.4) - capistrano-tada-defaults! - captastic! - captastic-foreman! - captastic-nginx! - captastic-subdomains! - foreman (~> 0.87.0) - -BUNDLED WITH - 1.17.2 \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index b6a699bf2..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,213 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "365")) - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - APP_IMAGE_REPO_DOMAIN = credentials("reopt-api-app-image-repo-domain") - DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain") - DEVELOPMENT_TEMP_BASE_DOMAIN = credentials("reopt-api-development-temp-base-domain") - STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain") - STAGING_TEMP_BASE_DOMAIN = credentials("reopt-api-staging-temp-base-domain") - PRODUCTION_DOMAIN = credentials("reopt-api-production-domain") - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - parameters { - booleanParam( - name: "DEVELOPMENT_DEPLOY", - defaultValue: false, - description: "Development Deploy: Deploy to development.", - ) - - booleanParam( - name: "STAGING_DEPLOY", - defaultValue: false, - description: "Staging Deploy: Deploy to staging for a non-master branch (master will always be deployed).", - ) - } - - stages { - stage("app-agent") { - agent { - dockerfile { - filename "Dockerfile" - additionalBuildArgs "--pull --build-arg XPRESS_LICENSE_HOST='${XPRESS_LICENSE_HOST}' --build-arg NREL_ROOT_CERT_URL_ROOT='${NREL_ROOT_CERT_URL_ROOT}'" - } - } - - stages { - stage("tests") { - steps { - sh "echo 'Container built and running.'" - } - } - } - } - - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_REPO = "${APP_IMAGE_REPO_DOMAIN}/tada/reopt-api" - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - LICENSESERVER_URL = credentials("reopt-api-xpress-licenseserver-url") - XPRESS_INSTALLED = "False" - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("deploy") { - stages { - // stage("solver setup") { - // steps { - // dir("julia_src/licenseserver") { - // git url: env.LICENSESERVER_URL - // } - // sh "cp julia_src/licenseserver/Dockerfile.xpress julia_src/" - // } - // } - - stage("lint") { - steps { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh "werf helm lint" - } - } - } - - stage("build") { - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - sh "werf build" - } - } - } - - stage("deploy-development") { - when { expression { params.DEVELOPMENT_DEPLOY } } - - environment { - DEPLOY_ENV = "development" - DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT = "6" - DEPLOY_APP_NAMESPACE_POD_LIMIT = "20" - } - - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfNamespaces(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh """ - werf converge \ - --values=./.helm/values.deploy.yaml \ - --values=./.helm/values.${DEPLOY_ENV}.yaml \ - --secret-values=./.helm/secret-values.${DEPLOY_ENV}.yaml \ - --set='ecrAwsAccessKeyId=${AWS_ACCESS_KEY_ID}' \ - --set='ecrAwsSecretAccessKey=${AWS_SECRET_ACCESS_KEY}' \ - --set='branchName=${BRANCH_NAME}' \ - --set='ingressHost=${DEPLOY_BRANCH_DOMAIN}' \ - --set='tempIngressHost=${tadaDeployBranchDomain(baseDomain: env.DEVELOPMENT_TEMP_BASE_DOMAIN, primaryBranch: "master")}' \ - --set='dbName=${DEPLOY_BRANCH_DB_NAME}' - """ - } - } - } - } - } - } - } - - stage("deploy-staging") { - when { expression { params.STAGING_DEPLOY || env.BRANCH_NAME == "master" } } - - environment { - DEPLOY_ENV = "staging" - DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT = "6" - DEPLOY_APP_NAMESPACE_POD_LIMIT = "20" - } - - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfNamespaces(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh """ - werf converge \ - --values=./.helm/values.deploy.yaml \ - --values=./.helm/values.${DEPLOY_ENV}.yaml \ - --secret-values=./.helm/secret-values.${DEPLOY_ENV}.yaml \ - --set='ecrAwsAccessKeyId=${AWS_ACCESS_KEY_ID}' \ - --set='ecrAwsSecretAccessKey=${AWS_SECRET_ACCESS_KEY}' \ - --set='branchName=${BRANCH_NAME}' \ - --set='ingressHost=${DEPLOY_BRANCH_DOMAIN}' \ - --set='tempIngressHost=${tadaDeployBranchDomain(baseDomain: env.STAGING_TEMP_BASE_DOMAIN, primaryBranch: "master")}' \ - --set='dbName=${DEPLOY_BRANCH_DB_NAME}' - """ - } - } - } - } - } - } - } - - stage("deploy-production") { - when { branch "master" } - - environment { - DEPLOY_ENV = "production" - DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT = "6" - DEPLOY_APP_NAMESPACE_POD_LIMIT = "70" - } - - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) { - tadaWithWerfNamespaces(rancherProject: "reopt-api-production", primaryBranch: "master") { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh """ - werf converge \ - --values=./.helm/values.deploy.yaml \ - --values=./.helm/values.${DEPLOY_ENV}.yaml \ - --secret-values=./.helm/secret-values.${DEPLOY_ENV}.yaml \ - --set='ecrAwsAccessKeyId=${AWS_ACCESS_KEY_ID}' \ - --set='ecrAwsSecretAccessKey=${AWS_SECRET_ACCESS_KEY}' \ - --set='ingressHost=${PRODUCTION_DOMAIN}' - """ - } - } - } - } - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-deploy b/Jenkinsfile-deploy deleted file mode 100644 index ec9fed809..000000000 --- a/Jenkinsfile-deploy +++ /dev/null @@ -1,77 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - choice( - name: "PARAM_STAGE", - choices: "development\nstaging\nproduction", - description: "Where do you want to deploy to?" - ), - - [ - $class: "GitParameterDefinition", - name: "PARAM_BRANCH", - type: "PT_BRANCH", - defaultValue: "origin/master", - sortMode: "ASCENDING_SMART", - selectedValue: "DEFAULT", - quickFilterEnabled: true, - ], - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - DEV_URL = credentials("reopt-api-dev-url") - STAGE_URL = credentials("reopt-api-stage-url") - STAGE1_URL = credentials("reopt-api1-stage-url") - STAGE2_URL = credentials("reopt-api2-stage-url") - STAGE_BASEDOMAIN_URL = credentials("reopt-api-stage-base-url") - PROD_URL = credentials("reopt-api-prod-url") - PROD1_URL = credentials("reopt-api1-prod-url") - PROD2_URL = credentials("reopt-api2-prod-url") - XPRESSDIR = "/opt/xpressmp" - } - - stages { - stage("checkout-deploy-branch") { - steps { - tadaCheckoutDeployBranch("https://github.com/NREL/REopt_API.git") - } - } - - stage("deploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branch: $PARAM_BRANCH" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - if(env.PARAM_STAGE == "development") { - // TODO: Remove this if we setup branched deployments for - // development (with real DNS subdomain support). - sh "bundle exec cap ${PARAM_STAGE} deploy --trace DEV_BRANCH=${PARAM_BRANCH} DEBUG_DEPLOY=true" - } else { - sh "bundle exec cap ${PARAM_STAGE} deploy --trace BRANCH=${PARAM_BRANCH} DEBUG_DEPLOY=true" - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-deploy-c110p b/Jenkinsfile-deploy-c110p deleted file mode 100644 index 68f7136f4..000000000 --- a/Jenkinsfile-deploy-c110p +++ /dev/null @@ -1,60 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - [ - $class: "GitParameterDefinition", - name: "PARAM_BRANCH", - type: "PT_BRANCH", - defaultValue: "origin/develop", - sortMode: "ASCENDING_SMART", - selectedValue: "DEFAULT", - quickFilterEnabled: true, - description: "Which branch do you want to deploy?" - ], - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - C110P_URL = credentials("reopt-api-c110p-url") - XPRESSDIR = "/opt/xpressmp" - PARAM_STAGE = "internal_c110p" - } - - stages { - stage("checkout-deploy-branch") { - steps { - tadaCheckoutDeployBranch("https://github.com/NREL/REopt_API.git") - } - } - - stage("deploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branch: $PARAM_BRANCH" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - sh "bundle exec cap ${PARAM_STAGE} deploy --trace BRANCH=${PARAM_BRANCH} DEBUG_DEPLOY=true" - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-image-cleanup b/Jenkinsfile-image-cleanup deleted file mode 100644 index 10b5c0951..000000000 --- a/Jenkinsfile-image-cleanup +++ /dev/null @@ -1,59 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "30")) - } - triggers { - cron(env.BRANCH_NAME == "main" ? "H H(0-5) * * *" : "") - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - APP_IMAGE_REPO_DOMAIN = credentials("reopt-api-app-image-repo-domain") - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_REPO = "${APP_IMAGE_REPO_DOMAIN}/tada/reopt-api" - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("cleanup") { - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - tadaRancherAllProjectNamespacesKubeConfig(projects: [[credentialsId: "kubeconfig-nrel-reopt-prod4", rancherProject: "reopt-api-dev"], [credentialsId: "kubeconfig-nrel-reopt-prod4", rancherProject: "reopt-api-staging"], [credentialsId: "kubeconfig-nrel-reopt-prod5", rancherProject: "reopt-api-production"]]) { - withCredentials([gitUsernamePassword(credentialsId: "github-nrel-gov-admin")]) { - sh "werf cleanup --scan-context-namespace-only" - } - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia deleted file mode 100644 index a463d10dc..000000000 --- a/Jenkinsfile-restart-celery-julia +++ /dev/null @@ -1,114 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent any - environment { - TZ = 'America/Denver' // MST/MDT (handles daylight saving automatically) - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain") - DEVELOPMENT_TEMP_BASE_DOMAIN = credentials("reopt-api-development-temp-base-domain") - STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain") - STAGING_TEMP_BASE_DOMAIN = credentials("reopt-api-staging-temp-base-domain") - PRODUCTION_DOMAIN = credentials("reopt-api-production-domain") - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - triggers { - cron('0 1 * * *') // 1:00 AM MST/MDT - } - - parameters { - booleanParam( - name: "DEVELOPMENT_DEPLOY", - defaultValue: false, - description: "Development Deploy: Deploy to development.", - ) - - booleanParam( - name: "STAGING_DEPLOY", - defaultValue: false, - description: "Staging Deploy: Deploy to staging for a non-master branch (master will always be deployed).", - ) - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - } - - stages { - stage('Rolling Restart Celery and Julia') { - stages { - stage("deploy-development") { - when { expression { params.DEVELOPMENT_DEPLOY } } - - environment { - DEPLOY_ENV = "development" - } - - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfEnv(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") { - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" - - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" - } - } - } - } - - stage("deploy-staging") { - when { expression { params.STAGING_DEPLOY || env.BRANCH_NAME == "master" } } - - environment { - DEPLOY_ENV = "staging" - } - - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfEnv(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") { - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" - - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" - } - } - } - } - - stage("deploy-production") { - when { branch "master" } - - environment { - DEPLOY_ENV = "production" - } - - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) { - tadaWithWerfEnv(rancherProject: "reopt-api-production", primaryBranch: "master") { - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" - - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" - } - } - } - } - } - } - } - } - } -} diff --git a/Jenkinsfile-undeploy-c110p b/Jenkinsfile-undeploy-c110p deleted file mode 100644 index 8ff8d304e..000000000 --- a/Jenkinsfile-undeploy-c110p +++ /dev/null @@ -1,48 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - string( - name: "PARAM_BRANCHES", - description: "Enter a space-deliminted list of branches you wish to undeploy." - ), - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - C110P_URL = credentials("reopt-api-c110p-url") - XPRESSDIR = "/opt/xpressmp" - PARAM_STAGE = "internal_c110p" - } - - stages { - stage("undeploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branches: $PARAM_BRANCHES" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - sh "for branch_name in ${PARAM_BRANCHES}; do bundle exec cap ${PARAM_STAGE} undeploy --trace BRANCH=\$branch_name CONFIRM_UNDEPLOY=true DEBUG_DEPLOY=true; done" - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-undeploy-develop b/Jenkinsfile-undeploy-develop deleted file mode 100644 index b0c9f9a86..000000000 --- a/Jenkinsfile-undeploy-develop +++ /dev/null @@ -1,76 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "30")) - } - triggers { - cron(env.BRANCH_NAME == "master" ? "H * * * *" : "") - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - } - - parameters { - booleanParam( - name: "UNDEPLOY_DELETED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been deleted from develop.", - ) - booleanParam( - name: "UNDEPLOY_MERGED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been merged into the master branch (but not deleted) from develop.", - ) - string( - name: "UNDEPLOY_BRANCH_NAMES", - defaultValue: "", - description: "Undeploy the listed branch names (space separated for multiple branches) from develop.", - ) - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - DEPLOY_ENV = "development" - RANCHER_PROJECT = "reopt-api-dev" - DB_BASE_NAME = "reopt_api_develop" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("undeploy-branches") { - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaUndeployEachBranch(rancherProject: env.RANCHER_PROJECT, undeployDeletedBranches: params.UNDEPLOY_DELETED_BRANCHES, undeployMergedBranches: params.UNDEPLOY_MERGED_BRANCHES, undeployBranchNames: params.UNDEPLOY_BRANCH_NAMES, primaryBranch: "master") { - tadaWithWerfEnv(rancherProject: env.RANCHER_PROJECT, dbBaseName: env.DB_BASE_NAME) { - tadaUndeployBranch() - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-undeploy-rhel b/Jenkinsfile-undeploy-rhel deleted file mode 100644 index ade1bbb90..000000000 --- a/Jenkinsfile-undeploy-rhel +++ /dev/null @@ -1,57 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - choice( - name: "PARAM_STAGE", - choices: "development\nstaging", - description: "Where do you want to deploy to?" - ), - - string( - name: "PARAM_BRANCHES", - description: "Enter a space-deliminted list of branches you wish to undeploy." - ), - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - DEV_URL = credentials("reopt-api-dev-url") - STAGE_URL = credentials("reopt-api-stage-url") - STAGE1_URL = credentials("reopt-api1-stage-url") - STAGE2_URL = credentials("reopt-api2-stage-url") - STAGE_BASEDOMAIN_URL = credentials("reopt-api-stage-base-url") - XPRESSDIR = "/opt/xpressmp" - } - - stages { - stage("undeploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branches: $PARAM_BRANCHES" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - sh "for branch_name in ${PARAM_BRANCHES}; do bundle exec cap ${PARAM_STAGE} undeploy --trace BRANCH=\$branch_name CONFIRM_UNDEPLOY=true DEBUG_DEPLOY=true; done" - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-undeploy-staging b/Jenkinsfile-undeploy-staging deleted file mode 100644 index 548e0045b..000000000 --- a/Jenkinsfile-undeploy-staging +++ /dev/null @@ -1,76 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "30")) - } - triggers { - cron(env.BRANCH_NAME == "master" ? "H * * * *" : "") - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - } - - parameters { - booleanParam( - name: "UNDEPLOY_DELETED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been deleted from staging.", - ) - booleanParam( - name: "UNDEPLOY_MERGED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been merged into the master branch (but not deleted) from staging.", - ) - string( - name: "UNDEPLOY_BRANCH_NAMES", - defaultValue: "", - description: "Undeploy the listed branch names (space separated for multiple branches) from staging.", - ) - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - DEPLOY_ENV = "staging" - RANCHER_PROJECT = "reopt-api-staging" - DB_BASE_NAME = "reopt_api_staging" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("undeploy-branches") { - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaUndeployEachBranch(rancherProject: env.RANCHER_PROJECT, undeployDeletedBranches: params.UNDEPLOY_DELETED_BRANCHES, undeployMergedBranches: params.UNDEPLOY_MERGED_BRANCHES, undeployBranchNames: params.UNDEPLOY_BRANCH_NAMES, primaryBranch: "master") { - tadaWithWerfEnv(rancherProject: env.RANCHER_PROJECT, dbBaseName: env.DB_BASE_NAME) { - tadaUndeployBranch() - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/checkdb.py b/checkdb.py index 8dbe50f31..684edf8fb 100644 --- a/checkdb.py +++ b/checkdb.py @@ -4,32 +4,13 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT env = os.getenv('APP_ENV') -dbname = os.getenv("DB_NAME") - -if env == 'development': - dbhost = dev_database_host - dbuser = dev_user - dbpass = dev_user_password -elif env == 'staging': - dbhost = staging_database_host - dbuser = staging_user - dbpass = staging_user_password -elif env == 'production': - dbhost = prod_database_host - dbuser = production_user - dbpass = production_user_password -else: - dbname = dev_database_name - dbhost = dev_database_host - dbuser = dev_user - dbpass = dev_user_password if env != 'production': conn = psycopg2.connect( dbname="postgres", - user=dbuser, - password=dbpass, - host=dbhost, + user=db_username, + password=db_password, + host=db_host, ) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) @@ -40,20 +21,20 @@ for dbn in cur.fetchall(): dbnames.append(dbn[0]) - if dbname not in dbnames: - cur.execute("CREATE DATABASE {};".format(dbname)) + if db_name not in dbnames: + cur.execute("CREATE DATABASE {};".format(db_name)) conn.commit() cur.close() conn.close() conn = psycopg2.connect( - dbname=dbname, - user=dbuser, - password=dbpass, - host=dbhost, + dbname=db_name, + user=db_username, + password=db_password, + host=db_host, ) cur = conn.cursor() cur.execute("CREATE SCHEMA reopt_api;") - cur.execute("ALTER SCHEMA reopt_api OWNER TO {};".format(dbuser)) + cur.execute("ALTER SCHEMA reopt_api OWNER TO {};".format(db_username)) conn.commit() cur.close() diff --git a/config/deploy.rb b/config/deploy.rb deleted file mode 100644 index 30995a2d4..000000000 --- a/config/deploy.rb +++ /dev/null @@ -1,55 +0,0 @@ -# config valid only for current version of Capistrano -lock "~> 3.12.0" - -set :application, "reopt_api" -set :repo_url, "https://github.com/NREL/REopt_API.git" - -# Set the base deployment directory. -set :deploy_to_base, "/srv/data/apps" - -# Set the user the web app runs as. -set :foreman_user, "www-data-local" -set :file_permissions_users, ["www-data-local"] - -# Symlink other directories across deploys. -set :linked_dirs, fetch(:linked_dirs, []).push("static/files", "tmp") - -# Allow the web user to write files for Xpress -set :file_permissions_paths, fetch(:file_permissions_paths, []).push("static/files") - -# Allow the web user to write files for Wind Data -set :file_permissions_paths, fetch(:file_permissions_paths, []).push("input_files") - -namespace :app do - task :pip_install do - on roles(:app) do - within release_path do - execute "virtualenv", "env", "--python=/bin/python3" - execute "./env/bin/pip3", "install", "-r", "requirements.txt" - execute "./env/bin/python", "-c", "'import julia; julia.install()'" - execute ". /opt/xpressmp/bin/xpvars.sh && julia --project='#{release_path}/julia_envs/Xpress/' -e 'import Pkg; Pkg.instantiate(); include(\"#{release_path}/julia_envs/Xpress/build_julia_image.jl\"); build_julia_image(\"#{release_path}\")'" - end - end - end - - task :keys do - on roles(:app) do - execute "ln", "-snf", "/etc/reopt-api-secrets/keys.py", "#{release_path}/keys.py" - end - end - - task :migrate do - on roles(:db) do - within release_path do - with "PATH" => "#{release_path}/env/bin:$PATH", "VIRTUAL_ENV" => "#{release_path}/env", "DJANGO_SETTINGS_MODULE" => fetch(:django_settings_module) do - execute "./env/bin/python", "manage.py", "migrate", "--noinput" - execute "./env/bin/python", "manage.py", "collectstatic", "--noinput" - end - end - end - end - - before "deploy:updated", "app:pip_install" - before "deploy:updated", "app:keys" - after "deploy:updated", "app:migrate" -end \ No newline at end of file diff --git a/config/deploy/development.rb b/config/deploy/development.rb deleted file mode 100644 index fc224d491..000000000 --- a/config/deploy/development.rb +++ /dev/null @@ -1,9 +0,0 @@ -server ENV.fetch("DEV_URL"), :user => "deploy", :roles => ["web", "app", "db"] - -set :app_env, "development" -set :django_settings_module, "reopt_api.dev_settings" -set :base_domain, ENV.fetch("DEV_URL") - -# TODO: Remove this if we get more flexible branched deployments setup (with -# our normal DNS subdomain support). -set :branch, ENV.fetch("DEV_BRANCH").gsub("origin/", "") \ No newline at end of file diff --git a/config/deploy/internal_c110p.rb b/config/deploy/internal_c110p.rb deleted file mode 100644 index 14344b71e..000000000 --- a/config/deploy/internal_c110p.rb +++ /dev/null @@ -1,5 +0,0 @@ -server ENV.fetch("C110P_URL"), :user => "deploy", :roles => ["web", "app", "db"] - -set :app_env, "internal_c110p" -set :django_settings_module, "reopt_api.internal_c110p_settings" -set :base_domain, ENV.fetch("C110P_URL") \ No newline at end of file diff --git a/config/deploy/production.rb b/config/deploy/production.rb deleted file mode 100644 index bcf8e4f1f..000000000 --- a/config/deploy/production.rb +++ /dev/null @@ -1,7 +0,0 @@ -server ENV.fetch("PROD1_URL"), :user => "deploy", :roles => ["web", "app", "db"] -server ENV.fetch("PROD2_URL"), :user => "deploy", :roles => ["web", "app"] - -set :app_env, "production" -set :django_settings_module, "reopt_api.production_settings" -set :base_domain, ENV.fetch("PROD_URL") -set :base_domain_aliases, [ENV.fetch("PROD1_URL"), ENV.fetch("PROD2_URL")] \ No newline at end of file diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb deleted file mode 100644 index fd85d18cd..000000000 --- a/config/deploy/staging.rb +++ /dev/null @@ -1,6 +0,0 @@ -server ENV.fetch("STAGE1_URL"), :user => "deploy", :roles => ["web", "app", "db"] - -set :app_env, "staging" -set :django_settings_module, "reopt_api.staging_settings" -set :base_domain, ENV.fetch("STAGE_BASEDOMAIN_URL") -set :base_domain_aliases, [ENV.fetch("STAGE_URL"), ENV.fetch("STAGE1_URL"), ENV.fetch("STAGE2_URL")] \ No newline at end of file diff --git a/config/deploy/templates/foreman/env.erb b/config/deploy/templates/foreman/env.erb deleted file mode 100644 index f4d8de2b3..000000000 --- a/config/deploy/templates/foreman/env.erb +++ /dev/null @@ -1,12 +0,0 @@ -APP_ENV=<%= fetch(:app_env) %> -# Set the queue name so that it's unique for separate branches on staging and -# development (by using "deploy_name") and so it's also separate for each -# individual server (by using "hostname"). This ensures branches have their own -# queues and also servers have their own queues (since currently the processing -# has to stay on a single server). -APP_QUEUE_NAME="<%= fetch(:deploy_name) %>-<%= host.hostname %>" -DEPLOY_CURRENT_PATH=<%= current_path %> -LD_PRELOAD=/usr/local/julia-1.3.1/lib/julia/libstdc++.so.6 -SOLVER=xpress -XPRESSDIR=/opt/xpressmp -JULIA_PROJECT="<%= current_path %>/julia_envs/Xpress" \ No newline at end of file diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index 879818060..b4d421ab2 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -37,14 +37,4 @@ # Set the appropriate DJANGO_SETTINGS_MODULE environment variable based on the # current environment. -env = os.environ['APP_ENV'] -if env == 'development': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.dev_settings'] -elif env == 'staging': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.staging_settings'] -elif env == 'production': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.production_settings'] -elif env == 'internal_c110p': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.internal_c110p_settings'] -else: - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.dev_settings'] +raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.settings'] diff --git a/docker-compose.nginx.yml b/docker-compose.nginx.yml index 78648fbc7..50a99d7fa 100644 --- a/docker-compose.nginx.yml +++ b/docker-compose.nginx.yml @@ -31,11 +31,15 @@ services: "celery -A reopt_api worker -l info" environment: - APP_ENV=local - - SQL_HOST=db-nginx - - SQL_PORT=5432 + - DB_HOST=db-nginx + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis-nginx - SOLVER=xpress - JULIA_HOST=julia-nginx + - NLR_API_KEY volumes: - .:/opt/reopt depends_on: @@ -51,11 +55,15 @@ services: && /opt/reopt/bin/wait-for-it.bash -t 0 julia-nginx:8081 -- python manage.py runserver 0.0.0.0:8000" environment: - APP_ENV=local - - SQL_HOST=db-nginx - - SQL_PORT=5432 + - DB_HOST=db-nginx + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis-nginx - SOLVER=xpress - JULIA_HOST=julia-nginx + - NLR_API_KEY depends_on: - db-nginx - redis-nginx diff --git a/docker-compose.nojulia.yml b/docker-compose.nojulia.yml index 556fe0b9e..0aee07a1a 100644 --- a/docker-compose.nojulia.yml +++ b/docker-compose.nojulia.yml @@ -32,10 +32,14 @@ services: "celery -A reopt_api worker -l info" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis - JULIA_HOST=host.docker.internal + - NLR_API_KEY volumes: - .:/opt/reopt depends_on: @@ -50,10 +54,14 @@ services: && python manage.py runserver 0.0.0.0:8000" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis - JULIA_HOST=host.docker.internal + - NLR_API_KEY depends_on: - db - redis diff --git a/docker-compose.yml b/docker-compose.yml index 49d3a3d32..730aa3d70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,9 +25,13 @@ services: "celery -A reopt_api worker -l info" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis + - NLR_API_KEY volumes: - .:/opt/reopt depends_on: @@ -42,9 +46,13 @@ services: && /opt/reopt/bin/wait-for-it.bash -t 0 julia:8081 -- python manage.py runserver 0.0.0.0:8000" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis + - NLR_API_KEY depends_on: - db - redis diff --git a/keys.py b/keys.py new file mode 100644 index 000000000..e568613d8 --- /dev/null +++ b/keys.py @@ -0,0 +1,18 @@ + +rollbar_access_token = os.getenv('SECRET_ROLLBAR_ACCESS_TOKEN', 'test') + +#get your individual key from here: https://developer.nrel.gov/docs/api-key/ and replace the DEMO_KEY with that +pvwatts_api_key = os.getenv('SECRET_PVWATTS_NLR_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) +developer_nrel_gov_key = os.getenv('SECRET_DEVELOPER_NLR_GOV_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) +ashrae_tmy_key = os.getenv('SECRET_ASHRAE_TMY_NLR_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) + + +secret_key_ = os.getenv('SECRET_DJANGO_SECRET_KEY', 'secret_key_test') + +db_host = os.getenv('DB_HOST', os.getenv('SECRET_DB_HOST', 'localhost')) +db_name = os.getenv('DB_NAME', os.getenv('SECRET_DB_NAME', 'reopt')) +db_username = os.getenv('DB_USERNAME', os.getenv('SECRET_DB_USERNAME', 'reopt_api')) +db_password = os.getenv('DB_PASSWORD', os.getenv('SECRET_DB_PASSWORD', 'reopt')) +db_search_path = os.getenv('DB_SEARCH_PATH', os.getenv('SECRET_DB_SEARCH_PATH', 'reopt_api')) +redis_host = os.getenv('REDIS_HOST', os.getenv('SECRET_REDIS_HOST', 'localhost')) +redis_password = os.getenv('REDIS_PASSWORD', os.getenv('SECRET_REDIS_PASSWORD', 'password')) diff --git a/manage.py b/manage.py index 4dec556fe..e00f2ce5e 100644 --- a/manage.py +++ b/manage.py @@ -4,7 +4,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.dev_settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.settings") from django.core.management import execute_from_command_line diff --git a/reopt_api/celery.py b/reopt_api/celery.py index e76436d3e..429cc2faf 100644 --- a/reopt_api/celery.py +++ b/reopt_api/celery.py @@ -7,45 +7,7 @@ from keys import * # set the default Django settings module for the 'celery' program. -try: - env = os.environ['APP_ENV'] - - if env == 'internal_c110p': - raw_env = 'reopt_api.internal_c110p_settings' - redis_host = ':' + dev_redis_password + '@localhost' - elif env == 'development': - raw_env = 'reopt_api.dev_settings' - if os.environ.get('K8S_DEPLOY') is None: - redis_host = ':' + dev_redis_password + '@' + dev_database_host - else: - redis_host = ':' + dev_redis_password + '@' + dev_redis_host - elif env == 'staging': - raw_env = 'reopt_api.staging_settings' - if os.environ.get('K8S_DEPLOY') is None: - redis_host = ':' + staging_redis_password + '@' + staging_database_host - else: - redis_host = ':' + staging_redis_password + '@' + staging_redis_host - elif env == 'production': - raw_env = 'reopt_api.production_settings' - if os.environ.get('K8S_DEPLOY') is None: - redis_host = ':' + production_redis_password + '@' + prod_database_host - else: - redis_host = ':' + production_redis_password + '@' + production_redis_host - else: - raw_env = 'reopt_api.dev_settings' - redis_host = os.environ.get('REDIS_HOST', 'localhost') - -except KeyError: - """ - This catch is necessary for running celery from command line when testing/developing locally. - APP_ENV is defined in config/deploy/[development, production, staging].rb files for servers. - For testing and local development, APP_ENV *can* be defined in .env file (see README.md), - which `honcho` or `foreman` loads before running Procfile. - """ - raw_env = 'reopt_api.dev_settings' - redis_host = os.environ.get('REDIS_HOST', 'localhost') - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', raw_env) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'reopt_api.settings') app = Celery('reopt_api') @@ -58,7 +20,11 @@ # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') -app.conf.broker_url = 'redis://' + redis_host + ':6379/0' +redis_auth = '' +if redis_password: + redis_auth = ':' + redis_password + '@' + +app.conf.broker_url = 'redis://' + redis_auth + redis_host + ':6379/0' # Create separate queues for each server (naming each queue after the server's # hostname). Since the worker jobs currently all have to be processes on the diff --git a/reopt_api/dev_settings.py b/reopt_api/dev_settings.py deleted file mode 100644 index ffb94d8a4..000000000 --- a/reopt_api/dev_settings.py +++ /dev/null @@ -1,165 +0,0 @@ -# REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * -import sys -import os -import django -import rollbar -""" -Django settings for reopt_api project. - -Generated by 'django-admin startproject' using Django 2.2. - -For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = secret_key_ - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False - -ALLOWED_HOSTS = ['*'] - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'reo', - 'summary', - 'tastypie', - 'proforma', - 'resilience_stats', - 'futurecosts', - 'django_celery_results', - 'django_extensions', - 'reoptjl', - 'ghpghx' - ) - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'rollbar.contrib.django.middleware.RollbarNotifierMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = 'reopt_api.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - - -WSGI_APPLICATION = 'reopt_api.wsgi.application' - -ROLLBAR = { - 'access_token': rollbar_access_token, - 'environment': 'development', - 'root': BASE_DIR, - 'enabled':True -} - -# Database -if 'test' in sys.argv or os.environ.get('APP_ENV') == 'local': - ROLLBAR['enabled'] = False - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'reopt', - 'USER': 'reopt', - 'PASSWORD': 'reopt', - 'OPTIONS': { - 'options': '-c search_path=public' - }, - "HOST": os.environ.get("SQL_HOST", "localhost"), - "PORT": os.environ.get("SQL_PORT", "5432"), - } -} -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': dev_database_host, - 'NAME': dev_database_name, - 'OPTIONS': { - 'options': '-c search_path=reopt_api' - }, - 'USER': dev_user, - 'PASSWORD': dev_user_password, - } -} - -# Internationalization -# https://docs.djangoproject.com/en/1.8/topics/i18n/ -rollbar.init(**ROLLBAR) - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Results backend -CELERY_RESULT_BACKEND = 'django-db' - -# celery task registration -CELERY_IMPORTS = ( - 'reo.api', - 'reo.scenario', - 'reo.process_results', - 'reo.src.run_jump_model', - 'resilience_stats.outage_simulator_LF', - 'futurecosts.api', - 'futurecosts.tasks', - 'reoptjl.api', - 'reoptjl.src.run_jump_model' -) - -if 'test' in sys.argv: - CELERY_TASK_ALWAYS_EAGER = True - CELERY_TASK_EAGER_PROPAGATES_EXCEPTIONS = False - -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 4 GB - -# Static files (used for Proforma xlsx) -STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/static/' - -APPEND_SLASH = False -TASTYPIE_ALLOW_MISSING_SLASH = True -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.dev_settings") -django.setup() diff --git a/reopt_api/internal_c110p_settings.py b/reopt_api/internal_c110p_settings.py deleted file mode 100755 index d0f41995e..000000000 --- a/reopt_api/internal_c110p_settings.py +++ /dev/null @@ -1,135 +0,0 @@ -# REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * -import sys -""" -Django settings for reopt_api project. - -Generated by 'django-admin startproject' using Django 1.8. - -For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -import os -import django - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = secret_key_ - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - -# Application definition -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'reo', - 'summary', - 'tastypie', - 'proforma', - 'resilience_stats', - 'django_celery_results', - 'django_extensions', - 'ghpghx' - ) - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = 'reopt_api.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - - -WSGI_APPLICATION = 'reopt_api.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/2.2/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': 'localhost', - 'NAME': 'reopt_development', - 'OPTIONS': { - 'options': '-c search_path=reopt_api' - }, - 'USER': dev_user, - 'PASSWORD': dev_user_password, - } -} - -# Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Results backend -CELERY_RESULT_BACKEND = 'django-db' -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 6000000 # 6 GB - -# celery task registration -CELERY_IMPORTS = ( - 'reo.api', - 'reo.scenario', - 'reo.process_results', - 'reo.src.run_jump_model', - 'resilience_stats.outage_simulator_LF', - 'django_extensions', - 'ghpghx' -) - -if 'test' in sys.argv: - CELERY_TASK_ALWAYS_EAGER = True - CELERY_TASK_EAGER_PROPAGATES_EXCEPTIONS = False - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/2.2/howto/static-files/ -STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/static/' - -APPEND_SLASH = False -TASTYPIE_ALLOW_MISSING_SLASH = True - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.internal_c110p_settings") -django.setup() diff --git a/reopt_api/production_settings.py b/reopt_api/settings.py similarity index 79% rename from reopt_api/production_settings.py rename to reopt_api/settings.py index 2574c6557..0b5da339a 100644 --- a/reopt_api/production_settings.py +++ b/reopt_api/settings.py @@ -16,6 +16,8 @@ https://docs.djangoproject.com/en/2.2/ref/settings/ """ +APP_ENV = os.getenv('APP_ENV', os.getenv('DEPLOY_ENV', 'development')) + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -87,15 +89,23 @@ DATABASES = { 'default':{ 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': prod_database_host, - 'NAME': prod_database_name, + 'HOST': db_host, + 'NAME': db_name, 'OPTIONS': { - 'options': '-c search_path=reopt_api' + 'options': '-c search_path=' + db_search_path }, - 'USER': production_user, - 'PASSWORD': production_user_password, + 'USER': db_username, + 'PASSWORD': db_password, } } +if 'test' in sys.argv or APP_ENV == 'local': + DATABASES['default']['NAME'] = 'reopt' + DATABASES['default']['USER'] = 'reopt' + DATABASES['default']['PASSWORD'] = 'reopt' + DATABASES['default']['OPTIONS'] = { + 'options': '-c search_path=public' + } + # Internationalization # https://docs.djangoproject.com/en/2.2/topics/i18n/ @@ -113,7 +123,15 @@ # Results backend CELERY_RESULT_BACKEND = 'django-db' -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 4 GB +if APP_ENV == 'staging' + CELERY_WORKER_MAX_MEMORY_PER_CHILD = 6000000 # 6 GB +else + CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 4 GB + +if 'test' in sys.argv: + CELERY_TASK_ALWAYS_EAGER = True + CELERY_TASK_EAGER_PROPAGATES_EXCEPTIONS = False + # we have been resetting celery workers by max memory due to a memory growth problem. # this problem may be fixed by removing PyJulia, but can't view Rancher metrics yet. # Once we can confirm that we no longer have a memory grwoth issue we can disable this setting. @@ -141,14 +159,19 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.2/howto/static-files/ STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/' +if APP_ENV == 'development': + STATIC_URL = '/static/' +else: + STATIC_URL = '/' ROLLBAR = { 'access_token': rollbar_access_token, - 'environment': 'production', + 'environment': APP_ENV, 'root': BASE_DIR, 'branch': os.environ.get('BRANCH_NAME') } +if 'test' in sys.argv or APP_ENV == 'local': + ROLLBAR['enabled'] = False rollbar.init(**ROLLBAR) @@ -156,5 +179,5 @@ TASTYPIE_ALLOW_MISSING_SLASH = True DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.production_settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.settings") django.setup() diff --git a/reopt_api/staging_settings.py b/reopt_api/staging_settings.py deleted file mode 100755 index af4f61d4b..000000000 --- a/reopt_api/staging_settings.py +++ /dev/null @@ -1,159 +0,0 @@ -# REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * -import os -import django -import rollbar -""" -Django settings for reopt_api project. - -Generated by 'django-admin startproject' using Django 2.2. - -For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = secret_key_ - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False - -ALLOWED_HOSTS = ['*'] - - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'reo', - 'summary', - 'tastypie', - 'proforma', - 'resilience_stats', - 'futurecosts', - 'django_celery_results', - 'django_extensions', - 'reoptjl', - 'ghpghx' -) - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'rollbar.contrib.django.middleware.RollbarNotifierMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = 'reopt_api.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - - -WSGI_APPLICATION = 'reopt_api.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/2.2/ref/settings/#databases - -DATABASES = { - 'default':{ - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': staging_database_host, - 'NAME': staging_database_name, - 'OPTIONS': { - 'options': '-c search_path=reopt_api' - }, - 'USER': staging_user, - 'PASSWORD': staging_user_password, - } -} - -# Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Results backend -CELERY_RESULT_BACKEND = 'django-db' - -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 6 GB -# we have been resetting celery workers by max memory due to a memory growth problem. -# this problem may be fixed by removing PyJulia, but can't view Rancher metrics yet. -# Once we can confirm that we no longer have a memory grwoth issue we can disable this setting. -# It has to be set according to the RAM available. - -# limit number of concurrent workers -CELERY_WORKER_CONCURRENCY = 1 -# controlling number of celery workers with number of celery pods - -# celery task registration -CELERY_IMPORTS = ( - 'reo.api', - 'reo.scenario', - 'reo.process_results', - 'reo.src.run_jump_model', - 'resilience_stats.outage_simulator_LF', - 'futurecosts.api', - 'futurecosts.tasks', - 'django_extensions', - 'reoptjl.api', - 'reoptjl.src.run_jump_model', - 'ghpghx' -) - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/2.2/howto/static-files/ -STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/' - -ROLLBAR = { - 'access_token': rollbar_access_token, - 'environment': 'staging', - 'root': BASE_DIR, - 'branch': os.environ.get('BRANCH_NAME') -} - -rollbar.init(**ROLLBAR) - -APPEND_SLASH = False -TASTYPIE_ALLOW_MISSING_SLASH = True -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.staging_settings") -django.setup() diff --git a/transcrypt/.editorconfig b/transcrypt/.editorconfig deleted file mode 100644 index a35ef6e8a..000000000 --- a/transcrypt/.editorconfig +++ /dev/null @@ -1,16 +0,0 @@ -root = true - -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true - -[*.md] -indent_size = 4 -indent_style = space -trim_trailing_whitespace = false - -[transcrypt] -indent_style = tab -tab_width = 4 diff --git a/transcrypt/.gitattributes b/transcrypt/.gitattributes deleted file mode 100644 index db8bb76a8..000000000 --- a/transcrypt/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -sensitive_file filter=crypt diff=crypt diff --git a/transcrypt/INSTALL.md b/transcrypt/INSTALL.md deleted file mode 100644 index 2e7350bd4..000000000 --- a/transcrypt/INSTALL.md +++ /dev/null @@ -1,61 +0,0 @@ -Install transcrypt -================== - -The requirements to run transcrypt are minimal: - -* Bash -* Git -* OpenSSL - -You also need access to the _transcrypt_ script itself... - -Manual Installation -------------------- - -You can add transcrypt directly to your repository, or just put it somewhere in -your $PATH: - - $ git clone https://github.com/elasticdog/transcrypt.git - $ cd transcrypt/ - $ sudo ln -s ${PWD}/transcrypt /usr/local/bin/transcrypt - -Installation via Packages -------------------------- - -A number of packages are available for installing transcrypt directly on your -system via its native package manager. Some of these packages also include man -page documentation as well as shell auto-completion scripts. - -### Arch Linux - -If you're on Arch Linux, you can build/install transcrypt using the -[provided PKGBUILD](https://github.com/elasticdog/transcrypt/blob/master/contrib/packaging/pacman/PKGBUILD): - - $ git clone https://github.com/elasticdog/transcrypt.git - $ cd transcrypt/contrib/packaging/pacman/ - $ makepkg -sic - -### Heroku - -If you're running software on Heroku, you can integrate transcrypt into your -slug compilation phase by using the -[transcrypt buildpack](https://github.com/perplexes/heroku-buildpack-transcrypt), -developed by [Colin Curtin](https://github.com/perplexes). - -### NixOS - -If you're on NixOS, you can install transcrypt directly via -[Nix](https://nixos.org/nix/): - - $ nix-env -iA nixos.gitAndTools.transcrypt - -> _**Note:** -> The [transcrypt derivation](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/version-management/git-and-tools/transcrypt/default.nix) -> was added in Oct 2015, so it is not available on the 15.09 channel._ - -### OS X - -If you're on OS X, you can install transcrypt directly via -[Homebrew](http://brew.sh/): - - $ brew install transcrypt diff --git a/transcrypt/LICENSE b/transcrypt/LICENSE deleted file mode 100644 index 5ed94e0bb..000000000 --- a/transcrypt/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2014-2019, Aaron Bull Schaefer -Copyright (c) 2011, Woody Gilk - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/transcrypt/README.md b/transcrypt/README.md deleted file mode 100644 index 15136f1ce..000000000 --- a/transcrypt/README.md +++ /dev/null @@ -1,296 +0,0 @@ -# transcrypt - -A script to configure transparent encryption of sensitive files stored in a Git -repository. Files that you choose will be automatically encrypted when you -commit them, and automatically decrypted when you check them out. The process -will degrade gracefully, so even people without your encryption password can -safely commit changes to the repository's non-encrypted files. - -transcrypt protects your data when it's pushed to remotes that you may not -directly control (e.g., GitHub, Dropbox clones, etc.), while still allowing you -to work normally on your local working copy. You can conveniently store things -like passwords and private keys within your repository and not have to share -them with your entire team or complicate your workflow. - -## Overview - -transcrypt is in the same vein as existing projects like -[git-crypt](https://github.com/AGWA/git-crypt) and -[git-encrypt](https://github.com/shadowhand/git-encrypt), which follow Git's -documentation regarding the use of clean/smudge filters for encryption. In -comparison to those other projects, transcrypt makes substantial improvements in -the areas of usability and safety. - -- transcrypt is just a Bash script and does not require compilation -- transcrypt uses OpenSSL's symmetric cipher routines rather than implementing - its own crypto -- transcrypt does not have to remain installed after the initial repository - configuration -- transcrypt generates a unique salt for each encrypted file -- transcrypt uses safety checks to avoid clobbering or duplicating configuration - data -- transcrypt facilitates setting up additional clones as well as rekeying -- transcrypt adds an alias `git ls-crypt` to list all encrypted files - -### Salt Generation - -The _decryption -> encryption_ process on an unchanged file must be -deterministic for everything to work transparently. To do that, the same salt -must be used each time we encrypt the same file. Rather than use a static salt -common to all files, transcrypt first has OpenSSL generate an HMAC-SHA256 -cryptographic hash-based message authentication code for each decrypted file -(keyed with a combination of the filename and transcrypt password), and then -uses the last 16 bytes of that HMAC for the file's unique salt. When the content -of the file changes, so does the salt. Since an -[HMAC has been proven to be a PRF](http://cseweb.ucsd.edu/~mihir/papers/hmac-new.html), -this method of salt selection does not leak information about the original -contents, but is still deterministic. - -## Usage - -The requirements to run transcrypt are minimal: - -- Bash -- Git -- OpenSSL - -...and optionally: - -- GnuPG - for secure configuration import/export - -You also need access to the _transcrypt_ script itself. You can add it directly -to your repository, or just put it somewhere in your \$PATH: - - $ git clone https://github.com/elasticdog/transcrypt.git - $ cd transcrypt/ - $ sudo ln -s ${PWD}/transcrypt /usr/local/bin/transcrypt - -#### Installation via Packages - -A number of packages are available for installing transcrypt directly on your -system via its native package manager. Some of these packages also include man -page documentation as well as shell auto-completion scripts. - -- Arch Linux -- Heroku (via [Buildpacks](https://devcenter.heroku.com/articles/buildpacks)) -- NixOS -- OS X (via [Homebrew](http://brew.sh/)) - -...see the [INSTALL document](INSTALL.md) for more details. - -### Initialize an Unconfigured Repository - -transcrypt will interactively prompt you for the required information, all you -have to do run the script within a Git repository: - - $ cd / - $ transcrypt - -If you already know the values you want to use, you can specify them directly -using the command line options. Run `transcrypt --help` for more details. - -### Designate a File to be Encrypted - -Once a repository has been configured with transcrypt, you can designate for -files to be encrypted by applying the "crypt" filter and diff to a -[pattern](https://www.kernel.org/pub/software/scm/git/docs/gitignore.html#_pattern_format) -in the top-level _[.gitattributes](http://git-scm.com/docs/gitattributes)_ -config. If that pattern matches a file in your repository, the file will be -transparently encrypted once you stage and commit it: - - $ cd / - $ echo 'sensitive_file filter=crypt diff=crypt' >> .gitattributes - $ git add .gitattributes sensitive_file - $ git commit -m 'Add encrypted version of a sensitive file' - -The _.gitattributes_ file should be committed and tracked along with everything -else in your repository so clones will be aware of what is encrypted. Make sure -you don't accidentally add a pattern that would encrypt this file :-) - -> For your reference, if you find the above description confusing, you'll find -> that this repository has been configured following these exact steps. - -### Listing the Currently Encrypted Files - -For convenience, transcrypt also adds a Git alias to allow you to list all of -the currently encrypted files in a repository: - - $ git ls-crypt - sensitive_file - -Alternatively, you can use the `--list` command line option: - - $ transcrypt --list - sensitive_file - -You can also use this to verify your _.gitattributes_ patterns when designating -new files to be encrypted, as the alias will list pattern matches as long as -everything has been staged (via `git add`). - -After committing things, but before you push to a remote repository, you can -validate that files are encrypted as expected by viewing them in their raw form: - - $ git show HEAD: --no-textconv - -The `` in the above command must be relative to the _top-level_ of -the repository. Alternatively, you can use the `--show-raw` command line option -and provide a path relative to your current directory: - - $ transcrypt --show-raw sensitive_file - -### Initialize a Clone of a Configured Repository - -If you have just cloned a repository containing files that are encrypted, you'll -want to configure transcrypt with the same cipher and password as the origin -repository. The owner of the origin repository can dump the credentials for you -by running the `--display` command line option: - - $ transcrypt --display - The current repository was configured using transcrypt v0.2.0 - and has the following configuration: - - CIPHER: aes-256-cbc - PASSWORD: correct horse battery staple - - Copy and paste the following command to initialize a cloned repository: - - transcrypt -c aes-256-cbc -p 'correct horse battery staple' - -Once transcrypt has stored the matching credentials, it will force a checkout of -any exising encrypted files in order to decrypt them. - -### Rekeying - -Periodically, you may want to change the encryption cipher or password used to -encrypt the files in your repository. You can do that easily with transcrypt's -rekey option: - - $ transcrypt --rekey - -> As a warning, rekeying will remove your ability to see historical diffs of the -> encrypted files in plain text. Changes made with the new key will still be -> visible, and you can always see the historical diffs in encrypted form by -> disabling the text conversion filters: -> -> $ git log --patch --no-textconv - -After rekeying, all clones of your repository should flush their transcrypt -credentials, fetch and merge the new encrypted files via Git, and then -re-configure transcrypt with the new credentials. - - $ transcrypt --flush-credentials - $ git fetch origin - $ git merge origin/master - $ transcrypt -c aes-256-cbc -p 'the-new-password' - -### Command Line Options - -Completion scripts for both Bash and Zsh are included in the _contrib/_ -directory. - - transcrypt [option...] - - -c, --cipher=CIPHER - the symmetric cipher to utilize for encryption; - defaults to aes-256-cbc - - -p, --password=PASSWORD - the password to derive the key from; - defaults to 30 random base64 characters - - -y, --yes - assume yes and accept defaults for non-specified options - - -d, --display - display the current repository's cipher and password - - -r, --rekey - re-encrypt all encrypted files using new credentials - - -f, --flush-credentials - remove the locally cached encryption credentials and re-encrypt - any files that had been previously decrypted - - -F, --force - ignore whether the git directory is clean, proceed with the - possibility that uncommitted changes are overwritten - - -u, --uninstall - remove all transcrypt configuration from the repository and - leave files in the current working copy decrypted - - -l, --list - list all of the transparently encrypted files in the repository, - relative to the top-level directory - - -s, --show-raw=FILE - show the raw file as stored in the git commit object; use this - to check if files are encrypted as expected - - -e, --export-gpg=RECIPIENT - export the repository's cipher and password to a file encrypted - for a gpg recipient - - -i, --import-gpg=FILE - import the password and cipher from a gpg encrypted file - - -v, --version - print the version information - - -h, --help - view this help message - -## Caveats - -### Overhead - -The method of using filters to selectively encrypt/decrypt files does add some -overhead to Git by regularly forking OpenSSL processes and removing Git's -ability to efficiently cache file changes. That said, it's not too different -from tracking binary files, and when used as intended, transcrypt should not -noticeably impact performance. There are much better options if your goal is to -encrypt the entire repository. - -### Localhost - -Note that the configuration and encryption information is stored in plain text -within the repository's _.git/config_ file. This prevents them from being -transferred to remote clones, but they are not protected from inquisitive users -on your local machine. - -For safety, you may prefer to only have the credentials stored when actually -updating encrypted files, and then flush them with `--flush-credentials` once -you're done (make sure you have the credentials backed up elsewhere!). This will -also revert any decrypted files back to their encrypted form in your local -working copy. - -### Cipher Selection - -Last up, regarding the default cipher choice of `aes-256-cbc`...there aren't any -fantastic alternatives without pulling in outside dependencies. Ideally, we -would use an authenticated cipher mode like `id-aes256-GCM` by default, but -there are a couple of issues: - -1. I'd like to support OS X out of the box, and unfortunately they are the - lowest common denominator when it comes to OpenSSL. For whatever reason, they - still include OpenSSL 0.9.8y rather than a newer release. Unfortunately, - GCM-based ciphers weren't added until OpenSSL 1.0.1 (back in early 2012). - -2. Even with newer versions of OpenSSL, the authenticated cipher modes - [don't work exactly right](http://openssl.6102.n7.nabble.com/id-aes256-GCM-command-line-encrypt-decrypt-fail-td27187.html) - when utilizing the command line `openssl enc`. - -I'm contemplating if transcrypt should append an HMAC to the `aes-256-cbc` -ciphertext to provide authentication, or if we should live with the -[malleability issues](http://www.jakoblell.com/blog/2013/12/22/practical-malleability-attack-against-cbc-encrypted-luks-partitions/) -as a known limitation. Essentially, malicious comitters without the transcrypt -password could potentially manipulate the plaintext in limited ways (given that -the attacker knows the original plaintext). Honestly, I'm not sure if the added -complexity here would be worth it given transcrypt's use case. - -## License - -transcrypt is provided under the terms of the -[MIT License](https://en.wikipedia.org/wiki/MIT_License). - -Copyright © 2014-2019, [Aaron Bull Schaefer](mailto:aaron@elasticdog.com). diff --git a/transcrypt/contrib/bash/transcrypt b/transcrypt/contrib/bash/transcrypt deleted file mode 100644 index 9a49f485a..000000000 --- a/transcrypt/contrib/bash/transcrypt +++ /dev/null @@ -1,55 +0,0 @@ -# completion script for transcrypt - -_files_and_dirs() { - local IFS=$'\n' - local LASTCHAR=' ' - - COMPREPLY=( $(compgen -o plusdirs -f -- "${COMP_WORDS[COMP_CWORD]}") ) - - if [[ ${#COMPREPLY[@]} -eq 1 ]]; then - [[ -d "$COMPREPLY" ]] && LASTCHAR='/' - COMPREPLY=$(printf '%q%s' "$COMPREPLY" "$LASTCHAR") - else - for ((i=0; i < ${#COMPREPLY[@]}; i++)); do - [[ -d "${COMPREPLY[$i]}" ]] && COMPREPLY[$i]=${COMPREPLY[$i]}/ - done - fi -} - -_transcrypt() { - local cur prev opts - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - opts="-c -p -y -d -r -f -F -u -l -s -e -i -v -h \ - --cipher --password --yes --display --rekey --flush-credentials --force --uninstall --list --show-raw --export-gpg --import-gpg --version --help" - - case "${prev}" in - -c | --cipher) - local ciphers=$(openssl list-cipher-commands) - COMPREPLY=( $(compgen -W "${ciphers}" -- ${cur}) ) - return 0 - ;; - -p | --password) - return 0 - ;; - -s | --show-raw) - _files_and_dirs - return 0 - ;; - -e | --export-gpg) - return 0 - ;; - -i | --import-gpg) - _files_and_dirs - return 0 - ;; - *) - ;; - esac - - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - COMPREPLY=$(printf '%q%s' "$COMPREPLY" ' ') -} - -complete -o nospace -F _transcrypt transcrypt diff --git a/transcrypt/contrib/packaging/pacman/.gitignore b/transcrypt/contrib/packaging/pacman/.gitignore deleted file mode 100644 index 11fca2d4d..000000000 --- a/transcrypt/contrib/packaging/pacman/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# ignore everything -* - -# except these files -!.gitignore -!PKGBUILD diff --git a/transcrypt/contrib/packaging/pacman/PKGBUILD b/transcrypt/contrib/packaging/pacman/PKGBUILD deleted file mode 100644 index 92976650d..000000000 --- a/transcrypt/contrib/packaging/pacman/PKGBUILD +++ /dev/null @@ -1,23 +0,0 @@ -# Maintainer: Aaron Bull Schaefer -pkgname=transcrypt -pkgver=2.0.0 -pkgrel=1 -pkgdesc='A script to configure transparent encryption of files within a Git repository' -arch=('any') -url='https://github.com/elasticdog/transcrypt' -license=('MIT') -depends=('git' 'openssl') -optdepends=('gnupg: config import/export support') -source=("https://github.com/elasticdog/${pkgname}/archive/v${pkgver}.tar.gz") -sha256sums=('12b891bcee50c71f5ee00c3c3e992c591ad6146ece3d3c5efa065d966a010d65') - -package() { - cd "${pkgname}-${pkgver}/" - - install -m 755 -D transcrypt "${pkgdir}/usr/bin/transcrypt" - install -m 644 -D man/transcrypt.1 "${pkgdir}/usr/share/man/man1/transcrypt.1" - install -m 644 -D LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" - - install -m 644 -D contrib/bash/transcrypt "${pkgdir}/usr/share/bash-completion/completions/transcrypt" - install -m 644 -D contrib/zsh/_transcrypt "${pkgdir}/usr/share/zsh/site-functions/_transcrypt" -} diff --git a/transcrypt/contrib/zsh/_transcrypt b/transcrypt/contrib/zsh/_transcrypt deleted file mode 100644 index 6bf4a92a4..000000000 --- a/transcrypt/contrib/zsh/_transcrypt +++ /dev/null @@ -1,37 +0,0 @@ -#compdef transcrypt - -_transcrypt() { - local curcontext="$curcontext" state line - typeset -A opt_args - - _arguments \ - '(- 1 *)'{-l,--list}'[list encrypted files]' \ - '(- 1 *)'{-s,--show-raw=}'[show raw file]:file:->file' \ - '(- 1 *)'{-e,--export-gpg=}'[export config to gpg recipient]:recipient:' \ - '(- 1 *)'{-v,--version}'[print version]' \ - '(- 1 *)'{-h,--help}'[view help message]' \ - '(-c --cipher -d --display -f --flush-credentials -u --uninstall)'{-c,--cipher=}'[specify encryption cipher]:cipher:->cipher' \ - '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ - '(-y --yes)'{-y,--yes}'[assume yes and accept defaults]' \ - '(-d --display -p --password -c --cipher -r --rekey -u --uninstall)'{-d,--display}'[display current credentials]' \ - '(-r --rekey -d --display -f --flush-credentials -u --uninstall)'{-r,--rekey}'[rekey all encrypted files]' \ - '(-f --flush-credentials -c --cipher -p --password -r --rekey -u --uninstall)'{-f,--flush-credentials}'[flush cached credentials]' \ - '(-F --force -d --display -u --uninstall)'{-F,--force}'[ignore repository clean state]' \ - '(-u --uninstall -c --cipher -d --display -f --flush-credentials -p --password -r --rekey)'{-u,--uninstall}'[uninstall transcrypt]' \ - '(-i --import-gpg -c --cipher -p --password -d --display -f --flush-credentials -u --uninstall)'{-i,--import-gpg=}'[import config from gpg file]:file:->file' \ - && return 0 - - case $state in - cipher) - ciphers=( ${(f)"$(_call_program available-ciphers openssl list-cipher-commands)"} ) - _describe -t available-ciphers 'available ciphers' ciphers - ;; - file) - _path_files - ;; - esac -} - -_transcrypt "$@" - -return 1 diff --git a/transcrypt/man/transcrypt.1 b/transcrypt/man/transcrypt.1 deleted file mode 100644 index 7b1d9798a..000000000 --- a/transcrypt/man/transcrypt.1 +++ /dev/null @@ -1,118 +0,0 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "TRANSCRYPT" "1" "August 2016" "" "" -. -.SH "NAME" -\fBtranscrypt\fR \- transparently encrypt files within a git repository -. -.SH "SYNOPSIS" -\fBtranscrypt\fR [\fIoptions\fR\.\.\.] -. -.SH "DESCRIPTION" -transcrypt will configure a Git repository to support the transparent encryption/decryption of files by utilizing OpenSSL\'s symmetric cipher routines and Git\'s built\-in clean/smudge filters\. It will also add a Git alias "ls\-crypt" to list all transparently encrypted files within the repository\. -. -.P -The transcrypt source code and full documentation may be downloaded from \fIhttps://github\.com/elasticdog/transcrypt\fR\. -. -.SH "OPTIONS" -. -.TP -\fB\-c\fR, \fB\-\-cipher\fR=\fIcipher\fR -the symmetric cipher to utilize for encryption; defaults to aes\-256\-cbc -. -.TP -\fB\-p\fR, \fB\-\-password\fR=\fIpassword\fR -the password to derive the key from; defaults to 30 random base64 characters -. -.TP -\fB\-y\fR, \fB\-\-yes\fR -assume yes and accept defaults for non\-specified options -. -.TP -\fB\-d\fR, \fB\-\-display\fR -display the current repository\'s cipher and password -. -.TP -\fB\-r\fR, \fB\-\-rekey\fR -re\-encrypt all encrypted files using new credentials -. -.TP -\fB\-f\fR, \fB\-\-flush\-credentials\fR -remove the locally cached encryption credentials and re\-encrypt any files that had been previously decrypted -. -.TP -\fB\-F\fR, \fB\-\-force\fR -ignore whether the git directory is clean, proceed with the possibility that uncommitted changes are overwritten -. -.TP -\fB\-u\fR, \fB\-\-uninstall\fR -remove all transcrypt configuration from the repository and leave files in the current working copy decrypted -. -.TP -\fB\-l\fR, \fB\-\-list\fR -list all of the transparently encrypted files in the repository, relative to the top\-level directory -. -.TP -\fB\-s\fR, \fB\-\-show\-raw\fR=\fIfile\fR -show the raw file as stored in the git commit object; use this to check if files are encrypted as expected -. -.TP -\fB\-e\fR, \fB\-\-export\-gpg\fR=\fIrecipient\fR -export the repository\'s cipher and password to a file encrypted for a gpg recipient -. -.TP -\fB\-i\fR, \fB\-\-import\-gpg\fR=\fIfile\fR -import the password and cipher from a gpg encrypted file -. -.TP -\fB\-v\fR, \fB\-\-version\fR -print the version information -. -.TP -\fB\-h\fR, \fB\-\-help\fR -view this help message -. -.SH "EXAMPLES" -To initialize a Git repository to support transparent encryption, just change into the repo and run the transcrypt script\. transcrypt will prompt you interactively for all required information if the corresponding option flags were not given\. -. -.IP "" 4 -. -.nf - -$ cd / -$ transcrypt -. -.fi -. -.IP "" 0 -. -.P -Once a repository has been configured with transcrypt, you can transparently encrypt files by applying the "crypt" filter and diff to a pattern in the top\-level \fI\.gitattributes\fR config\. If that pattern matches a file in your repository, the file will be transparently encrypted once you stage and commit it: -. -.IP "" 4 -. -.nf - -$ echo \'sensitive_file filter=crypt diff=crypt\' >> \.gitattributes -$ git add \.gitattributes sensitive_file -$ git commit \-m \'Add encrypted version of a sensitive file\' -. -.fi -. -.IP "" 0 -. -.P -See the gitattributes(5) man page for more information\. -. -.P -If you have just cloned a repository containing files that are encrypted, you\'ll want to configure transcrypt with the same cipher and password as the origin repository\. Once transcrypt has stored the matching credentials, it will force a checkout of any existing encrypted files in order to decrypt them\. -. -.P -If the origin repository has just rekeyed, all clones should flush their transcrypt credentials, fetch and merge the new encrypted files via Git, and then re\-configure transcrypt with the new credentials\. -. -.SH "AUTHOR" -Aaron Bull Schaefer -. -.SH "SEE ALSO" -enc(1), gitattributes(5) diff --git a/transcrypt/man/transcrypt.1.ronn b/transcrypt/man/transcrypt.1.ronn deleted file mode 100644 index fb6a7d03b..000000000 --- a/transcrypt/man/transcrypt.1.ronn +++ /dev/null @@ -1,107 +0,0 @@ -transcrypt(1) -- transparently encrypt files within a git repository -==================================================================== - -## SYNOPSIS - -`transcrypt` [...] - -## DESCRIPTION - -transcrypt will configure a Git repository to support the transparent -encryption/decryption of files by utilizing OpenSSL's symmetric cipher routines -and Git's built-in clean/smudge filters. It will also add a Git alias -"ls-crypt" to list all transparently encrypted files within the repository. - -The transcrypt source code and full documentation may be downloaded from -. - -## OPTIONS - - * `-c`, `--cipher`=: - the symmetric cipher to utilize for encryption; - defaults to aes-256-cbc - - * `-p`, `--password`=: - the password to derive the key from; - defaults to 30 random base64 characters - - * `-y`, `--yes`: - assume yes and accept defaults for non-specified options - - * `-d`, `--display`: - display the current repository's cipher and password - - * `-r`, `--rekey`: - re-encrypt all encrypted files using new credentials - - * `-f`, `--flush-credentials`: - remove the locally cached encryption credentials - and re-encrypt any files that had been previously decrypted - - * `-F`, `--force`: - ignore whether the git directory is clean, proceed with the - possibility that uncommitted changes are overwritten - - * `-u`, `--uninstall`: - remove all transcrypt configuration from the repository - and leave files in the current working copy decrypted - - * `-l`, `--list`: - list all of the transparently encrypted files in the repository, - relative to the top-level directory - - * `-s`, `--show-raw`=: - show the raw file as stored in the git commit object; - use this to check if files are encrypted as expected - - * `-e`, `--export-gpg`=: - export the repository's cipher and password to a file encrypted - for a gpg recipient - - * `-i`, `--import-gpg`=: - import the password and cipher from a gpg encrypted file - - * `-v`, `--version`: - print the version information - - * `-h`, `--help`: - view this help message - -## EXAMPLES - -To initialize a Git repository to support transparent encryption, just change -into the repo and run the transcrypt script. transcrypt will prompt you -interactively for all required information if the corresponding option flags -were not given. - - $ cd / - $ transcrypt - -Once a repository has been configured with transcrypt, you can transparently -encrypt files by applying the "crypt" filter and diff to a pattern in the -top-level _.gitattributes_ config. If that pattern matches a file in your -repository, the file will be transparently encrypted once you stage and commit -it: - - $ echo 'sensitive_file filter=crypt diff=crypt' >> .gitattributes - $ git add .gitattributes sensitive_file - $ git commit -m 'Add encrypted version of a sensitive file' - -See the gitattributes(5) man page for more information. - -If you have just cloned a repository containing files that are encrypted, -you'll want to configure transcrypt with the same cipher and password as the -origin repository. Once transcrypt has stored the matching credentials, it will -force a checkout of any existing encrypted files in order to decrypt them. - -If the origin repository has just rekeyed, all clones should flush their -transcrypt credentials, fetch and merge the new encrypted files via Git, and -then re-configure transcrypt with the new credentials. - -## AUTHOR - -Aaron Bull Schaefer <aaron@elasticdog.com> - -## SEE ALSO - -enc(1), gitattributes(5) diff --git a/transcrypt/transcrypt b/transcrypt/transcrypt deleted file mode 100755 index ef5261a79..000000000 --- a/transcrypt/transcrypt +++ /dev/null @@ -1,887 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# -# transcrypt - https://github.com/elasticdog/transcrypt -# -# A script to configure transparent encryption of sensitive files stored in -# a Git repository. It utilizes OpenSSL's symmetric cipher routines and follows -# the gitattributes(5) man page regarding the use of filters. -# -# Copyright (c) 2014-2019 Aaron Bull Schaefer -# This source code is provided under the terms of the MIT License -# that can be be found in the LICENSE file. -# - -##### CONSTANTS - -# the release version of this script -readonly VERSION='2.0.0' - -# the default cipher to utilize -readonly DEFAULT_CIPHER='aes-256-cbc' - -##### FUNCTIONS - -# print a canonicalized absolute pathname -realpath() { - local path=$1 - - # make path absolute - local abspath=$path - if [[ -n ${abspath##/*} ]]; then - abspath=$(pwd -P)/$abspath - fi - - # canonicalize path - local dirname= - if [[ -d $abspath ]]; then - dirname=$(cd "$abspath" && pwd -P) - abspath=$dirname - elif [[ -e $abspath ]]; then - dirname=$(cd "${abspath%/*}/" 2>/dev/null && pwd -P) - abspath=$dirname/${abspath##*/} - fi - - if [[ -d $dirname && -e $abspath ]]; then - printf '%s\n' "$abspath" - else - printf 'invalid path: %s\n' "$path" >&2 - exit 1 - fi -} - -# establish repository metadata and directory handling -gather_repo_metadata() { - # whether or not transcrypt is already configured - readonly CONFIGURED=$(git config --get --local transcrypt.version 2>/dev/null) - - # the current git repository's top-level directory - readonly REPO=$(git rev-parse --show-toplevel 2>/dev/null) - - # whether or not a HEAD revision exists - readonly HEAD_EXISTS=$(git rev-parse --verify --quiet HEAD 2>/dev/null) - - # https://github.com/RichiH/vcsh - # whether or not the git repository is running under vcsh - readonly IS_VCSH=$(git config --get --local --bool vcsh.vcsh 2>/dev/null) - - # whether or not the git repository is bare - readonly IS_BARE=$(git rev-parse --is-bare-repository 2>/dev/null || printf 'false') - - # the current git repository's .git directory - local RELATIVE_GIT_DIR - RELATIVE_GIT_DIR=$(git rev-parse --git-dir 2>/dev/null || printf '') - readonly GIT_DIR=$(realpath "$RELATIVE_GIT_DIR" 2>/dev/null) - - # the current git repository's gitattributes file - local CORE_ATTRIBUTES - CORE_ATTRIBUTES=$(git config --get --local --path core.attributesFile 2>/dev/null || printf '') - if [[ $CORE_ATTRIBUTES ]]; then - readonly GIT_ATTRIBUTES=$CORE_ATTRIBUTES - elif [[ $IS_BARE == 'true' ]] || [[ $IS_VCSH == 'true' ]]; then - readonly GIT_ATTRIBUTES="${GIT_DIR}/info/attributes" - else - readonly GIT_ATTRIBUTES="${REPO}/.gitattributes" - fi -} - -# print a message to stderr -warn() { - local fmt="$1" - shift - # shellcheck disable=SC2059 - printf "transcrypt: $fmt\n" "$@" >&2 -} - -# print a message to stderr and exit with either -# the given status or that of the most recent command -die() { - local st="$?" - if [[ "$1" != *[^0-9]* ]]; then - st="$1" - shift - fi - warn "$@" - exit "$st" -} - -# verify that all requirements have been met -run_safety_checks() { - # validate that we're in a git repository - [[ $GIT_DIR ]] || die 'you are not currently in a git repository; did you forget to run "git init"?' - - # exit if transcrypt is not in the required state - if [[ $requires_existing_config ]] && [[ ! $CONFIGURED ]]; then - die 1 'the current repository is not configured' - elif [[ ! $requires_existing_config ]] && [[ $CONFIGURED ]]; then - die 1 'the current repository is already configured; see --display' - fi - - # check for dependencies - for cmd in {column,grep,mktemp,openssl,sed,tee}; do - command -v $cmd >/dev/null || die 'required command "%s" was not found' "$cmd" - done - - # ensure the repository is clean (if it has a HEAD revision) so we can force - # checkout files without the destruction of uncommitted changes - if [[ $requires_clean_repo ]] && [[ $HEAD_EXISTS ]] && [[ $IS_BARE == 'false' ]]; then - # check if the repo is dirty - if ! git diff-index --quiet HEAD --; then - die 1 'the repo is dirty; commit or stash your changes before running transcrypt' - fi - fi -} - -# unset the cipher variable if it is not supported by openssl -validate_cipher() { - local list_cipher_commands - if openssl list-cipher-commands &>/dev/null; then - # OpenSSL < v1.1.0 - list_cipher_commands='openssl list-cipher-commands' - else - # OpenSSL >= v1.1.0 - list_cipher_commands='openssl list -cipher-commands' - fi - - local supported - supported=$($list_cipher_commands | tr -s ' ' '\n' | grep --line-regexp "$cipher") || true - if [[ ! $supported ]]; then - if [[ $interactive ]]; then - printf '"%s" is not a valid cipher; choose one of the following:\n\n' "$cipher" - $list_cipher_commands | column -c 80 - printf '\n' - cipher='' - else - # shellcheck disable=SC2016 - die 1 '"%s" is not a valid cipher; see `%s`' "$cipher" "$list_cipher_commands" - fi - fi -} - -# ensure we have a cipher to encrypt with -get_cipher() { - while [[ ! $cipher ]]; do - local answer= - if [[ $interactive ]]; then - printf 'Encrypt using which cipher? [%s] ' "$DEFAULT_CIPHER" - read -r answer - fi - - # use the default cipher if the user gave no answer; - # otherwise verify the given cipher is supported by openssl - if [[ ! $answer ]]; then - cipher=$DEFAULT_CIPHER - else - cipher=$answer - validate_cipher - fi - done -} - -# ensure we have a password to encrypt with -get_password() { - while [[ ! $password ]]; do - local answer= - if [[ $interactive ]]; then - printf 'Generate a random password? [Y/n] ' - read -r -n 1 -s answer - printf '\n' - fi - - # generate a random password if the user answered yes; - # otherwise prompt the user for a password - if [[ $answer =~ $YES_REGEX ]] || [[ ! $answer ]]; then - local password_length=30 - local random_base64 - random_base64=$(openssl rand -base64 $password_length) - password=$random_base64 - else - printf 'Password: ' - read -r password - [[ $password ]] || printf 'no password was specified\n' - fi - done -} - -# confirm the transcrypt configuration -confirm_configuration() { - local answer= - - printf '\nRepository metadata:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" - printf 'The following configuration will be saved:\n\n' - printf ' CIPHER: %s\n' "$cipher" - printf ' PASSWORD: %s\n\n' "$password" - printf 'Does this look correct? [Y/n] ' - read -r -n 1 -s answer - - # exit if the user did not confirm - if [[ $answer =~ $YES_REGEX ]] || [[ ! $answer ]]; then - printf '\n\n' - else - printf '\n' - die 1 'configuration has been aborted' - fi -} - -# confirm the rekey configuration -confirm_rekey() { - local answer= - - printf '\nRepository metadata:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" - printf 'The following configuration will be saved:\n\n' - printf ' CIPHER: %s\n' "$cipher" - printf ' PASSWORD: %s\n\n' "$password" - printf 'You are about to re-encrypt all encrypted files using new credentials.\n' - printf 'Once you do this, their historical diffs will no longer display in plain text.\n\n' - printf 'Proceed with rekey? [y/N] ' - read -r answer - - # only rekey if the user explicitly confirmed - if [[ $answer =~ $YES_REGEX ]]; then - printf '\n' - else - die 1 'rekeying has been aborted' - fi -} - -# automatically stage rekeyed files in preparation for the user to commit them -stage_rekeyed_files() { - local encrypted_files - encrypted_files=$(git ls-crypt) - if [[ $encrypted_files ]] && [[ $IS_BARE == 'false' ]]; then - # touch all encrypted files to prevent stale stat info - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - # shellcheck disable=SC2086 - touch $encrypted_files - # shellcheck disable=SC2086 - git update-index --add -- $encrypted_files - - printf '*** rekeyed files have been staged ***\n' - printf '*** COMMIT THESE CHANGES RIGHT AWAY! ***\n\n' - fi -} - -# save helper scripts under the repository's git directory -save_helper_scripts() { - mkdir -p "${GIT_DIR}/crypt" - - # The `decryption -> encryption` process on an unchanged file must be - # deterministic for everything to work transparently. To do that, the same - # salt must be used each time we encrypt the same file. An HMAC has been - # proven to be a PRF, so we generate an HMAC-SHA256 for each decrypted file - # (keyed with a combination of the filename and transcrypt password), and - # then use the last 16 bytes of that HMAC for the file's unique salt. - - cat <<-'EOF' >"${GIT_DIR}/crypt/clean" - #!/usr/bin/env bash - filename=$1 - # ignore empty files - if [[ -s $filename ]]; then - # cache STDIN to test if it's already encrypted - tempfile=$(mktemp 2>/dev/null || mktemp -t tmp) - trap 'rm -f "$tempfile"' EXIT - tee "$tempfile" &>/dev/null - # the first bytes of an encrypted file are always "Salted" in Base64 - read -n 8 firstbytes <"$tempfile" - if [[ $firstbytes == "U2FsdGVk" ]]; then - cat "$tempfile" - else - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - salt=$(openssl dgst -hmac "${filename}:${password}" -sha256 "$filename" | tr -d '\r\n' | tail -c 16) - ENC_PASS=$password openssl enc -$cipher -md MD5 -pass env:ENC_PASS -e -a -S "$salt" -in "$tempfile" - fi - fi - EOF - - cat <<-'EOF' >"${GIT_DIR}/crypt/smudge" - #!/usr/bin/env bash - tempfile=$(mktemp 2>/dev/null || mktemp -t tmp) - trap 'rm -f "$tempfile"' EXIT - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - tee "$tempfile" | ENC_PASS=$password openssl enc -$cipher -md MD5 -pass env:ENC_PASS -d -a 2>/dev/null || cat "$tempfile" - EOF - - cat <<-'EOF' >"${GIT_DIR}/crypt/textconv" - #!/usr/bin/env bash - filename=$1 - # ignore empty files - if [[ -s $filename ]]; then - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - ENC_PASS=$password openssl enc -$cipher -md MD5 -pass env:ENC_PASS -d -a -in "$filename" 2>/dev/null || cat "$filename" - fi - EOF - - # make scripts executable - for script in {clean,smudge,textconv}; do - chmod 0755 "${GIT_DIR}/crypt/${script}" - done -} - -# write the configuration to the repository's git config -save_configuration() { - save_helper_scripts - - # write the encryption info - git config transcrypt.version "$VERSION" - git config transcrypt.cipher "$cipher" - git config transcrypt.password "$password" - - # write the filter settings - if [[ -d $(git rev-parse --git-common-dir) ]]; then - # this allows us to support multiple working trees via git-worktree - # ...but the --git-common-dir flag was only added in November 2014 - # shellcheck disable=SC2016 - git config filter.crypt.clean '"$(git rev-parse --git-common-dir)"/crypt/clean %f' - # shellcheck disable=SC2016 - git config filter.crypt.smudge '"$(git rev-parse --git-common-dir)"/crypt/smudge' - # shellcheck disable=SC2016 - git config diff.crypt.textconv '"$(git rev-parse --git-common-dir)"/crypt/textconv' - else - # shellcheck disable=SC2016 - git config filter.crypt.clean '"$(git rev-parse --git-dir)"/crypt/clean %f' - # shellcheck disable=SC2016 - git config filter.crypt.smudge '"$(git rev-parse --git-dir)"/crypt/smudge' - # shellcheck disable=SC2016 - git config diff.crypt.textconv '"$(git rev-parse --git-dir)"/crypt/textconv' - fi - git config filter.crypt.required 'true' - git config diff.crypt.cachetextconv 'true' - git config diff.crypt.binary 'true' - git config merge.renormalize 'true' - - # add a git alias for listing encrypted files - git config alias.ls-crypt "!git ls-files | git check-attr --stdin filter | awk 'BEGIN { FS = \":\" }; /crypt$/{ print \$1 }'" -} - -# display the current configuration settings -display_configuration() { - local current_cipher - current_cipher=$(git config --get --local transcrypt.cipher) - local current_password - current_password=$(git config --get --local transcrypt.password) - local escaped_password=${current_password//\'/\'\\\'\'} - - printf 'The current repository was configured using transcrypt version %s\n' "$CONFIGURED" - printf 'and has the following configuration:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" - printf ' CIPHER: %s\n' "$current_cipher" - printf ' PASSWORD: %s\n\n' "$current_password" - printf 'Copy and paste the following command to initialize a cloned repository:\n\n' - printf " transcrypt -c %s -p '%s'\n" "$current_cipher" "$escaped_password" -} - -# remove transcrypt-related settings from the repository's git config -clean_gitconfig() { - git config --remove-section transcrypt 2>/dev/null || true - git config --remove-section filter.crypt 2>/dev/null || true - git config --remove-section diff.crypt 2>/dev/null || true - git config --unset merge.renormalize - - # remove the merge section if it's now empty - local merge_values - merge_values=$(git config --get-regex --local 'merge\..*') || true - if [[ ! $merge_values ]]; then - git config --remove-section merge 2>/dev/null || true - fi -} - -# force the checkout of any files with the crypt filter applied to them; -# this will decrypt existing encrypted files if you've just cloned a repository, -# or it will encrypt locally decrypted files if you've just flushed the credentials -force_checkout() { - # make sure a HEAD revision exists - if [[ $HEAD_EXISTS ]] && [[ $IS_BARE == 'false' ]]; then - # this would normally delete uncommitted changes in the working directory, - # but we already made sure the repo was clean during the safety checks - local encrypted_files - encrypted_files=$(git ls-crypt) - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - IFS=$'\n' - for file in $encrypted_files; do - rm "$file" - git checkout --force HEAD -- "$file" >/dev/null - done - unset IFS - fi -} - -# remove the locally cached encryption credentials and -# re-encrypt any files that had been previously decrypted -flush_credentials() { - local answer= - - if [[ $interactive ]]; then - printf 'You are about to flush the local credentials; make sure you have saved them elsewhere.\n' - printf 'All previously decrypted files will revert to their encrypted form.\n\n' - printf 'Proceed with credential flush? [y/N] ' - read -r answer - printf '\n' - else - # although destructive, we should support the --yes option - answer='y' - fi - - # only flush if the user explicitly confirmed - if [[ $answer =~ $YES_REGEX ]]; then - clean_gitconfig - - # re-encrypt any files that had been previously decrypted - force_checkout - - printf 'The local transcrypt credentials have been successfully flushed.\n' - else - die 1 'flushing of credentials has been aborted' - fi -} - -# remove all transcrypt configuration from the repository -uninstall_transcrypt() { - local answer= - - if [[ $interactive ]]; then - printf 'You are about to remove all transcrypt configuration from your repository.\n' - printf 'All previously encrypted files will remain decrypted in this working copy.\n\n' - printf 'Proceed with uninstall? [y/N] ' - read -r answer - printf '\n' - else - # although destructive, we should support the --yes option - answer='y' - fi - - # only uninstall if the user explicitly confirmed - if [[ $answer =~ $YES_REGEX ]]; then - clean_gitconfig - - # remove helper scripts - for script in {clean,smudge,textconv}; do - [[ ! -f "${GIT_DIR}/crypt/${script}" ]] || rm "${GIT_DIR}/crypt/${script}" - done - [[ ! -d "${GIT_DIR}/crypt" ]] || rmdir "${GIT_DIR}/crypt" - - # touch all encrypted files to prevent stale stat info - local encrypted_files - encrypted_files=$(git ls-crypt) - if [[ $encrypted_files ]] && [[ $IS_BARE == 'false' ]]; then - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - # shellcheck disable=SC2086 - touch $encrypted_files - fi - - # remove the `git ls-crypt` alias - git config --unset alias.ls-crypt - - # remove the alias section if it's now empty - local alias_values - alias_values=$(git config --get-regex --local 'alias\..*') || true - if [[ ! $alias_values ]]; then - git config --remove-section alias 2>/dev/null || true - fi - - # remove any defined crypt patterns in gitattributes - case $OSTYPE in - darwin*) - /usr/bin/sed -i '' '/filter=crypt diff=crypt[ \t]*$/d' "$GIT_ATTRIBUTES" - ;; - linux*) - sed -i '/filter=crypt diff=crypt[ \t]*$/d' "$GIT_ATTRIBUTES" - ;; - esac - - printf 'The transcrypt configuration has been completely removed from the repository.\n' - else - die 1 'uninstallation has been aborted' - fi -} - -# list all of the currently encrypted files in the repository -list_files() { - if [[ $IS_BARE == 'false' ]]; then - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - git ls-files | git check-attr --stdin filter | awk 'BEGIN { FS = ":" }; /crypt$/{ print $1 }' - fi -} - -# show the raw file as stored in the git commit object -show_raw_file() { - if [[ -f $show_file ]]; then - # ensure the file is currently being tracked - local escaped_file=${show_file//\//\\\/} - if git ls-files --others -- "$show_file" | awk "/${escaped_file}/{ exit 1 }"; then - file_paths=$(git ls-tree --name-only --full-name HEAD "$show_file") - else - die 1 'the file "%s" is not currently being tracked by git' "$show_file" - fi - elif [[ $show_file == '*' ]]; then - file_paths=$(git ls-crypt) - else - die 1 'the file "%s" does not exist' "$show_file" - fi - - IFS=$'\n' - for file in $file_paths; do - printf '==> %s <==\n' "$file" >&2 - git --no-pager show HEAD:"$file" --no-textconv - printf '\n' >&2 - done - unset IFS -} - -# export password and cipher to a gpg encrypted file -export_gpg() { - # check for dependencies - command -v gpg >/dev/null || die 'required command "gpg" was not found' - - # ensure the recipient key exists - if ! gpg --list-keys "$gpg_recipient" 2>/dev/null; then - die 1 'GPG recipient key "%s" does not exist' "$gpg_recipient" - fi - - local current_cipher - current_cipher=$(git config --get --local transcrypt.cipher) - local current_password - current_password=$(git config --get --local transcrypt.password) - mkdir -p "${GIT_DIR}/crypt" - - local gpg_encrypt_cmd="gpg --batch --recipient $gpg_recipient --trust-model always --yes --armor --quiet --encrypt -" - printf 'password=%s\ncipher=%s\n' "$current_password" "$current_cipher" | $gpg_encrypt_cmd >"${GIT_DIR}/crypt/${gpg_recipient}.asc" - printf "The transcrypt configuration has been encrypted and exported to:\n%s/crypt/%s.asc\n" "$GIT_DIR" "$gpg_recipient" -} - -# import password and cipher from a gpg encrypted file -import_gpg() { - # check for dependencies - command -v gpg >/dev/null || die 'required command "gpg" was not found' - - local path - if [[ -f "${GIT_DIR}/crypt/${gpg_import_file}" ]]; then - path="${GIT_DIR}/crypt/${gpg_import_file}" - elif [[ -f "${GIT_DIR}/crypt/${gpg_import_file}.asc" ]]; then - path="${GIT_DIR}/crypt/${gpg_import_file}.asc" - elif [[ ! -f $gpg_import_file ]]; then - die 1 'the file "%s" does not exist' "$gpg_import_file" - else - path="$gpg_import_file" - fi - - local configuration='' - local safety_counter=0 # fix for intermittent 'no secret key' decryption failures - while [[ ! $configuration ]]; do - configuration=$(gpg --batch --quiet --decrypt "$path") - - safety_counter=$((safety_counter + 1)) - if [[ $safety_counter -eq 3 ]]; then - die 1 'unable to decrypt the file "%s"' "$path" - fi - done - - cipher=$(printf '%s' "$configuration" | grep '^cipher' | cut -d'=' -f 2-) - password=$(printf '%s' "$configuration" | grep '^password' | cut -d'=' -f 2-) -} - -# print this script's usage message to stderr -usage() { - cat <<-EOF >&2 - usage: transcrypt [-c CIPHER] [-p PASSWORD] [-h] - EOF -} - -# print this script's help message to stdout -help() { - cat <<-EOF - NAME - transcrypt -- transparently encrypt files within a git repository - - SYNOPSIS - transcrypt [options...] - - DESCRIPTION - - transcrypt will configure a Git repository to support the transparent - encryption/decryption of files by utilizing OpenSSL's symmetric cipher - routines and Git's built-in clean/smudge filters. It will also add a - Git alias "ls-crypt" to list all transparently encrypted files within - the repository. - - The transcrypt source code and full documentation may be downloaded - from https://github.com/elasticdog/transcrypt. - - OPTIONS - -c, --cipher=CIPHER - the symmetric cipher to utilize for encryption; - defaults to aes-256-cbc - - -p, --password=PASSWORD - the password to derive the key from; - defaults to 30 random base64 characters - - -y, --yes - assume yes and accept defaults for non-specified options - - -d, --display - display the current repository's cipher and password - - -r, --rekey - re-encrypt all encrypted files using new credentials - - -f, --flush-credentials - remove the locally cached encryption credentials and re-encrypt - any files that had been previously decrypted - - -F, --force - ignore whether the git directory is clean, proceed with the - possibility that uncommitted changes are overwritten - - -u, --uninstall - remove all transcrypt configuration from the repository and - leave files in the current working copy decrypted - - -l, --list - list all of the transparently encrypted files in the repository, - relative to the top-level directory - - -s, --show-raw=FILE - show the raw file as stored in the git commit object; use this - to check if files are encrypted as expected - - -e, --export-gpg=RECIPIENT - export the repository's cipher and password to a file encrypted - for a gpg recipient - - -i, --import-gpg=FILE - import the password and cipher from a gpg encrypted file - - -v, --version - print the version information - - -h, --help - view this help message - - EXAMPLES - - To initialize a Git repository to support transparent encryption, just - change into the repo and run the transcrypt script. transcrypt will - prompt you interactively for all required information if the corre- - sponding option flags were not given. - - $ cd / - $ transcrypt - - Once a repository has been configured with transcrypt, you can trans- - parently encrypt files by applying the "crypt" filter and diff to a - pattern in the top-level .gitattributes config. If that pattern matches - a file in your repository, the file will be transparently encrypted - once you stage and commit it: - - $ echo 'sensitive_file filter=crypt diff=crypt' >> .gitattributes - $ git add .gitattributes sensitive_file - $ git commit -m 'Add encrypted version of a sensitive file' - - See the gitattributes(5) man page for more information. - - If you have just cloned a repository containing files that are - encrypted, you'll want to configure transcrypt with the same cipher and - password as the origin repository. Once transcrypt has stored the - matching credentials, it will force a checkout of any existing - encrypted files in order to decrypt them. - - If the origin repository has just rekeyed, all clones should flush - their transcrypt credentials, fetch and merge the new encrypted files - via Git, and then re-configure transcrypt with the new credentials. - - AUTHOR - Aaron Bull Schaefer - - SEE ALSO - enc(1), gitattributes(5) - EOF -} - -##### MAIN - -# reset all variables that might be set -cipher='' -display_config='' -flush_creds='' -gpg_import_file='' -gpg_recipient='' -interactive='true' -list='' -password='' -rekey='' -show_file='' -uninstall='' - -# used to bypass certain safety checks -requires_existing_config='' -requires_clean_repo='true' - -# parse command line options -while [[ "${1:-}" != '' ]]; do - case $1 in - -c | --cipher) - cipher=$2 - shift - ;; - --cipher=*) - cipher=${1#*=} - ;; - -p | --password) - password=$2 - shift - ;; - --password=*) - password=${1#*=} - ;; - -y | --yes) - interactive='' - ;; - -d | --display) - display_config='true' - requires_existing_config='true' - requires_clean_repo='' - ;; - -r | --rekey) - rekey='true' - requires_existing_config='true' - ;; - -f | --flush-credentials) - flush_creds='true' - requires_existing_config='true' - ;; - -F | --force) - requires_clean_repo='' - ;; - -u | --uninstall) - uninstall='true' - requires_existing_config='true' - requires_clean_repo='' - ;; - -l | --list) - list='true' - ;; - -s | --show-raw) - show_file=$2 - show_raw_file - exit 0 - ;; - --show-raw=*) - show_file=${1#*=} - show_raw_file - exit 0 - ;; - -e | --export-gpg) - gpg_recipient=$2 - requires_existing_config='true' - requires_clean_repo='' - shift - ;; - --export-gpg=*) - gpg_recipient=${1#*=} - requires_existing_config='true' - requires_clean_repo='' - ;; - -i | --import-gpg) - gpg_import_file=$2 - shift - ;; - --import-gpg=*) - gpg_import_file=${1#*=} - ;; - -v | --version) - printf 'transcrypt %s\n' "$VERSION" - exit 0 - ;; - -h | --help | -\?) - help - exit 0 - ;; - --*) - warn 'unknown option -- %s' "${1#--}" - usage - exit 1 - ;; - *) - warn 'unknown option -- %s' "${1#-}" - usage - exit 1 - ;; - esac - shift -done - -gather_repo_metadata - -# always run our safety checks -run_safety_checks - -# regular expression used to test user input -readonly YES_REGEX='^[Yy]$' - -# in order to keep behavior consistent no matter what order the options were -# specified in, we must run these here rather than in the case statement above -if [[ $list ]]; then - list_files - exit 0 -elif [[ $uninstall ]]; then - uninstall_transcrypt - exit 0 -elif [[ $display_config ]] && [[ $flush_creds ]]; then - display_configuration - printf '\n' - flush_credentials - exit 0 -elif [[ $display_config ]]; then - display_configuration - exit 0 -elif [[ $flush_creds ]]; then - flush_credentials - exit 0 -elif [[ $gpg_recipient ]]; then - export_gpg - exit 0 -elif [[ $gpg_import_file ]]; then - import_gpg -elif [[ $cipher ]]; then - validate_cipher -fi - -# perform function calls to configure transcrypt -get_cipher -get_password - -if [[ $rekey ]] && [[ $interactive ]]; then - confirm_rekey -elif [[ $interactive ]]; then - confirm_configuration -fi - -save_configuration - -if [[ $rekey ]]; then - stage_rekeyed_files -else - force_checkout -fi - -# ensure the git attributes file exists -if [[ ! -f $GIT_ATTRIBUTES ]]; then - mkdir -p "${GIT_ATTRIBUTES%/*}" - printf '#pattern filter=crypt diff=crypt\n' >"$GIT_ATTRIBUTES" -fi - -printf 'The repository has been successfully configured by transcrypt.\n' - -exit 0 diff --git a/werf-giterminism.yaml b/werf-giterminism.yaml deleted file mode 100644 index f1802bc8a..000000000 --- a/werf-giterminism.yaml +++ /dev/null @@ -1,12 +0,0 @@ -giterminismConfigVersion: 1 -config: - goTemplateRendering: - allowEnvVariables: - - XPRESS_LICENSE_HOST - - NREL_ROOT_CERT_URL_ROOT - # dockerfile: - # allowUncommitted: - # - julia_src/Dockerfile.xpress - # allowContextAddFiles: - # - julia_src/Dockerfile.xpress - # - julia_src/licenseserver diff --git a/werf.yaml b/werf.yaml deleted file mode 100644 index 0e9b7a396..000000000 --- a/werf.yaml +++ /dev/null @@ -1,19 +0,0 @@ -project: reopt-api -configVersion: 1 ---- -image: reopt-api -context: . -dockerfile: Dockerfile -args: - XPRESS_LICENSE_HOST: {{ env "XPRESS_LICENSE_HOST" | quote }} - NREL_ROOT_CERT_URL_ROOT: {{ env "NREL_ROOT_CERT_URL_ROOT" | quote }} ---- -image: julia-api -context: julia_src/ -dockerfile: Dockerfile -# contextAddFiles: -# - Dockerfile.xpress -# - licenseserver -args: - XPRESS_LICENSE_HOST: {{ env "XPRESS_LICENSE_HOST" | quote }} - NREL_ROOT_CERT_URL_ROOT: {{ env "NREL_ROOT_CERT_URL_ROOT" | quote }} From 82509443c4bc9368be4d8db252c4ac3b9c88a24b Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:02:50 -0600 Subject: [PATCH 02/66] Fixes for initial keys/settings revamp work. Fixes various syntax errors and missing imports. Also renames `keys.py` to `keys_env.py` just so we don't end up conflicting with people's existing `keys.py` that was gitinored previously. Also sets up docker to use optional `.env` files which should simplify the setup instructions and migration for existing users with custom keys.py files. --- .gitignore | 2 ++ checkdb.py | 2 +- docker-compose.nginx.yml | 8 ++++++++ docker-compose.nojulia.yml | 8 ++++++++ docker-compose.yml | 8 ++++++++ keys.py.template | 29 ----------------------------- keys.py => keys_env.py | 1 + reo/src/pvwatts.py | 4 ++-- reo/src/wind_resource.py | 2 +- reopt_api/celery.py | 2 +- reopt_api/settings.py | 7 ++++--- reoptjl/api.py | 4 ++-- 12 files changed, 38 insertions(+), 39 deletions(-) delete mode 100755 keys.py.template rename keys.py => keys_env.py (99%) diff --git a/.gitignore b/.gitignore index e9a3e7c50..54d7fcad8 100755 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ __pycache__/ *.py[cod] *$py.class +keys.py +.env .idea input_files/* !input_files/**/ diff --git a/checkdb.py b/checkdb.py index 684edf8fb..c66eef155 100644 --- a/checkdb.py +++ b/checkdb.py @@ -1,6 +1,6 @@ import psycopg2 import os -from keys import * +from keys_env import * from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT env = os.getenv('APP_ENV') diff --git a/docker-compose.nginx.yml b/docker-compose.nginx.yml index 50a99d7fa..02c2f1163 100644 --- a/docker-compose.nginx.yml +++ b/docker-compose.nginx.yml @@ -37,9 +37,13 @@ services: - DB_PASSWORD=reopt - DB_SEARCH_PATH=public - REDIS_HOST=redis-nginx + - REDIS_PASSWORD= - SOLVER=xpress - JULIA_HOST=julia-nginx - NLR_API_KEY + env_file: + - path: .env + required: false volumes: - .:/opt/reopt depends_on: @@ -61,9 +65,13 @@ services: - DB_PASSWORD=reopt - DB_SEARCH_PATH=public - REDIS_HOST=redis-nginx + - REDIS_PASSWORD= - SOLVER=xpress - JULIA_HOST=julia-nginx - NLR_API_KEY + env_file: + - path: .env + required: false depends_on: - db-nginx - redis-nginx diff --git a/docker-compose.nojulia.yml b/docker-compose.nojulia.yml index 0aee07a1a..f78c9603d 100644 --- a/docker-compose.nojulia.yml +++ b/docker-compose.nojulia.yml @@ -38,8 +38,12 @@ services: - DB_PASSWORD=reopt - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= - JULIA_HOST=host.docker.internal - NLR_API_KEY + env_file: + - path: .env + required: false volumes: - .:/opt/reopt depends_on: @@ -60,8 +64,12 @@ services: - DB_PASSWORD=reopt - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= - JULIA_HOST=host.docker.internal - NLR_API_KEY + env_file: + - path: .env + required: false depends_on: - db - redis diff --git a/docker-compose.yml b/docker-compose.yml index 730aa3d70..ed3796cb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,11 @@ services: - DB_PASSWORD=reopt - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= - NLR_API_KEY + env_file: + - path: .env + required: false volumes: - .:/opt/reopt depends_on: @@ -52,7 +56,11 @@ services: - DB_PASSWORD=reopt - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= - NLR_API_KEY + env_file: + - path: .env + required: false depends_on: - db - redis diff --git a/keys.py.template b/keys.py.template deleted file mode 100755 index 7c72f9f33..000000000 --- a/keys.py.template +++ /dev/null @@ -1,29 +0,0 @@ - -rollbar_access_token = 'test' - -#get your individual key from here: https://developer.nrel.gov/docs/api-key/ and replace the DEMO_KEY with that -pvwatts_api_key = 'DEMO_KEY' -developer_nrel_gov_key = 'DEMO_KEY' - - -secret_key_='secret_key_test' - -dev_database_host = 'localhost' -dev_database_name = 'reopt' -dev_user = 'reopt_api' -dev_user_password = 'reopt' -dev_redis_host = 'localhost' -dev_redis_password = 'password' - -# items that don't have to be changed by users (needed for NREL API servers) -staging_redis_host = "" -staging_redis_password = "" -staging_database_host = "" -staging_database_name = "" -production_redis_host = "" -production_redis_password = "" -prod_database_host = "" -prod_database_name = "" -production_user = "" -production_user_password = "" - diff --git a/keys.py b/keys_env.py similarity index 99% rename from keys.py rename to keys_env.py index e568613d8..c61bb6c5e 100644 --- a/keys.py +++ b/keys_env.py @@ -1,3 +1,4 @@ +import os rollbar_access_token = os.getenv('SECRET_ROLLBAR_ACCESS_TOKEN', 'test') diff --git a/reo/src/pvwatts.py b/reo/src/pvwatts.py index 83e98b3d0..29faf4947 100644 --- a/reo/src/pvwatts.py +++ b/reo/src/pvwatts.py @@ -1,7 +1,7 @@ # REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. import requests import json -import keys +import keys_env import logging from reo.exceptions import PVWattsDownloadError log = logging.getLogger(__name__) @@ -39,7 +39,7 @@ class PVWatts: def __init__(self, url_base="https://developer.nrel.gov/api/pvwatts/v6.json", - key=keys.developer_nrel_gov_key, + key=keys_env.developer_nrel_gov_key, azimuth=180, system_capacity=1, losses=0.14, diff --git a/reo/src/wind_resource.py b/reo/src/wind_resource.py index a8d041a89..5d4df21c1 100644 --- a/reo/src/wind_resource.py +++ b/reo/src/wind_resource.py @@ -10,7 +10,7 @@ from requests.adapters import HTTPAdapter import json -from keys import developer_nrel_gov_key +from keys_env import developer_nrel_gov_key import logging log = logging.getLogger(__name__) """ diff --git a/reopt_api/celery.py b/reopt_api/celery.py index 429cc2faf..c8b889734 100644 --- a/reopt_api/celery.py +++ b/reopt_api/celery.py @@ -4,7 +4,7 @@ import logging from celery import Celery from celery.signals import after_setup_logger -from keys import * +from keys_env import * # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'reopt_api.settings') diff --git a/reopt_api/settings.py b/reopt_api/settings.py index 0b5da339a..035ee1ade 100644 --- a/reopt_api/settings.py +++ b/reopt_api/settings.py @@ -1,8 +1,9 @@ # REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * +from keys_env import * import os import django import rollbar +import sys """ Django settings for reopt_api project. @@ -123,9 +124,9 @@ # Results backend CELERY_RESULT_BACKEND = 'django-db' -if APP_ENV == 'staging' +if APP_ENV == 'staging': CELERY_WORKER_MAX_MEMORY_PER_CHILD = 6000000 # 6 GB -else +else: CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 4 GB if 'test' in sys.argv: diff --git a/reoptjl/api.py b/reoptjl/api.py index 6d75a30b0..3508941ec 100644 --- a/reoptjl/api.py +++ b/reoptjl/api.py @@ -17,7 +17,7 @@ from ghpghx.models import GHPGHXInputs from django.core.exceptions import ValidationError from reoptjl.models import APIMeta -import keys +import keys_env log = logging.getLogger(__name__) @@ -74,7 +74,7 @@ def obj_get_list(self, bundle, **kwargs): def obj_create(self, bundle, **kwargs): run_uuid = str(uuid.uuid4()) # Attempt to get POSTed api_key assigned to APIMeta.api_key (or try method below for user_uuid) - api_key = keys.developer_nrel_gov_key #bundle.request.GET.get("api_key", "") + api_key = keys_env.developer_nrel_gov_key #bundle.request.GET.get("api_key", "") if 'user_uuid' in bundle.data.keys(): if type(bundle.data['user_uuid']) == str: From 226596d0fc309e33bcc6601abff5fef1351da1ea Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 22 Apr 2026 20:38:00 -0600 Subject: [PATCH 03/66] Trying to setup new GitHub Actions based deployment approach. This leverages GitHub Actions, along with other changes we've adopted, like Vault for secrets, Pkl for Kubernetes configuration management, and Carvel for deployments. This also removes some areas that I believe are now unused, like the nginx proxy layer, since it doesn't seem like that was previously being used in production any longer. --- .github/workflows/deploy.yml | 112 +++++ .github/workflows/prune-deploy-images.yml | 52 +++ .github/workflows/pull_request_tests.yml | 2 +- .github/workflows/push_tests.yml | 2 +- .github/workflows/undeploy-closed-prs.yml | 52 +++ .gitignore | 3 + .gomplate.yaml | 3 + Procfile | 2 - bin/ready-wait | 30 -- config/deploy/AppKbldConfig.pkl | 56 +++ config/deploy/PklProject | 8 + config/deploy/PklProject.deps.json | 24 ++ config/deploy/WebDeployment.pkl | 404 ++++++++++++++++++ config/deploy/render | 62 +++ .../templates/nginx/proxy_settings.conf.erb | 26 -- config/deploy/templates/nginx/site.conf.erb | 139 ------ config/deploy/vendir.lock.yml | 9 + config/deploy/vendir.yml | 9 + config/gunicorn.conf.py | 19 +- config/vault_secrets.json.tmpl | 52 +++ docker-compose.nginx.yml | 106 ----- nginx/Dockerfile | 4 - nginx/nginx.conf | 18 - 23 files changed, 853 insertions(+), 341 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/prune-deploy-images.yml create mode 100644 .github/workflows/undeploy-closed-prs.yml create mode 100644 .gomplate.yaml delete mode 100644 Procfile delete mode 100755 bin/ready-wait create mode 100644 config/deploy/AppKbldConfig.pkl create mode 100644 config/deploy/PklProject create mode 100644 config/deploy/PklProject.deps.json create mode 100644 config/deploy/WebDeployment.pkl create mode 100755 config/deploy/render delete mode 100644 config/deploy/templates/nginx/proxy_settings.conf.erb delete mode 100644 config/deploy/templates/nginx/site.conf.erb create mode 100644 config/deploy/vendir.lock.yml create mode 100644 config/deploy/vendir.yml create mode 100644 config/vault_secrets.json.tmpl delete mode 100644 docker-compose.nginx.yml delete mode 100644 nginx/Dockerfile delete mode 100644 nginx/nginx.conf diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..86805d1e0 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,112 @@ +name: Deploy + +on: + push: + branches: "**" + pull_request: + types: + - closed + - labeled + - reopened + - unlabeled + workflow_dispatch: + +concurrency: + # Concurrency group is more complicated in this case because: + # 1. This gets triggered by both `push` and `pull_request` label events, so + # both should use the same git head ref (and not `github.ref`, which may + # be different for PRs). + # 2. Our own deploy process may trigger an `unlabeled` event for removing the + # db-restore label, so separate that so that doesn't cause the previous + # deploy that was finishing up to be canceled. + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}${{ github.event.action == 'unlabeled' && github.event.label.name == 'deploy-db-restore' && '-removing-ephemeral-label' || '' }} + cancel-in-progress: true + +jobs: + deploy-metadata: + name: Deploy Metadata + runs-on: self-hosted + outputs: + staging-perform-deploy: ${{ steps.staging-metadata.outputs.perform-deploy }} + staging-perform-undeploy: ${{ steps.staging-metadata.outputs.perform-undeploy }} + staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} + production-perform-deploy: ${{ steps.production-metadata.outputs.perform-deploy }} + production-metadata: ${{ toJSON(steps.production-metadata.outputs) }} + steps: + - name: Import vault secrets + id: vault_secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_URL }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + exportEnv: false + secrets: | + secret/data/reopt-api/ci/deploy container_registry ; + secret/data/reopt-api/ci/deploy production_rancher_project_id ; + secret/data/reopt-api/ci/deploy staging_rancher_project_id ; + + - name: Staging Metadata + id: staging-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: staging + app-name: reopt-api + rancher-project-id: ${{ steps.vault_secrets.outputs.staging_rancher_project_id }} + registry: ${{ steps.vault_secrets.outputs.container_registry }} + branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} + branch-db-name-base: reopt_api_staging + + - name: Production Metadata + id: production-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: production + app-name: reopt-api + rancher-project-id: ${{ steps.vault_secrets.outputs.production_rancher_project_id }} + registry: ${{ steps.vault_secrets.outputs.container_registry }} + url-host: ${{ secrets.PRODUCTION_URL_HOST }} + + undeploy-staging: + name: Undeploy Staging + needs: + - deploy-metadata + if: ${{ needs.deploy-metadata.outputs.staging-perform-undeploy == 'true' }} + uses: TADA/deploy-action/.github/workflows/undeploy-branch.yml@v2 + with: + metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + vault-db-superuser-path: secret/data/reopt-db/staging/db-superuser + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + deploy-staging: + name: Deploy Staging + needs: + - deploy-metadata + if: ${{ needs.deploy-metadata.outputs.staging-perform-deploy == 'true' }} + uses: TADA/deploy-action/.github/workflows/deploy.yml@v2 + with: + metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} + vault-registry-credentials-path: secret/data/deploy/common/aws-ecr + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + deploy-production: + name: Deploy Production + needs: + - deploy-metadata + - deploy-staging + if: ${{ needs.deploy-metadata.outputs.production-perform-deploy == 'true' }} + uses: TADA/deploy-action/.github/workflows/deploy.yml@v2 + with: + metadata: ${{ needs.deploy-metadata.outputs.production-metadata }} + vault-registry-credentials-path: secret/data/deploy/common/aws-ecr + vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-production + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/prune-deploy-images.yml b/.github/workflows/prune-deploy-images.yml new file mode 100644 index 000000000..c3f11d236 --- /dev/null +++ b/.github/workflows/prune-deploy-images.yml @@ -0,0 +1,52 @@ +name: Prune Deploy Images + +on: + schedule: + - cron: "6 6 * * *" # Every day at 11:06 PM MST / 12:06 AM MDT + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + vault-secrets: + name: Vault Secrets + runs-on: self-hosted + outputs: + container_registry: ${{ steps.vault_secrets.outputs.container_registry }} + production_rancher_project_id: ${{ steps.vault_secrets.outputs.production_rancher_project_id }} + staging_rancher_project_id: ${{ steps.vault_secrets.outputs.staging_rancher_project_id }} + steps: + - name: Import vault secrets + id: vault_secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_URL }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + exportEnv: false + secrets: | + secret/data/reopt-api/ci/deploy container_registry ; + secret/data/reopt-api/ci/deploy production_rancher_project_id ; + secret/data/reopt-api/ci/deploy staging_rancher_project_id ; + + prune-images: + name: Prune Deploy Images + uses: TADA/deploy-action/.github/workflows/prune-deploy-images.yml@v2 + needs: + - vault-secrets + with: + vault-registry-credentials-path: secret/data/deploy/common/aws-ecr + registry: ${{ needs.vault-secrets.outputs.container_registry }} + images: | + tada/reopt-api + clusters: | + - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + rancher-project-id: ${{ needs.vault-secrets.outputs.staging_rancher_project_id }} + - vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-production + rancher-project-id: ${{ needs.vault-secrets.outputs.production_rancher_project_id }} + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/pull_request_tests.yml b/.github/workflows/pull_request_tests.yml index 308e72fe0..ab241cd22 100644 --- a/.github/workflows/pull_request_tests.yml +++ b/.github/workflows/pull_request_tests.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 - name: Build containers run: docker compose up -d env: diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index f2c174ba9..3ddecd0d1 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 - name: Build containers run: docker compose up -d env: diff --git a/.github/workflows/undeploy-closed-prs.yml b/.github/workflows/undeploy-closed-prs.yml new file mode 100644 index 000000000..36ff026db --- /dev/null +++ b/.github/workflows/undeploy-closed-prs.yml @@ -0,0 +1,52 @@ +name: Undeploy Closed PRs + +on: + schedule: + - cron: "23 16 * * *" # Every day at 09:23 AM MST / 10:23 AM MDT + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + deploy-metadata: + name: Deploy Metadata + runs-on: self-hosted + outputs: + staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} + steps: + - name: Import vault secrets + id: vault_secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_URL }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + exportEnv: false + secrets: | + secret/data/reopt-api/ci/deploy staging_rancher_project_id ; + + - name: Staging Metadata + id: staging-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: staging + app-name: reopt-api + rancher-project-id: ${{ steps.vault_secrets.outputs.staging_rancher_project_id }} + branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} + branch-db-name-base: reopt_api_staging + + undeploy-staging: + name: Undeploy Staging + needs: + - deploy-metadata + uses: TADA/deploy-action/.github/workflows/undeploy-closed-prs.yml@v2 + with: + base-metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + vault-db-superuser-path: secret/data/reopt-db/staging/db-superuser + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.gitignore b/.gitignore index 54d7fcad8..94edd4b03 100755 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,6 @@ julia_src/Dockerfile.xpress docker-compose.xpress.yml julia_src/solver_setup.sh compare_run_*.json + +/config/deploy/external +/config/deploy/tmp diff --git a/.gomplate.yaml b/.gomplate.yaml new file mode 100644 index 000000000..89659b153 --- /dev/null +++ b/.gomplate.yaml @@ -0,0 +1,3 @@ +datasources: + vault: + url: "vault:///secret/data" diff --git a/Procfile b/Procfile deleted file mode 100644 index ed6b995fe..000000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: $DEPLOY_CURRENT_PATH/bin/server -worker: $DEPLOY_CURRENT_PATH/bin/worker \ No newline at end of file diff --git a/bin/ready-wait b/bin/ready-wait deleted file mode 100755 index 9146cf996..000000000 --- a/bin/ready-wait +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -Eeuxo pipefail - -# Used for Kubernetes initContainers to wait for the database to be migrated -# before allowing the app to spin up and be part of the live pods. -while true; do - # Look for pending migrations by viewing the migration plan and looking for - # any pending migrations identified by starting with "[ ]" (already run - # migrations will be identified by "[X]". Keep waiting until all migrations - # are run, and there are no more pending migrations. - # - # A little hacky, but based on info at https://stackoverflow.com/a/31847406 - # - # If the app is upgraded to Django 3.1, then this could maybe be made simpler - # with the new "migrate --check" support: - # https://docs.djangoproject.com/en/3.1/ref/django-admin/#cmdoption-migrate-check - migrations=$(python manage.py showmigrations --plan) - { set +x; } 2>/dev/null - pending_migrations=$(echo "$migrations" | grep '\[ \]' || true) - if [ -z "$pending_migrations" ]; then - echo "Migrations up to date." - break - else - echo "Waiting for migrations to complete:" - echo "$pending_migrations" - sleep 3 - fi - set -x -done diff --git a/config/deploy/AppKbldConfig.pkl b/config/deploy/AppKbldConfig.pkl new file mode 100644 index 000000000..50ee5e41f --- /dev/null +++ b/config/deploy/AppKbldConfig.pkl @@ -0,0 +1,56 @@ +extends "@tadaSharedKube/KbldConfig.pkl" + +import "pkl:json" + +local deployMetadata = import("@tadaSharedKube/tadaDeployMetadata.pkl").deployMetadata + +local const secretsString = read?("env:DEPLOY_SECRETS") +local const secrets = new json.Parser {}.parse(secretsString ?? "{}") + +// Define the docker images that need to be built and the registries to push +// them to. For integration with `kbld` during deployments. +sources { + new { + image = "reopt-api" + path = "." + docker { + build { + file = "Dockerfile" + pull = true + buildkit = true + rawOptions { + "--build-arg" + "NREL_ROOT_CERT_URL_ROOT=\(secrets.`SECRET_NLR_ROOT_CERT_URL_ROOT`)" + } + } + } + } + + new { + image = "julia-api" + path = "julia_src" + docker { + build { + file = "Dockerfile" + pull = true + buildkit = true + rawOptions { + "--build-arg" + "NREL_ROOT_CERT_URL_ROOT=\(secrets.`SECRET_NLR_ROOT_CERT_URL_ROOT`)" + } + } + } + } +} + +destinations { + new { + ["image"] = "reopt-api" + ["newImage"] = secrets.`SECRET_CONTAINER_REGISTRY` + } + + new { + ["image"] = "julia-api" + ["newImage"] = secrets.`SECRET_CONTAINER_REGISTRY` + } +} diff --git a/config/deploy/PklProject b/config/deploy/PklProject new file mode 100644 index 000000000..fcef1daad --- /dev/null +++ b/config/deploy/PklProject @@ -0,0 +1,8 @@ +amends "pkl:Project" + +dependencies { + ["tadaSharedKube"] = import("./external/tada-shared-kube/PklProject") + ["k8s"] { + uri = "package://pkg.pkl-lang.org/pkl-k8s/k8s@1.4.1" + } +} diff --git a/config/deploy/PklProject.deps.json b/config/deploy/PklProject.deps.json new file mode 100644 index 000000000..41f0d1adb --- /dev/null +++ b/config/deploy/PklProject.deps.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": 1, + "resolvedDependencies": { + "package://pkg.pkl-lang.org/pkl-k8s/k8s@1": { + "type": "remote", + "uri": "projectpackage://pkg.pkl-lang.org/pkl-k8s/k8s@1.4.1", + "checksums": { + "sha256": "0520f82219c074d2f4b69ab9c57a4a1c1d0b3c4acc1e5d12749e72e0d567d9bf" + } + }, + "package://github.nrel.gov/TADA/tada-shared-kube@0": { + "type": "local", + "uri": "projectpackage://github.nrel.gov/TADA/tada-shared-kube@0.1.0", + "path": "external/tada-shared-kube" + }, + "package://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.deepToTyped@1": { + "type": "remote", + "uri": "projectpackage://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.deepToTyped@1.0.1", + "checksums": { + "sha256": "3a1fce17191cab6bcce6cca3cd86b2d50a3489927678fdffb4551a6afedeac32" + } + } + } +} diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl new file mode 100644 index 000000000..ea1792321 --- /dev/null +++ b/config/deploy/WebDeployment.pkl @@ -0,0 +1,404 @@ +import "./AppKbldConfig.pkl" +import "@k8s/K8sResource.pkl" +import "@k8s/api/core/v1/EnvFromSource.pkl" +import "@k8s/api/apps/v1/StatefulSet.pkl" +import "@tadaSharedKube/TadaAppNamespace.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewConfigMap.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewCronJob.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewInitialSetupJob.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewRoleBinding.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewRole.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewSecrets.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewServiceAccount.pkl" +import "@tadaSharedKube/TadaWebConfigMap.pkl" +import "@tadaSharedKube/TadaWebDbMigrateJob.pkl" +import "@tadaSharedKube/TadaWebDbMigrateSecrets.pkl" +import "@tadaSharedKube/TadaWebDeployment.pkl" +import "@tadaSharedKube/TadaWebIngress.pkl" +import "@tadaSharedKube/TadaWebPodDisruptionBudget.pkl" +import "@tadaSharedKube/TadaWebSecrets.pkl" +import "@tadaSharedKube/TadaWebService.pkl" + +local deployMetadata = import("@tadaSharedKube/tadaDeployMetadata.pkl").deployMetadata +local containerImage = "reopt-api" + +resources: Listing = new { + new TadaAppNamespace { + when (deployMetadata.`deploy-env` == "production") { + tadaRancherResourceQuotaPodLimit = 70 + } else { + tadaRancherResourceQuotaPodLimit = 20 + } + } + + new AppKbldConfig {} + + // To be able to pull images from our ECR repos in our on-premise (non-AWS) + // clusters. + local ecrLoginRenewConfigMapResource = new TadaEcrLoginRenewConfigMap {} + ecrLoginRenewConfigMapResource + new TadaEcrLoginRenewCronJob {} + new TadaEcrLoginRenewInitialSetupJob {} + new TadaEcrLoginRenewRoleBinding {} + new TadaEcrLoginRenewRole {} + new TadaEcrLoginRenewSecrets {} + new TadaEcrLoginRenewServiceAccount {} + + local webConfigMap = new TadaWebConfigMap { + data { + ["APP_ENV"] = deployMetadata.`deploy-env` + ["BRANCH_NAME"] = deployMetadata.`branch-name` + ["JULIA_HOST"] = "julia-service.svc.cluster.local" + ["REDIS_HOST"] = "redis-service.svc.cluster.local" + ["K8S_DEPLOY"] = "true" + + when (deployMetadata.`branch-is-primary` == "false") { + ["DB_NAME"] = deployMetadata.`branch-db-name` + // Use the database name as the queue name so that separate branches or + // databases on staging remain in separate queues. + ["APP_QUEUE_NAME"] = deployMetadata.`branch-db-name` + } else { + // In production (or where the database name isn't explicitly + // overridden), just use the app env as the queue name. + ["APP_QUEUE_NAME"] = deployMetadata.`deploy-env` + } + } + } + webConfigMap + + local webSecrets = new TadaWebSecrets {} + webSecrets + + new TadaWebDbMigrateSecrets {} + + new TadaWebDbMigrateJob { + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + spec { + template { + spec { + containers { + [[ name == "web" ]] { + args = new { + "bin/migrate" + } + } + } + } + } + } + } + + // Deployment for the Django APIs. + local djangoDeployment = new TadaWebDeployment { + tadaName = "django" + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + tadaResourceLimitCpu = "2000m" + tadaResourceLimitMemory = 2000.mib + + when (deployMetadata.`deploy-env` == "production") { + tadaReplicaCount = 10 + tadaResourceRequestCpu = "1000m" + tadaResourceRequestMemory = 2000.mib + } else { + tadaResourceRequestCpu = "500m" + tadaResourceRequestMemory = 700.mib + tadaResourceLimitMemory = 700.mib + } + + tadaContainerPort = 8000 + tadaHealthProbePath = "/_health" + + spec { + template { + spec { + containers { + [[ name == tadaName ]] { + args = new { + "bin/server" + } + } + } + } + } + } + } + djangoDeployment + + // Deployment for the standalone Julia containers. These have lower memory + // limits since they're only used for lighterweight API requests from the + // Django app. The primary Julia containers are separate pods on the Celery + // deployment. + local juliaDeployment = new TadaWebDeployment { + tadaName = "julia" + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + tadaResourceLimitCpu = "4000m" + tadaResourceLimitMemory = 2000.mib + tadaResourceRequestMemory = 2000.mib + + when (deployMetadata.`deploy-env` == "production") { + tadaReplicaCount = 5 + tadaResourceRequestCpu = "2000m" + } else { + tadaResourceRequestCpu = "500m" + } + + tadaContainerPort = 8081 + tadaHealthProbePath = "/health" + + spec { + template { + spec { + containers { + [[ name == tadaName ]] { + args = new { + "julia" "--project=/opt/julia_src" "http.jl" + } + + // Overwrite to remove default web-secrets from list, since Julia + // container doesn't need secrets access. + envFrom = new Listing { + new { + configMapRef { + name = webConfigMap.metadata.name + } + } + } + + env { + new { + name = "JULIA_NUM_THREADS" + value = "4" + } + } + } + } + } + } + } + } + juliaDeployment + + // Deployment for the Celery workers, which also includes Julia pods deployed + // in tandem so there's always a one-to-one relationship between Celery + // worker pods and a dedicated Julia pod. + new TadaWebDeployment { + tadaName = "celery" + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + tadaResourceLimitCpu = "1000m" + tadaResourceRequestCpu = "100m" + tadaResourceLimitMemory = 1600.mib + tadaResourceRequestMemory = 1600.mib + + when (deployMetadata.`deploy-env` == "production") { + tadaReplicaCount = 10 + } + + spec { + template { + spec { + containers { + [[ name == tadaName ]] { + args = new { + "bin/worker" + } + + env { + new { + name = "JULIA_HOST" + value = "localhost" + } + } + + startupProbe { + httpGet = null + exec { + command = new { + "pgrep" "-f" "bin/celery" + } + } + } + + readinessProbe { + httpGet = null + exec { + command = new { + "pgrep" "-f" "bin/celery" + } + } + } + + livenessProbe { + httpGet = null + exec { + command = new { + "pgrep" "-f" "bin/celery" + } + } + } + } + + // Add an extra pod for the Julia container. We will leverage the + // standalone Julia deployment's container definition, but tweaked + // for different resource limits, since these are the more resource + // intenstive areas. + (juliaDeployment.spec.template.spec.containers[0]) { + resources { + requests { + when (deployMetadata.`deploy-env` == "production") { + ["memory"] = 12000.mib + } else { + ["memory"] = 6000.mib + } + } + + limits { + when (deployMetadata.`deploy-env` == "production") { + ["memory"] = 12000.mib + } else { + ["memory"] = 6000.mib + } + } + } + } + } + } + } + } + } + + new StatefulSet { + metadata { + name = "redis-stateful-set" + + labels { + ["app"] = "redis" + } + + annotations { + ["kapp.k14s.io/change-group"] = "change-group/deployment" + } + } + + spec { + selector { + matchLabels { + ["app"] = "redis" + } + } + + replicas = 1 + + template { + metadata { + labels { + ["app"] = "redis" + } + } + + spec { + imagePullSecrets { + new { + name = ecrLoginRenewConfigMapResource.imagePullSecretName + } + } + + containers { + new { + name = "redis" + + image = "public.ecr.aws/docker/library/redis:6.2.21-alpine" + + args = new { + "sh" "-c" "redis-server --requirepass $$SECRET_REDIS_PASSWORD --save 900 1 --save 300 10 --save 60 10000 --appendonly yes --maxmemory 2048mb --maxmemory-policy allkeys-lru" + } + + env { + new { + name = "SECRET_REDIS_PASSWORD" + valueFrom { + secretKeyRef { + name = webSecrets.metadata.name + key = "SECRET_REDIS_PASSWORD" + } + } + } + } + } + } + } + } + } + } + + local djangoService = new TadaWebService { + tadaName = "django" + tadaAppLabel = "django" + tadaPort = 8000 + } + djangoService + + local juliaService = new TadaWebService { + tadaName = "julia" + tadaAppLabel = "julia" + tadaPort = 8081 + } + juliaService + + new TadaWebService { + tadaName = "redis" + tadaAppLabel = "redis" + tadaPort = 6379 + } + + new TadaWebIngress { + tadaServiceName = djangoService.metadata.name + tadaServicePort = 8000 + + // Configure the ingress on `main` to listen on the old `*.nrel.gov` + // domains (in addition to the new `nlr.gov` domains). We can remove this + // once the nrel.gov domain is fully gone. + when (deployMetadata.`deploy-env` == "production" || (deployMetadata.`deploy-env` == "staging" && deployMetadata.`branch-is-primary` == "true")) { + local nrelHost = deployMetadata.`url-host`.replaceLast(".nlr.gov", ".nrel.gov") + spec { + rules { + new { + host = nrelHost!! + http { + paths { + new { + path = "/" + pathType = "Prefix" + backend { + service { + name = djangoService.metadata.name + port { + number = 8000 + } + } + } + } + } + } + } + } + } + } + } + + new TadaWebPodDisruptionBudget {} +} + +output { + value = resources + renderer = (K8sResource.output.renderer as YamlRenderer) { + isStream = true + } +} diff --git a/config/deploy/render b/config/deploy/render new file mode 100755 index 000000000..ad5afac88 --- /dev/null +++ b/config/deploy/render @@ -0,0 +1,62 @@ +#!/usr/bin/env ruby + +require "json" +require "open3" +require "rake" +require "shellwords" + +# Install dependencies. +sh "vendir sync --chdir #{Shellwords.escape(__dir__)} --locked 1>&2" +sh "pkl project resolve --working-dir #{Shellwords.escape(__dir__)} 1>&2" + +# Parse metadata generated from GitHub Actions. +deploy_metadata = JSON.parse(ENV.fetch("DEPLOY_METADATA", "{}")) +deploy_env = deploy_metadata.fetch("deploy-env", "development") + +# Generate decrypted secrets file. +output, status = Open3.capture2( + {"DEPLOY_ENV" => deploy_env}, + "gomplate", + "--config", File.expand_path("../../.gomplate.yaml", __dir__), + "--file", File.expand_path("../../config/vault_secrets.json.tmpl", __dir__) +) +raise "gomplate error: #{output.inspect}" unless status.success? +ENV["DEPLOY_SECRETS"] = output + +# Generate decrypted secrets file for the database migration environment (which +# contain extra values we don't normally want to expose). +output, status = Open3.capture2( + {"DEPLOY_ENV" => deploy_env, "DB_MIGRATE" => "true"}, + "gomplate", + "--config", File.expand_path("../../.gomplate.yaml", __dir__), + "--file", File.expand_path("../../config/vault_secrets.json.tmpl", __dir__) +) +raise "gomplate error: #{output.inspect}" unless status.success? +ENV["DEPLOY_DB_MIGRATE_SECRETS"] = output + +# Generate decrypted secrets file for the database migration environment (which +# contain extra values we don't normally want to expose). +output, status = Open3.capture2( + {"DEPLOY_ENV" => deploy_env, "ECR_LOGIN_RENEW" => "true"}, + "gomplate", + "--config", File.expand_path("../../.gomplate.yaml", __dir__), + "--file", File.expand_path("../../config/vault_secrets.json.tmpl", __dir__) +) +raise "gomplate error: #{output.inspect}" unless status.success? +ENV["DEPLOY_ECR_LOGIN_RENEW_SECRETS"] = output + +# Render the Kubernetes YAML output. +case ENV["RENDER_JOB"] +when "run-job" + sh "pkl", "eval", + "--working-dir", __dir__, + "--format", "yaml", + File.expand_path("./RunJob.pkl", __dir__) +when nil + sh "pkl", "eval", + "--working-dir", __dir__, + "--format", "yaml", + File.expand_path("./WebDeployment.pkl", __dir__) +else + raise "Unknown RENDER_JOB=#{ENV["RENDER_JOB"].inspect}" +end diff --git a/config/deploy/templates/nginx/proxy_settings.conf.erb b/config/deploy/templates/nginx/proxy_settings.conf.erb deleted file mode 100644 index c5aa774b2..000000000 --- a/config/deploy/templates/nginx/proxy_settings.conf.erb +++ /dev/null @@ -1,26 +0,0 @@ -# Allow response streaming -proxy_buffering off; - -# Set original IP -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - -# Proxy over HTTP 1.1 so keepalive connections to the backend are supported -proxy_http_version 1.1; -proxy_set_header Connection ""; - -# Pass along the original Host header, rather than the upstream name used in -# proxy_pass. -proxy_set_header Host $host; - -# Disable redirect rewriting. -proxy_redirect off; - -# Increase timeout for longer response times. -# -# This value should be be kept in sync with the xpress/mosel run timeout -# (defined in reo/models.py), and the gunicorn timeout (defined in -# config/gunicorn.conf.py). -# -# This timeout should be greater than the gunicorn timeout, to give the app an -# opportunity to handle timeouts more gracefully. -proxy_read_timeout 450s; \ No newline at end of file diff --git a/config/deploy/templates/nginx/site.conf.erb b/config/deploy/templates/nginx/site.conf.erb deleted file mode 100644 index 1a44c6c6d..000000000 --- a/config/deploy/templates/nginx/site.conf.erb +++ /dev/null @@ -1,139 +0,0 @@ -# Security headers -# https://www.owasp.org/index.php/OWASP_Secure_Headers_Project#tab=Headers -<% if(fetch(:app_env) == "production") %> - more_set_headers "Strict-Transport-Security: max-age=31536000; includeSubDomains; preload"; -<% else %> - more_set_headers "Strict-Transport-Security: max-age=300; preload"; -<% end %> - -more_set_headers "X-XSS-Protection: 1; mode=block"; -more_set_headers "X-Frame-Options: DENY"; -more_set_headers "X-Content-Type-Options: nosniff"; - -upstream <%= fetch(:deploy_name) %>-http { - server 127.0.0.1:80; - keepalive 10; -} - -upstream <%= fetch(:deploy_name) %>-django_backend { - server unix:<%= current_path %>/tmp/gunicorn.sock; - keepalive 10; -} - -# The following settings can only be set once, globally in the http scope, so -# don't apply them to branched deployments. -<% if(!fetch(:subdomain)) %> -# Set far-future cache expiration headers for all JS, CSS, and image -# responses. -# -# This assumes all of these resources should be cache-busted by changing the -# filename when the contents change (which everything should be doing). -map $sent_http_content_type $expires { - default off; - <% if(fetch(:app_env) != "development") %> - application/javascript max; - text/css max; - ~image/ max; - <% end %> -} -expires $expires; -<% end %> - -# SSL Terminator. -server { - listen 443 ssl; - listen [::]:443 ssl; - server_name <%= fetch(:all_domains).join(" ") %>; - - <% if(fetch(:base_domain) =~ /vagrant$/) %> - ssl_certificate /etc/ssl/vagrant.crt; - ssl_certificate_key /etc/ssl/vagrant.key; - <% elsif(fetch(:stage) == :internal_c110p) %> - <% if(fetch(:subdomain)) %> - ssl_certificate /etc/ssl/star_<%= host.hostname %>.crt; - ssl_certificate_key /etc/ssl/star_<%= host.hostname %>.key; - <% else %> - ssl_certificate /etc/ssl/<%= host.hostname %>.crt; - ssl_certificate_key /etc/ssl/<%= host.hostname %>.key; - <% end %> - <% else %> - ssl_certificate /etc/httpd/ssl/<%= host.hostname %>.pem; - ssl_certificate_key /etc/httpd/ssl/<%= host.hostname %>.key; - <% end %> - - access_log <%= current_path %>/log/access.log combined_extended; - error_log <%= current_path %>/log/error.log; - - # Default proxy settings. - include <%= fetch(:compiled_config_dir) %>/nginx/proxy_settings.conf; - - # Pass along original HTTPS protocol and port information. - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Port $server_port; - - location / { - proxy_pass http://<%= fetch(:deploy_name) %>-http; - } -} - -server { - listen 80; - listen [::]:80; - server_name <%= fetch(:all_domains).join(" ") %>; - - # Redirect everything to HTTPS. - # - # Since we're sitting behind an SSL terminator all of our requests will be on - # port 80, so we need to check the forwarded information. - if ($http_x_forwarded_proto != "https") { - return 301 https://$host$request_uri; - } - - root <%= current_path %>/static; - - access_log <%= current_path %>/log/access.log combined_extended; - error_log <%= current_path %>/log/error.log; - - # gzip configuration - gzip on; - gzip_comp_level 2; - gzip_http_version 1.0; - gzip_types text/xml application/xml text/javascript application/javascript application/x-javascript application/json text/csv; - gzip_proxied any; - - # Default proxy settings. - include <%= fetch(:compiled_config_dir) %>/nginx/proxy_settings.conf; - - # Set original IP and forwarded details - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; - proxy_set_header X-Forwarded-Port $http_x_forwarded_port; - - # Workaround for Django requiring a trailing slash on the POST endpoint. - rewrite ^/v1/job$ /v1/job/ last; - - <% if(fetch(:app_env) != "production") %> - # Don't allow search engine indexing of our staging sites. - location = /robots.txt { - return 200 "User-agent: *\nDisallow: /"; - } - <% end %> - - location / { - # Serve static files with nginx, everything else with Puma. - try_files $uri/index.html $uri.html $uri @app; - } - - location @app { - <% if(fetch(:app_env) != "production") %> - set $lazy_hydra_start_script "sudo supervisorctl start '<%= fetch(:deploy_name) %>:*'"; - set $lazy_hydra_stop_script "sudo supervisorctl stop '<%= fetch(:deploy_name) %>:*'"; - set $lazy_hydra_health_url "unix:<%= current_path %>/tmp/gunicorn.sock:/"; - set $lazy_hydra_startup_timeout 90; # 90 seconds - this app can take longer than normal to spin up. - set $lazy_hydra_inactivity_timeout 10800; # 3 hours - include "lazy-hydra/hook.conf"; - <% end %> - - proxy_pass http://<%= fetch(:deploy_name) %>-django_backend; - } -} \ No newline at end of file diff --git a/config/deploy/vendir.lock.yml b/config/deploy/vendir.lock.yml new file mode 100644 index 000000000..bcca8c3ad --- /dev/null +++ b/config/deploy/vendir.lock.yml @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - git: + commitTitle: Allow for easier customization of service and ingress name. + sha: 2ad60887c5a5c3f6766a43cb394cb890a50b43f1 + path: tada-shared-kube + path: external +kind: LockConfig diff --git a/config/deploy/vendir.yml b/config/deploy/vendir.yml new file mode 100644 index 000000000..a830b6cd6 --- /dev/null +++ b/config/deploy/vendir.yml @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: external + contents: + - path: tada-shared-kube + git: + url: https://github.nrel.gov/TADA/tada-shared-kube.git + ref: origin/ports diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index b4d421ab2..695d13aa3 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -1,20 +1,13 @@ import os import multiprocessing -# Bind to unix socket that nginx will proxy to. if os.environ.get('TEST') is None: - if os.environ.get('K8S_DEPLOY') is None: - bind = 'unix:' + os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'tmp/gunicorn.sock') - else: - bind = "0.0.0.0:8000" + bind = "0.0.0.0:8000" else: bind = "127.0.0.1:8000" # Based the number of workers on the number of CPU cores. -if os.environ.get('K8S_DEPLOY') is None: - workers = multiprocessing.cpu_count() -else: - workers = 4 +workers = 4 # Note that the app currently has threading issues, so we explicitly want a # non-thread worker process model. @@ -27,12 +20,10 @@ # Increase timeout for longer response times. # # This value should be be kept in sync with the xpress/mosel run timeout -# (defined in reo/models.py), and the nginx timeout (defined in -# config/deploy/templates/nginx/proxy_settings.conf.erb). +# (defined in reo/models.py). # -# This timeout should be greater than the xpress timeout, but less than the -# nginx timeout, to give the app an opportunity to handle timeouts more -# gracefully. +# This timeout should be greater than the xpress timeout to give the app an +# opportunity to handle timeouts more gracefully. timeout = 435 # Set the appropriate DJANGO_SETTINGS_MODULE environment variable based on the diff --git a/config/vault_secrets.json.tmpl b/config/vault_secrets.json.tmpl new file mode 100644 index 000000000..9c656fbda --- /dev/null +++ b/config/vault_secrets.json.tmpl @@ -0,0 +1,52 @@ +{{/* Build a JSON file containing secrets fetched from our Vault server. The needed secrets may vary depending on the environment (eg, different secrets are needed for development vs production). */}} + +{{ $deploy_env := (env.Getenv "DEPLOY_ENV" (env.Getenv "APP_ENV" "development")) }} +{{ $secrets := coll.Dict }} + +{{ if (ne (env.Getenv "ECR_LOGIN_RENEW") "true") }} + {{ with (datasource "vault" "reopt-api/common/build").data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_NLR_ROOT_CERT_URL_ROOT" .nlr_root_cert_url_root + ) $secrets }} + {{ end }} + + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_CONTAINER_REGISTRY" .container_registry + ) $secrets }} + {{ end }} + + {{ if (or (eq $deploy_env "staging") (eq $deploy_env "production")) }} + {{ with (datasource "vault" (printf "reopt-api/%s/web" $deploy_env)).data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_ASHRAE_TMY_NLR_API_KEY" .ashrae_tmy_nlr_api_key + "SECRET_DB_HOST" .db_host + "SECRET_DB_NAME" .db_name + "SECRET_DEVELOPER_NLR_GOV_API_KEY" .developer_nlr_gov_api_key + "SECRET_DJANGO_SECRET_KEY" .django_secret_key + "SECRET_PVWATTS_NLR_API_KEY" .pvwatts_nlr_api_key + "SECRET_REDIS_PASSWORD" .redis_password + "SECRET_ROLLBAR_ACCESS_TOKEN" .rollbar_access_token + ) $secrets }} + {{ end }} + + {{ with (datasource "vault" (printf "reopt-api/%s/db-migrate" $deploy_env)).data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_DB_PASSWORD" .db_password + "SECRET_DB_USERNAME" .db_username + ) $secrets }} + {{ end }} + {{ end }} +{{ end }} + +{{ if (eq (env.Getenv "ECR_LOGIN_RENEW") "true") }} + {{ with (datasource "vault" "deploy/common/aws-ecr").data }} + {{ $secrets = coll.Merge (coll.Dict + "AWS_ACCESS_KEY_ID" .aws_access_key_id + "AWS_SECRET_ACCESS_KEY" .aws_secret_access_key + "AWS_REGION" .aws_region + ) $secrets }} + {{ end }} +{{ end }} + +{{ $secrets | data.ToJSON }} diff --git a/docker-compose.nginx.yml b/docker-compose.nginx.yml deleted file mode 100644 index 02c2f1163..000000000 --- a/docker-compose.nginx.yml +++ /dev/null @@ -1,106 +0,0 @@ -version: "2.1" - -services: - - redis-nginx: - container_name: redis-nginx - image: redis - command: redis-server - expose: - - 6379 - - db-nginx: - container_name: db-nginx - image: postgres - restart: always - volumes: - - pgdata:/var/lib/postgresql/data - environment: - - POSTGRES_USER=reopt - - POSTGRES_PASSWORD=reopt - - POSTGRES_DB=reopt - ports: - - 5432:5432 - - celery-nginx: - container_name: celery-nginx - build: - context: . - image: base-api-image:latest - command: > - "celery -A reopt_api worker -l info" - environment: - - APP_ENV=local - - DB_HOST=db-nginx - - DB_NAME=reopt - - DB_USERNAME=reopt - - DB_PASSWORD=reopt - - DB_SEARCH_PATH=public - - REDIS_HOST=redis-nginx - - REDIS_PASSWORD= - - SOLVER=xpress - - JULIA_HOST=julia-nginx - - NLR_API_KEY - env_file: - - path: .env - required: false - volumes: - - .:/opt/reopt - depends_on: - - db-nginx - - redis-nginx - - julia-nginx - - django-nginx: - image: base-api-image:latest - container_name: django-nginx - command: > - "python manage.py migrate - && /opt/reopt/bin/wait-for-it.bash -t 0 julia-nginx:8081 -- python manage.py runserver 0.0.0.0:8000" - environment: - - APP_ENV=local - - DB_HOST=db-nginx - - DB_NAME=reopt - - DB_USERNAME=reopt - - DB_PASSWORD=reopt - - DB_SEARCH_PATH=public - - REDIS_HOST=redis-nginx - - REDIS_PASSWORD= - - SOLVER=xpress - - JULIA_HOST=julia-nginx - - NLR_API_KEY - env_file: - - path: .env - required: false - depends_on: - - db-nginx - - redis-nginx - - julia-nginx - - celery-nginx - expose: - - 8000 - volumes: - - .:/opt/reopt - - nginx: - build: ./nginx - ports: - - 80:80 - depends_on: - - django-nginx - restart: "on-failure" - - julia-nginx: - container_name: julia-nginx - build: - context: julia_src/ - command: bash ./run_julia_servers.sh 8 - environment: - - JULIA_NUM_THREADS=2 - expose: - - 8081 - volumes: - - ./julia_src:/opt/julia_src - -volumes: - pgdata: diff --git a/nginx/Dockerfile b/nginx/Dockerfile deleted file mode 100644 index c86307f32..000000000 --- a/nginx/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:1.19.0-alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d diff --git a/nginx/nginx.conf b/nginx/nginx.conf deleted file mode 100644 index f5ab4d720..000000000 --- a/nginx/nginx.conf +++ /dev/null @@ -1,18 +0,0 @@ -upstream reopt_api { - server django-nginx:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://reopt_api; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - client_max_body_size 30M; - proxy_read_timeout 600s; - } - -} From cc173d6e87b4e7931ecacd47739d374c2ec1eab7 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:55:36 -0600 Subject: [PATCH 04/66] Try workaround for extracting non-sensitive secrets. Since these values were output in the metadata's own output, GitHub Actions would fail, since it would prevent the metadata's output from occurring since it thought it contained masked secrets. --- .github/workflows/deploy.yml | 36 ++++++++++---------- .github/workflows/prune-deploy-images.yml | 40 +++++++++-------------- .github/workflows/undeploy-closed-prs.yml | 26 +++++++-------- 3 files changed, 46 insertions(+), 56 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 86805d1e0..dcd99c1be 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,9 +23,23 @@ concurrency: cancel-in-progress: true jobs: + vault-nonsensitive-secrets: + name: Vault Non-sensitive Secrets + uses: TADA/vault-action/.github/workflows/nonsensitive-secrets.yml@nonsensitive + with: + nonsensitive-secrets: | + secret/data/reopt-api/ci/deploy container_registry ; + secret/data/reopt-api/ci/deploy production_rancher_project_id ; + secret/data/reopt-api/ci/deploy staging_rancher_project_id ; + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + deploy-metadata: name: Deploy Metadata runs-on: self-hosted + needs: + - vault-nonsensitive-secrets outputs: staging-perform-deploy: ${{ steps.staging-metadata.outputs.perform-deploy }} staging-perform-undeploy: ${{ steps.staging-metadata.outputs.perform-undeploy }} @@ -33,28 +47,14 @@ jobs: production-perform-deploy: ${{ steps.production-metadata.outputs.perform-deploy }} production-metadata: ${{ toJSON(steps.production-metadata.outputs) }} steps: - - name: Import vault secrets - id: vault_secrets - uses: hashicorp/vault-action@v3 - with: - url: ${{ secrets.VAULT_URL }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - exportEnv: false - secrets: | - secret/data/reopt-api/ci/deploy container_registry ; - secret/data/reopt-api/ci/deploy production_rancher_project_id ; - secret/data/reopt-api/ci/deploy staging_rancher_project_id ; - - name: Staging Metadata id: staging-metadata uses: TADA/deploy-action/metadata@v2 with: deploy-env: staging app-name: reopt-api - rancher-project-id: ${{ steps.vault_secrets.outputs.staging_rancher_project_id }} - registry: ${{ steps.vault_secrets.outputs.container_registry }} + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} branch-db-name-base: reopt_api_staging @@ -64,8 +64,8 @@ jobs: with: deploy-env: production app-name: reopt-api - rancher-project-id: ${{ steps.vault_secrets.outputs.production_rancher_project_id }} - registry: ${{ steps.vault_secrets.outputs.container_registry }} + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} + registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} url-host: ${{ secrets.PRODUCTION_URL_HOST }} undeploy-staging: diff --git a/.github/workflows/prune-deploy-images.yml b/.github/workflows/prune-deploy-images.yml index c3f11d236..c086b473d 100644 --- a/.github/workflows/prune-deploy-images.yml +++ b/.github/workflows/prune-deploy-images.yml @@ -10,43 +10,33 @@ concurrency: cancel-in-progress: true jobs: - vault-secrets: - name: Vault Secrets - runs-on: self-hosted - outputs: - container_registry: ${{ steps.vault_secrets.outputs.container_registry }} - production_rancher_project_id: ${{ steps.vault_secrets.outputs.production_rancher_project_id }} - staging_rancher_project_id: ${{ steps.vault_secrets.outputs.staging_rancher_project_id }} - steps: - - name: Import vault secrets - id: vault_secrets - uses: hashicorp/vault-action@v3 - with: - url: ${{ secrets.VAULT_URL }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - exportEnv: false - secrets: | - secret/data/reopt-api/ci/deploy container_registry ; - secret/data/reopt-api/ci/deploy production_rancher_project_id ; - secret/data/reopt-api/ci/deploy staging_rancher_project_id ; + vault-nonsensitive-secrets: + name: Vault Non-sensitive Secrets + uses: TADA/vault-action/.github/workflows/nonsensitive-secrets.yml@nonsensitive + with: + nonsensitive-secrets: | + secret/data/reopt-api/ci/deploy container_registry ; + secret/data/reopt-api/ci/deploy production_rancher_project_id ; + secret/data/reopt-api/ci/deploy staging_rancher_project_id ; + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} prune-images: name: Prune Deploy Images uses: TADA/deploy-action/.github/workflows/prune-deploy-images.yml@v2 needs: - - vault-secrets + - vault-nonsensitive-secrets with: vault-registry-credentials-path: secret/data/deploy/common/aws-ecr - registry: ${{ needs.vault-secrets.outputs.container_registry }} + registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} images: | tada/reopt-api clusters: | - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test - rancher-project-id: ${{ needs.vault-secrets.outputs.staging_rancher_project_id }} + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} - vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-production - rancher-project-id: ${{ needs.vault-secrets.outputs.production_rancher_project_id }} + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/undeploy-closed-prs.yml b/.github/workflows/undeploy-closed-prs.yml index 36ff026db..7d8ea366f 100644 --- a/.github/workflows/undeploy-closed-prs.yml +++ b/.github/workflows/undeploy-closed-prs.yml @@ -10,31 +10,31 @@ concurrency: cancel-in-progress: true jobs: + vault-nonsensitive-secrets: + name: Vault Non-sensitive Secrets + uses: TADA/vault-action/.github/workflows/nonsensitive-secrets.yml@nonsensitive + with: + nonsensitive-secrets: | + secret/data/reopt-api/ci/deploy staging_rancher_project_id ; + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + deploy-metadata: name: Deploy Metadata runs-on: self-hosted + needs: + - vault-nonsensitive-secrets outputs: staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} steps: - - name: Import vault secrets - id: vault_secrets - uses: hashicorp/vault-action@v3 - with: - url: ${{ secrets.VAULT_URL }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - exportEnv: false - secrets: | - secret/data/reopt-api/ci/deploy staging_rancher_project_id ; - - name: Staging Metadata id: staging-metadata uses: TADA/deploy-action/metadata@v2 with: deploy-env: staging app-name: reopt-api - rancher-project-id: ${{ steps.vault_secrets.outputs.staging_rancher_project_id }} + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} branch-db-name-base: reopt_api_staging From edcf19008111ee810b4504ee6f6d23218deecb3a Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 22 Apr 2026 22:45:19 -0600 Subject: [PATCH 05/66] Try different approach to extracting non-sensitive secrets. --- .github/workflows/deploy.yml | 37 ++++++++++++----------- .github/workflows/prune-deploy-images.yml | 29 ++++++++++++------ .github/workflows/undeploy-closed-prs.yml | 27 +++++++++-------- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index dcd99c1be..2c9d05f72 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,23 +23,9 @@ concurrency: cancel-in-progress: true jobs: - vault-nonsensitive-secrets: - name: Vault Non-sensitive Secrets - uses: TADA/vault-action/.github/workflows/nonsensitive-secrets.yml@nonsensitive - with: - nonsensitive-secrets: | - secret/data/reopt-api/ci/deploy container_registry ; - secret/data/reopt-api/ci/deploy production_rancher_project_id ; - secret/data/reopt-api/ci/deploy staging_rancher_project_id ; - secrets: - vault-role-id: ${{ secrets.VAULT_ROLE_ID }} - vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} - deploy-metadata: name: Deploy Metadata runs-on: self-hosted - needs: - - vault-nonsensitive-secrets outputs: staging-perform-deploy: ${{ steps.staging-metadata.outputs.perform-deploy }} staging-perform-undeploy: ${{ steps.staging-metadata.outputs.perform-undeploy }} @@ -47,14 +33,29 @@ jobs: production-perform-deploy: ${{ steps.production-metadata.outputs.perform-deploy }} production-metadata: ${{ toJSON(steps.production-metadata.outputs) }} steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@nonsensitive + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "container_registry" .container_registry + "production_rancher_project_id" .production_rancher_project_id + "staging_rancher_project_id" .staging_rancher_project_id + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + - name: Staging Metadata id: staging-metadata uses: TADA/deploy-action/metadata@v2 with: deploy-env: staging app-name: reopt-api - rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} - registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} branch-db-name-base: reopt_api_staging @@ -64,8 +65,8 @@ jobs: with: deploy-env: production app-name: reopt-api - rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} - registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} url-host: ${{ secrets.PRODUCTION_URL_HOST }} undeploy-staging: diff --git a/.github/workflows/prune-deploy-images.yml b/.github/workflows/prune-deploy-images.yml index c086b473d..6910b1e68 100644 --- a/.github/workflows/prune-deploy-images.yml +++ b/.github/workflows/prune-deploy-images.yml @@ -11,16 +11,25 @@ concurrency: jobs: vault-nonsensitive-secrets: - name: Vault Non-sensitive Secrets - uses: TADA/vault-action/.github/workflows/nonsensitive-secrets.yml@nonsensitive - with: - nonsensitive-secrets: | - secret/data/reopt-api/ci/deploy container_registry ; - secret/data/reopt-api/ci/deploy production_rancher_project_id ; - secret/data/reopt-api/ci/deploy staging_rancher_project_id ; - secrets: - vault-role-id: ${{ secrets.VAULT_ROLE_ID }} - vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + name: Vault Non-Sensitive Secrets + runs-on: self-hosted + outputs: + nonsensitive-secrets: ${{ steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets }} + steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@nonsensitive + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "container_registry" .container_registry + "production_rancher_project_id" .production_rancher_project_id + "staging_rancher_project_id" .staging_rancher_project_id + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} prune-images: name: Prune Deploy Images diff --git a/.github/workflows/undeploy-closed-prs.yml b/.github/workflows/undeploy-closed-prs.yml index 7d8ea366f..e7a677140 100644 --- a/.github/workflows/undeploy-closed-prs.yml +++ b/.github/workflows/undeploy-closed-prs.yml @@ -10,31 +10,32 @@ concurrency: cancel-in-progress: true jobs: - vault-nonsensitive-secrets: - name: Vault Non-sensitive Secrets - uses: TADA/vault-action/.github/workflows/nonsensitive-secrets.yml@nonsensitive - with: - nonsensitive-secrets: | - secret/data/reopt-api/ci/deploy staging_rancher_project_id ; - secrets: - vault-role-id: ${{ secrets.VAULT_ROLE_ID }} - vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} - deploy-metadata: name: Deploy Metadata runs-on: self-hosted - needs: - - vault-nonsensitive-secrets outputs: staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@nonsensitive + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "staging_rancher_project_id" .staging_rancher_project_id + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + - name: Staging Metadata id: staging-metadata uses: TADA/deploy-action/metadata@v2 with: deploy-env: staging app-name: reopt-api - rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} branch-db-name-base: reopt_api_staging From 4726ab9d39e5b881abee1552ab2ce330d5acdfe3 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 22 Apr 2026 23:13:28 -0600 Subject: [PATCH 06/66] Fix staging config rendering. --- config/deploy/WebDeployment.pkl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index ea1792321..e4d4abf64 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -97,11 +97,11 @@ resources: Listing = new { tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName tadaResourceLimitCpu = "2000m" - tadaResourceLimitMemory = 2000.mib when (deployMetadata.`deploy-env` == "production") { tadaReplicaCount = 10 tadaResourceRequestCpu = "1000m" + tadaResourceLimitMemory = 2000.mib tadaResourceRequestMemory = 2000.mib } else { tadaResourceRequestCpu = "500m" From 35135abfec453dc1ad5dce63d70f2fdce6d2b992 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 22 Apr 2026 23:21:47 -0600 Subject: [PATCH 07/66] Fix image name during build --- config/deploy/AppKbldConfig.pkl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/deploy/AppKbldConfig.pkl b/config/deploy/AppKbldConfig.pkl index 50ee5e41f..53e750ed5 100644 --- a/config/deploy/AppKbldConfig.pkl +++ b/config/deploy/AppKbldConfig.pkl @@ -46,11 +46,11 @@ sources { destinations { new { ["image"] = "reopt-api" - ["newImage"] = secrets.`SECRET_CONTAINER_REGISTRY` + ["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api" } new { ["image"] = "julia-api" - ["newImage"] = secrets.`SECRET_CONTAINER_REGISTRY` + ["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api" } } From 1b27d5eb440eda180ecfa962d9a4fbf6e3fdd02b Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 07:15:41 -0600 Subject: [PATCH 08/66] Update vault kubeconfig paths for reopt api cluster. --- .github/workflows/deploy.yml | 6 +++--- .github/workflows/prune-deploy-images.yml | 4 ++-- .github/workflows/undeploy-closed-prs.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2c9d05f72..891451495 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -77,7 +77,7 @@ jobs: uses: TADA/deploy-action/.github/workflows/undeploy-branch.yml@v2 with: metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt vault-db-superuser-path: secret/data/reopt-db/staging/db-superuser secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} @@ -92,7 +92,7 @@ jobs: with: metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} vault-registry-credentials-path: secret/data/deploy/common/aws-ecr - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} @@ -107,7 +107,7 @@ jobs: with: metadata: ${{ needs.deploy-metadata.outputs.production-metadata }} vault-registry-credentials-path: secret/data/deploy/common/aws-ecr - vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-production + vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/prune-deploy-images.yml b/.github/workflows/prune-deploy-images.yml index 6910b1e68..5c190a950 100644 --- a/.github/workflows/prune-deploy-images.yml +++ b/.github/workflows/prune-deploy-images.yml @@ -42,9 +42,9 @@ jobs: images: | tada/reopt-api clusters: | - - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} - - vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-production + - vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} diff --git a/.github/workflows/undeploy-closed-prs.yml b/.github/workflows/undeploy-closed-prs.yml index e7a677140..2dd201df1 100644 --- a/.github/workflows/undeploy-closed-prs.yml +++ b/.github/workflows/undeploy-closed-prs.yml @@ -46,7 +46,7 @@ jobs: uses: TADA/deploy-action/.github/workflows/undeploy-closed-prs.yml@v2 with: base-metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt vault-db-superuser-path: secret/data/reopt-db/staging/db-superuser secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} From 2218e0feca2d9250fbde7d3dcfaf0faf4eb21a3a Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 07:28:39 -0600 Subject: [PATCH 09/66] Improve docker cacheablity of build process. --- .dockerignore | 2 ++ Dockerfile | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2fa840fb8..c9a1728d8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,5 @@ Jenkinsfile werf.yaml **/*.dylib **/*.dll +Dockerfile* +/config/deploy diff --git a/Dockerfile b/Dockerfile index c5c064371..07f5ee0a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,19 +8,21 @@ ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV SRC_DIR=/opt/reopt/reo/src ENV LD_LIBRARY_PATH="/opt/reopt/reo/src:${LD_LIBRARY_PATH}" -# Copy all code -ENV PYTHONDONTWRITEBYTECODE=1 -COPY . /opt/reopt - # Install python packages +ENV PYTHONDONTWRITEBYTECODE=1 +COPY requirements.txt /opt/reopt/ WORKDIR /opt/reopt RUN ["pip", "install", "-r", "requirements.txt"] # Conditionally install EVI-EnLitePy and pydantic (dependency) if EVI-EnLitePy has been cloned via git submodule +COPY EVI-EnLitePy /opt/reopt/ RUN if [ -d "/opt/reopt/EVI-EnLitePy" ] && [ "$(ls -A /opt/reopt/EVI-EnLitePy)" ]; then \ cd /opt/reopt/EVI-EnLitePy && pip install -e .; \ pip install pydantic; \ fi +# Copy the rest of the app code. +COPY . /opt/reopt + EXPOSE 8000 ENTRYPOINT ["/bin/bash", "-c"] From 29f52d6d47397e1a2669d4e8f3951a36453214e9 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 07:31:40 -0600 Subject: [PATCH 10/66] Fix julia deployment using incorrect container. --- config/deploy/WebDeployment.pkl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index e4d4abf64..6d71e718a 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -134,7 +134,7 @@ resources: Listing = new { // deployment. local juliaDeployment = new TadaWebDeployment { tadaName = "julia" - tadaContainerImage = containerImage + tadaContainerImage = "julia-api" tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName tadaResourceLimitCpu = "4000m" From 5c53ff5bd0747b4c30b86abe17358b1e0af3eed2 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 07:32:21 -0600 Subject: [PATCH 11/66] Fix service URLs --- config/deploy/WebDeployment.pkl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index 6d71e718a..94bdbd1b1 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -48,8 +48,8 @@ resources: Listing = new { data { ["APP_ENV"] = deployMetadata.`deploy-env` ["BRANCH_NAME"] = deployMetadata.`branch-name` - ["JULIA_HOST"] = "julia-service.svc.cluster.local" - ["REDIS_HOST"] = "redis-service.svc.cluster.local" + ["JULIA_HOST"] = "julia-service.\(deployMetadata.`app-namespace`).svc.cluster.local" + ["REDIS_HOST"] = "redis-service.\(deployMetadata.`app-namespace`).svc.cluster.local" ["K8S_DEPLOY"] = "true" when (deployMetadata.`branch-is-primary` == "false") { From f379f43d60b864face256269601eaa79210c79bb Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 07:56:02 -0600 Subject: [PATCH 12/66] Additional dockerignore rules for better build caching. --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index c9a1728d8..624bac97e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ .git +.git* env/ *.pyc *.rdb @@ -15,4 +16,5 @@ werf.yaml **/*.dylib **/*.dll Dockerfile* +.dockerignore /config/deploy From 2e5dbcca0ac59ac6e2c56acae980f145e94fc880 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 08:02:49 -0600 Subject: [PATCH 13/66] Fix missing URL hosts for deployments. --- .github/workflows/deploy.yml | 6 ++-- .github/workflows/undeploy-closed-prs.yml | 3 +- config/deploy/WebDeployment.pkl | 38 +++++++++++------------ 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 891451495..53e53aee4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -42,7 +42,9 @@ jobs: {{ $secrets = coll.Merge (coll.Dict "container_registry" .container_registry "production_rancher_project_id" .production_rancher_project_id + "production_url_host" .production_url_host "staging_rancher_project_id" .staging_rancher_project_id + "staging_url_host_base" .staging_url_host_base ) $secrets }} {{ end }} vault-role-id: ${{ secrets.VAULT_ROLE_ID }} @@ -56,7 +58,7 @@ jobs: app-name: reopt-api rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} - branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_url_host_base }} branch-db-name-base: reopt_api_staging - name: Production Metadata @@ -67,7 +69,7 @@ jobs: app-name: reopt-api rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} - url-host: ${{ secrets.PRODUCTION_URL_HOST }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_url_host }} undeploy-staging: name: Undeploy Staging diff --git a/.github/workflows/undeploy-closed-prs.yml b/.github/workflows/undeploy-closed-prs.yml index 2dd201df1..5bbaca25e 100644 --- a/.github/workflows/undeploy-closed-prs.yml +++ b/.github/workflows/undeploy-closed-prs.yml @@ -24,6 +24,7 @@ jobs: {{ with (datasource "vault" "reopt-api/ci/deploy").data }} {{ $secrets = coll.Merge (coll.Dict "staging_rancher_project_id" .staging_rancher_project_id + "staging_url_host_base" .staging_url_host_base ) $secrets }} {{ end }} vault-role-id: ${{ secrets.VAULT_ROLE_ID }} @@ -36,7 +37,7 @@ jobs: deploy-env: staging app-name: reopt-api rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} - branch-url-host-base: ${{ secrets.STAGING_URL_HOST_BASE }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_url_host_base }} branch-db-name-base: reopt_api_staging undeploy-staging: diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index 94bdbd1b1..a815fe4b1 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -362,26 +362,24 @@ resources: Listing = new { tadaServiceName = djangoService.metadata.name tadaServicePort = 8000 - // Configure the ingress on `main` to listen on the old `*.nrel.gov` - // domains (in addition to the new `nlr.gov` domains). We can remove this - // once the nrel.gov domain is fully gone. - when (deployMetadata.`deploy-env` == "production" || (deployMetadata.`deploy-env` == "staging" && deployMetadata.`branch-is-primary` == "true")) { - local nrelHost = deployMetadata.`url-host`.replaceLast(".nlr.gov", ".nrel.gov") - spec { - rules { - new { - host = nrelHost!! - http { - paths { - new { - path = "/" - pathType = "Prefix" - backend { - service { - name = djangoService.metadata.name - port { - number = 8000 - } + // Configure the ingress to listen on the old `*.nrel.gov` domains (in + // addition to the new `nlr.gov` domains). We can remove this once the + // nrel.gov domain is fully gone. + local nrelHost = deployMetadata.`url-host`.replaceLast(".nlr.gov", ".nrel.gov") + spec { + rules { + new { + host = nrelHost!! + http { + paths { + new { + path = "/" + pathType = "Prefix" + backend { + service { + name = djangoService.metadata.name + port { + number = 8000 } } } From 111b9a17bcb4457742d0d32a1906bf8ba587762e Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 08:23:57 -0600 Subject: [PATCH 14/66] Use new settings to deduplicate some things. --- config/deploy/WebDeployment.pkl | 5 +---- config/deploy/vendir.lock.yml | 5 +++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index a815fe4b1..1c25de812 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -340,26 +340,23 @@ resources: Listing = new { local djangoService = new TadaWebService { tadaName = "django" - tadaAppLabel = "django" tadaPort = 8000 } djangoService local juliaService = new TadaWebService { tadaName = "julia" - tadaAppLabel = "julia" tadaPort = 8081 } juliaService new TadaWebService { tadaName = "redis" - tadaAppLabel = "redis" tadaPort = 6379 } new TadaWebIngress { - tadaServiceName = djangoService.metadata.name + tadaName = "django" tadaServicePort = 8000 // Configure the ingress to listen on the old `*.nrel.gov` domains (in diff --git a/config/deploy/vendir.lock.yml b/config/deploy/vendir.lock.yml index bcca8c3ad..92862eece 100644 --- a/config/deploy/vendir.lock.yml +++ b/config/deploy/vendir.lock.yml @@ -2,8 +2,9 @@ apiVersion: vendir.k14s.io/v1alpha1 directories: - contents: - git: - commitTitle: Allow for easier customization of service and ingress name. - sha: 2ad60887c5a5c3f6766a43cb394cb890a50b43f1 + commitTitle: Make ingress match service name by default when custom name is + defined + sha: 033995789e8e137b581d26040acd0934b193ffbb path: tada-shared-kube path: external kind: LockConfig From a5c306d8922f8a217217e7094441aa34064d4e63 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 09:22:27 -0600 Subject: [PATCH 15/66] Increase staging django memory limit. This appears to be an existing staging issue, but it looks like django keeps bumping into this limit and restarting workers inside the container. --- config/deploy/WebDeployment.pkl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index 1c25de812..73e73eeee 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -105,8 +105,8 @@ resources: Listing = new { tadaResourceRequestMemory = 2000.mib } else { tadaResourceRequestCpu = "500m" - tadaResourceRequestMemory = 700.mib - tadaResourceLimitMemory = 700.mib + tadaResourceRequestMemory = 800.mib + tadaResourceLimitMemory = 1000.mib } tadaContainerPort = 8000 From e355daaddee740061bc1a78843666400442923ab Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 09:44:32 -0600 Subject: [PATCH 16/66] Tweaks to use stable versions of dependencies we've merged updates to. --- .dockerignore | 1 - .github/workflows/deploy.yml | 2 +- .github/workflows/prune-deploy-images.yml | 2 +- .github/workflows/undeploy-closed-prs.yml | 2 +- config/deploy/vendir.lock.yml | 5 ++--- config/deploy/vendir.yml | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.dockerignore b/.dockerignore index 624bac97e..5509e8f7a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ -.git .git* env/ *.pyc diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 53e53aee4..1e8dfcfae 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Import vault nonsensitive secrets id: vault-nonsensitive-secrets - uses: TADA/vault-action/nonsensitive-secrets@nonsensitive + uses: TADA/vault-action/nonsensitive-secrets@v1 with: template: | {{ with (datasource "vault" "reopt-api/ci/deploy").data }} diff --git a/.github/workflows/prune-deploy-images.yml b/.github/workflows/prune-deploy-images.yml index 5c190a950..3791dc3f4 100644 --- a/.github/workflows/prune-deploy-images.yml +++ b/.github/workflows/prune-deploy-images.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Import vault nonsensitive secrets id: vault-nonsensitive-secrets - uses: TADA/vault-action/nonsensitive-secrets@nonsensitive + uses: TADA/vault-action/nonsensitive-secrets@v1 with: template: | {{ with (datasource "vault" "reopt-api/ci/deploy").data }} diff --git a/.github/workflows/undeploy-closed-prs.yml b/.github/workflows/undeploy-closed-prs.yml index 5bbaca25e..1bbd76d9e 100644 --- a/.github/workflows/undeploy-closed-prs.yml +++ b/.github/workflows/undeploy-closed-prs.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Import vault nonsensitive secrets id: vault-nonsensitive-secrets - uses: TADA/vault-action/nonsensitive-secrets@nonsensitive + uses: TADA/vault-action/nonsensitive-secrets@v1 with: template: | {{ with (datasource "vault" "reopt-api/ci/deploy").data }} diff --git a/config/deploy/vendir.lock.yml b/config/deploy/vendir.lock.yml index 92862eece..d8b107d81 100644 --- a/config/deploy/vendir.lock.yml +++ b/config/deploy/vendir.lock.yml @@ -2,9 +2,8 @@ apiVersion: vendir.k14s.io/v1alpha1 directories: - contents: - git: - commitTitle: Make ingress match service name by default when custom name is - defined - sha: 033995789e8e137b581d26040acd0934b193ffbb + commitTitle: 'Merge pull request #29 from TADA/main...' + sha: 2cd9ca01cdf325f2daaa608b8b91c7de4a5d2b8c path: tada-shared-kube path: external kind: LockConfig diff --git a/config/deploy/vendir.yml b/config/deploy/vendir.yml index a830b6cd6..9bbc9b46b 100644 --- a/config/deploy/vendir.yml +++ b/config/deploy/vendir.yml @@ -6,4 +6,4 @@ directories: - path: tada-shared-kube git: url: https://github.nrel.gov/TADA/tada-shared-kube.git - ref: origin/ports + ref: origin/v2 From b3095ed4c6077a4410211be164d8fe786222a4c6 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 14:04:53 -0600 Subject: [PATCH 17/66] Provide .env example file and update instructions --- .env.example | 3 +++ keys_env.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..f20c768a2 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +# Get your individual key from here: https://developer.nlr.gov/signup/ and +# replace the DEMO_KEY with that +NLR_API_KEY=DEMO_KEY diff --git a/keys_env.py b/keys_env.py index c61bb6c5e..eae9d776d 100644 --- a/keys_env.py +++ b/keys_env.py @@ -1,13 +1,12 @@ import os -rollbar_access_token = os.getenv('SECRET_ROLLBAR_ACCESS_TOKEN', 'test') +# To adjust settings, set environment variables (either through a .env file, +# docker-compose.yml or other means). -#get your individual key from here: https://developer.nrel.gov/docs/api-key/ and replace the DEMO_KEY with that pvwatts_api_key = os.getenv('SECRET_PVWATTS_NLR_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) developer_nrel_gov_key = os.getenv('SECRET_DEVELOPER_NLR_GOV_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) ashrae_tmy_key = os.getenv('SECRET_ASHRAE_TMY_NLR_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) - secret_key_ = os.getenv('SECRET_DJANGO_SECRET_KEY', 'secret_key_test') db_host = os.getenv('DB_HOST', os.getenv('SECRET_DB_HOST', 'localhost')) @@ -17,3 +16,4 @@ db_search_path = os.getenv('DB_SEARCH_PATH', os.getenv('SECRET_DB_SEARCH_PATH', 'reopt_api')) redis_host = os.getenv('REDIS_HOST', os.getenv('SECRET_REDIS_HOST', 'localhost')) redis_password = os.getenv('REDIS_PASSWORD', os.getenv('SECRET_REDIS_PASSWORD', 'password')) +rollbar_access_token = os.getenv('SECRET_ROLLBAR_ACCESS_TOKEN', 'test') From 4e231cc4a31612021d962a64e27acb2992fee026 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 23 Apr 2026 14:06:58 -0600 Subject: [PATCH 18/66] Setup restart celery/julia restart job in GitHub Actions. This brings back the restart job that used to be implemented in Jenkins. --- .github/workflows/restart-celery-julia.yml | 118 +++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 .github/workflows/restart-celery-julia.yml diff --git a/.github/workflows/restart-celery-julia.yml b/.github/workflows/restart-celery-julia.yml new file mode 100644 index 000000000..a02d36990 --- /dev/null +++ b/.github/workflows/restart-celery-julia.yml @@ -0,0 +1,118 @@ +name: Restart Celery & Julia + +on: + schedule: + - cron: "23 8 * * *" # Every day at 01:23 AM MST / 02:23 AM MDT + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + deploy-metadata: + name: Deploy Metadata + runs-on: self-hosted + outputs: + ci-deploy-image: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).ci_deploy_image }} + staging-perform-deploy: ${{ steps.staging-metadata.outputs.perform-deploy }} + staging-perform-undeploy: ${{ steps.staging-metadata.outputs.perform-undeploy }} + staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} + production-perform-deploy: ${{ steps.production-metadata.outputs.perform-deploy }} + production-metadata: ${{ toJSON(steps.production-metadata.outputs) }} + steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@v1 + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "ci_deploy_image" .ci_deploy_image + "container_registry" .container_registry + "production_rancher_project_id" .production_rancher_project_id + "production_url_host" .production_url_host + "staging_rancher_project_id" .staging_rancher_project_id + "staging_url_host_base" .staging_url_host_base + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + - name: Staging Metadata + id: staging-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: staging + app-name: reopt-api + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_url_host_base }} + branch-db-name-base: reopt_api_staging + + - name: Production Metadata + id: production-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: production + app-name: reopt-api + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_url_host }} + + restart-staging: + name: Restart Staging + needs: + - deploy-metadata + if: ${{ needs.deploy-metadata.outputs.staging-perform-deploy == 'true' }} + runs-on: self-hosted + container: + image: ${{ needs.deploy-metadata.outputs.ci-deploy-image }} + env: + NODE_OPTIONS: --use-openssl-ca + steps: + - name: Kubernetes config setup + uses: TADA/deploy-action/kubeconfig@v2 + with: + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + rancher-project-id: ${{ fromJSON(needs.deploy-metadata.outputs.staging-metadata).rancher-project-id }} + - name: Rollout restart + env: + app_namespace: "${{ fromJSON(needs.deploy-metadata.outputs.staging-metadata).app-namespace }}" + run: | + set -x + kubectl -n "$app_namespace" rollout restart deployment/celery-deployment + kubectl -n "$app_namespace" rollout status deployment/celery-deployment --timeout=10m + kubectl -n "$app_namespace" rollout restart deployment/julia-deployment + kubectl -n "$app_namespace" rollout status deployment/julia-deployment --timeout=10m + + restart-production: + name: Restart Production + needs: + - deploy-metadata + - restart-staging + if: ${{ needs.deploy-metadata.outputs.production-perform-deploy == 'true' }} + runs-on: self-hosted + container: + image: ${{ needs.deploy-metadata.outputs.ci-deploy-image }} + env: + NODE_OPTIONS: --use-openssl-ca + steps: + - name: Kubernetes config setup + uses: TADA/deploy-action/kubeconfig@v2 + with: + vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + rancher-project-id: ${{ fromJSON(needs.deploy-metadata.outputs.production-metadata).rancher-project-id }} + - name: Rollout restart + env: + app_namespace: "${{ fromJSON(needs.deploy-metadata.outputs.production-metadata).app-namespace }}" + run: | + set -x + kubectl -n "$app_namespace" rollout restart deployment/celery-deployment + kubectl -n "$app_namespace" rollout status deployment/celery-deployment --timeout=10m + kubectl -n "$app_namespace" rollout restart deployment/julia-deployment + kubectl -n "$app_namespace" rollout status deployment/julia-deployment --timeout=10m From 86a8e5de58bdbcf10b7a5ac0dbdd79d364ae9ff9 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 27 May 2026 11:17:56 -0400 Subject: [PATCH 19/66] Test PostgreSQL v18 deployment --- config/vault_secrets.json.tmpl | 18 ++++++++++++++---- docker-compose.yml | 27 +++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/config/vault_secrets.json.tmpl b/config/vault_secrets.json.tmpl index 9c656fbda..85738b083 100644 --- a/config/vault_secrets.json.tmpl +++ b/config/vault_secrets.json.tmpl @@ -16,12 +16,22 @@ ) $secrets }} {{ end }} + {{ if (eq (env.Getenv "DB_SYNCER") "true") }} + {{ with (datasource "vault" "reopt-web/common/s3").data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_S3_AWS_ACCESS_KEY_ID" .aws_access_key_id + "SECRET_S3_AWS_REGION" .aws_region + "SECRET_S3_AWS_SECRET_ACCESS_KEY" .aws_secret_access_key + ) $secrets }} + {{ end }} + {{ end }} + {{ if (or (eq $deploy_env "staging") (eq $deploy_env "production")) }} {{ with (datasource "vault" (printf "reopt-api/%s/web" $deploy_env)).data }} {{ $secrets = coll.Merge (coll.Dict "SECRET_ASHRAE_TMY_NLR_API_KEY" .ashrae_tmy_nlr_api_key - "SECRET_DB_HOST" .db_host - "SECRET_DB_NAME" .db_name + "SECRET_DB_HOST" .db_host_v18 + "SECRET_DB_NAME" .db_name_v18 "SECRET_DEVELOPER_NLR_GOV_API_KEY" .developer_nlr_gov_api_key "SECRET_DJANGO_SECRET_KEY" .django_secret_key "SECRET_PVWATTS_NLR_API_KEY" .pvwatts_nlr_api_key @@ -32,8 +42,8 @@ {{ with (datasource "vault" (printf "reopt-api/%s/db-migrate" $deploy_env)).data }} {{ $secrets = coll.Merge (coll.Dict - "SECRET_DB_PASSWORD" .db_password - "SECRET_DB_USERNAME" .db_username + "SECRET_DB_PASSWORD" .db_password_v18 + "SECRET_DB_USERNAME" .db_username_v18 ) $secrets }} {{ end }} {{ end }} diff --git a/docker-compose.yml b/docker-compose.yml index ed3796cb6..142ce7de2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,8 +7,10 @@ services: - 6379 db: - image: postgres + image: postgres:18 restart: always + volumes: + - postgres_data:/var/lib/postgresql environment: - POSTGRES_USER=reopt - POSTGRES_PASSWORD=reopt @@ -29,7 +31,6 @@ services: - DB_NAME=reopt - DB_USERNAME=reopt - DB_PASSWORD=reopt - - DB_SEARCH_PATH=public - REDIS_HOST=redis - REDIS_PASSWORD= - NLR_API_KEY @@ -54,7 +55,6 @@ services: - DB_NAME=reopt - DB_USERNAME=reopt - DB_PASSWORD=reopt - - DB_SEARCH_PATH=public - REDIS_HOST=redis - REDIS_PASSWORD= - NLR_API_KEY @@ -70,7 +70,7 @@ services: - 8000:8000 volumes: - .:/opt/reopt - + julia: container_name: julia_api build: @@ -85,3 +85,22 @@ services: - "8081:8081" volumes: - ./julia_src:/opt/julia_src + + db-syncer: + build: + context: db-syncer + args: + BUNDLE_FROZEN: "false" + volumes: + - .:/app + - ~/.vault-token:/root/.vault-token:ro + depends_on: + - db + environment: + - DEV_DB_HOST=db + - DEV_DB_NAME=reopt_api + - DEV_DB_USERNAME=reopt + - DEV_DB_PASSWORD=reopt + +volumes: + postgres_data: From 2beae8d114f5f17bb10b15fdb6ae35dc97616fc5 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 27 May 2026 11:23:26 -0400 Subject: [PATCH 20/66] Use our db-syncer Ruby gem for providing db dump/restore tasks. While this is a bit funky to shoehorn this into this Python app, we have other components of our deployment process that use this library to provide easy database restores, so by leveraging this library, this is the easiest way to integrate that process with this app. --- .dockerignore | 4 + .github/workflows/db-sync-dump.yml | 51 +++ .github/workflows/deploy.yml | 4 + .gitignore | 6 + config/deploy/AppKbldConfig.pkl | 21 ++ config/deploy/RunJob.pkl | 54 +++ config/deploy/WebDeployment.pkl | 8 + db-syncer/.gomplate.yaml | 3 + db-syncer/Dockerfile | 34 ++ db-syncer/Gemfile | 13 + db-syncer/Gemfile.lock | 325 ++++++++++++++++++ db-syncer/Rakefile | 3 + db-syncer/bin/rails | 4 + db-syncer/bin/rake | 4 + db-syncer/config/application.rb | 19 + db-syncer/config/boot.rb | 3 + db-syncer/config/database.yml | 36 ++ db-syncer/config/environment.rb | 5 + db-syncer/config/environments/development.rb | 9 + db-syncer/config/environments/production.rb | 7 + db-syncer/config/environments/staging.rb | 1 + db-syncer/config/environments/test.rb | 10 + .../config/initializers/vault_env_secrets.rb | 1 + db-syncer/lib/tasks/db_data.rake | 213 ++++++++++++ docker-compose.nojulia.yml | 2 - docker-compose.yml | 5 +- 26 files changed, 841 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/db-sync-dump.yml create mode 100644 config/deploy/RunJob.pkl create mode 100644 db-syncer/.gomplate.yaml create mode 100644 db-syncer/Dockerfile create mode 100644 db-syncer/Gemfile create mode 100644 db-syncer/Gemfile.lock create mode 100644 db-syncer/Rakefile create mode 100755 db-syncer/bin/rails create mode 100755 db-syncer/bin/rake create mode 100644 db-syncer/config/application.rb create mode 100644 db-syncer/config/boot.rb create mode 100644 db-syncer/config/database.yml create mode 100644 db-syncer/config/environment.rb create mode 100644 db-syncer/config/environments/development.rb create mode 100644 db-syncer/config/environments/production.rb create mode 100644 db-syncer/config/environments/staging.rb create mode 100644 db-syncer/config/environments/test.rb create mode 100644 db-syncer/config/initializers/vault_env_secrets.rb create mode 100644 db-syncer/lib/tasks/db_data.rake diff --git a/.dockerignore b/.dockerignore index 5509e8f7a..526a07a69 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,3 +17,7 @@ werf.yaml Dockerfile* .dockerignore /config/deploy + +/db-syncer/tmp +/db-syncer/log +!/db-syncer/.gomplate.yaml diff --git a/.github/workflows/db-sync-dump.yml b/.github/workflows/db-sync-dump.yml new file mode 100644 index 000000000..702c3e1d1 --- /dev/null +++ b/.github/workflows/db-sync-dump.yml @@ -0,0 +1,51 @@ +name: DB Sync Dump + +on: + schedule: + - cron: "40 5 * * *" # Every day at 10:40 PM MST / 11:40 PM MDT + workflow_dispatch: + push: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + vault-nonsensitive-secrets: + name: Vault Non-Sensitive Secrets + runs-on: self-hosted + outputs: + nonsensitive-secrets: ${{ steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets }} + steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@v1 + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "container_registry" .container_registry + "staging_rancher_project_id" .staging_rancher_project_id + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + dump: + name: Dump + uses: TADA/deploy-action/.github/workflows/run-job.yml@v2 + needs: + - vault-nonsensitive-secrets + with: + job-name: reopt-api-db-sync-dump + job-command: "DB_SYNCER_PERFORM_UPLOAD=true rails db:data:dump --trace" + deploy-env: production + render-config-command: "DB_OWNER_AUTH=true DB_SYNCER=true RENDER_JOB=run-job RUN_JOB_CONTAINER_IMAGE=db-syncer ./config/deploy/render" + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + vault-registry-credentials-path: secret/data/deploy/common/aws-ecr + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + force-run: true + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1e8dfcfae..fe42733d0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,6 +10,10 @@ on: - reopened - unlabeled workflow_dispatch: + inputs: + db-sync-restore: + description: "Staging Only: Restore this branch's database with a recent snapshot from production." + type: boolean concurrency: # Concurrency group is more complicated in this case because: diff --git a/.gitignore b/.gitignore index 94edd4b03..7d14051ba 100755 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,9 @@ compare_run_*.json /config/deploy/external /config/deploy/tmp +docker-compose.override.yml + +!/db-syncer/lib/ +/db-syncer/db/structure.sql +/db-syncer/log +/db-syncer/tmp diff --git a/config/deploy/AppKbldConfig.pkl b/config/deploy/AppKbldConfig.pkl index 53e750ed5..550b493c9 100644 --- a/config/deploy/AppKbldConfig.pkl +++ b/config/deploy/AppKbldConfig.pkl @@ -41,6 +41,22 @@ sources { } } } + + new { + image = "db-syncer" + path = "." + docker { + build { + file = "db-syncer/Dockerfile" + pull = true + buildkit = true + rawOptions { + "--build-arg" + "NREL_ROOT_CERT_URL_ROOT=\(secrets.`SECRET_NLR_ROOT_CERT_URL_ROOT`)" + } + } + } + } } destinations { @@ -53,4 +69,9 @@ destinations { ["image"] = "julia-api" ["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api" } + + new { + ["image"] = "db-syncer" + ["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api" + } } diff --git a/config/deploy/RunJob.pkl b/config/deploy/RunJob.pkl new file mode 100644 index 000000000..bb0743ffc --- /dev/null +++ b/config/deploy/RunJob.pkl @@ -0,0 +1,54 @@ +import "./AppKbldConfig.pkl" +import "@k8s/K8sResource.pkl" +import "@tadaSharedKube/TadaAppNamespace.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewConfigMap.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewCronJob.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewInitialSetupJob.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewRole.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewRoleBinding.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewSecrets.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewServiceAccount.pkl" +import "@tadaSharedKube/TadaWebConfigMap.pkl" +import "@tadaSharedKube/TadaWebDbMigrateSecrets.pkl" +import "@tadaSharedKube/TadaWebRunJob.pkl" +import "@tadaSharedKube/TadaWebSecrets.pkl" + +local containerImage = read?("env:RUN_JOB_CONTAINER_IMAGE") ?? "reopt-api" + +resources: Listing = new { + new TadaAppNamespace { + tadaRancherResourceQuotaPodLimit = 1 + } + + new AppKbldConfig {} + + // To be able to pull images from our ECR repos in our on-premise (non-AWS) + // clusters. + local ecrLoginRenewConfigMapResource = new TadaEcrLoginRenewConfigMap {} + ecrLoginRenewConfigMapResource + new TadaEcrLoginRenewCronJob {} + new TadaEcrLoginRenewInitialSetupJob {} + new TadaEcrLoginRenewRoleBinding {} + new TadaEcrLoginRenewRole {} + new TadaEcrLoginRenewSecrets {} + new TadaEcrLoginRenewServiceAccount {} + + new TadaWebConfigMap {} + + local secretResource = new TadaWebSecrets {} + secretResource + + new TadaWebRunJob { + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + tadaJobCommand = read("env:RUN_JOB_COMMAND") + tadaSecretName = secretResource.metadata.name + } +} + +output { + value = resources + renderer = (K8sResource.output.renderer as YamlRenderer) { + isStream = true + } +} diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index 73e73eeee..afbdc38a7 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -13,6 +13,7 @@ import "@tadaSharedKube/TadaEcrLoginRenewServiceAccount.pkl" import "@tadaSharedKube/TadaWebConfigMap.pkl" import "@tadaSharedKube/TadaWebDbMigrateJob.pkl" import "@tadaSharedKube/TadaWebDbMigrateSecrets.pkl" +import "@tadaSharedKube/TadaWebDbSyncRestoreJob.pkl" import "@tadaSharedKube/TadaWebDeployment.pkl" import "@tadaSharedKube/TadaWebIngress.pkl" import "@tadaSharedKube/TadaWebPodDisruptionBudget.pkl" @@ -71,6 +72,13 @@ resources: Listing = new { new TadaWebDbMigrateSecrets {} + when (deployMetadata.`perform-db-sync-restore` == "true") { + new TadaWebDbSyncRestoreJob { + tadaContainerImage = "db-syncer" + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + } + } + new TadaWebDbMigrateJob { tadaContainerImage = containerImage tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName diff --git a/db-syncer/.gomplate.yaml b/db-syncer/.gomplate.yaml new file mode 100644 index 000000000..89659b153 --- /dev/null +++ b/db-syncer/.gomplate.yaml @@ -0,0 +1,3 @@ +datasources: + vault: + url: "vault:///secret/data" diff --git a/db-syncer/Dockerfile b/db-syncer/Dockerfile new file mode 100644 index 000000000..9f761feb7 --- /dev/null +++ b/db-syncer/Dockerfile @@ -0,0 +1,34 @@ +FROM public.ecr.aws/docker/library/ruby:4.0-trixie + +# Install NLR root certs for machines running on NLR's network. +ARG NREL_ROOT_CERT_URL_ROOT="" +RUN set -x && if [ -n "$NREL_ROOT_CERT_URL_ROOT" ]; then curl -fsSLk -o /usr/local/share/ca-certificates/nrel_root.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_root.pem" && curl -fsSLk -o /usr/local/share/ca-certificates/nrel_xca1.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_xca1.pem" && update-ca-certificates; fi + +# Postgresql +ARG POSTGRESQL_MAJOR_VERSION=18 +RUN set -x && \ + distro=$(. /etc/os-release && echo "$VERSION_CODENAME") && \ + curl -fsSL -o /usr/share/keyrings/pgdg.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc && \ + echo "deb [signed-by=/usr/share/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ ${distro}-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + apt-get update && \ + apt-get -y --no-install-recommends install "postgresql-client-${POSTGRESQL_MAJOR_VERSION}" rclone zstd && \ + rm -rf /var/lib/apt/lists/* /var/lib/dpkg/*-old /var/cache/* /var/log/* + +ARG TARGETARCH + +# Config templates. +ARG GOMPLATE_VERSION=3.11.8 +RUN curl -fsSL -o /usr/local/bin/gomplate "https://github.com/hairyhenderson/gomplate/releases/download/v${GOMPLATE_VERSION}/gomplate_linux-${TARGETARCH}" && \ + chmod +x /usr/local/bin/gomplate + +WORKDIR /app/db-syncer + +# Install gems +COPY ./db-syncer/Gemfile ./db-syncer/Gemfile.lock /app/db-syncer/ +ARG BUNDLE_FROZEN="true" +ENV BUNDLE_FROZEN=$BUNDLE_FROZEN +RUN bundle install + +# Copy the rest of the app. +COPY ./db-syncer /app/db-syncer +COPY ./config/vault_secrets.json.tmpl /app/config/ diff --git a/db-syncer/Gemfile b/db-syncer/Gemfile new file mode 100644 index 000000000..014f4f858 --- /dev/null +++ b/db-syncer/Gemfile @@ -0,0 +1,13 @@ +source "https://rubygems.org" + +# Syncing production/staging data back to local environments. +gem "db-syncer", :require => false, :git => "https://github.nrel.gov/TADA/db-syncer.git", :branch => "main" + +# Vault secrets as environment variables for development. +gem "vault_env_secrets", "~> 2.0.0" + +# This isn't really a full Rails app, we're just using it for the db-syncer gem +# above to handle database dumps/restores for staging and development +# environments. +gem "rails", "~> 8.1.3" +gem "pg", "~> 1.6.3" diff --git a/db-syncer/Gemfile.lock b/db-syncer/Gemfile.lock new file mode 100644 index 000000000..eccb8f844 --- /dev/null +++ b/db-syncer/Gemfile.lock @@ -0,0 +1,325 @@ +GIT + remote: https://github.nrel.gov/TADA/db-syncer.git + revision: 1cf266b305701dc0cd84d8f6c4c8e86789abc330 + branch: main + specs: + db-syncer (0.2.0) + highline + rest-client + +GEM + remote: https://rubygems.org/ + specs: + action_text-trix (2.1.19) + railties + actioncable (8.1.3) + actionpack (= 8.1.3) + activesupport (= 8.1.3) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + zeitwerk (~> 2.6) + actionmailbox (8.1.3) + actionpack (= 8.1.3) + activejob (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) + mail (>= 2.8.0) + actionmailer (8.1.3) + actionpack (= 8.1.3) + actionview (= 8.1.3) + activejob (= 8.1.3) + activesupport (= 8.1.3) + mail (>= 2.8.0) + rails-dom-testing (~> 2.2) + actionpack (8.1.3) + actionview (= 8.1.3) + activesupport (= 8.1.3) + nokogiri (>= 1.8.5) + rack (>= 2.2.4) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + useragent (~> 0.16) + actiontext (8.1.3) + action_text-trix (~> 2.1.15) + actionpack (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) + globalid (>= 0.6.0) + nokogiri (>= 1.8.5) + actionview (8.1.3) + activesupport (= 8.1.3) + builder (~> 3.1) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activejob (8.1.3) + activesupport (= 8.1.3) + globalid (>= 0.3.6) + activemodel (8.1.3) + activesupport (= 8.1.3) + activerecord (8.1.3) + activemodel (= 8.1.3) + activesupport (= 8.1.3) + timeout (>= 0.4.0) + activestorage (8.1.3) + actionpack (= 8.1.3) + activejob (= 8.1.3) + activerecord (= 8.1.3) + activesupport (= 8.1.3) + marcel (~> 1.0) + activesupport (8.1.3) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) + minitest (>= 5.1) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) + base64 (0.3.0) + bigdecimal (4.1.2) + builder (3.3.0) + concurrent-ruby (1.3.6) + connection_pool (3.0.2) + crass (1.0.6) + date (3.5.1) + domain_name (0.6.20240107) + drb (2.2.3) + erb (6.0.4) + erubi (1.13.1) + globalid (1.3.0) + activesupport (>= 6.1) + highline (3.1.2) + reline + http-accept (1.7.0) + http-cookie (1.1.6) + domain_name (~> 0.5) + i18n (1.14.8) + concurrent-ruby (~> 1.0) + io-console (0.8.2) + irb (1.18.0) + pp (>= 0.6.0) + prism (>= 1.3.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.19.5) + logger (1.7.0) + loofah (2.25.1) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mail (2.9.0) + logger + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp + marcel (1.2.1) + mime-types (3.7.0) + logger + mime-types-data (~> 3.2025, >= 3.2025.0507) + mime-types-data (3.2026.0414) + mini_mime (1.1.5) + mini_portile2 (2.8.9) + minitest (6.0.6) + drb (~> 2.0) + prism (~> 1.5) + net-imap (0.6.4) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.5.1) + net-protocol + netrc (0.11.0) + nio4r (2.7.5) + nokogiri (1.19.3) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.3-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.3-x86_64-linux-gnu) + racc (~> 1.4) + pg (1.6.3) + pg (1.6.3-aarch64-linux) + pg (1.6.3-x86_64-linux) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.9.0) + psych (5.3.1) + date + stringio + racc (1.8.1) + rack (3.2.6) + rack-session (2.1.2) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rackup (2.3.1) + rack (>= 3) + rails (8.1.3) + actioncable (= 8.1.3) + actionmailbox (= 8.1.3) + actionmailer (= 8.1.3) + actionpack (= 8.1.3) + actiontext (= 8.1.3) + actionview (= 8.1.3) + activejob (= 8.1.3) + activemodel (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) + bundler (>= 1.15.0) + railties (= 8.1.3) + rails-dom-testing (2.3.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.7.0) + loofah (~> 2.25) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (8.1.3) + actionpack (= 8.1.3) + activesupport (= 8.1.3) + irb (~> 1.13) + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + tsort (>= 0.2) + zeitwerk (~> 2.6) + rake (13.4.2) + rdoc (7.2.0) + erb + psych (>= 4.0.0) + tsort + reline (0.6.3) + io-console (~> 0.5) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + securerandom (0.4.1) + stringio (3.2.0) + thor (1.5.0) + timeout (0.6.1) + tsort (0.2.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + uri (1.1.1) + useragent (0.16.11) + vault_env_secrets (2.0.0) + websocket-driver (0.8.0) + base64 + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) + zeitwerk (2.8.2) + +PLATFORMS + aarch64-linux + ruby + x86_64-linux + +DEPENDENCIES + db-syncer! + pg (~> 1.6.3) + rails (~> 8.1.3) + vault_env_secrets (~> 2.0.0) + +CHECKSUMS + action_text-trix (2.1.19) sha256=7012f59421009cf284aa651294896414d653a61a2417c9b8714c8476d2f74009 + actioncable (8.1.3) sha256=e5bc7f75e44e6a22de29c4f43176927c3a9ce4824464b74ed18d8226e75a80f0 + actionmailbox (8.1.3) sha256=df7da474eaa0e70df4ed5a6fef66eb3b3b0f2dbf7f14518deee8d77f1b4aae59 + actionmailer (8.1.3) sha256=831f724891bb70d0aaa4d76581a6321124b6a752cb655c9346aae5479318448d + actionpack (8.1.3) sha256=af998cae4d47c5d581a2cc363b5c77eb718b7c4b45748d81b1887b25621c29a3 + actiontext (8.1.3) sha256=d291019c00e1ea9e6463011fa214f6081a56d7b9a1d224e7d3f6384c1dafc7d2 + actionview (8.1.3) sha256=1347c88c7f3edb38100c5ce0e9fb5e62d7755f3edc1b61cce2eb0b2c6ea2fd5d + activejob (8.1.3) sha256=a149b1766aa8204c3c3da7309e4becd40fcd5529c348cffbf6c9b16b565fe8d3 + activemodel (8.1.3) sha256=90c05cbe4cef3649b8f79f13016191ea94c4525ce4a5c0fb7ef909c4b91c8219 + activerecord (8.1.3) sha256=8003be7b2466ba0a2a670e603eeb0a61dd66058fccecfc49901e775260ac70ab + activestorage (8.1.3) sha256=0564ce9309143951a67615e1bb4e090ee54b8befed417133cae614479b46384d + activesupport (8.1.3) sha256=21a5e0dfbd4c3ddd9e1317ec6a4d782fa226e7867dc70b0743acda81a1dca20e + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a + crass (1.0.6) sha256=dc516022a56e7b3b156099abc81b6d2b08ea1ed12676ac7a5657617f012bd45d + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + db-syncer (0.2.0) + domain_name (0.6.20240107) sha256=5f693b2215708476517479bf2b3802e49068ad82167bcd2286f899536a17d933 + drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373 + erb (6.0.4) sha256=38e3803694be357fe2bfe312487c74beaf9fb4e5beb3e22498952fe1645b95d9 + erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9 + globalid (1.3.0) sha256=05c639ad6eb4594522a0b07983022f04aa7254626ab69445a0e493aa3786ff11 + highline (3.1.2) sha256=67cbd34d19f6ef11a7ee1d82ffab5d36dfd5b3be861f450fc1716c7125f4bb4a + http-accept (1.7.0) sha256=c626860682bfbb3b46462f8c39cd470fd7b0584f61b3cc9df5b2e9eb9972a126 + http-cookie (1.1.6) sha256=ba4b82be64de61dc281243dac70e3c382c45142f20268ed9276a3670c93feaa9 + i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5 + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 + json (2.19.5) sha256=218a18553e4801d579ca7e0f5bc72bafd776d7397238a1fb4e74db5b0a812c59 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + loofah (2.25.1) sha256=d436c73dbd0c1147b16c4a41db097942d217303e1f7728704b37e4df9f6d2e04 + mail (2.9.0) sha256=6fa6673ecd71c60c2d996260f9ee3dd387d4673b8169b502134659ece6d34941 + marcel (1.2.1) sha256=1678e9360e32f9eafa917c80029e2f6d10b2715c66a4b87b6d0da9b9cd1f859f + mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 + mime-types-data (3.2026.0414) sha256=461c4c655373a44bd6c5fe54bcf5b7776026ea96e808144b1ec465c4b99148cc + mini_mime (1.1.5) sha256=8681b7e2e4215f2a159f9400b5816d85e9d8c6c6b491e96a12797e798f8bccef + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1 + net-imap (0.6.4) sha256=9a5598c67a3022c284d98430ef1d4948e7dbdb62596f61081ea8ca933270a02b + net-pop (0.1.2) sha256=848b4e982013c15b2f0382792268763b748cce91c9e91e36b0f27ed26420dff3 + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + net-smtp (0.5.1) sha256=ed96a0af63c524fceb4b29b0d352195c30d82dd916a42f03c62a3a70e5b70736 + netrc (0.11.0) sha256=de1ce33da8c99ab1d97871726cba75151113f117146becbe45aa85cb3dabee3f + nio4r (2.7.5) sha256=6c90168e48fb5f8e768419c93abb94ba2b892a1d0602cb06eef16d8b7df1dca1 + nokogiri (1.19.3) sha256=78312cbac32a40c812780d9678221b79d51288eec00054c1a8d15f7ce05960e8 + nokogiri (1.19.3-aarch64-linux-gnu) sha256=46b89e5d7b9e844c2ee360794240c6ea2a4e6fa0c5892a4ed487db621224b639 + nokogiri (1.19.3-x86_64-linux-gnu) sha256=2f5078620fe12e83669b5b17311b32532a8153d02eee7ad06948b926d6080976 + pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99 + pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea + pg (1.6.3-x86_64-linux) sha256=5d9e188c8f7a0295d162b7b88a768d8452a899977d44f3274d1946d67920ae8d + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + rack (3.2.6) sha256=5ed78e1f73b2e25679bec7d45ee2d4483cc4146eb1be0264fc4d94cb5ef212c2 + rack-session (2.1.2) sha256=595434f8c0c3473ae7d7ac56ecda6cc6dfd9d37c0b2b5255330aa1576967ffe8 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rackup (2.3.1) sha256=6c79c26753778e90983761d677a48937ee3192b3ffef6bc963c0950f94688868 + rails (8.1.3) sha256=6d017ba5348c98fc909753a8169b21d44de14d2a0b92d140d1a966834c3c9cd3 + rails-dom-testing (2.3.0) sha256=8acc7953a7b911ca44588bf08737bc16719f431a1cc3091a292bca7317925c1d + rails-html-sanitizer (1.7.0) sha256=28b145cceaf9cc214a9874feaa183c3acba036c9592b19886e0e45efc62b1e89 + railties (8.1.3) sha256=913eb0e0cb520aac687ffd74916bd726d48fa21f47833c6292576ef6a286de22 + rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 + rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rest-client (2.1.0) sha256=35a6400bdb14fae28596618e312776c158f7ebbb0ccad752ff4fa142bf2747e3 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73 + timeout (0.6.1) sha256=78f57368a7e7bbadec56971f78a3f5ecbcfb59b7fcbb0a3ed6ddc08a5094accb + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + useragent (0.16.11) sha256=700e6413ad4bb954bb63547fa098dddf7b0ebe75b40cc6f93b8d54255b173844 + vault_env_secrets (2.0.0) sha256=6c5bb1e29440e43ace29ffd43f55a318846e5eb663f15ab2bee7675701a428db + websocket-driver (0.8.0) sha256=ed0dba4b943c22f17f9a734817e808bc84cdce6a7e22045f5315aa57676d4962 + websocket-extensions (0.1.5) sha256=1c6ba63092cda343eb53fc657110c71c754c56484aad42578495227d717a8241 + zeitwerk (2.8.2) sha256=7212a61311083c604184b1ea2574b9aa05cd14f855a0841c06985cabe9181d12 + +BUNDLED WITH + 4.0.10 diff --git a/db-syncer/Rakefile b/db-syncer/Rakefile new file mode 100644 index 000000000..d1baef069 --- /dev/null +++ b/db-syncer/Rakefile @@ -0,0 +1,3 @@ +require_relative "config/application" + +Rails.application.load_tasks diff --git a/db-syncer/bin/rails b/db-syncer/bin/rails new file mode 100755 index 000000000..efc037749 --- /dev/null +++ b/db-syncer/bin/rails @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +APP_PATH = File.expand_path("../config/application", __dir__) +require_relative "../config/boot" +require "rails/commands" diff --git a/db-syncer/bin/rake b/db-syncer/bin/rake new file mode 100755 index 000000000..4fbf10b96 --- /dev/null +++ b/db-syncer/bin/rake @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +require_relative "../config/boot" +require "rake" +Rake.application.run diff --git a/db-syncer/config/application.rb b/db-syncer/config/application.rb new file mode 100644 index 000000000..bd562e5f2 --- /dev/null +++ b/db-syncer/config/application.rb @@ -0,0 +1,19 @@ +require_relative "boot" + +require "rails" +require "active_record/railtie" + +Bundler.require(*Rails.groups) + +module ReoptApi + class Application < Rails::Application + config.load_defaults 8.1 + config.autoload_lib(ignore: %w[assets tasks]) + + # Use SQL dumps (db/structure.sql) instead of the default Ruby dumps + # (db/schema.rb). This provides easier dumping of multiple Postgres schemas + # and other special Postgres types. + config.active_record.schema_format = :sql + config.active_record.dump_schemas = :all + end +end diff --git a/db-syncer/config/boot.rb b/db-syncer/config/boot.rb new file mode 100644 index 000000000..282011619 --- /dev/null +++ b/db-syncer/config/boot.rb @@ -0,0 +1,3 @@ +ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) + +require "bundler/setup" # Set up gems listed in the Gemfile. diff --git a/db-syncer/config/database.yml b/db-syncer/config/database.yml new file mode 100644 index 000000000..8eb91f52b --- /dev/null +++ b/db-syncer/config/database.yml @@ -0,0 +1,36 @@ +default: &default + adapter: postgresql + encoding: unicode + schema_search_path: reopt_api,public + +development: &development + <<: *default + host: <%= ENV["DEV_DB_HOST"] || ENV["DB_HOST"] %> + port: <%= ENV["DEV_DB_PORT"] || ENV["DB_PORT"] %> + database: <%= ENV["DEV_DB_NAME"] || ENV["DB_NAME"] || "reopt_api_development" %> + username: <%= ENV["DEV_DB_USERNAME"] || ENV["DB_USERNAME"] %> + password: <%= ENV["DEV_DB_PASSWORD"] || ENV["DB_PASSWORD"] %> + min_messages: notice + +# Warning: The database defined as "test" will be erased and +# re-generated from your development database when you run "rake". +# Do not set this db to the same as development or production. +test: + <<: *development + database: <%= ENV["DB_TEST_NAME"] || "reopt_api_test" %> + +staging: + <<: *default + host: <%= ENV["DB_HOST"] || ENV["SECRET_DB_HOST"] %> + port: <%= ENV["DB_PORT"] || ENV["SECRET_DB_PORT"] %> + database: <%= ENV["DB_NAME"] || ENV["SECRET_DB_NAME"] %> + username: <%= ENV["DB_USERNAME"] || ENV["SECRET_DB_USERNAME"] %> + password: <%= ENV["DB_PASSWORD"] || ENV["SECRET_DB_PASSWORD"] %> + +production: + <<: *default + host: <%= ENV["DB_HOST"] || ENV["SECRET_DB_HOST"] %> + port: <%= ENV["DB_PORT"] || ENV["SECRET_DB_PORT"] %> + database: <%= ENV["DB_NAME"] || ENV["SECRET_DB_NAME"] %> + username: <%= ENV["DB_USERNAME"] || ENV["SECRET_DB_USERNAME"] %> + password: <%= ENV["DB_PASSWORD"] || ENV["SECRET_DB_PASSWORD"] %> diff --git a/db-syncer/config/environment.rb b/db-syncer/config/environment.rb new file mode 100644 index 000000000..cac531577 --- /dev/null +++ b/db-syncer/config/environment.rb @@ -0,0 +1,5 @@ +# Load the Rails application. +require_relative "application" + +# Initialize the Rails application. +Rails.application.initialize! diff --git a/db-syncer/config/environments/development.rb b/db-syncer/config/environments/development.rb new file mode 100644 index 000000000..439d5819a --- /dev/null +++ b/db-syncer/config/environments/development.rb @@ -0,0 +1,9 @@ +require "active_support/core_ext/integer/time" + +Rails.application.configure do + # Make code changes take effect immediately without server restart. + config.enable_reloading = true + + # Do not eager load code on boot. + config.eager_load = false +end diff --git a/db-syncer/config/environments/production.rb b/db-syncer/config/environments/production.rb new file mode 100644 index 000000000..54a0cb7e7 --- /dev/null +++ b/db-syncer/config/environments/production.rb @@ -0,0 +1,7 @@ +Rails.application.configure do + # Code is not reloaded between requests. + config.enable_reloading = false + + # Eager load code on boot for better performance and memory savings (ignored by Rake tasks). + config.eager_load = true +end diff --git a/db-syncer/config/environments/staging.rb b/db-syncer/config/environments/staging.rb new file mode 100644 index 000000000..d89e30cf1 --- /dev/null +++ b/db-syncer/config/environments/staging.rb @@ -0,0 +1 @@ +require_relative "production" diff --git a/db-syncer/config/environments/test.rb b/db-syncer/config/environments/test.rb new file mode 100644 index 000000000..1d6ae6c4a --- /dev/null +++ b/db-syncer/config/environments/test.rb @@ -0,0 +1,10 @@ +Rails.application.configure do + # While tests run files are not watched, reloading is not necessary. + config.enable_reloading = false + + # Eager loading loads your entire application. When running a single test locally, + # this is usually not necessary, and can slow down your test suite. However, it's + # recommended that you enable it in continuous integration systems to ensure eager + # loading is working properly before deploying your code. + config.eager_load = ENV["CI"].present? +end diff --git a/db-syncer/config/initializers/vault_env_secrets.rb b/db-syncer/config/initializers/vault_env_secrets.rb new file mode 100644 index 000000000..666683f17 --- /dev/null +++ b/db-syncer/config/initializers/vault_env_secrets.rb @@ -0,0 +1 @@ +VaultEnvSecrets.template_path = Rails.root.join("../config/vault_secrets.json.tmpl") diff --git a/db-syncer/lib/tasks/db_data.rake b/db-syncer/lib/tasks/db_data.rake new file mode 100644 index 000000000..94ae7c4f2 --- /dev/null +++ b/db-syncer/lib/tasks/db_data.rake @@ -0,0 +1,213 @@ +# frozen_string_literal: true + +namespace :db do + namespace :data do + desc "Dump database data for this application from production." + task dump: :environment do + require "db-syncer" + dump_env = ENV["DUMP_ENV"] || Rails.env + VaultEnvSecrets.load(env: {"DB_SYNCER" => "true", "APP_ENV" => dump_env}) + dump = DbSyncer::Dump.new(dump_env, upload: ":s3:nrel-tada-reopt-files/private/ci/api/db_data_dump.tar") + + # Exclude all table data by default, since we will only import subsets of + # data. + exclude_data_tables = [ + "reopt_api.*", + ] + + # Subset the API run data that we include in the dump so that it's a more + # mangeable size for local development and staging restores. + # + # Include API runs (and all of the associated data for those runs) for: + # + # - All runs from the past 15 days. + # - From the past year, runs from every 26th day (this roughly + # corresponds to another half month worth of data, while picking a + # non-7 based interaval ensures coverage over different days of the + # week throughout the year). + subset_all_days = 15 + subset_sample_days = 365 + subset_sample_frequency = 26 + subset_data = [] + begin + # Create a temporary table with all of the subsetted run IDs to dump. + # Since the subset process runs in separate transactions, we'll dump + # all of these to a real table (that we'll later drop) that the other + # queries can reference. + begin + original_env = Rails.env + ActiveRecord::Base.establish_connection(dump_env.to_sym) + ActiveRecord::Base.connection.execute <<~SQL + CREATE TABLE public.temp_db_syncer_subset_apimeta AS ( + SELECT id, run_uuid + FROM reopt_api.reoptjl_apimeta + WHERE + created >= date_trunc('day', now() - interval '#{subset_all_days} days') + OR ( + created >= date_trunc('day', now() - interval '#{subset_sample_days} days') + AND extract(doy FROM created)::integer % #{subset_sample_frequency} = 1 + ) + ) + SQL + ensure + ActiveRecord::Base.establish_connection(original_env.to_sym) + end + + # Dump the `reoptjl_apimeta` table itself. + subset_tables = [ + "reoptjl_apimeta", + ] + subset_data += subset_tables.map do |table_name| + { + :table => "reopt_api.#{table_name}", + :query => <<~SQL + SELECT #{table_name}.* + FROM reopt_api.#{table_name} + WHERE #{table_name}.id IN (SELECT id FROM public.temp_db_syncer_subset_apimeta) + SQL + } + end + + # Dump all of the child tables that are tied to the `reoptjl_apimeta` via + # a `meta_id` foreign key. + subset_meta_tables = [ + "reoptjl_absorptionchillerinputs", + "reoptjl_absorptionchilleroutputs", + "reoptjl_ashpspaceheaterinputs", + "reoptjl_ashpspaceheateroutputs", + "reoptjl_ashpwaterheaterinputs", + "reoptjl_ashpwaterheateroutputs", + "reoptjl_boilerinputs", + "reoptjl_boileroutputs", + "reoptjl_chpinputs", + "reoptjl_chpoutputs", + "reoptjl_coldthermalstorageinputs", + "reoptjl_coldthermalstorageoutputs", + "reoptjl_coolingloadinputs", + "reoptjl_coolingloadoutputs", + "reoptjl_cstinputs", + "reoptjl_cstoutputs", + "reoptjl_domestichotwaterloadinputs", + "reoptjl_electricheaterinputs", + "reoptjl_electricheateroutputs", + "reoptjl_electricloadinputs", + "reoptjl_electricloadoutputs", + "reoptjl_electricstorageinputs", + "reoptjl_electricstorageoutputs", + "reoptjl_electrictariffinputs", + "reoptjl_electrictariffoutputs", + "reoptjl_electricutilityinputs", + "reoptjl_electricutilityoutputs", + "reoptjl_existingboilerinputs", + "reoptjl_existingboileroutputs", + "reoptjl_existingchillerinputs", + "reoptjl_existingchilleroutputs", + "reoptjl_financialinputs", + "reoptjl_financialoutputs", + "reoptjl_generatorinputs", + "reoptjl_generatoroutputs", + "reoptjl_ghpinputs", + "reoptjl_ghpoutputs", + "reoptjl_heatingloadoutputs", + "reoptjl_hightempthermalstorageinputs", + "reoptjl_hightempthermalstorageoutputs", + "reoptjl_hotthermalstorageinputs", + "reoptjl_hotthermalstorageoutputs", + "reoptjl_message", + "reoptjl_outageoutputs", + "reoptjl_processheatloadinputs", + "reoptjl_pvinputs", + "reoptjl_pvoutputs", + "reoptjl_reoptjlmessageoutputs", + "reoptjl_settings", + "reoptjl_siteinputs", + "reoptjl_siteoutputs", + "reoptjl_spaceheatingloadinputs", + "reoptjl_steamturbineinputs", + "reoptjl_steamturbineoutputs", + "reoptjl_userprovidedmeta", + "reoptjl_windinputs", + "reoptjl_windoutputs", + ] + subset_data += subset_meta_tables.map do |table_name| + { + :table => "reopt_api.#{table_name}", + :query => <<~SQL + SELECT #{table_name}.* + FROM reopt_api.#{table_name} + WHERE #{table_name}.meta_id IN (SELECT id FROM public.temp_db_syncer_subset_apimeta) + SQL + } + end + + # Dump all of the child tables that are tied to the `reoptjl_apimeta` via + # a `run_uuid` foreign key. + subset_meta_uuid_tables = [ + "reoptjl_portfoliounlinkedruns", + "reoptjl_userunlinkedruns", + ] + subset_data += subset_meta_uuid_tables.map do |table_name| + { + :table => "reopt_api.#{table_name}", + :query => <<~SQL + SELECT #{table_name}.* + FROM reopt_api.#{table_name} + WHERE #{table_name}.run_uuid IN (SELECT run_uuid FROM public.temp_db_syncer_subset_apimeta) + SQL + } + end + + dump.dump({ + schemas: ["reopt_api"], + exclude_data_tables: exclude_data_tables, + subset_data: subset_data, + }) + ensure + # Cleanup the temporary table of subsetted IDs. + begin + original_env = Rails.env + ActiveRecord::Base.establish_connection(dump_env.to_sym) + ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS public.temp_db_syncer_subset_apimeta" + ensure + ActiveRecord::Base.establish_connection(original_env.to_sym) + end + end + end + + desc "Restore a production database dump to the local environment." + task :restore do + require "db-syncer" + VaultEnvSecrets.load(env: {"DB_SYNCER" => "true"}) + DbSyncer::Restore.new(download: ":s3:nrel-tada-reopt-files/private/ci/api/db_data_dump.tar") do |restore| + # Load schemas, dropping any previous tables. + restore.restore_dump(File.join(restore.dir, "schemas.pgdump.zst"), + exclude_list: [ + # Exclude extensions that might be present in the production dump, + # but aren't necessary for development (and might not be present in + # everyone's dev environment). + "EXTENSION - pg_repack", + "COMMENT - EXTENSION pg_repack", + "EXTENSION - pg_stat_statements", + "COMMENT - EXTENSION pg_stat_statements", + "bucardo_", + "USER MAPPING - USER MAPPING postgres", + ], + ) + + restore.restore_sql(File.join(restore.dir, "subset_data.sql.zst")) + end + end + end +end + +# Since this isn't running in a normal, full Rails environment, fake some tasks +# that the db-syncer gem expects to exist so they don't perform anything. +[ + "db:migrate", + "db:seed", + "db:structure:dump", + "db:test:prepare", +].each do |task_name| + Rake::Task.define_task(task_name) + Rake::Task[task_name].clear +end diff --git a/docker-compose.nojulia.yml b/docker-compose.nojulia.yml index f78c9603d..fcf4ad528 100644 --- a/docker-compose.nojulia.yml +++ b/docker-compose.nojulia.yml @@ -36,7 +36,6 @@ services: - DB_NAME=reopt - DB_USERNAME=reopt - DB_PASSWORD=reopt - - DB_SEARCH_PATH=public - REDIS_HOST=redis - REDIS_PASSWORD= - JULIA_HOST=host.docker.internal @@ -62,7 +61,6 @@ services: - DB_NAME=reopt - DB_USERNAME=reopt - DB_PASSWORD=reopt - - DB_SEARCH_PATH=public - REDIS_HOST=redis - REDIS_PASSWORD= - JULIA_HOST=host.docker.internal diff --git a/docker-compose.yml b/docker-compose.yml index 142ce7de2..facad8fa5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -88,11 +88,12 @@ services: db-syncer: build: - context: db-syncer + context: . + dockerfile: db-syncer/Dockerfile args: BUNDLE_FROZEN: "false" volumes: - - .:/app + # - .:/app - ~/.vault-token:/root/.vault-token:ro depends_on: - db From 55b3d1c77e75b31079096fdfacdd40df0a8575de Mon Sep 17 00:00:00 2001 From: lixiangk1 <72464565+lixiangk1@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:21:25 -0600 Subject: [PATCH 21/66] Add initial MPC implementation --- julia_src/http.jl | 52 +- julia_src/mpc.jl | 655 ++++++++++++++++++ .../migrations/0117_merge_20260530_1640.py | 14 + 3 files changed, 720 insertions(+), 1 deletion(-) create mode 100644 julia_src/mpc.jl create mode 100644 reoptjl/migrations/0117_merge_20260530_1640.py diff --git a/julia_src/http.jl b/julia_src/http.jl index 9eab6396d..e6f94a42c 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -1,4 +1,4 @@ -using HTTP, JSON, JuMP +using HTTP, JSON, JuMP using HiGHS, Cbc, SCIP using GhpGhx import REopt as reoptjl # For REopt.jl, needed because we still have local REopt.jl module for V1/V2 @@ -8,6 +8,7 @@ DotEnv.load!() const test_nrel_developer_api_key = ENV["NREL_DEVELOPER_API_KEY"] ENV["NREL_DEVELOPER_EMAIL"] = "reopt@nlr.gov" +include("mpc.jl") include("os_solvers.jl") @@ -64,6 +65,27 @@ function reopt(req::HTTP.Request) ENV["NREL_DEVELOPER_API_KEY"] = test_nrel_developer_api_key delete!(d, "api_key") end + + # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- + # When ElectricStorage.dispatch_options == "daily_foresight_optimized", first run the MPC rolling-horizon loop to get an SOC profile. + # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. + electric_storage = get(d, "ElectricStorage", Dict()) + if get(electric_storage, "dispatch_options", nothing) == "daily_foresight_optimized" + try + @info "Running MPC to obtain daily foresight optimized battery dispatch profile." + mpc_results = get_mpc_results(d) + soc = mpc_results["dispatch"]["ElectricStorage"]["soc_series_fraction"] + d["ElectricStorage"]["fixed_soc_series_fraction"] = soc + d["ElectricStorage"]["dispatch_options"] = "custom" + catch e + @error "MPC pre-solve failed" exception=(e, catch_backtrace()) + return HTTP.Response(500, JSON.json(Dict( + "error" => "MPC pre-solve failed: " * sprint(showerror, e), + "reopt_version" => string(pkgversion(reoptjl)), + ))) + end + end + settings = d["Settings"] solver_name = get(settings, "solver_name", "HiGHS") if solver_name == "Xpress" && !(xpress_installed=="True") @@ -810,6 +832,33 @@ function job_no_xpress(req::HTTP.Request) return HTTP.Response(500, JSON.json(error_response)) end + +function mpc(req::HTTP.Request) + d = JSON.parse(String(req.body)) + error_response = Dict() + results = Dict() + try + if !isempty(get(d, "api_key", "")) + ENV["NREL_DEVELOPER_API_KEY"] = pop!(d, "api_key") + else + ENV["NREL_DEVELOPER_API_KEY"] = test_nrel_developer_api_key + delete!(d, "api_key") + end + results = get_mpc_results(d) + catch e + @error "MPC failed" exception=(e, catch_backtrace()) + error_response["error"] = sprint(showerror, e) + error_response["reopt_version"] = string(pkgversion(reoptjl)) + end + GC.gc() + if isempty(error_response) + @info "MPC ran successfully." + return HTTP.Response(200, JSON.json(results)) + else + return HTTP.Response(500, JSON.json(error_response)) + end +end + # define REST endpoints to dispatch to "service" functions const ROUTER = HTTP.Router() @@ -820,6 +869,7 @@ else end HTTP.register!(ROUTER, "POST", "/reopt", reopt) HTTP.register!(ROUTER, "POST", "/erp", erp) +HTTP.register!(ROUTER, "POST", "/mpc", mpc) HTTP.register!(ROUTER, "POST", "/ghpghx", ghpghx) HTTP.register!(ROUTER, "GET", "/chp_defaults", chp_defaults) HTTP.register!(ROUTER, "GET", "/avert_emissions_profile", avert_emissions_profile) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl new file mode 100644 index 000000000..21cb9510a --- /dev/null +++ b/julia_src/mpc.jl @@ -0,0 +1,655 @@ +# REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NatLabRockies/REopt_API/blob/master/LICENSE. +# ============================================================================ +# MPC (Model Predictive Control) endpoint +# ---------------------------------------------------------------------------- +# Rolling-horizon dispatch with one day look-ahead using REopt.run_mpc. Assumes: +# * PV and ElectricStorage are the only available technologies (perfect forecast) +# * Only perfect forecast scenarios are modeled (no forecast errors) +# * Sizing: Perform a sizing run first if PV or ElectricStorage sizes are not provided (storage sizes must be greater than zero) +# * PV: Use PVWatts if `PV.production_factor_series` is not provided +# * ElectricLoad: Use commercial reference profiles if `ElectricLoad.loads_kw` is not provided +# * Code currently wraps with Jan 1 data to determine Dec 31 dispatch (allows leap-year data, which avoids the need to wrap) +# * MPC settings are currently hard coded (e.g., forecast horizon, control horizon, optimization horizon) +# ============================================================================ + +""" + get_month_transition_timesteps(time_steps_per_hour) + +Return the 1-based timestep index marking the START of each month for a +non-leap year beginning Jan 1 00:00. Length 12; first element == 1. +The last "transition" (end of December) is implicitly `length_of_data + 1` +and is handled by the caller. +""" +function get_month_transition_timesteps(time_steps_per_hour::Int) + # hours in each month for a non-leap year + hours_per_month = [744, 672, 744, 720, 744, 720, 744, 744, 720, 744, 720, 744] + starts = Vector{Int}(undef, 12) + starts[1] = 1 + for m in 2:12 + starts[m] = starts[m-1] + hours_per_month[m-1] * time_steps_per_hour + end + return starts +end + +""" + _mpc_slice_with_wrap(arr, idx, end_idx) + +Return `arr[idx:end_idx]` but wrap around to the start of `arr` if +`end_idx > length(arr)`. The wrap threshold is the actual array length, +not `length_of_data`, so a leap-year-length input (8784*tsh) naturally +supplies the extra day for the year-end look-ahead window before any +wrap is needed. +""" +function _mpc_slice_with_wrap(arr::AbstractVector, idx::Int, end_idx::Int) + n = length(arr) + if end_idx <= n + return arr[idx:end_idx] + else + wrap_len = end_idx - n + return vcat(arr[idx:n], arr[1:wrap_len]) + end +end + +""" + _mpc_check_series_length(name, series, time_steps_per_hour, length_of_data) + +Validate that a user-supplied annual time series has either +`8760 * time_steps_per_hour` entries (standard year) or +`8784 * time_steps_per_hour` entries (leap year). Returns the series +unchanged. + +The MPC loop iterates over `length_of_data = 8760 * time_steps_per_hour` +timesteps; if a leap-year-length series is supplied, the extra day is +used as additional look-ahead data by `_mpc_slice_with_wrap` (which wraps +on the actual array length, not on `length_of_data`). Any other length is +a hard error — REopt.jl's own `check_and_adjust_load_length` would +silently repeat/pad, masking the mistake. +""" +function _mpc_check_series_length(name::String, series::AbstractVector, + time_steps_per_hour::Int, length_of_data::Int) + n = length(series) + n == length_of_data && return series + if n == 8784 * time_steps_per_hour + @info "MPC: $name has leap-year length ($n); the extra day will be used " * + "for year-end look-ahead instead of wrapping to Jan 1." + return series + end + # Diagnostic: if `n` is consistent with 8760 hours at a *different* + # sub-hourly resolution, the user probably mis-declared + # `Settings.time_steps_per_hour`. + tsh_hint = "" + for tsh_guess in (1, 2, 4) + tsh_guess == time_steps_per_hour && continue + if n == 8760 * tsh_guess || n == 8784 * tsh_guess + tsh_hint = " (length is consistent with time_steps_per_hour=$(tsh_guess); " * + "check Settings.time_steps_per_hour)." + break + end + end + error("MPC: $name length $n != " * + "8760 * time_steps_per_hour ($length_of_data) " * + "and != 8784 * time_steps_per_hour ($(8784 * time_steps_per_hour))." * + tsh_hint) +end + +""" + _mpc_build_tariff_arrays(et_input, year, time_steps_per_hour, length_of_data) + +Construct a `REopt.ElectricTariff` from the user's ElectricTariff input dict +(which uses the real REopt input schema: `urdb_label`, `urdb_response`, +`urdb_utility_name`+`urdb_rate_name`, `tou_energy_rates_per_kwh`, +`monthly_energy_rates`, `monthly_demand_rates`, `blended_annual_energy_rate`, +`blended_annual_demand_rate`, etc.) and extract the processed arrays MPC +needs for its per-window posts. + +Returns a NamedTuple with: + * `energy_rates::Vector{Float64}` (length `length_of_data`) + * `monthly_demand_rates::Vector{Float64}` (length 12, all zeros if not set) + * `tou_demand_rates::Vector{Float64}` (one per ratchet; empty for non-URDB) + * `tou_demand_time_steps::Vector{Vector{Int}}` (full-year indices per ratchet) + +Multi-tier rates are flattened to a single tier (MPC does not model tiers). +""" +function _mpc_build_tariff_arrays(et_input::Dict, year::Union{Int,Nothing}, + time_steps_per_hour::Int, length_of_data::Int) + # Only forward known ElectricTariff kwargs so unrelated keys (e.g. a + # parsed API artifact) don't trip the constructor. + known_kwargs = (:urdb_label, :urdb_response, :urdb_utility_name, + :urdb_rate_name, :urdb_metadata, + :wholesale_rate, :export_rate_beyond_net_metering_limit, + :monthly_energy_rates, :monthly_demand_rates, + :blended_annual_energy_rate, :blended_annual_demand_rate, + :add_monthly_rates_to_urdb_rate, + :tou_energy_rates_per_kwh, :add_tou_energy_rates_to_urdb_rate, + :remove_tiers, :demand_lookback_months, + :demand_lookback_percent, :demand_lookback_range, + :coincident_peak_load_active_time_steps, + :coincident_peak_load_charge_per_kw) + kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in et_input + if Symbol(k) in known_kwargs) + # Force single-tier arrays so MPC sees a flat schedule; without this the + # constructor could return `length_of_data x n_tiers` and we'd silently + # only use one tier (and inconsistently across energy vs demand). + kwargs[:remove_tiers] = get(kwargs, :remove_tiers, true) + + tariff = reoptjl.ElectricTariff(; year = year, + time_steps_per_hour = time_steps_per_hour, + kwargs...) + + # All arrays are single-tier (we forced `remove_tiers=true`). + energy_rates = vec(Float64.(tariff.energy_rates[:, 1])) + length(energy_rates) == length_of_data || error( + "MPC: derived energy_rates length $(length(energy_rates)) != length_of_data $(length_of_data).") + + monthly_demand_rates = isempty(tariff.monthly_demand_rates) ? + zeros(Float64, 12) : vec(Float64.(tariff.monthly_demand_rates[:, 1])) + length(monthly_demand_rates) == 12 || error( + "MPC: derived monthly_demand_rates length $(length(monthly_demand_rates)) != 12.") + + tou_demand_rates = isempty(tariff.tou_demand_rates) ? + Float64[] : vec(Float64.(tariff.tou_demand_rates[:, 1])) + tou_demand_time_steps = [Int.(v) for v in tariff.tou_demand_ratchet_time_steps] + + return (energy_rates = energy_rates, + monthly_demand_rates = monthly_demand_rates, + tou_demand_rates = tou_demand_rates, + tou_demand_time_steps = tou_demand_time_steps) +end + +""" + _mpc_generate_pv_production_factor(d, time_steps_per_hour) + +Derive a PV production factor series via `REopt.get_production_factor` +(PVWatts) using `Site.latitude`/`Site.longitude` and the PV inputs in `d`. +Returns a `Vector{Float64}`. Throws if lat/lon are missing. +""" +function _mpc_generate_pv_production_factor(d::Dict, time_steps_per_hour::Int) + site = get(d, "Site", Dict()) + haskey(site, "latitude") || error( + "MPC: Site.latitude is required to derive PV.production_factor_series via PVWatts.") + haskey(site, "longitude") || error( + "MPC: Site.longitude is required to derive PV.production_factor_series via PVWatts.") + lat = Float64(site["latitude"]) + lon = Float64(site["longitude"]) + + @info "MPC: PV.production_factor_series not provided; deriving via REopt.jl PVWatts (lat=$(lat), lon=$(lon))." + + # Only forward known REopt.PV kwargs that affect the production factor. + pv = get(d, "PV", Dict()) + pv_pf_kwargs = (:array_type, :tilt, :module_type, :losses, :azimuth, :gcr, + :radius, :name, :location, :dc_ac_ratio, :inv_eff) + kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in pv + if Symbol(k) in pv_pf_kwargs) + pv_struct = reoptjl.PV(; latitude = lat, kwargs...) + derived = reoptjl.get_production_factor(pv_struct, lat, lon; + time_steps_per_hour = time_steps_per_hour) + return collect(Float64.(derived)) +end + +""" + _mpc_generate_loads_kw(d, time_steps_per_hour) + +Derive an electric load profile via `REopt.ElectricLoad` from the user's +standard ElectricLoad inputs (`doe_reference_name`+`city` / +`blended_doe_reference_names`+`blended_doe_reference_percents`, +`annual_kwh`, `monthly_totals_kwh`, `monthly_peaks_kw`, `year`, etc.) — +the same DOE CRB derivation path the main REopt run would take. Returns a +`Vector{Float64}`. Requires `Site.latitude`/`Site.longitude`. +""" +function _mpc_generate_loads_kw(d::Dict, time_steps_per_hour::Int) + site = get(d, "Site", Dict()) + haskey(site, "latitude") || error( + "MPC: Site.latitude is required to derive ElectricLoad.loads_kw from a DOE CRB profile.") + haskey(site, "longitude") || error( + "MPC: Site.longitude is required to derive ElectricLoad.loads_kw from a DOE CRB profile.") + lat = Float64(site["latitude"]) + lon = Float64(site["longitude"]) + + @info "MPC: ElectricLoad.loads_kw not provided; deriving via REopt.jl ElectricLoad (lat=$(lat), lon=$(lon))." + + # Only forward known ElectricLoad kwargs. + eload = get(d, "ElectricLoad", Dict()) + known_kwargs = (:normalize_and_scale_load_profile_input, + :path_to_csv, :doe_reference_name, + :blended_doe_reference_names, :blended_doe_reference_percents, + :year, :city, :annual_kwh, :monthly_totals_kwh, + :monthly_peaks_kw, :critical_loads_kw, :loads_kw_is_net, + :critical_loads_kw_is_net, :critical_load_fraction, + :operating_reserve_required_fraction, + :min_load_met_annual_fraction, :off_grid_flag) + kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in eload + if Symbol(k) in known_kwargs) + load_struct = reoptjl.ElectricLoad(; latitude = lat, longitude = lon, + time_steps_per_hour = time_steps_per_hour, + kwargs...) + return collect(Float64.(load_struct.loads_kw)) +end + +""" + _mpc_resolve_sizes!(d) + +Determines PV/ElectricStorage sizes for the MPC loop, using the actual +REopt input schema (`min_kw`/`max_kw`, `min_kwh`/`max_kwh`). + +Rules: + * If `min_kw == max_kw` for PV AND `min_kw == max_kw` AND + `min_kwh == max_kwh` for ElectricStorage → sizes are already fixed; use + those values directly. No sizing run. + * Otherwise → run a regular REopt sizing optimization, then mutate `d` to + set `min_kw = max_kw = sized_kw` (and similarly for kWh on storage). + Locking sizes in `d` ensures the downstream main REopt run uses the + same sizes the MPC SOC trajectory was computed against. + +Returns the tuple `(pv_kw, bess_kw, bess_kwh)` for the caller to use when +building per-window MPC posts. +""" +function _mpc_resolve_sizes!(d::Dict) + pv = get!(d, "PV", Dict()) + bess = get!(d, "ElectricStorage", Dict()) + + function _is_fixed(dct, lo_key, hi_key) + lo = get(dct, lo_key, nothing) + hi = get(dct, hi_key, nothing) + return lo !== nothing && hi !== nothing && Float64(lo) == Float64(hi) + end + + # Guard: daily_foresight_optimized requires a non-zero ElectricStorage. + # If the user has pinned kW or kWh to zero (min == max == 0), fail fast + # with a clear message instead of running a pointless sizing/MPC pass. + if (_is_fixed(bess, "min_kw", "max_kw") && Float64(get(bess, "min_kw", 0)) == 0.0) || + (_is_fixed(bess, "min_kwh", "max_kwh") && Float64(get(bess, "min_kwh", 0)) == 0.0) + error("ElectricStorage power (kW) and energy (kWh) must both be greater than zero to run the daily_foresight_optimized dispatch option. Got min_kw=$(get(bess, "min_kw", nothing)), max_kw=$(get(bess, "max_kw", nothing)), min_kwh=$(get(bess, "min_kwh", nothing)), max_kwh=$(get(bess, "max_kwh", nothing)).") + end + + pv_fixed = _is_fixed(pv, "min_kw", "max_kw") + bess_fixed = _is_fixed(bess, "min_kw", "max_kw") && + _is_fixed(bess, "min_kwh", "max_kwh") + + if pv_fixed && bess_fixed + pv_kw = Float64(pv["min_kw"]) + bess_kw = Float64(bess["min_kw"]) + bess_kwh = Float64(bess["min_kwh"]) + @info "MPC: sizes already fixed via min_kw==max_kw. PV=$(pv_kw) kW, BESS=$(bess_kw) kW / $(bess_kwh) kWh." + return (pv_kw = pv_kw, bess_kw = bess_kw, bess_kwh = bess_kwh) + end + + @info "MPC: PV and/or ElectricStorage sizes not fixed — running REopt sizing first." + sizing_post = deepcopy(d) + # Strip any API-only sentinels before sizing. + if haskey(sizing_post, "ElectricStorage") + delete!(sizing_post["ElectricStorage"], "dispatch_options") + delete!(sizing_post["ElectricStorage"], "fixed_soc_series_fraction") + end + settings = get!(sizing_post, "Settings", Dict()) + settings["run_bau"] = false + settings["solver_name"] = get(settings, "solver_name", "HiGHS") + settings["timeout_seconds"] = get(settings, "timeout_seconds", 420) + settings["optimality_tolerance"] = get(settings, "optimality_tolerance", 0.001) + + solver_attributes = SolverAttributes( + settings["timeout_seconds"], settings["optimality_tolerance"]) + m = get_solver_model(get_solver_model_type(settings["solver_name"]), solver_attributes) + + model_inputs = reoptjl.REoptInputs(sizing_post) + sizing_results = reoptjl.run_reopt(m, model_inputs) + + if get(sizing_results, "status", "") != "optimal" + error("MPC sizing pre-step did not solve to optimality (status = $(get(sizing_results, "status", "unknown"))).") + end + + pv_kw = Float64(get(get(sizing_results, "PV", Dict()), "size_kw", 0.0)) + bess_kw = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kw", 0.0)) + bess_kwh = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kwh", 0.0)) + + # Lock the resolved sizes in `d` so the downstream main REopt run uses + # the SAME sizes that the MPC SOC trajectory is consistent with. + pv["min_kw"] = pv_kw + pv["max_kw"] = pv_kw + bess["min_kw"] = bess_kw + bess["max_kw"] = bess_kw + bess["min_kwh"] = bess_kwh + bess["max_kwh"] = bess_kwh + + @info "MPC: sized PV=$(pv_kw) kW, BESS=$(bess_kw) kW / $(bess_kwh) kWh (locked into d via min==max)." + return (pv_kw = pv_kw, bess_kw = bess_kw, bess_kwh = bess_kwh) +end + +# Core MPC computation. Takes the already-parsed (and api-key-stripped) request +# dict `d` and returns the results dict (dispatch series, costs, etc.). +# May throw; callers handle error responses. +# Called by both /mpc (standalone) and /reopt (when dispatch_options = +# "daily_foresight_optimized"). +function get_mpc_results(d::Dict)::Dict + # ---- Shared settings (with sensible defaults) ---- + settings = get(d, "Settings", Dict()) + solver_name = get(settings, "solver_name", "HiGHS") + if solver_name == "Xpress" && xpress_installed != "True" + solver_name = "HiGHS" + @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. " * + "Next time specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'." + end + + optimality_tolerance = Float64(get(settings, "optimality_tolerance", 0.001)) + + # ---- Hard-coded MPC config ---- + # Full-year, 24-h horizon, year-end wrap-around, 60 s/iter solver cap. + time_steps_per_hour = Int(get(settings, "time_steps_per_hour", 1)) + length_of_data = 8760 * time_steps_per_hour + horizon = 24 * time_steps_per_hour + per_iter_timeout_s = 60.0 + + # ---- Guarantee presence of the tech / load sub-dicts ---- + # Done up front so PV/load resolution and sizing can mutate them in + # place, and downstream code can use plain indexing. + pv = get!(d, "PV", Dict()) + bess = get!(d, "ElectricStorage", Dict()) + eload = get!(d, "ElectricLoad", Dict()) + et = get(d, "ElectricTariff", Dict()) + eutil = get(d, "ElectricUtility", Dict()) + + # ---- Resolve PV production factor series ---- + # If not user-provided, derive via REopt.jl PVWatts and cache into `d` + # so both the sizing pre-step and downstream main REopt run reuse it + # without a second PVWatts call. + pv_prod_factor = if haskey(pv, "production_factor_series") && !isempty(pv["production_factor_series"]) + # Broadcast-convert: handles Vector{Any} from JSON parsing + # (convert(Vector{Float64}, ::Vector{Any}) would throw). + Float64.(pv["production_factor_series"]) + else + series = _mpc_generate_pv_production_factor(d, time_steps_per_hour) + pv["production_factor_series"] = series + series + end + + # ---- Resolve electric load series ---- + # Mirror the PV pattern: if user-provided, use it; else derive via + # REopt.jl ElectricLoad (DOE CRB path) and cache into `d`. + load_series = if haskey(eload, "loads_kw") && !isempty(eload["loads_kw"]) + Float64.(eload["loads_kw"]) + else + series = _mpc_generate_loads_kw(d, time_steps_per_hour) + eload["loads_kw"] = series + series + end + + # ---- Length checks (8760 or 8784 * tsh) ---- + # No mutation of user-supplied series. A leap-year-length input + # (8784*tsh) is accepted as-is; `_mpc_slice_with_wrap` reads the extra + # day from it for year-end look-ahead instead of wrapping to Jan 1. + pv_prod_factor = _mpc_check_series_length("PV.production_factor_series", + pv_prod_factor, time_steps_per_hour, length_of_data) + load_series = _mpc_check_series_length("ElectricLoad.loads_kw", + load_series, time_steps_per_hour, length_of_data) + + # ---- Resolve PV and ElectricStorage sizes ---- + # Runs AFTER PV/load resolution so the (possibly internal) sizing + # REopt run reuses the cached series instead of re-deriving them. + # Either pulls sizes from min_kw==max_kw / min_kwh==max_kwh, or runs + # a sizing REopt and locks the resolved values into `d` so the + # downstream main REopt run uses the SAME sizes. + sizes = _mpc_resolve_sizes!(d) + pv_kw = sizes.pv_kw + bess_kw = sizes.bess_kw + bess_kwh = sizes.bess_kwh + + soc_init = Float64(get(bess, "soc_init_fraction", 0.5)) + soc_min = Float64(get(bess, "soc_min_fraction", 0.2)) + # REopt ElectricStorage exposes three component efficiencies; MPC's + # MPCElectricStorage uses combined charge/discharge round-trip halves. + # Match REopt's own composition: charge = rectifier * sqrt(internal), + # discharge = inverter * sqrt(internal). Defaults from REopt.jl. + rect_eff = Float64(get(bess, "rectifier_efficiency_fraction", 0.96)) + inv_eff = Float64(get(bess, "inverter_efficiency_fraction", 0.96)) + int_eff = Float64(get(bess, "internal_efficiency_fraction", 0.975)) + charge_eff = rect_eff * sqrt(int_eff) + discharge_eff = inv_eff * sqrt(int_eff) + + # Year for URDB schedule decoding is sourced from ElectricLoad.year + # (the canonical year input in REopt; ElectricTariff's `year` kwarg is + # an internal pass-through, not a user input). + tariff_year = haskey(eload, "year") ? Int(eload["year"]) : nothing + tariff = _mpc_build_tariff_arrays(et, tariff_year, time_steps_per_hour, length_of_data) + energy_rates = tariff.energy_rates + tou_demand_rates = tariff.tou_demand_rates # one per ratchet, $/kW + tou_demand_ts_all = tariff.tou_demand_time_steps # full-year indices per ratchet + monthly_demand_rates = tariff.monthly_demand_rates # length 12, $/kW per month + + # Optional emissions series — broadcast zero if not provided. + emissions = haskey(eutil, "emissions_factor_series_lb_CO2_per_kwh") ? + Float64.(eutil["emissions_factor_series_lb_CO2_per_kwh"]) : + zeros(Float64, length_of_data) + emissions = _mpc_check_series_length("ElectricUtility.emissions_factor_series_lb_CO2_per_kwh", + emissions, time_steps_per_hour, length_of_data) + + # ---- Demand tracking state ---- + # Reverse indices into the calendar: each timestep maps to its + # 1-based month (always defined) and 1-based TOU ratchet (0 if no + # ratchet applies). Both are vector lookups (O(1)) used inside the + # rolling-horizon loop to build window-local time-step groupings + # and to attribute realized peaks to the right month/tier. + month_starts = get_month_transition_timesteps(time_steps_per_hour) + ts_to_month = Vector{Int}(undef, length_of_data) + for m in 1:12 + s = month_starts[m] + e = m < 12 ? month_starts[m+1] - 1 : length_of_data + ts_to_month[s:e] .= m + end + + ts_to_tier = zeros(Int, length_of_data) + for (t, ratchet_ts) in enumerate(tou_demand_ts_all), g in ratchet_ts + if 1 <= g <= length_of_data + ts_to_tier[g] = t + end + end + + n_tou = length(tou_demand_rates) + # `monthly_previous_peak_demands` and `tou_previous_peak_demands` are + # passed into every MPC post AND mutated after each solve so that + # subsequent windows "see" the realized peaks so far. Both reset at + # month boundaries (matches REopt billing semantics; multi-month + # ratchet/lookback support would require a different reset cadence). + tou_previous_peak_demands = zeros(Float64, n_tou) + monthly_previous_peak_demands = zeros(Float64, 12) + monthly_demand_cost_total = 0.0 + tou_demand_cost_total = 0.0 + tou_peaks_by_month = Vector{Vector{Float64}}() + monthly_peaks = Float64[] + + # ---- Dispatch accumulators (1 value per executed timestep) ---- + dispatch = Dict( + "PV" => Dict( + "electric_to_load_series_kw" => Float64[], + "electric_to_storage_series_kw" => Float64[], + "electric_to_grid_series_kw" => Float64[], + "electric_curtailed_series_kw" => Float64[], + ), + "ElectricStorage" => Dict( + "storage_to_load_series_kw" => Float64[], + "soc_series_fraction" => Float64[], + ), + "ElectricUtility" => Dict( + "electric_to_load_series_kw" => Float64[], + "electric_to_storage_series_kw" => Float64[], + "emissions_series_lb_CO2" => Float64[], + ), + "ElectricLoad" => Dict( + "load_series_kw" => Float64[], + ), + ) + cost_series = Float64[] + total_energy_cost = 0.0 + bess_soc_init_fraction = soc_init + + # ---- Build the (reused) MPC post template ---- + # Note: no Settings block here; the solver is constructed below per + # window and passed directly to run_mpc(model, post). All four demand + # inputs (TOU + monthly rates and previous-peak vectors) are passed + # so the optimizer's objective accounts for *both* charges; omitting + # `monthly_*` would silently optimize without the monthly demand + # contribution and over-state savings under monthly-demand tariffs. + function build_mpc_post(window_pv, window_load, window_rates, window_emissions, + window_tou_ts, window_monthly_ts, + tou_prev_peaks, monthly_prev_peaks, soc_init_frac) + return Dict( + "PV" => Dict( + "size_kw" => pv_kw, + "production_factor_series" => window_pv, + ), + "ElectricStorage" => Dict( + "size_kw" => bess_kw, + "size_kwh" => bess_kwh, + "charge_efficiency" => charge_eff, + "discharge_efficiency" => discharge_eff, + "soc_init_fraction" => soc_init_frac, + "soc_min_fraction" => soc_min, + ), + "ElectricLoad" => Dict( + "loads_kw" => window_load, + ), + "ElectricTariff" => Dict( + "energy_rates" => window_rates, + "tou_demand_rates" => tou_demand_rates, + "tou_demand_time_steps" => window_tou_ts, + "tou_previous_peak_demands" => tou_prev_peaks, + "monthly_demand_rates" => monthly_demand_rates, + "time_steps_monthly" => window_monthly_ts, + "monthly_previous_peak_demands" => monthly_prev_peaks, + ), + "ElectricUtility" => Dict( + "emissions_factor_series_lb_CO2_per_kwh" => window_emissions, + ), + ) + end + + @info "MPC: starting rolling-horizon loop ($(length_of_data) iterations, horizon=$(horizon))" + for idx in 1:length_of_data + end_ts = idx + horizon - 1 + window_len = end_ts - idx + 1 + + window_pv = _mpc_slice_with_wrap(pv_prod_factor, idx, end_ts) + window_load = _mpc_slice_with_wrap(load_series, idx, end_ts) + window_rates = _mpc_slice_with_wrap(energy_rates, idx, end_ts) + window_emiss = _mpc_slice_with_wrap(emissions, idx, end_ts) + + # Window-local time-step groupings, both built from a single pass + # over the horizon. TOU buckets are 1 vector per ratchet (length + # n_tou); monthly buckets are 1 vector per month (length 12). + # Global indices wrap on `length_of_data` (8760·tsh) so the month + # / tier of a wrapped step matches its calendar position. + window_tou_ts = [Int[] for _ in 1:n_tou] + window_monthly_ts = [Int[] for _ in 1:12] + for k in 1:window_len + g = idx + k - 1 + if g > length_of_data + g -= length_of_data + end + push!(window_monthly_ts[ts_to_month[g]], k) + tier = ts_to_tier[g] + if tier > 0 + push!(window_tou_ts[tier], k) + end + end + + post = build_mpc_post(window_pv, window_load, window_rates, window_emiss, + window_tou_ts, window_monthly_ts, + tou_previous_peak_demands, monthly_previous_peak_demands, + bess_soc_init_fraction) + + model = get_solver_model(get_solver_model_type(solver_name), + SolverAttributes(per_iter_timeout_s, optimality_tolerance)) + result = reoptjl.run_mpc(model, post) + + # ---- Extract first-timestep dispatch (perfect-forecast) ---- + pv_res = result["PV"] + bess_res = result["ElectricStorage"] + util_res = result["ElectricUtility"] + + pv_to_load = pv_res["electric_to_load_series_kw"][1] + pv_to_bess = pv_res["electric_to_storage_series_kw"][1] + pv_to_grid = haskey(pv_res, "electric_to_grid_series_kw") ? pv_res["electric_to_grid_series_kw"][1] : 0.0 + pv_curtailed = haskey(pv_res, "electric_curtailed_series_kw") ? pv_res["electric_curtailed_series_kw"][1] : 0.0 + bess_to_load = bess_res["storage_to_load_series_kw"][1] + bess_soc = bess_res["soc_series_fraction"][1] + util_to_load = util_res["electric_to_load_series_kw"][1] + util_to_bess = util_res["electric_to_storage_series_kw"][1] + grid_power = max(util_to_load + util_to_bess, 0.0) + + push!(dispatch["PV"]["electric_to_load_series_kw"], pv_to_load) + push!(dispatch["PV"]["electric_to_storage_series_kw"], pv_to_bess) + push!(dispatch["PV"]["electric_to_grid_series_kw"], pv_to_grid) + push!(dispatch["PV"]["electric_curtailed_series_kw"], pv_curtailed) + push!(dispatch["ElectricStorage"]["storage_to_load_series_kw"], bess_to_load) + push!(dispatch["ElectricStorage"]["soc_series_fraction"], round(bess_soc, digits=6)) + push!(dispatch["ElectricUtility"]["electric_to_load_series_kw"], util_to_load) + push!(dispatch["ElectricUtility"]["electric_to_storage_series_kw"], util_to_bess) + push!(dispatch["ElectricUtility"]["emissions_series_lb_CO2"], + emissions[idx] * grid_power / time_steps_per_hour) + push!(dispatch["ElectricLoad"]["load_series_kw"], load_series[idx]) + + # ---- Energy cost ---- + step_energy_cost = grid_power * energy_rates[idx] / time_steps_per_hour + push!(cost_series, step_energy_cost) + total_energy_cost += step_energy_cost + + # ---- Carry state ---- + bess_soc_init_fraction = bess_soc + + # ---- Demand peak tracking (per-tier and per-month) ---- + # Attribute realized peak grid power to its TOU ratchet (if any) + # and to its calendar month. Identity comes from the reverse + # indices, NOT from rate values — so coincident-rate ratchets + # and equal-rate months remain separate billing pools. + m_now = ts_to_month[idx] + monthly_previous_peak_demands[m_now] = + max(grid_power, monthly_previous_peak_demands[m_now]) + + if n_tou > 0 + tier_now = ts_to_tier[idx] + if tier_now > 0 + tou_previous_peak_demands[tier_now] = + max(grid_power, tou_previous_peak_demands[tier_now]) + end + end + + # ---- Month transition: close out the just-ended month ---- + # Both monthly and TOU peaks reset at month boundaries (matches + # REopt billing semantics). The last timestep of month m is + # detected by m_now != ts_to_month[idx+1]; we handle Dec via + # idx == length_of_data. + is_month_end = (idx == length_of_data) || (ts_to_month[idx + 1] != m_now) + if is_month_end + push!(monthly_peaks, monthly_previous_peak_demands[m_now]) + monthly_demand_cost_total += + monthly_previous_peak_demands[m_now] * monthly_demand_rates[m_now] + monthly_previous_peak_demands[m_now] = 0.0 + + if n_tou > 0 + push!(tou_peaks_by_month, copy(tou_previous_peak_demands)) + tou_demand_cost_total += sum(tou_previous_peak_demands .* tou_demand_rates) + fill!(tou_previous_peak_demands, 0.0) + end + end + end + + @info "MPC: loop complete. total_energy_cost=\$$(round(total_energy_cost, digits=2))" + + return Dict( + "MPC" => Dict( + "time_steps_per_hour" => time_steps_per_hour, + "horizon" => horizon, + ), + "PV" => Dict("size_kw" => pv_kw), + "ElectricStorage" => Dict("size_kw" => bess_kw, "size_kwh" => bess_kwh), + "dispatch" => dispatch, + "ElectricTariff" => Dict( + "total_energy_cost" => total_energy_cost, + "energy_cost_series_per_timestep" => cost_series, + "total_tou_demand_cost" => tou_demand_cost_total, + "total_monthly_demand_cost" => monthly_demand_cost_total, + "tou_peaks_by_month_kw" => tou_peaks_by_month, + "monthly_peaks_kw" => monthly_peaks, + ), + "status" => "optimal", + "reopt_version" => string(pkgversion(reoptjl)), + ) +end diff --git a/reoptjl/migrations/0117_merge_20260530_1640.py b/reoptjl/migrations/0117_merge_20260530_1640.py new file mode 100644 index 000000000..0302ff8e6 --- /dev/null +++ b/reoptjl/migrations/0117_merge_20260530_1640.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.26 on 2026-05-30 16:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('reoptjl', '0116_alter_apimeta_api_key_alter_apimeta_job_type'), + ('reoptjl', '0116_rename_state_of_health_electricstorageoutputs_state_of_health_series_fraction_and_more'), + ] + + operations = [ + ] From e64ed5e0fe3b35969f1fc4a9ffb09d3c3ac77445 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:35:31 -0600 Subject: [PATCH 22/66] Rely on kubernetes secrets in deployed environment. --- db-syncer/config/initializers/vault_env_secrets.rb | 5 +++++ docker-compose.yml | 1 + 2 files changed, 6 insertions(+) diff --git a/db-syncer/config/initializers/vault_env_secrets.rb b/db-syncer/config/initializers/vault_env_secrets.rb index 666683f17..efc6cf1b6 100644 --- a/db-syncer/config/initializers/vault_env_secrets.rb +++ b/db-syncer/config/initializers/vault_env_secrets.rb @@ -1 +1,6 @@ +# Read secrets from main app config. VaultEnvSecrets.template_path = Rails.root.join("../config/vault_secrets.json.tmpl") + +# Read Vault secrets into environment variables for local development (in +# production, these will be handled via Kubernetes secrets). +VaultEnvSecrets.enabled = (ENV["LOAD_VAULT_ENV_SECRETS"] == "true") diff --git a/docker-compose.yml b/docker-compose.yml index facad8fa5..e297d4515 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,6 +98,7 @@ services: depends_on: - db environment: + - LOAD_VAULT_ENV_SECRETS=true - DEV_DB_HOST=db - DEV_DB_NAME=reopt_api - DEV_DB_USERNAME=reopt From 589b68489e4603d039e39829328f23977cbf0a30 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:14:03 -0600 Subject: [PATCH 23/66] Add secret value so db-syncer process can spin up. --- config/vault_secrets.json.tmpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/vault_secrets.json.tmpl b/config/vault_secrets.json.tmpl index 85738b083..a5688dda7 100644 --- a/config/vault_secrets.json.tmpl +++ b/config/vault_secrets.json.tmpl @@ -23,6 +23,11 @@ "SECRET_S3_AWS_REGION" .aws_region "SECRET_S3_AWS_SECRET_ACCESS_KEY" .aws_secret_access_key ) $secrets }} + + {{/* Dummy secret value for the db-syncer Rails app to spin up */}} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_KEY_BASE" "dummy_value" + ) $secrets }} {{ end }} {{ end }} From 9e597e3a27a0101df6aa638367ba711cf266968f Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 3 Jun 2026 18:28:01 -0600 Subject: [PATCH 24/66] Don't run on every push. --- .github/workflows/db-sync-dump.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/db-sync-dump.yml b/.github/workflows/db-sync-dump.yml index 702c3e1d1..8ccc7d7bb 100644 --- a/.github/workflows/db-sync-dump.yml +++ b/.github/workflows/db-sync-dump.yml @@ -4,7 +4,6 @@ on: schedule: - cron: "40 5 * * *" # Every day at 10:40 PM MST / 11:40 PM MDT workflow_dispatch: - push: concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 438707db08a70b709aa23ab5e57ad178454c4c44 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 3 Jun 2026 19:00:04 -0600 Subject: [PATCH 25/66] Try to fix restores during staging deploys. --- config/vault_secrets.json.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/vault_secrets.json.tmpl b/config/vault_secrets.json.tmpl index a5688dda7..3717dfb4f 100644 --- a/config/vault_secrets.json.tmpl +++ b/config/vault_secrets.json.tmpl @@ -16,7 +16,7 @@ ) $secrets }} {{ end }} - {{ if (eq (env.Getenv "DB_SYNCER") "true") }} + {{ if (or (eq (env.Getenv "DB_SYNCER") "true") (eq (env.Getenv "DB_MIGRATE") "true")) }} {{ with (datasource "vault" "reopt-web/common/s3").data }} {{ $secrets = coll.Merge (coll.Dict "SECRET_S3_AWS_ACCESS_KEY_ID" .aws_access_key_id From 8e48a6f6048f35db2fdbb05bdf0267bdadb1273e Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 3 Jun 2026 19:38:26 -0600 Subject: [PATCH 26/66] Try to fix permissions used during restores. --- config/vault_secrets.json.tmpl | 9 +++++++++ db-syncer/config/database.yml | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/config/vault_secrets.json.tmpl b/config/vault_secrets.json.tmpl index 3717dfb4f..483cdcba8 100644 --- a/config/vault_secrets.json.tmpl +++ b/config/vault_secrets.json.tmpl @@ -51,6 +51,15 @@ "SECRET_DB_USERNAME" .db_username_v18 ) $secrets }} {{ end }} + + {{ if (eq (env.Getenv "DB_MIGRATE") "true") }} + {{ with (datasource "vault" (printf "reopt-db/%s/db-superuser" $deploy_env)).data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_DB_SUPERUSER_PASSWORD" .db_password_v18 + "SECRET_DB_SUPERUSER_USERNAME" .db_username_v18 + ) $secrets }} + {{ end }} + {{ end }} {{ end }} {{ end }} diff --git a/db-syncer/config/database.yml b/db-syncer/config/database.yml index 8eb91f52b..f8530bb6f 100644 --- a/db-syncer/config/database.yml +++ b/db-syncer/config/database.yml @@ -24,13 +24,13 @@ staging: host: <%= ENV["DB_HOST"] || ENV["SECRET_DB_HOST"] %> port: <%= ENV["DB_PORT"] || ENV["SECRET_DB_PORT"] %> database: <%= ENV["DB_NAME"] || ENV["SECRET_DB_NAME"] %> - username: <%= ENV["DB_USERNAME"] || ENV["SECRET_DB_USERNAME"] %> - password: <%= ENV["DB_PASSWORD"] || ENV["SECRET_DB_PASSWORD"] %> + username: <%= ENV["DB_USERNAME"] || ENV["SECRET_DB_SUPERUSER_USERNAME"] || ENV["SECRET_DB_USERNAME"] %> + password: <%= ENV["DB_PASSWORD"] || ENV["SECRET_DB_SUPERUSER_PASSWORD"] || ENV["SECRET_DB_PASSWORD"] %> production: <<: *default host: <%= ENV["DB_HOST"] || ENV["SECRET_DB_HOST"] %> port: <%= ENV["DB_PORT"] || ENV["SECRET_DB_PORT"] %> database: <%= ENV["DB_NAME"] || ENV["SECRET_DB_NAME"] %> - username: <%= ENV["DB_USERNAME"] || ENV["SECRET_DB_USERNAME"] %> - password: <%= ENV["DB_PASSWORD"] || ENV["SECRET_DB_PASSWORD"] %> + username: <%= ENV["DB_USERNAME"] || ENV["SECRET_DB_SUPERUSER_USERNAME"] || ENV["SECRET_DB_USERNAME"] %> + password: <%= ENV["DB_PASSWORD"] || ENV["SECRET_DB_SUPERUSER_PASSWORD"] || ENV["SECRET_DB_PASSWORD"] %> From 07a2bb868926003c96ba743de4c69b349eb9d3a8 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 3 Jun 2026 20:15:13 -0600 Subject: [PATCH 27/66] Include django_migrations in database dumps. --- db-syncer/lib/tasks/db_data.rake | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/db-syncer/lib/tasks/db_data.rake b/db-syncer/lib/tasks/db_data.rake index 94ae7c4f2..727a7de41 100644 --- a/db-syncer/lib/tasks/db_data.rake +++ b/db-syncer/lib/tasks/db_data.rake @@ -9,10 +9,18 @@ namespace :db do VaultEnvSecrets.load(env: {"DB_SYNCER" => "true", "APP_ENV" => dump_env}) dump = DbSyncer::Dump.new(dump_env, upload: ":s3:nrel-tada-reopt-files/private/ci/api/db_data_dump.tar") - # Exclude all table data by default, since we will only import subsets of + # Exclude nearly all table data by default (except we want to include the + # django metadata/migration tables), since we will only import subsets of # data. exclude_data_tables = [ - "reopt_api.*", + "reopt_api.delete_run_uuids", + "reopt_api.django_celery*", + "reopt_api.future*", + "reopt_api.ghp*", + "reopt_api.proforma*", + "reopt_api.reo*", + "reopt_api.resilience*", + "reopt_api.summary*", ] # Subset the API run data that we include in the dump so that it's a more From 9a8a8d8c8dc3176d5fe57617b4ac6371cb49a9d0 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 3 Jun 2026 21:33:20 -0600 Subject: [PATCH 28/66] Fix permissions on restored tables. --- db-syncer/Gemfile | 7 +- db-syncer/Gemfile.lock | 47 ++++++++- .../initializers/rails_sql_tasks_enhancer.rb | 2 + db-syncer/db/grants.sql.erb | 10 ++ db-syncer/db/setup.sql.erb | 38 +++++++ db-syncer/db/task_functions.sql.erb | 99 +++++++++++++++++++ db-syncer/db/task_grants.sql.erb | 9 ++ db-syncer/db/task_setup.sql.erb | 16 +++ docker-compose.yml | 9 +- 9 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 db-syncer/config/initializers/rails_sql_tasks_enhancer.rb create mode 100644 db-syncer/db/grants.sql.erb create mode 100644 db-syncer/db/setup.sql.erb create mode 100644 db-syncer/db/task_functions.sql.erb create mode 100644 db-syncer/db/task_grants.sql.erb create mode 100644 db-syncer/db/task_setup.sql.erb diff --git a/db-syncer/Gemfile b/db-syncer/Gemfile index 014f4f858..2fb154c86 100644 --- a/db-syncer/Gemfile +++ b/db-syncer/Gemfile @@ -1,7 +1,10 @@ -source "https://rubygems.org" +source "https://rubygems.org", cooldown: 7 # Syncing production/staging data back to local environments. -gem "db-syncer", :require => false, :git => "https://github.nrel.gov/TADA/db-syncer.git", :branch => "main" +gem "db-syncer", git: "https://github.nrel.gov/TADA/db-syncer.git", branch: "main", require: false + +# Database setup tasks. +gem "rails-sql-tasks-enhancer", git: "https://github.nrel.gov/TADA/rails-sql-tasks-enhancer.git", branch: "main" # Vault secrets as environment variables for development. gem "vault_env_secrets", "~> 2.0.0" diff --git a/db-syncer/Gemfile.lock b/db-syncer/Gemfile.lock index eccb8f844..536b5152e 100644 --- a/db-syncer/Gemfile.lock +++ b/db-syncer/Gemfile.lock @@ -7,6 +7,14 @@ GIT highline rest-client +GIT + remote: https://github.nrel.gov/TADA/rails-sql-tasks-enhancer.git + revision: 0e460e536e33b5228597c83af4a00d6ca8ef1ec5 + branch: main + specs: + rails-sql-tasks-enhancer (0.2.0) + rails (>= 6.1) + GEM remote: https://rubygems.org/ specs: @@ -147,11 +155,27 @@ GEM racc (~> 1.4) nokogiri (1.19.3-aarch64-linux-gnu) racc (~> 1.4) + nokogiri (1.19.3-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.19.3-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.3-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.19.3-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.3-x86_64-darwin) + racc (~> 1.4) nokogiri (1.19.3-x86_64-linux-gnu) racc (~> 1.4) + nokogiri (1.19.3-x86_64-linux-musl) + racc (~> 1.4) pg (1.6.3) pg (1.6.3-aarch64-linux) + pg (1.6.3-aarch64-linux-musl) + pg (1.6.3-arm64-darwin) + pg (1.6.3-x86_64-darwin) pg (1.6.3-x86_64-linux) + pg (1.6.3-x86_64-linux-musl) pp (0.6.3) prettyprint prettyprint (0.2.0) @@ -228,13 +252,22 @@ GEM PLATFORMS aarch64-linux + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin ruby + x86_64-darwin x86_64-linux + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES db-syncer! pg (~> 1.6.3) rails (~> 8.1.3) + rails-sql-tasks-enhancer! vault_env_secrets (~> 2.0.0) CHECKSUMS @@ -253,6 +286,7 @@ CHECKSUMS base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + bundler (4.0.13) sha256=19f08be7f27022cf0b89f27da0b044ae075e8270a9ef44ad248a932614e1ca3b concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a crass (1.0.6) sha256=dc516022a56e7b3b156099abc81b6d2b08ea1ed12676ac7a5657617f012bd45d @@ -287,10 +321,20 @@ CHECKSUMS nio4r (2.7.5) sha256=6c90168e48fb5f8e768419c93abb94ba2b892a1d0602cb06eef16d8b7df1dca1 nokogiri (1.19.3) sha256=78312cbac32a40c812780d9678221b79d51288eec00054c1a8d15f7ce05960e8 nokogiri (1.19.3-aarch64-linux-gnu) sha256=46b89e5d7b9e844c2ee360794240c6ea2a4e6fa0c5892a4ed487db621224b639 + nokogiri (1.19.3-aarch64-linux-musl) sha256=8392dfdcd21be7a94dbbe9ccc138dea01b97b24cb2dc02a114ca98bfb1d9a0b7 + nokogiri (1.19.3-arm-linux-gnu) sha256=3919d5ffc334ad778a4a9eb88fda7dcb8b1fb58c8a52ac640c6dcd2f038e774f + nokogiri (1.19.3-arm-linux-musl) sha256=9ce1cb6346bb9c67b1550eb537aa183ead91e4b6eadb2f36ade02d8dd2a79fb6 + nokogiri (1.19.3-arm64-darwin) sha256=71b9bd424b1b7abc18b05052a1a3cfd3627abdca62be280854cc411791357e42 + nokogiri (1.19.3-x86_64-darwin) sha256=77f3fba57d46c53ab31e62fc6c28f705109d1bf6264356c76f132b2be5728d4d nokogiri (1.19.3-x86_64-linux-gnu) sha256=2f5078620fe12e83669b5b17311b32532a8153d02eee7ad06948b926d6080976 + nokogiri (1.19.3-x86_64-linux-musl) sha256=248c906d2166eca5efb56d52fdee5f9a1f51d69a72e2b64fdac647b4ce39ea3f pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99 pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea + pg (1.6.3-aarch64-linux-musl) sha256=06a75f4ea04b05140146f2a10550b8e0d9f006a79cdaf8b5b130cde40e3ecc2c + pg (1.6.3-arm64-darwin) sha256=7240330b572e6355d7c75a7de535edb5dfcbd6295d9c7777df4d9dddfb8c0e5f + pg (1.6.3-x86_64-darwin) sha256=ee2e04a17c0627225054ffeb43e31a95be9d7e93abda2737ea3ce4a62f2729d6 pg (1.6.3-x86_64-linux) sha256=5d9e188c8f7a0295d162b7b88a768d8452a899977d44f3274d1946d67920ae8d + pg (1.6.3-x86_64-linux-musl) sha256=9c9c90d98c72f78eb04c0f55e9618fe55d1512128e411035fe229ff427864009 pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 @@ -303,6 +347,7 @@ CHECKSUMS rails (8.1.3) sha256=6d017ba5348c98fc909753a8169b21d44de14d2a0b92d140d1a966834c3c9cd3 rails-dom-testing (2.3.0) sha256=8acc7953a7b911ca44588bf08737bc16719f431a1cc3091a292bca7317925c1d rails-html-sanitizer (1.7.0) sha256=28b145cceaf9cc214a9874feaa183c3acba036c9592b19886e0e45efc62b1e89 + rails-sql-tasks-enhancer (0.2.0) railties (8.1.3) sha256=913eb0e0cb520aac687ffd74916bd726d48fa21f47833c6292576ef6a286de22 rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192 @@ -322,4 +367,4 @@ CHECKSUMS zeitwerk (2.8.2) sha256=7212a61311083c604184b1ea2574b9aa05cd14f855a0841c06985cabe9181d12 BUNDLED WITH - 4.0.10 + 4.0.13 diff --git a/db-syncer/config/initializers/rails_sql_tasks_enhancer.rb b/db-syncer/config/initializers/rails_sql_tasks_enhancer.rb new file mode 100644 index 000000000..dcf373a09 --- /dev/null +++ b/db-syncer/config/initializers/rails_sql_tasks_enhancer.rb @@ -0,0 +1,2 @@ +RailsSqlTasksEnhancer.add(:setup, "task_setup.sql.erb", after_load: true, before_migrate: Rails.env.local?) +RailsSqlTasksEnhancer.add(:setup_grants, "task_grants.sql.erb", after_load: true, after_migrate: true) diff --git a/db-syncer/db/grants.sql.erb b/db-syncer/db/grants.sql.erb new file mode 100644 index 000000000..d31787492 --- /dev/null +++ b/db-syncer/db/grants.sql.erb @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION pg_temp.perform_grants() +RETURNS void +PARALLEL UNSAFE +LANGUAGE plpgsql +AS $$ + BEGIN + -- Ensure tables, sequences, etc are owned by schema-owning user. + PERFORM pg_temp.ensure_schema_owner('reopt_api', 'reopt_api'); + END; +$$; diff --git a/db-syncer/db/setup.sql.erb b/db-syncer/db/setup.sql.erb new file mode 100644 index 000000000..dbfa37d2a --- /dev/null +++ b/db-syncer/db/setup.sql.erb @@ -0,0 +1,38 @@ +<% +if Rails.env.local? + db_passwords = { + :reopt_api => "dev_password", + :reopt_api_app_user => "dev_password", + } +end +%> + +CREATE OR REPLACE FUNCTION pg_temp.perform_setup() +RETURNS void +PARALLEL UNSAFE +LANGUAGE plpgsql +AS $$ + BEGIN + <% if Rails.env.local? %> + -- Create schema-owning user. + PERFORM pg_temp.upsert_role('reopt_api', 'NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT LOGIN NOREPLICATION NOBYPASSRLS', '<%= PG::Connection.escape(db_passwords.fetch(:reopt_api)) %>'); + + -- Create application user. + PERFORM pg_temp.upsert_role('reopt_api_app_user', 'NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT LOGIN NOREPLICATION NOBYPASSRLS', '<%= PG::Connection.escape(db_passwords.fetch(:reopt_api_app_user)) %>'); + + -- Create application group + PERFORM pg_temp.upsert_role('reopt_api_app_group', 'NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOLOGIN NOREPLICATION NOBYPASSRLS'); + <% end %> + + -- Grant schema-owner privileges to DBA accounts. + PERFORM pg_temp.tuple_concurrency_retry('GRANT reopt_api TO postgres'); + + -- Grant the app group to the users. + PERFORM pg_temp.tuple_concurrency_retry('GRANT reopt_api_app_group TO reopt_api'); + PERFORM pg_temp.tuple_concurrency_retry('GRANT reopt_api_app_group TO reopt_api_app_user'); + + -- Create schemas. + CREATE SCHEMA IF NOT EXISTS reopt_api; + ALTER SCHEMA reopt_api OWNER TO reopt_api; + END; +$$; diff --git a/db-syncer/db/task_functions.sql.erb b/db-syncer/db/task_functions.sql.erb new file mode 100644 index 000000000..b09b20b73 --- /dev/null +++ b/db-syncer/db/task_functions.sql.erb @@ -0,0 +1,99 @@ +-- Lock to prevent `tuple concurrently updated` errors with parallel ALTER +-- ROLEs or GRANTs. +-- https://www.postgresql.org/message-id/E1TLyjV-0008JC-OV@wrigleys.postgresql.org +CREATE OR REPLACE FUNCTION pg_temp.setup_concurrency_lock() +RETURNS void +PARALLEL UNSAFE +LANGUAGE plpgsql +AS $$ + BEGIN + <% if Rails.env.test? %> + -- This performs a global, cluster-wide lock (since users are shared + -- across the cluster). + LOCK TABLE pg_catalog.pg_user IN SHARE UPDATE EXCLUSIVE MODE; + <% else %> + -- AWS RDS doesn't give us permissions to perform the cluster-wide + -- pg_catalog lock above, so instead we'll perform an advisory lock + -- (using an arbitrary ID that we'll reserve for these types of scripts) + -- to at least perform a database-wide lock. This won't necessarily help + -- with cluster-wide ROLE updates, but may still help with possibly + -- concurrency for database-specific tasks. + PERFORM pg_advisory_xact_lock(-999876); + <% end %> + END; +$$; + +-- Retry queries until they don't lead to a `tuple concurrently updated` error +-- to help with cluster-wide race conditions. +-- +-- While the `setup_concurrency_lock` helps prevent some concurrency issues, +-- we've still observed concurrency errors performing certain SQL queries (ALTER +-- ROLE and GRANT). This function can be used to execute the SQL and retry until +-- there are no concurrency conflicts. +CREATE OR REPLACE FUNCTION pg_temp.tuple_concurrency_retry(sql_query TEXT) +RETURNS void +PARALLEL UNSAFE +LANGUAGE plpgsql +AS $$ + BEGIN + LOOP + BEGIN + EXECUTE sql_query; + EXIT; + EXCEPTION + WHEN internal_error THEN + IF sqlerrm = 'tuple concurrently updated' THEN + RAISE NOTICE 'tuple concurrency error, retrying: %', sqlerrm; + ELSE + RAISE; + END IF; + END; + END LOOP; + END; +$$; + +-- Create or update a database role. +CREATE FUNCTION pg_temp.upsert_role(role_name TEXT, options TEXT, password TEXT = NULL) +RETURNS void +PARALLEL UNSAFE +LANGUAGE plpgsql +AS $$ + BEGIN + BEGIN + PERFORM pg_temp.tuple_concurrency_retry(format('CREATE ROLE %I WITH %s PASSWORD %L', role_name, options, password)); + EXCEPTION + WHEN duplicate_object THEN + PERFORM pg_temp.tuple_concurrency_retry(format('ALTER ROLE %I WITH %s PASSWORD %L', role_name, options, password)); + END; + END; +$$; + +-- Set all resources inside of a schema to be owned by a specific user. +CREATE OR REPLACE FUNCTION pg_temp.ensure_schema_owner(schema_name TEXT, role_name TEXT) +RETURNS void +PARALLEL UNSAFE +LANGUAGE plpgsql +AS $$ + DECLARE + r record; + BEGIN + FOR r IN EXECUTE format('SELECT n.nspname, c.relname FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON c.relnamespace = n.oid WHERE c.relkind = ''r'' AND n.nspname = %L AND pg_get_userbyid(c.relowner) != %L', schema_name, role_name) LOOP + EXECUTE format('ALTER TABLE %I.%I OWNER TO %I', r.nspname, r.relname, role_name); + END LOOP; + FOR r IN EXECUTE format('SELECT n.nspname, c.relname FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON c.relnamespace = n.oid WHERE c.relkind = ''v'' AND n.nspname = %L AND pg_get_userbyid(c.relowner) != %L', schema_name, role_name) LOOP + EXECUTE format('ALTER VIEW %I.%I OWNER TO %I', r.nspname, r.relname, role_name); + END LOOP; + FOR r IN EXECUTE format('SELECT n.nspname, c.relname FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON c.relnamespace = n.oid WHERE c.relkind = ''m'' AND n.nspname = %L AND pg_get_userbyid(c.relowner) != %L', schema_name, role_name) LOOP + EXECUTE format('ALTER MATERIALIZED VIEW %I.%I OWNER TO %I', r.nspname, r.relname, role_name); + END LOOP; + FOR r IN EXECUTE format('SELECT n.nspname, c.relname FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON c.relnamespace = n.oid WHERE c.relkind = ''i'' AND n.nspname = %L AND pg_get_userbyid(c.relowner) != %L', schema_name, role_name) LOOP + EXECUTE format('ALTER INDEX %I.%I OWNER TO %I', r.nspname, r.relname, role_name); + END LOOP; + FOR r IN EXECUTE format('SELECT n.nspname, c.relname FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON c.relnamespace = n.oid WHERE c.relkind = ''S'' AND n.nspname = %L AND pg_get_userbyid(c.relowner) != %L', schema_name, role_name) LOOP + EXECUTE format('ALTER SEQUENCE %I.%I OWNER TO %I', r.nspname, r.relname, role_name); + END LOOP; + FOR r IN EXECUTE format('SELECT p.oid::regprocedure AS function_signature FROM pg_catalog.pg_proc AS p JOIN pg_catalog.pg_namespace AS n ON p.pronamespace = n.oid WHERE n.nspname = %L AND pg_get_userbyid(p.proowner) != %L', schema_name, role_name) LOOP + EXECUTE format('ALTER FUNCTION %s OWNER TO %I', r.function_signature, role_name); + END LOOP; + END; +$$; diff --git a/db-syncer/db/task_grants.sql.erb b/db-syncer/db/task_grants.sql.erb new file mode 100644 index 000000000..aa5164fab --- /dev/null +++ b/db-syncer/db/task_grants.sql.erb @@ -0,0 +1,9 @@ +<%= ERB.new(File.read(Rails.root.join("db/grants.sql.erb"))).result %> +<%= ERB.new(File.read(Rails.root.join("db/task_functions.sql.erb"))).result %> + +DO $$ + BEGIN + PERFORM pg_temp.setup_concurrency_lock(); + PERFORM pg_temp.perform_grants(); + END; +$$; diff --git a/db-syncer/db/task_setup.sql.erb b/db-syncer/db/task_setup.sql.erb new file mode 100644 index 000000000..b2d5ba310 --- /dev/null +++ b/db-syncer/db/task_setup.sql.erb @@ -0,0 +1,16 @@ +<%= ERB.new(File.read(Rails.root.join("db/grants.sql.erb"))).result %> +<%= ERB.new(File.read(Rails.root.join("db/setup.sql.erb"))).result %> +<%= ERB.new(File.read(Rails.root.join("db/task_functions.sql.erb"))).result %> + +DO $$ + BEGIN + PERFORM pg_temp.setup_concurrency_lock(); + PERFORM pg_temp.perform_setup(); + + -- Ensure grants are always performed inside the same transaction as setup, + -- in case setup dropped/recreated any resources (like foreign data tables) + -- so that grants don't go missing (since setup is often run before + -- migrations and grants aren't run until after migrations). + PERFORM pg_temp.perform_grants(); + END; +$$; diff --git a/docker-compose.yml b/docker-compose.yml index e297d4515..b6ca96043 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,6 @@ services: volumes: - postgres_data:/var/lib/postgresql environment: - - POSTGRES_USER=reopt - POSTGRES_PASSWORD=reopt - POSTGRES_DB=reopt expose: @@ -29,7 +28,7 @@ services: - APP_ENV=local - DB_HOST=db - DB_NAME=reopt - - DB_USERNAME=reopt + - DB_USERNAME=postgres - DB_PASSWORD=reopt - REDIS_HOST=redis - REDIS_PASSWORD= @@ -53,7 +52,7 @@ services: - APP_ENV=local - DB_HOST=db - DB_NAME=reopt - - DB_USERNAME=reopt + - DB_USERNAME=postgres - DB_PASSWORD=reopt - REDIS_HOST=redis - REDIS_PASSWORD= @@ -93,7 +92,7 @@ services: args: BUNDLE_FROZEN: "false" volumes: - # - .:/app + - .:/app - ~/.vault-token:/root/.vault-token:ro depends_on: - db @@ -101,7 +100,7 @@ services: - LOAD_VAULT_ENV_SECRETS=true - DEV_DB_HOST=db - DEV_DB_NAME=reopt_api - - DEV_DB_USERNAME=reopt + - DEV_DB_USERNAME=postgres - DEV_DB_PASSWORD=reopt volumes: From 54def5fd6b52368877f163e5a5e768f296716d5e Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 4 Jun 2026 08:10:17 -0600 Subject: [PATCH 29/66] Shift db-syncer to separate docker compose profile Since this is an optional container only used in local development, shift it to a profile so that it doesn't start by default. This container will only work for NLR developers, so we don't want it starting/building by default in the CI environment or for other users. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b6ca96043..aefe8ee6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,6 +86,8 @@ services: - ./julia_src:/opt/julia_src db-syncer: + profiles: + - db-syncer build: context: . dockerfile: db-syncer/Dockerfile From b51e8a7a8d4b9b3855c836374c593fb52449c316 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 4 Jun 2026 09:15:39 -0600 Subject: [PATCH 30/66] Have tests use standard database credentials from environment variables Also update the nojulia docker compose variant to align with the main docker compose file that uses the default postgres superuser now (to better align with other environments). --- docker-compose.nojulia.yml | 5 ++--- reopt_api/settings.py | 7 ------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docker-compose.nojulia.yml b/docker-compose.nojulia.yml index fcf4ad528..b37743f44 100644 --- a/docker-compose.nojulia.yml +++ b/docker-compose.nojulia.yml @@ -16,7 +16,6 @@ services: volumes: - pgdata:/var/lib/postgresql/data environment: - - POSTGRES_USER=reopt - POSTGRES_PASSWORD=reopt - POSTGRES_DB=reopt expose: @@ -34,7 +33,7 @@ services: - APP_ENV=local - DB_HOST=db - DB_NAME=reopt - - DB_USERNAME=reopt + - DB_USERNAME=postgres - DB_PASSWORD=reopt - REDIS_HOST=redis - REDIS_PASSWORD= @@ -59,7 +58,7 @@ services: - APP_ENV=local - DB_HOST=db - DB_NAME=reopt - - DB_USERNAME=reopt + - DB_USERNAME=postgres - DB_PASSWORD=reopt - REDIS_HOST=redis - REDIS_PASSWORD= diff --git a/reopt_api/settings.py b/reopt_api/settings.py index 4e0e492b3..aa45895ac 100644 --- a/reopt_api/settings.py +++ b/reopt_api/settings.py @@ -99,13 +99,6 @@ 'PASSWORD': db_password, } } -if 'test' in sys.argv or APP_ENV == 'local': - DATABASES['default']['NAME'] = 'reopt' - DATABASES['default']['USER'] = 'reopt' - DATABASES['default']['PASSWORD'] = 'reopt' - DATABASES['default']['OPTIONS'] = { - 'options': '-c search_path=public' - } # Internationalization From 8fd53c2302b359d91e8d12c3b8f5f2d0f9b2beb8 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:30:20 -0600 Subject: [PATCH 31/66] Fix to create database schema for test environment. --- reopt_api/settings.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/reopt_api/settings.py b/reopt_api/settings.py index aa45895ac..2efa87d06 100644 --- a/reopt_api/settings.py +++ b/reopt_api/settings.py @@ -4,6 +4,9 @@ import django import rollbar import sys +from django.db import connections +from django.db.models.signals import pre_migrate + """ Django settings for reopt_api project. @@ -173,5 +176,14 @@ TASTYPIE_ALLOW_MISSING_SLASH = True DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +# Create the default PostgreSQL schema before any migrations run if it doesn't +# already exist. +def create_postgres_schema(sender, **kwargs): + using = kwargs.get('using', 'default') + connection = connections[using] + with connection.cursor() as cursor: + cursor.execute(f"CREATE SCHEMA IF NOT EXISTS reopt_api;") +pre_migrate.connect(create_postgres_schema) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.settings") django.setup() From bfcef2a4c9faf5ae6ea8a2d10f8e0c65cdf1d8e2 Mon Sep 17 00:00:00 2001 From: lixiangk1 <72464565+lixiangk1@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:45:04 -0600 Subject: [PATCH 32/66] Update MPC modeling --- julia_src/http.jl | 28 +- julia_src/mpc.jl | 688 ++++++++++++++++++---------------------------- 2 files changed, 294 insertions(+), 422 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index e6f94a42c..b03a742f8 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -66,14 +66,23 @@ function reopt(req::HTTP.Request) delete!(d, "api_key") end + settings = d["Settings"] + solver_name = get(settings, "solver_name", "HiGHS") + if solver_name == "Xpress" && !(xpress_installed=="True") + solver_name = "HiGHS" + @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. Next time + Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" + end + # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- # When ElectricStorage.dispatch_options == "daily_foresight_optimized", first run the MPC rolling-horizon loop to get an SOC profile. # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. + # Runs after the Xpress availability check so the resolved solver_name is passed in. electric_storage = get(d, "ElectricStorage", Dict()) if get(electric_storage, "dispatch_options", nothing) == "daily_foresight_optimized" try @info "Running MPC to obtain daily foresight optimized battery dispatch profile." - mpc_results = get_mpc_results(d) + mpc_results = get_mpc_results(d; solver_name=solver_name) soc = mpc_results["dispatch"]["ElectricStorage"]["soc_series_fraction"] d["ElectricStorage"]["fixed_soc_series_fraction"] = soc d["ElectricStorage"]["dispatch_options"] = "custom" @@ -86,13 +95,7 @@ function reopt(req::HTTP.Request) end end - settings = d["Settings"] - solver_name = get(settings, "solver_name", "HiGHS") - if solver_name == "Xpress" && !(xpress_installed=="True") - solver_name = "HiGHS" - @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. Next time - Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" - end + #TODO: What timeout and optimality tolerance should MPC use? timeout_seconds = pop!(settings, "timeout_seconds") optimality_tolerance = pop!(settings, "optimality_tolerance") solver_attributes = SolverAttributes(timeout_seconds, optimality_tolerance) @@ -844,7 +847,14 @@ function mpc(req::HTTP.Request) ENV["NREL_DEVELOPER_API_KEY"] = test_nrel_developer_api_key delete!(d, "api_key") end - results = get_mpc_results(d) + settings = get(d, "Settings", Dict()) + solver_name = get(settings, "solver_name", "HiGHS") + if solver_name == "Xpress" && !(xpress_installed=="True") + solver_name = "HiGHS" + @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. + Next time specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'." + end + results = get_mpc_results(d; solver_name=solver_name) catch e @error "MPC failed" exception=(e, catch_backtrace()) error_response["error"] = sprint(showerror, e) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 21cb9510a..af376cdbe 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -3,22 +3,19 @@ # MPC (Model Predictive Control) endpoint # ---------------------------------------------------------------------------- # Rolling-horizon dispatch with one day look-ahead using REopt.run_mpc. Assumes: -# * PV and ElectricStorage are the only available technologies (perfect forecast) +# * PV and ElectricStorage are the only available technologies # * Only perfect forecast scenarios are modeled (no forecast errors) -# * Sizing: Perform a sizing run first if PV or ElectricStorage sizes are not provided (storage sizes must be greater than zero) +# * Sizing: Perform a sizing run first if PV or ElectricStorage sizes are not provided (storage sizes must be > 0) # * PV: Use PVWatts if `PV.production_factor_series` is not provided # * ElectricLoad: Use commercial reference profiles if `ElectricLoad.loads_kw` is not provided -# * Code currently wraps with Jan 1 data to determine Dec 31 dispatch (allows leap-year data, which avoids the need to wrap) +# * Code wraps with Jan 1 data to determine Dec 31 dispatch (leap-year inputs are not supported) # * MPC settings are currently hard coded (e.g., forecast horizon, control horizon, optimization horizon) # ============================================================================ """ get_month_transition_timesteps(time_steps_per_hour) -Return the 1-based timestep index marking the START of each month for a -non-leap year beginning Jan 1 00:00. Length 12; first element == 1. -The last "transition" (end of December) is implicitly `length_of_data + 1` -and is handled by the caller. +Return an array of length 12 specifying the index marking the start of each month in a non-leap year """ function get_month_transition_timesteps(time_steps_per_hour::Int) # hours in each month for a non-leap year @@ -32,15 +29,11 @@ function get_month_transition_timesteps(time_steps_per_hour::Int) end """ - _mpc_slice_with_wrap(arr, idx, end_idx) + slice_data(arr, idx, end_idx) -Return `arr[idx:end_idx]` but wrap around to the start of `arr` if -`end_idx > length(arr)`. The wrap threshold is the actual array length, -not `length_of_data`, so a leap-year-length input (8784*tsh) naturally -supplies the extra day for the year-end look-ahead window before any -wrap is needed. +Returns arr[idx:end_idx] but wrap around to the start of arr if end_idx > length(arr). """ -function _mpc_slice_with_wrap(arr::AbstractVector, idx::Int, end_idx::Int) +function slice_data(arr::AbstractVector, idx::Int, end_idx::Int) n = length(arr) if end_idx <= n return arr[idx:end_idx] @@ -51,383 +44,286 @@ function _mpc_slice_with_wrap(arr::AbstractVector, idx::Int, end_idx::Int) end """ - _mpc_check_series_length(name, series, time_steps_per_hour, length_of_data) - -Validate that a user-supplied annual time series has either -`8760 * time_steps_per_hour` entries (standard year) or -`8784 * time_steps_per_hour` entries (leap year). Returns the series -unchanged. - -The MPC loop iterates over `length_of_data = 8760 * time_steps_per_hour` -timesteps; if a leap-year-length series is supplied, the extra day is -used as additional look-ahead data by `_mpc_slice_with_wrap` (which wraps -on the actual array length, not on `length_of_data`). Any other length is -a hard error — REopt.jl's own `check_and_adjust_load_length` would -silently repeat/pad, masking the mistake. + check_series_length(name, series, time_steps_per_hour, length_of_data) + +Validate that a user-entered time series is of length 8760 * time_steps_per_hour (cannot be a leap year). """ -function _mpc_check_series_length(name::String, series::AbstractVector, - time_steps_per_hour::Int, length_of_data::Int) +function check_series_length(name::String, series::AbstractVector, length_of_data::Int) n = length(series) - n == length_of_data && return series - if n == 8784 * time_steps_per_hour - @info "MPC: $name has leap-year length ($n); the extra day will be used " * - "for year-end look-ahead instead of wrapping to Jan 1." + if n == length_of_data return series end - # Diagnostic: if `n` is consistent with 8760 hours at a *different* - # sub-hourly resolution, the user probably mis-declared - # `Settings.time_steps_per_hour`. - tsh_hint = "" - for tsh_guess in (1, 2, 4) - tsh_guess == time_steps_per_hour && continue - if n == 8760 * tsh_guess || n == 8784 * tsh_guess - tsh_hint = " (length is consistent with time_steps_per_hour=$(tsh_guess); " * - "check Settings.time_steps_per_hour)." - break - end - end - error("MPC: $name length $n != " * - "8760 * time_steps_per_hour ($length_of_data) " * - "and != 8784 * time_steps_per_hour ($(8784 * time_steps_per_hour))." * - tsh_hint) + error("MPC: $name length $n != 8760 * time_steps_per_hour ($length_of_data).") end """ - _mpc_build_tariff_arrays(et_input, year, time_steps_per_hour, length_of_data) - -Construct a `REopt.ElectricTariff` from the user's ElectricTariff input dict -(which uses the real REopt input schema: `urdb_label`, `urdb_response`, -`urdb_utility_name`+`urdb_rate_name`, `tou_energy_rates_per_kwh`, -`monthly_energy_rates`, `monthly_demand_rates`, `blended_annual_energy_rate`, -`blended_annual_demand_rate`, etc.) and extract the processed arrays MPC -needs for its per-window posts. - -Returns a NamedTuple with: - * `energy_rates::Vector{Float64}` (length `length_of_data`) - * `monthly_demand_rates::Vector{Float64}` (length 12, all zeros if not set) - * `tou_demand_rates::Vector{Float64}` (one per ratchet; empty for non-URDB) - * `tou_demand_time_steps::Vector{Vector{Int}}` (full-year indices per ratchet) - -Multi-tier rates are flattened to a single tier (MPC does not model tiers). + get_tariff_inputs(electric_tariff, year, time_steps_per_hour) + +Call REopt.ElectricTariff to get energy_rates, monthly_demand_rates, tou_demand_rates, tou_demand_ratchet_time_steps +MPC does not currently model tiers (tiers are flattened), coincident peak charges, or demand lookback. """ -function _mpc_build_tariff_arrays(et_input::Dict, year::Union{Int,Nothing}, - time_steps_per_hour::Int, length_of_data::Int) - # Only forward known ElectricTariff kwargs so unrelated keys (e.g. a - # parsed API artifact) don't trip the constructor. - known_kwargs = (:urdb_label, :urdb_response, :urdb_utility_name, +function get_tariff_inputs(electric_tariff::Dict, year::Union{Int,Nothing}, time_steps_per_hour::Int) + + # TODO: Are all of these relevant? Any missing inputs? + # TODO: Implement lookback? (demand_lookback_months, demand_lookback_percent, demand_lookback_range) Ignoring coincident peak charges for now + # TODO: Need to think through NEM or passing back export values (wholesale_rate, export_rate_beyond_net_metering_limit) + tariff_kwargs = (:urdb_label, :urdb_response, :urdb_utility_name, :urdb_rate_name, :urdb_metadata, :wholesale_rate, :export_rate_beyond_net_metering_limit, :monthly_energy_rates, :monthly_demand_rates, :blended_annual_energy_rate, :blended_annual_demand_rate, :add_monthly_rates_to_urdb_rate, :tou_energy_rates_per_kwh, :add_tou_energy_rates_to_urdb_rate, - :remove_tiers, :demand_lookback_months, - :demand_lookback_percent, :demand_lookback_range, - :coincident_peak_load_active_time_steps, - :coincident_peak_load_charge_per_kw) - kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in et_input - if Symbol(k) in known_kwargs) - # Force single-tier arrays so MPC sees a flat schedule; without this the - # constructor could return `length_of_data x n_tiers` and we'd silently - # only use one tier (and inconsistently across energy vs demand). - kwargs[:remove_tiers] = get(kwargs, :remove_tiers, true) - - tariff = reoptjl.ElectricTariff(; year = year, - time_steps_per_hour = time_steps_per_hour, - kwargs...) - - # All arrays are single-tier (we forced `remove_tiers=true`). - energy_rates = vec(Float64.(tariff.energy_rates[:, 1])) - length(energy_rates) == length_of_data || error( - "MPC: derived energy_rates length $(length(energy_rates)) != length_of_data $(length_of_data).") + :demand_lookback_months, :demand_lookback_percent, :demand_lookback_range) + kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in electric_tariff if Symbol(k) in tariff_kwargs) - monthly_demand_rates = isempty(tariff.monthly_demand_rates) ? - zeros(Float64, 12) : vec(Float64.(tariff.monthly_demand_rates[:, 1])) - length(monthly_demand_rates) == 12 || error( - "MPC: derived monthly_demand_rates length $(length(monthly_demand_rates)) != 12.") + # MPC does not currently model tiered rates + kwargs[:remove_tiers] = true + tariff = reoptjl.ElectricTariff(; year = year, time_steps_per_hour = time_steps_per_hour, kwargs...) - tou_demand_rates = isempty(tariff.tou_demand_rates) ? - Float64[] : vec(Float64.(tariff.tou_demand_rates[:, 1])) - tou_demand_time_steps = [Int.(v) for v in tariff.tou_demand_ratchet_time_steps] + energy_rates = Float64.(tariff.energy_rates[:, 1]) + monthly_demand_rates = isempty(tariff.monthly_demand_rates) ? + zeros(Float64, 12) : Float64.(tariff.monthly_demand_rates[:, 1]) + tou_demand_rates = isempty(tariff.tou_demand_rates) ? Float64[] : Float64.(tariff.tou_demand_rates[:, 1]) + tou_demand_ratchet_time_steps = [Int.(v) for v in tariff.tou_demand_ratchet_time_steps] + # TODO: Track lookback variables - demand_lookback_months, demand_lookback_percent, demand_lookback_range? return (energy_rates = energy_rates, monthly_demand_rates = monthly_demand_rates, tou_demand_rates = tou_demand_rates, - tou_demand_time_steps = tou_demand_time_steps) + tou_demand_ratchet_time_steps = tou_demand_ratchet_time_steps) end """ - _mpc_generate_pv_production_factor(d, time_steps_per_hour) + generate_pv_production_factors(d, time_steps_per_hour) -Derive a PV production factor series via `REopt.get_production_factor` -(PVWatts) using `Site.latitude`/`Site.longitude` and the PV inputs in `d`. -Returns a `Vector{Float64}`. Throws if lat/lon are missing. +Generate a PV production factor series using PVWatts by calling REopt.get_production_factor """ -function _mpc_generate_pv_production_factor(d::Dict, time_steps_per_hour::Int) +function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) site = get(d, "Site", Dict()) - haskey(site, "latitude") || error( - "MPC: Site.latitude is required to derive PV.production_factor_series via PVWatts.") - haskey(site, "longitude") || error( - "MPC: Site.longitude is required to derive PV.production_factor_series via PVWatts.") + if !haskey(site, "latitude") + error("MPC: Site.latitude is required to generate PV.production_factor_series using PVWatts.") + end + if !haskey(site, "longitude") + error("MPC: Site.longitude is required to generate PV.production_factor_series using PVWatts.") + end lat = Float64(site["latitude"]) lon = Float64(site["longitude"]) - @info "MPC: PV.production_factor_series not provided; deriving via REopt.jl PVWatts (lat=$(lat), lon=$(lon))." + @info "MPC: PV.production_factor_series not provided; generating using PVWatts through REopt.jl (lat=$(lat), lon=$(lon))." - # Only forward known REopt.PV kwargs that affect the production factor. pv = get(d, "PV", Dict()) pv_pf_kwargs = (:array_type, :tilt, :module_type, :losses, :azimuth, :gcr, :radius, :name, :location, :dc_ac_ratio, :inv_eff) - kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in pv - if Symbol(k) in pv_pf_kwargs) + kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in pv if Symbol(k) in pv_pf_kwargs) pv_struct = reoptjl.PV(; latitude = lat, kwargs...) - derived = reoptjl.get_production_factor(pv_struct, lat, lon; - time_steps_per_hour = time_steps_per_hour) - return collect(Float64.(derived)) + pv_production_factor_series = reoptjl.get_production_factor(pv_struct, lat, lon; + time_steps_per_hour = time_steps_per_hour) + return Vector{Float64}(pv_production_factor_series) end """ - _mpc_generate_loads_kw(d, time_steps_per_hour) - -Derive an electric load profile via `REopt.ElectricLoad` from the user's -standard ElectricLoad inputs (`doe_reference_name`+`city` / -`blended_doe_reference_names`+`blended_doe_reference_percents`, -`annual_kwh`, `monthly_totals_kwh`, `monthly_peaks_kw`, `year`, etc.) — -the same DOE CRB derivation path the main REopt run would take. Returns a -`Vector{Float64}`. Requires `Site.latitude`/`Site.longitude`. + generate_loads_kw(d, time_steps_per_hour) + +Generate an electric load profile using DOE Commercial Reference buildings by calling REopt.ElectricLoad """ -function _mpc_generate_loads_kw(d::Dict, time_steps_per_hour::Int) +function generate_loads_kw(d::Dict, time_steps_per_hour::Int) site = get(d, "Site", Dict()) - haskey(site, "latitude") || error( - "MPC: Site.latitude is required to derive ElectricLoad.loads_kw from a DOE CRB profile.") - haskey(site, "longitude") || error( - "MPC: Site.longitude is required to derive ElectricLoad.loads_kw from a DOE CRB profile.") + if !haskey(site, "latitude") + error("MPC: Site.latitude is required to generate ElectricLoad.loads_kw using a DOE CRB profile.") + end + if !haskey(site, "longitude") + error("MPC: Site.longitude is required to generate ElectricLoad.loads_kw using a DOE CRB profile.") + end lat = Float64(site["latitude"]) lon = Float64(site["longitude"]) - @info "MPC: ElectricLoad.loads_kw not provided; deriving via REopt.jl ElectricLoad (lat=$(lat), lon=$(lon))." + @info "MPC: ElectricLoad.loads_kw not provided; generating via REopt.jl ElectricLoad (lat=$(lat), lon=$(lon))." - # Only forward known ElectricLoad kwargs. - eload = get(d, "ElectricLoad", Dict()) - known_kwargs = (:normalize_and_scale_load_profile_input, + electric_load = get(d, "ElectricLoad", Dict()) + + # TODO: Double check this list for relevance/missing inputs + load_kwargs = (:normalize_and_scale_load_profile_input, :path_to_csv, :doe_reference_name, :blended_doe_reference_names, :blended_doe_reference_percents, :year, :city, :annual_kwh, :monthly_totals_kwh, - :monthly_peaks_kw, :critical_loads_kw, :loads_kw_is_net, - :critical_loads_kw_is_net, :critical_load_fraction, - :operating_reserve_required_fraction, - :min_load_met_annual_fraction, :off_grid_flag) - kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in eload - if Symbol(k) in known_kwargs) - load_struct = reoptjl.ElectricLoad(; latitude = lat, longitude = lon, - time_steps_per_hour = time_steps_per_hour, - kwargs...) - return collect(Float64.(load_struct.loads_kw)) + :monthly_peaks_kw, :loads_kw_is_net) + kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in electric_load if Symbol(k) in load_kwargs) + load = reoptjl.ElectricLoad(; latitude = lat, longitude = lon, + time_steps_per_hour = time_steps_per_hour, kwargs...) + return Vector{Float64}(load.loads_kw) end """ - _mpc_resolve_sizes!(d) - -Determines PV/ElectricStorage sizes for the MPC loop, using the actual -REopt input schema (`min_kw`/`max_kw`, `min_kwh`/`max_kwh`). - -Rules: - * If `min_kw == max_kw` for PV AND `min_kw == max_kw` AND - `min_kwh == max_kwh` for ElectricStorage → sizes are already fixed; use - those values directly. No sizing run. - * Otherwise → run a regular REopt sizing optimization, then mutate `d` to - set `min_kw = max_kw = sized_kw` (and similarly for kWh on storage). - Locking sizes in `d` ensures the downstream main REopt run uses the - same sizes the MPC SOC trajectory was computed against. - -Returns the tuple `(pv_kw, bess_kw, bess_kwh)` for the caller to use when -building per-window MPC posts. + get_technology_sizes!(d) + +Determine PV and ElectricStorage sizes for the MPC loop. If min_kw != max_kw and/or min_kwh != max_kwh, +call REopt to size technologies. User input battery sizes must also be greater than zero. """ -function _mpc_resolve_sizes!(d::Dict) +function get_technology_sizes!(d::Dict) pv = get!(d, "PV", Dict()) - bess = get!(d, "ElectricStorage", Dict()) + batt = get!(d, "ElectricStorage", Dict()) - function _is_fixed(dct, lo_key, hi_key) + function is_fixed(dct, lo_key, hi_key) lo = get(dct, lo_key, nothing) hi = get(dct, hi_key, nothing) return lo !== nothing && hi !== nothing && Float64(lo) == Float64(hi) end - # Guard: daily_foresight_optimized requires a non-zero ElectricStorage. - # If the user has pinned kW or kWh to zero (min == max == 0), fail fast - # with a clear message instead of running a pointless sizing/MPC pass. - if (_is_fixed(bess, "min_kw", "max_kw") && Float64(get(bess, "min_kw", 0)) == 0.0) || - (_is_fixed(bess, "min_kwh", "max_kwh") && Float64(get(bess, "min_kwh", 0)) == 0.0) - error("ElectricStorage power (kW) and energy (kWh) must both be greater than zero to run the daily_foresight_optimized dispatch option. Got min_kw=$(get(bess, "min_kw", nothing)), max_kw=$(get(bess, "max_kw", nothing)), min_kwh=$(get(bess, "min_kwh", nothing)), max_kwh=$(get(bess, "max_kwh", nothing)).") + # Check storage sizes are greater than zero + if Float64(get(batt, "max_kw", 1.0)) <= 0.0 || Float64(get(batt, "max_kwh", 1.0)) <= 0.0 + error("ElectricStorage max_kw and max_kwh must both be greater than zero " * + "to run the daily_foresight_optimized dispatch option.") end - pv_fixed = _is_fixed(pv, "min_kw", "max_kw") - bess_fixed = _is_fixed(bess, "min_kw", "max_kw") && - _is_fixed(bess, "min_kwh", "max_kwh") + pv_fixed = is_fixed(pv, "min_kw", "max_kw") + batt_fixed = is_fixed(batt, "min_kw", "max_kw") && + is_fixed(batt, "min_kwh", "max_kwh") - if pv_fixed && bess_fixed + if pv_fixed && batt_fixed pv_kw = Float64(pv["min_kw"]) - bess_kw = Float64(bess["min_kw"]) - bess_kwh = Float64(bess["min_kwh"]) - @info "MPC: sizes already fixed via min_kw==max_kw. PV=$(pv_kw) kW, BESS=$(bess_kw) kW / $(bess_kwh) kWh." - return (pv_kw = pv_kw, bess_kw = bess_kw, bess_kwh = bess_kwh) + batt_kw = Float64(batt["min_kw"]) + batt_kwh = Float64(batt["min_kwh"]) + @info "MPC: fixed technology sizes entered, PV = $(pv_kw) kW, battery = $(batt_kw) kW / $(batt_kwh) kWh." + return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh) end - @info "MPC: PV and/or ElectricStorage sizes not fixed — running REopt sizing first." + @info "MPC: PV and/or ElectricStorage sizes are not specified — running REopt sizing first." sizing_post = deepcopy(d) - # Strip any API-only sentinels before sizing. + + # Delete inputs specific to the heuristic battery dispatch run if haskey(sizing_post, "ElectricStorage") delete!(sizing_post["ElectricStorage"], "dispatch_options") delete!(sizing_post["ElectricStorage"], "fixed_soc_series_fraction") end + settings = get!(sizing_post, "Settings", Dict()) settings["run_bau"] = false + + # TODO: Okay to hard code defaults here? solver_name check is redundant if function is only called from get_mpc_results settings["solver_name"] = get(settings, "solver_name", "HiGHS") settings["timeout_seconds"] = get(settings, "timeout_seconds", 420) settings["optimality_tolerance"] = get(settings, "optimality_tolerance", 0.001) - solver_attributes = SolverAttributes( - settings["timeout_seconds"], settings["optimality_tolerance"]) + solver_attributes = SolverAttributes(settings["timeout_seconds"], settings["optimality_tolerance"]) m = get_solver_model(get_solver_model_type(settings["solver_name"]), solver_attributes) model_inputs = reoptjl.REoptInputs(sizing_post) sizing_results = reoptjl.run_reopt(m, model_inputs) if get(sizing_results, "status", "") != "optimal" - error("MPC sizing pre-step did not solve to optimality (status = $(get(sizing_results, "status", "unknown"))).") + error("MPC sizing pre-step did not solve (status = $(get(sizing_results, "status", "unknown"))).") end pv_kw = Float64(get(get(sizing_results, "PV", Dict()), "size_kw", 0.0)) - bess_kw = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kw", 0.0)) - bess_kwh = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kwh", 0.0)) + batt_kw = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kw", 0.0)) + batt_kwh = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kwh", 0.0)) + + if batt_kw <= 0.0 || batt_kwh <= 0.0 + error("MPC: Optimal REopt sizing does not include ElectricStorage (size_kw=$(batt_kw), size_kwh=$(batt_kwh)). " * + "The daily_foresight_optimized dispatch option requires a non-zero battery size. " * + "Set ElectricStorage.min_kw == ElectricStorage.max_kw and " * + "ElectricStorage.min_kwh == ElectricStorage.max_kwh to fix sizes manually.") + end - # Lock the resolved sizes in `d` so the downstream main REopt run uses - # the SAME sizes that the MPC SOC trajectory is consistent with. pv["min_kw"] = pv_kw pv["max_kw"] = pv_kw - bess["min_kw"] = bess_kw - bess["max_kw"] = bess_kw - bess["min_kwh"] = bess_kwh - bess["max_kwh"] = bess_kwh + batt["min_kw"] = batt_kw + batt["max_kw"] = batt_kw + batt["min_kwh"] = batt_kwh + batt["max_kwh"] = batt_kwh - @info "MPC: sized PV=$(pv_kw) kW, BESS=$(bess_kw) kW / $(bess_kwh) kWh (locked into d via min==max)." - return (pv_kw = pv_kw, bess_kw = bess_kw, bess_kwh = bess_kwh) + @info "MPC: REopt sizing solved with PV = $(pv_kw) kW and battery = $(batt_kw) kW / $(batt_kwh) kWh." + return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh) end -# Core MPC computation. Takes the already-parsed (and api-key-stripped) request -# dict `d` and returns the results dict (dispatch series, costs, etc.). -# May throw; callers handle error responses. -# Called by both /mpc (standalone) and /reopt (when dispatch_options = -# "daily_foresight_optimized"). -function get_mpc_results(d::Dict)::Dict - # ---- Shared settings (with sensible defaults) ---- - settings = get(d, "Settings", Dict()) - solver_name = get(settings, "solver_name", "HiGHS") - if solver_name == "Xpress" && xpress_installed != "True" - solver_name = "HiGHS" - @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. " * - "Next time specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'." - end +function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict + """ + Run a full-year rolling-horizon MPC dispatch for PV + ElectricStorage by + calling `REopt.run_mpc` once per timestep with a 24-hour look-ahead. + + Inputs: + d::Dict, REopt inputs dictionary + solver_name::String, solver to use ("HiGHS", "Cbc", "SCIP", or "Xpress") + Returns a Dict with PV and BATT sizes, dispatch time series, and cost metrics + """ + + settings = get!(d, "Settings", Dict()) + settings["solver_name"] = solver_name + + # TODO: MPC timeout and optimality tolerance optimality_tolerance = Float64(get(settings, "optimality_tolerance", 0.001)) - # ---- Hard-coded MPC config ---- - # Full-year, 24-h horizon, year-end wrap-around, 60 s/iter solver cap. + # TODO: MPC horizons and timeout are currently hard coded time_steps_per_hour = Int(get(settings, "time_steps_per_hour", 1)) length_of_data = 8760 * time_steps_per_hour horizon = 24 * time_steps_per_hour - per_iter_timeout_s = 60.0 - - # ---- Guarantee presence of the tech / load sub-dicts ---- - # Done up front so PV/load resolution and sizing can mutate them in - # place, and downstream code can use plain indexing. - pv = get!(d, "PV", Dict()) - bess = get!(d, "ElectricStorage", Dict()) - eload = get!(d, "ElectricLoad", Dict()) - et = get(d, "ElectricTariff", Dict()) - eutil = get(d, "ElectricUtility", Dict()) - - # ---- Resolve PV production factor series ---- - # If not user-provided, derive via REopt.jl PVWatts and cache into `d` - # so both the sizing pre-step and downstream main REopt run reuse it - # without a second PVWatts call. - pv_prod_factor = if haskey(pv, "production_factor_series") && !isempty(pv["production_factor_series"]) - # Broadcast-convert: handles Vector{Any} from JSON parsing - # (convert(Vector{Float64}, ::Vector{Any}) would throw). - Float64.(pv["production_factor_series"]) + per_iter_timeout_s = 30.0 + + # TODO: Should MPC handle multiple PVs? + pv = get!(d, "PV", Dict()) + electric_storage = get!(d, "ElectricStorage", Dict()) + electric_load = get!(d, "ElectricLoad", Dict()) + electric_tariff = get(d, "ElectricTariff", Dict()) + electric_utility = get(d, "ElectricUtility", Dict()) + + # Read timeseries PV production factors or generate using PVWatts, save generated values to reduce PVWatts calls + if haskey(pv, "production_factor_series") && !isempty(pv["production_factor_series"]) + pv_prod_factor = Float64.(pv["production_factor_series"]) else - series = _mpc_generate_pv_production_factor(d, time_steps_per_hour) - pv["production_factor_series"] = series - series + # TODO: These production factors don't consider degradation, problem? + # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? + pv_prod_factor = generate_pv_production_factors(d, time_steps_per_hour) + pv["production_factor_series"] = pv_prod_factor end - # ---- Resolve electric load series ---- - # Mirror the PV pattern: if user-provided, use it; else derive via - # REopt.jl ElectricLoad (DOE CRB path) and cache into `d`. - load_series = if haskey(eload, "loads_kw") && !isempty(eload["loads_kw"]) - Float64.(eload["loads_kw"]) + # Read timeseries electric load inputs or generate using CRBs + if haskey(electric_load, "loads_kw") && !isempty(electric_load["loads_kw"]) + loads_kw = Float64.(electric_load["loads_kw"]) else - series = _mpc_generate_loads_kw(d, time_steps_per_hour) - eload["loads_kw"] = series - series + loads_kw = generate_loads_kw(d, time_steps_per_hour) + electric_load["loads_kw"] = loads_kw end - # ---- Length checks (8760 or 8784 * tsh) ---- - # No mutation of user-supplied series. A leap-year-length input - # (8784*tsh) is accepted as-is; `_mpc_slice_with_wrap` reads the extra - # day from it for year-end look-ahead instead of wrapping to Jan 1. - pv_prod_factor = _mpc_check_series_length("PV.production_factor_series", - pv_prod_factor, time_steps_per_hour, length_of_data) - load_series = _mpc_check_series_length("ElectricLoad.loads_kw", - load_series, time_steps_per_hour, length_of_data) - - # ---- Resolve PV and ElectricStorage sizes ---- - # Runs AFTER PV/load resolution so the (possibly internal) sizing - # REopt run reuses the cached series instead of re-deriving them. - # Either pulls sizes from min_kw==max_kw / min_kwh==max_kwh, or runs - # a sizing REopt and locks the resolved values into `d` so the - # downstream main REopt run uses the SAME sizes. - sizes = _mpc_resolve_sizes!(d) - pv_kw = sizes.pv_kw - bess_kw = sizes.bess_kw - bess_kwh = sizes.bess_kwh - - soc_init = Float64(get(bess, "soc_init_fraction", 0.5)) - soc_min = Float64(get(bess, "soc_min_fraction", 0.2)) - # REopt ElectricStorage exposes three component efficiencies; MPC's - # MPCElectricStorage uses combined charge/discharge round-trip halves. - # Match REopt's own composition: charge = rectifier * sqrt(internal), - # discharge = inverter * sqrt(internal). Defaults from REopt.jl. - rect_eff = Float64(get(bess, "rectifier_efficiency_fraction", 0.96)) - inv_eff = Float64(get(bess, "inverter_efficiency_fraction", 0.96)) - int_eff = Float64(get(bess, "internal_efficiency_fraction", 0.975)) + # Check data series lengths + # TODO: Do API inputs validation when calling the MPC endpoint directly? + # Should we even have an MPC endpoint? + pv_prod_factor = check_series_length("PV.production_factor_series", pv_prod_factor, length_of_data) + loads_kw = check_series_length("ElectricLoad.loads_kw", loads_kw, length_of_data) + + # MPC requires fixed PV and battery sizes. If not provided, call REopt first in a sizing run. + technology_sizes = get_technology_sizes!(d) + pv_kw = technology_sizes.pv_kw + batt_kw = technology_sizes.batt_kw + batt_kwh = technology_sizes.batt_kwh + + # TODO: Should these defaults be read from somewhere so that they don't have to be updated in various places? + soc_0 = Float64(get(electric_storage, "soc_init_fraction", 0.5)) + soc_min = Float64(get(electric_storage, "soc_min_fraction", 0.2)) + rect_eff = Float64(get(electric_storage, "rectifier_efficiency_fraction", 0.96)) + inv_eff = Float64(get(electric_storage, "inverter_efficiency_fraction", 0.96)) + int_eff = Float64(get(electric_storage, "internal_efficiency_fraction", 0.975)) charge_eff = rect_eff * sqrt(int_eff) discharge_eff = inv_eff * sqrt(int_eff) - # Year for URDB schedule decoding is sourced from ElectricLoad.year - # (the canonical year input in REopt; ElectricTariff's `year` kwarg is - # an internal pass-through, not a user input). - tariff_year = haskey(eload, "year") ? Int(eload["year"]) : nothing - tariff = _mpc_build_tariff_arrays(et, tariff_year, time_steps_per_hour, length_of_data) - energy_rates = tariff.energy_rates - tou_demand_rates = tariff.tou_demand_rates # one per ratchet, $/kW - tou_demand_ts_all = tariff.tou_demand_time_steps # full-year indices per ratchet - monthly_demand_rates = tariff.monthly_demand_rates # length 12, $/kW per month - - # Optional emissions series — broadcast zero if not provided. - emissions = haskey(eutil, "emissions_factor_series_lb_CO2_per_kwh") ? - Float64.(eutil["emissions_factor_series_lb_CO2_per_kwh"]) : - zeros(Float64, length_of_data) - emissions = _mpc_check_series_length("ElectricUtility.emissions_factor_series_lb_CO2_per_kwh", - emissions, time_steps_per_hour, length_of_data) - - # ---- Demand tracking state ---- - # Reverse indices into the calendar: each timestep maps to its - # 1-based month (always defined) and 1-based TOU ratchet (0 if no - # ratchet applies). Both are vector lookups (O(1)) used inside the - # rolling-horizon loop to build window-local time-step groupings - # and to attribute realized peaks to the right month/tier. + # Process utility rate + year = haskey(electric_load, "year") ? Int(electric_load["year"]) : nothing + tariff = get_tariff_inputs(electric_tariff, year, time_steps_per_hour) + energy_rates = tariff.energy_rates + tou_demand_rates = tariff.tou_demand_rates + tou_demand_ratchet_time_steps = tariff.tou_demand_ratchet_time_steps + monthly_demand_rates = tariff.monthly_demand_rates + + # TODO: MPC loop cherry picked one specific emissions type - remove or keep and add others? + # TODO: If keep, add function to pull defaults? Currently only allowing user upload + emissions = haskey(electric_utility, "emissions_factor_series_lb_CO2_per_kwh") ? + Float64.(electric_utility["emissions_factor_series_lb_CO2_per_kwh"]) : zeros(Float64, length_of_data) + emissions = check_series_length("ElectricUtility.emissions_factor_series_lb_CO2_per_kwh", emissions, length_of_data) + month_starts = get_month_transition_timesteps(time_steps_per_hour) + + # ts_to_month = 8760 array specifying which month each timestep falls in (1-12) ts_to_month = Vector{Int}(undef, length_of_data) for m in 1:12 s = month_starts[m] @@ -435,28 +331,21 @@ function get_mpc_results(d::Dict)::Dict ts_to_month[s:e] .= m end - ts_to_tier = zeros(Int, length_of_data) - for (t, ratchet_ts) in enumerate(tou_demand_ts_all), g in ratchet_ts + # ts_to_ratchet = 8760 array specifying which ratchet each timestep falls in + ts_to_ratchet = zeros(Int, length_of_data) + for (t, ratchet_ts) in enumerate(tou_demand_ratchet_time_steps), g in ratchet_ts if 1 <= g <= length_of_data - ts_to_tier[g] = t + ts_to_ratchet[g] = t end end - n_tou = length(tou_demand_rates) - # `monthly_previous_peak_demands` and `tou_previous_peak_demands` are - # passed into every MPC post AND mutated after each solve so that - # subsequent windows "see" the realized peaks so far. Both reset at - # month boundaries (matches REopt billing semantics; multi-month - # ratchet/lookback support would require a different reset cadence). - tou_previous_peak_demands = zeros(Float64, n_tou) - monthly_previous_peak_demands = zeros(Float64, 12) - monthly_demand_cost_total = 0.0 - tou_demand_cost_total = 0.0 - tou_peaks_by_month = Vector{Vector{Float64}}() - monthly_peaks = Float64[] - - # ---- Dispatch accumulators (1 value per executed timestep) ---- - dispatch = Dict( + # TODO: Monthly demand has never been tested + n_tou_ratchets = length(tou_demand_rates) # Number of TOU ratchets + tou_previous_peak_demands = zeros(Float64, n_tou_ratchets) # Tracks past TOU peak demand per ratchet + monthly_previous_peak_demands = zeros(Float64, 12) # Tracks past monthly peak demand + + # Saved dispatch series (first timestep of each MPC loop) + dispatch_series = Dict( "PV" => Dict( "electric_to_load_series_kw" => Float64[], "electric_to_storage_series_kw" => Float64[], @@ -476,178 +365,151 @@ function get_mpc_results(d::Dict)::Dict "load_series_kw" => Float64[], ), ) - cost_series = Float64[] + energy_cost_series = Float64[] total_energy_cost = 0.0 - bess_soc_init_fraction = soc_init - - # ---- Build the (reused) MPC post template ---- - # Note: no Settings block here; the solver is constructed below per - # window and passed directly to run_mpc(model, post). All four demand - # inputs (TOU + monthly rates and previous-peak vectors) are passed - # so the optimizer's objective accounts for *both* charges; omitting - # `monthly_*` would silently optimize without the monthly demand - # contribution and over-state savings under monthly-demand tariffs. - function build_mpc_post(window_pv, window_load, window_rates, window_emissions, - window_tou_ts, window_monthly_ts, - tou_prev_peaks, monthly_prev_peaks, soc_init_frac) + soc_init_frac = soc_0 + + # Build MPC post + function build_mpc_post(current_horizon_pv, current_horizon_load, current_horizon_energy_rates, + current_horizon_emissions, current_horizon_tou_ts, current_horizon_monthly_ts, + tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac) return Dict( "PV" => Dict( "size_kw" => pv_kw, - "production_factor_series" => window_pv, + "production_factor_series" => current_horizon_pv, ), "ElectricStorage" => Dict( - "size_kw" => bess_kw, - "size_kwh" => bess_kwh, + "size_kw" => batt_kw, + "size_kwh" => batt_kwh, "charge_efficiency" => charge_eff, "discharge_efficiency" => discharge_eff, "soc_init_fraction" => soc_init_frac, "soc_min_fraction" => soc_min, ), "ElectricLoad" => Dict( - "loads_kw" => window_load, + "loads_kw" => current_horizon_load, ), "ElectricTariff" => Dict( - "energy_rates" => window_rates, + "energy_rates" => current_horizon_energy_rates, "tou_demand_rates" => tou_demand_rates, - "tou_demand_time_steps" => window_tou_ts, - "tou_previous_peak_demands" => tou_prev_peaks, + "tou_demand_ratchet_time_steps" => current_horizon_tou_ts, + "tou_previous_peak_demands" => tou_previous_peak_demands, "monthly_demand_rates" => monthly_demand_rates, - "time_steps_monthly" => window_monthly_ts, - "monthly_previous_peak_demands" => monthly_prev_peaks, + "time_steps_monthly" => current_horizon_monthly_ts, + "monthly_previous_peak_demands" => monthly_previous_peak_demands, ), "ElectricUtility" => Dict( - "emissions_factor_series_lb_CO2_per_kwh" => window_emissions, + "emissions_factor_series_lb_CO2_per_kwh" => current_horizon_emissions, ), ) end - @info "MPC: starting rolling-horizon loop ($(length_of_data) iterations, horizon=$(horizon))" + @info "MPC: starting rolling-horizon optimization ($(length_of_data) iterations, horizon = $(horizon) timesteps)" for idx in 1:length_of_data end_ts = idx + horizon - 1 - window_len = end_ts - idx + 1 - - window_pv = _mpc_slice_with_wrap(pv_prod_factor, idx, end_ts) - window_load = _mpc_slice_with_wrap(load_series, idx, end_ts) - window_rates = _mpc_slice_with_wrap(energy_rates, idx, end_ts) - window_emiss = _mpc_slice_with_wrap(emissions, idx, end_ts) - - # Window-local time-step groupings, both built from a single pass - # over the horizon. TOU buckets are 1 vector per ratchet (length - # n_tou); monthly buckets are 1 vector per month (length 12). - # Global indices wrap on `length_of_data` (8760·tsh) so the month - # / tier of a wrapped step matches its calendar position. - window_tou_ts = [Int[] for _ in 1:n_tou] - window_monthly_ts = [Int[] for _ in 1:12] - for k in 1:window_len + + current_horizon_pv = slice_data(pv_prod_factor, idx, end_ts) + current_horizon_load = slice_data(loads_kw, idx, end_ts) + current_horizon_energy_rates = slice_data(energy_rates, idx, end_ts) + current_horizon_emissions = slice_data(emissions, idx, end_ts) + + # List of length n_tou_ratchets, specifies which ts of the current horizon are in each TOU ratchet + # by placing values 1 to horizon into the corresponding element of the array based on ratchet number + current_horizon_tou_ts = [Int[] for _ in 1:n_tou_ratchets] + + # 12 element list, each element for one month of the year. Specifies which timesteps of the current horizon are + # in each month by placing values 1 - horizon into the corresponding element of the array based on month number + current_horizon_monthly_ts = [Int[] for _ in 1:12] + for k in 1:horizon g = idx + k - 1 if g > length_of_data g -= length_of_data end - push!(window_monthly_ts[ts_to_month[g]], k) - tier = ts_to_tier[g] - if tier > 0 - push!(window_tou_ts[tier], k) + push!(current_horizon_monthly_ts[ts_to_month[g]], k) + ratchet = ts_to_ratchet[g] + if ratchet > 0 + push!(current_horizon_tou_ts[ratchet], k) end end - post = build_mpc_post(window_pv, window_load, window_rates, window_emiss, - window_tou_ts, window_monthly_ts, - tou_previous_peak_demands, monthly_previous_peak_demands, - bess_soc_init_fraction) + post = build_mpc_post(current_horizon_pv, current_horizon_load, current_horizon_energy_rates, + current_horizon_emissions, current_horizon_tou_ts, current_horizon_monthly_ts, + tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac) model = get_solver_model(get_solver_model_type(solver_name), SolverAttributes(per_iter_timeout_s, optimality_tolerance)) result = reoptjl.run_mpc(model, post) - # ---- Extract first-timestep dispatch (perfect-forecast) ---- + # Assume perfect forecast; save first timestep of results as the executed state pv_res = result["PV"] - bess_res = result["ElectricStorage"] + batt_res = result["ElectricStorage"] util_res = result["ElectricUtility"] pv_to_load = pv_res["electric_to_load_series_kw"][1] - pv_to_bess = pv_res["electric_to_storage_series_kw"][1] + pv_to_batt = pv_res["electric_to_storage_series_kw"][1] pv_to_grid = haskey(pv_res, "electric_to_grid_series_kw") ? pv_res["electric_to_grid_series_kw"][1] : 0.0 pv_curtailed = haskey(pv_res, "electric_curtailed_series_kw") ? pv_res["electric_curtailed_series_kw"][1] : 0.0 - bess_to_load = bess_res["storage_to_load_series_kw"][1] - bess_soc = bess_res["soc_series_fraction"][1] + batt_to_load = batt_res["storage_to_load_series_kw"][1] + batt_soc = batt_res["soc_series_fraction"][1] util_to_load = util_res["electric_to_load_series_kw"][1] - util_to_bess = util_res["electric_to_storage_series_kw"][1] - grid_power = max(util_to_load + util_to_bess, 0.0) - - push!(dispatch["PV"]["electric_to_load_series_kw"], pv_to_load) - push!(dispatch["PV"]["electric_to_storage_series_kw"], pv_to_bess) - push!(dispatch["PV"]["electric_to_grid_series_kw"], pv_to_grid) - push!(dispatch["PV"]["electric_curtailed_series_kw"], pv_curtailed) - push!(dispatch["ElectricStorage"]["storage_to_load_series_kw"], bess_to_load) - push!(dispatch["ElectricStorage"]["soc_series_fraction"], round(bess_soc, digits=6)) - push!(dispatch["ElectricUtility"]["electric_to_load_series_kw"], util_to_load) - push!(dispatch["ElectricUtility"]["electric_to_storage_series_kw"], util_to_bess) - push!(dispatch["ElectricUtility"]["emissions_series_lb_CO2"], + util_to_batt = util_res["electric_to_storage_series_kw"][1] + grid_power = max(util_to_load + util_to_batt, 0.0) + + push!(dispatch_series["PV"]["electric_to_load_series_kw"], pv_to_load) + push!(dispatch_series["PV"]["electric_to_storage_series_kw"], pv_to_batt) + push!(dispatch_series["PV"]["electric_to_grid_series_kw"], pv_to_grid) + push!(dispatch_series["PV"]["electric_curtailed_series_kw"], pv_curtailed) + push!(dispatch_series["ElectricStorage"]["storage_to_load_series_kw"], batt_to_load) + push!(dispatch_series["ElectricStorage"]["soc_series_fraction"], batt_soc) + push!(dispatch_series["ElectricUtility"]["electric_to_load_series_kw"], util_to_load) + push!(dispatch_series["ElectricUtility"]["electric_to_storage_series_kw"], util_to_batt) + push!(dispatch_series["ElectricUtility"]["emissions_series_lb_CO2"], emissions[idx] * grid_power / time_steps_per_hour) - push!(dispatch["ElectricLoad"]["load_series_kw"], load_series[idx]) + push!(dispatch_series["ElectricLoad"]["load_series_kw"], loads_kw[idx]) - # ---- Energy cost ---- + # Running energy costs step_energy_cost = grid_power * energy_rates[idx] / time_steps_per_hour - push!(cost_series, step_energy_cost) + push!(energy_cost_series, step_energy_cost) total_energy_cost += step_energy_cost - # ---- Carry state ---- - bess_soc_init_fraction = bess_soc - - # ---- Demand peak tracking (per-tier and per-month) ---- - # Attribute realized peak grid power to its TOU ratchet (if any) - # and to its calendar month. Identity comes from the reverse - # indices, NOT from rate values — so coincident-rate ratchets - # and equal-rate months remain separate billing pools. - m_now = ts_to_month[idx] - monthly_previous_peak_demands[m_now] = - max(grid_power, monthly_previous_peak_demands[m_now]) - - if n_tou > 0 - tier_now = ts_to_tier[idx] - if tier_now > 0 - tou_previous_peak_demands[tier_now] = - max(grid_power, tou_previous_peak_demands[tier_now]) - end - end + soc_init_frac = batt_soc + + # Update monthly and TOU peak demand as max(current ts grid_power, previous max) + current_month = ts_to_month[idx] + monthly_previous_peak_demands[current_month] = max(grid_power, monthly_previous_peak_demands[current_month]) - # ---- Month transition: close out the just-ended month ---- - # Both monthly and TOU peaks reset at month boundaries (matches - # REopt billing semantics). The last timestep of month m is - # detected by m_now != ts_to_month[idx+1]; we handle Dec via - # idx == length_of_data. - is_month_end = (idx == length_of_data) || (ts_to_month[idx + 1] != m_now) - if is_month_end - push!(monthly_peaks, monthly_previous_peak_demands[m_now]) - monthly_demand_cost_total += - monthly_previous_peak_demands[m_now] * monthly_demand_rates[m_now] - monthly_previous_peak_demands[m_now] = 0.0 - - if n_tou > 0 - push!(tou_peaks_by_month, copy(tou_previous_peak_demands)) - tou_demand_cost_total += sum(tou_previous_peak_demands .* tou_demand_rates) - fill!(tou_previous_peak_demands, 0.0) + if n_tou_ratchets > 0 + current_ratchet = ts_to_ratchet[idx] + if current_ratchet > 0 + tou_previous_peak_demands[current_ratchet] = max(grid_power, tou_previous_peak_demands[current_ratchet]) end end + end - @info "MPC: loop complete. total_energy_cost=\$$(round(total_energy_cost, digits=2))" + @info "MPC looping completed." + + # Calculate final demand costs + monthly_demand_cost_total = sum(monthly_previous_peak_demands .* monthly_demand_rates) + tou_demand_cost_total = n_tou_ratchets > 0 ? + sum(tou_previous_peak_demands .* tou_demand_rates) : 0.0 return Dict( "MPC" => Dict( "time_steps_per_hour" => time_steps_per_hour, "horizon" => horizon, ), - "PV" => Dict("size_kw" => pv_kw), - "ElectricStorage" => Dict("size_kw" => bess_kw, "size_kwh" => bess_kwh), - "dispatch" => dispatch, + "PV" => merge(Dict("size_kw" => pv_kw), dispatch_series["PV"]), + "ElectricStorage" => merge(Dict("size_kw" => batt_kw, "size_kwh" => batt_kwh), dispatch_series["ElectricStorage"]), + "ElectricUtility" => dispatch_series["ElectricUtility"], + "ElectricLoad" => dispatch_series["ElectricLoad"], "ElectricTariff" => Dict( "total_energy_cost" => total_energy_cost, - "energy_cost_series_per_timestep" => cost_series, + "energy_cost_series_per_timestep" => energy_cost_series, "total_tou_demand_cost" => tou_demand_cost_total, "total_monthly_demand_cost" => monthly_demand_cost_total, - "tou_peaks_by_month_kw" => tou_peaks_by_month, - "monthly_peaks_kw" => monthly_peaks, + "tou_peaks_by_ratchet_kw" => tou_previous_peak_demands, + "monthly_peaks_kw" => monthly_previous_peak_demands, ), "status" => "optimal", "reopt_version" => string(pkgversion(reoptjl)), From 3e695b30806950c7c7f7cdb7859e9d1b7b78da29 Mon Sep 17 00:00:00 2001 From: lixiangk1 <72464565+lixiangk1@users.noreply.github.com> Date: Thu, 11 Jun 2026 20:29:54 -0600 Subject: [PATCH 33/66] Update MPC variable names --- julia_src/http.jl | 9 +++++---- julia_src/mpc.jl | 7 +++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index b03a742f8..3f83c18d4 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -74,18 +74,19 @@ function reopt(req::HTTP.Request) Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" end + @info "HERE1" # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- - # When ElectricStorage.dispatch_options == "daily_foresight_optimized", first run the MPC rolling-horizon loop to get an SOC profile. + # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", first run the MPC rolling-horizon loop to get an SOC profile. # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. # Runs after the Xpress availability check so the resolved solver_name is passed in. electric_storage = get(d, "ElectricStorage", Dict()) - if get(electric_storage, "dispatch_options", nothing) == "daily_foresight_optimized" + if get(electric_storage, "dispatch_strategy", nothing) == "daily_foresight_optimized" try @info "Running MPC to obtain daily foresight optimized battery dispatch profile." mpc_results = get_mpc_results(d; solver_name=solver_name) - soc = mpc_results["dispatch"]["ElectricStorage"]["soc_series_fraction"] + soc = mpc_results["ElectricStorage"]["soc_series_fraction"] d["ElectricStorage"]["fixed_soc_series_fraction"] = soc - d["ElectricStorage"]["dispatch_options"] = "custom" + d["ElectricStorage"]["dispatch_strategy"] = "custom_soc" catch e @error "MPC pre-solve failed" exception=(e, catch_backtrace()) return HTTP.Response(500, JSON.json(Dict( diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index af376cdbe..f9916b955 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -193,7 +193,7 @@ function get_technology_sizes!(d::Dict) # Delete inputs specific to the heuristic battery dispatch run if haskey(sizing_post, "ElectricStorage") - delete!(sizing_post["ElectricStorage"], "dispatch_options") + delete!(sizing_post["ElectricStorage"], "dispatch_strategy") delete!(sizing_post["ElectricStorage"], "fixed_soc_series_fraction") end @@ -219,6 +219,7 @@ function get_technology_sizes!(d::Dict) batt_kw = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kw", 0.0)) batt_kwh = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kwh", 0.0)) + # TODO: If no battery is sized, just return the optimal REopt result if batt_kw <= 0.0 || batt_kwh <= 0.0 error("MPC: Optimal REopt sizing does not include ElectricStorage (size_kw=$(batt_kw), size_kwh=$(batt_kwh)). " * "The daily_foresight_optimized dispatch option requires a non-zero battery size. " * @@ -283,7 +284,9 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict loads_kw = Float64.(electric_load["loads_kw"]) else loads_kw = generate_loads_kw(d, time_steps_per_hour) - electric_load["loads_kw"] = loads_kw + + # Previously caching to save an extra CRB call but loads_kw conflicts with other load inputs + # electric_load["loads_kw"] = loads_kw end # Check data series lengths From 40d32dba8e85ba60200d162785ae390bdc829aa2 Mon Sep 17 00:00:00 2001 From: lixiangk1 <72464565+lixiangk1@users.noreply.github.com> Date: Thu, 11 Jun 2026 20:51:24 -0600 Subject: [PATCH 34/66] Remove print statement --- julia_src/http.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index 3f83c18d4..e13d04679 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -74,7 +74,6 @@ function reopt(req::HTTP.Request) Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" end - @info "HERE1" # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", first run the MPC rolling-horizon loop to get an SOC profile. # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. From dd45a05b17379f9b69efdf3faf25937096071ed4 Mon Sep 17 00:00:00 2001 From: lixiangk1 <72464565+lixiangk1@users.noreply.github.com> Date: Sat, 13 Jun 2026 13:55:20 -0600 Subject: [PATCH 35/66] Skip MPC dispatch if battery is not optimally sized --- julia_src/http.jl | 16 +++++++++++----- julia_src/mpc.jl | 49 +++++++++++++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index e13d04679..ee5b99cc4 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -76,16 +76,22 @@ function reopt(req::HTTP.Request) # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", first run the MPC rolling-horizon loop to get an SOC profile. - # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. - # Runs after the Xpress availability check so the resolved solver_name is passed in. + # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. If the user does not + # specify a fixed PV/battery size, optimally size the technologies using REopt (skip MPC dispatch if optimal battery size is 0). electric_storage = get(d, "ElectricStorage", Dict()) if get(electric_storage, "dispatch_strategy", nothing) == "daily_foresight_optimized" try @info "Running MPC to obtain daily foresight optimized battery dispatch profile." mpc_results = get_mpc_results(d; solver_name=solver_name) - soc = mpc_results["ElectricStorage"]["soc_series_fraction"] - d["ElectricStorage"]["fixed_soc_series_fraction"] = soc - d["ElectricStorage"]["dispatch_strategy"] = "custom_soc" + # TODO: Cache sizing run results and avoid a second call to REopt? Are those the same results? + if get(mpc_results, "skip_mpc", false) == true + @info "Cannot execute daily_foresight_optimized battery dispatch: optimal battery size is 0." + delete!(d["ElectricStorage"], "dispatch_strategy") + else + soc = mpc_results["ElectricStorage"]["soc_series_fraction"] + d["ElectricStorage"]["fixed_soc_series_fraction"] = soc + d["ElectricStorage"]["dispatch_strategy"] = "custom_soc" + end catch e @error "MPC pre-solve failed" exception=(e, catch_backtrace()) return HTTP.Response(500, JSON.json(Dict( diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index f9916b955..3798ee226 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -185,7 +185,7 @@ function get_technology_sizes!(d::Dict) batt_kw = Float64(batt["min_kw"]) batt_kwh = Float64(batt["min_kwh"]) @info "MPC: fixed technology sizes entered, PV = $(pv_kw) kW, battery = $(batt_kw) kW / $(batt_kwh) kWh." - return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh) + return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false) end @info "MPC: PV and/or ElectricStorage sizes are not specified — running REopt sizing first." @@ -197,34 +197,45 @@ function get_technology_sizes!(d::Dict) delete!(sizing_post["ElectricStorage"], "fixed_soc_series_fraction") end - settings = get!(sizing_post, "Settings", Dict()) - settings["run_bau"] = false + # TODO: Avoid redefining defaults here? + settings = get(sizing_post, "Settings", Dict()) + sizing_solver_name = get(settings, "solver_name", "HiGHS") + sizing_timeout = Float64(get(settings, "timeout_seconds", 420)) + sizing_opt_tol = Float64(get(settings, "optimality_tolerance", 0.001)) - # TODO: Okay to hard code defaults here? solver_name check is redundant if function is only called from get_mpc_results - settings["solver_name"] = get(settings, "solver_name", "HiGHS") - settings["timeout_seconds"] = get(settings, "timeout_seconds", 420) - settings["optimality_tolerance"] = get(settings, "optimality_tolerance", 0.001) + solver_attributes = SolverAttributes(sizing_timeout, sizing_opt_tol) + m = get_solver_model(get_solver_model_type(sizing_solver_name), solver_attributes) - solver_attributes = SolverAttributes(settings["timeout_seconds"], settings["optimality_tolerance"]) - m = get_solver_model(get_solver_model_type(settings["solver_name"]), solver_attributes) + # Delete Settings inputs specific to the API + api_only_settings_keys = ("timeout_seconds", "optimality_tolerance", "run_bau") + if haskey(sizing_post, "Settings") + for k in api_only_settings_keys + delete!(sizing_post["Settings"], k) + end + if isempty(sizing_post["Settings"]) + delete!(sizing_post, "Settings") + end + end model_inputs = reoptjl.REoptInputs(sizing_post) sizing_results = reoptjl.run_reopt(m, model_inputs) if get(sizing_results, "status", "") != "optimal" - error("MPC sizing pre-step did not solve (status = $(get(sizing_results, "status", "unknown"))).") + status = get(sizing_results, "status", "unknown") + msgs = get(sizing_results, "Messages", Dict()) + errs = get(msgs, "errors", []) + warns = get(msgs, "warnings", []) + error("MPC sizing pre-step did not solve (status = $(status)). " * + "REopt errors: $(errs). REopt warnings: $(warns).") end pv_kw = Float64(get(get(sizing_results, "PV", Dict()), "size_kw", 0.0)) batt_kw = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kw", 0.0)) batt_kwh = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kwh", 0.0)) - # TODO: If no battery is sized, just return the optimal REopt result + # Skip the MPC loop if no battery is sized if batt_kw <= 0.0 || batt_kwh <= 0.0 - error("MPC: Optimal REopt sizing does not include ElectricStorage (size_kw=$(batt_kw), size_kwh=$(batt_kwh)). " * - "The daily_foresight_optimized dispatch option requires a non-zero battery size. " * - "Set ElectricStorage.min_kw == ElectricStorage.max_kw and " * - "ElectricStorage.min_kwh == ElectricStorage.max_kwh to fix sizes manually.") + return (pv_kw = pv_kw, batt_kw = 0.0, batt_kwh = 0.0, skip_mpc = true) end pv["min_kw"] = pv_kw @@ -235,7 +246,7 @@ function get_technology_sizes!(d::Dict) batt["max_kwh"] = batt_kwh @info "MPC: REopt sizing solved with PV = $(pv_kw) kW and battery = $(batt_kw) kW / $(batt_kwh) kWh." - return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh) + return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false) end function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict @@ -297,6 +308,12 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict # MPC requires fixed PV and battery sizes. If not provided, call REopt first in a sizing run. technology_sizes = get_technology_sizes!(d) + + # Skip MPC if no battery is optimally sized + if technology_sizes.skip_mpc + return Dict("skip_mpc" => true) + end + pv_kw = technology_sizes.pv_kw batt_kw = technology_sizes.batt_kw batt_kwh = technology_sizes.batt_kwh From 3d259af522e92e2dcb3f8ec500c59052db2355f6 Mon Sep 17 00:00:00 2001 From: adfarth Date: Wed, 24 Jun 2026 09:40:48 -0600 Subject: [PATCH 36/66] Update Manifest.toml --- julia_src/Manifest.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/julia_src/Manifest.toml b/julia_src/Manifest.toml index dc825f89c..47ac83bda 100644 --- a/julia_src/Manifest.toml +++ b/julia_src/Manifest.toml @@ -948,7 +948,9 @@ version = "1.11.0" [[deps.REopt]] deps = ["ArchGDAL", "CSV", "CoolProp", "DataFrames", "Dates", "DelimitedFiles", "HTTP", "JLD", "JSON", "JuMP", "LinDistFlow", "LinearAlgebra", "Logging", "MathOptInterface", "Requires", "Roots", "Statistics", "TestEnv"] -git-tree-sha1 = "e807b62ab249fca7b59ac6ab7bb71725f504cc5f" +git-tree-sha1 = "1e54ef8f58ddcf9d66f392cb76fdd78123d7d2ad" +repo-rev = "dispatch-options" +repo-url = "https://github.com/NatLabRockies/REopt.jl.git" uuid = "d36ad4e8-d74a-4f7a-ace1-eaea049febf6" version = "0.59.2" From 855f18902f9189bd998f5da9664f924c44207002 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 11:15:25 -0600 Subject: [PATCH 37/66] validation and help text --- julia_src/http.jl | 42 +++++++++++++++++++++++++++------------- julia_src/mpc.jl | 49 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index ee5b99cc4..04a4bfa5e 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -75,9 +75,9 @@ function reopt(req::HTTP.Request) end # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- - # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", first run the MPC rolling-horizon loop to get an SOC profile. - # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. If the user does not - # specify a fixed PV/battery size, optimally size the technologies using REopt (skip MPC dispatch if optimal battery size is 0). + # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", if needed, first run REopt to get optimal sizing of PV and battery. + # Then run the MPC rolling-horizon loop to get a SOC profile (skip MPC dispatch if optimal battery size is 0). + # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. electric_storage = get(d, "ElectricStorage", Dict()) if get(electric_storage, "dispatch_strategy", nothing) == "daily_foresight_optimized" try @@ -85,8 +85,8 @@ function reopt(req::HTTP.Request) mpc_results = get_mpc_results(d; solver_name=solver_name) # TODO: Cache sizing run results and avoid a second call to REopt? Are those the same results? if get(mpc_results, "skip_mpc", false) == true - @info "Cannot execute daily_foresight_optimized battery dispatch: optimal battery size is 0." - delete!(d["ElectricStorage"], "dispatch_strategy") + @info "Cannot execute daily_foresight_optimized battery dispatch because optimal battery size is 0. Setting dispatch strategy to 'optimized'." + d["ElectricStorage"]["dispatch_strategy"] = "optimized" else soc = mpc_results["ElectricStorage"]["soc_series_fraction"] d["ElectricStorage"]["fixed_soc_series_fraction"] = soc @@ -841,7 +841,29 @@ function job_no_xpress(req::HTTP.Request) return HTTP.Response(500, JSON.json(error_response)) end +""" + mpc(req::HTTP.Request) +HTTP endpoint for rolling-horizon Model Predictive Control (MPC) dispatch optimization. + +This endpoint performs a full-year rolling-horizon MPC dispatch for PV + ElectricStorage systems, +optimizing daily dispatch using a 24-hour look-ahead window. Runs the MPC dispatch loop via +`get_mpc_results` and returns dispatch results and cost metrics as JSON. + +Arguments: + req::HTTP.Request: REopt inputs dictionary + +Returns JSON dictionary containing: + - MPC: Metadata (time_steps_per_hour, horizon) + - PV: Size and dispatch series (to load, storage, grid, curtailed) + - ElectricStorage: Sizes and state-of-charge series + - ElectricUtility: Grid dispatch series and emissions + - ElectricLoad: Load profile used + - ElectricTariff: Energy and demand costs, peak demands by month/ratchet + - status: "optimal" + - reopt_version: Version of REopt.jl used + +""" function mpc(req::HTTP.Request) d = JSON.parse(String(req.body)) error_response = Dict() @@ -853,14 +875,8 @@ function mpc(req::HTTP.Request) ENV["NREL_DEVELOPER_API_KEY"] = test_nrel_developer_api_key delete!(d, "api_key") end - settings = get(d, "Settings", Dict()) - solver_name = get(settings, "solver_name", "HiGHS") - if solver_name == "Xpress" && !(xpress_installed=="True") - solver_name = "HiGHS" - @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. - Next time specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'." - end - results = get_mpc_results(d; solver_name=solver_name) + + results = get_mpc_results(d) catch e @error "MPC failed" exception=(e, catch_backtrace()) error_response["error"] = sprint(showerror, e) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 3798ee226..d1b0a12e4 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -249,19 +249,60 @@ function get_technology_sizes!(d::Dict) return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false) end -function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict +function get_mpc_results(d::Dict)::Dict """ Run a full-year rolling-horizon MPC dispatch for PV + ElectricStorage by calling `REopt.run_mpc` once per timestep with a 24-hour look-ahead. Inputs: d::Dict, REopt inputs dictionary - solver_name::String, solver to use ("HiGHS", "Cbc", "SCIP", or "Xpress") - Returns a Dict with PV and BATT sizes, dispatch time series, and cost metrics + Returns a Dict with PV and ElectricStorage sizes, dispatch time series, and cost metrics """ - + + ## Validation on allowable inputs for MPC ## + # Error if any techs other than PV and ElectricStorage are provided + mpc_allowed_keys = Set(["PV", "ElectricStorage", "ElectricLoad", "ElectricTariff", "ElectricUtility", "Site", "Settings", "Financial"]) + unsupported_keys = setdiff(keys(d), mpc_allowed_keys) + if !isempty(unsupported_keys) + error("When using MPC (daily_foresight_optimized dispatch), only PV and ElectricStorage are supported technologies. " * + "Unsupported inputs found: $(join(unsupported_keys, ", ")).") + end + + # Error if unsupported CO2/renewable-fraction constraints are set + _site_input = get(d, "Site", Dict()) + if !isnothing(get(_site_input, "CO2_emissions_reduction_min_fraction", nothing)) + error("MPC: Site.CO2_emissions_reduction_min_fraction is not supported in MPC runs.") + end + if get(_site_input, "include_grid_renewable_fraction_in_RE_constraints", false) == true + error("MPC: Site.include_grid_renewable_fraction_in_RE_constraints is not supported in MPC runs.") + end + if get(_site_input, "include_exported_elec_emissions_in_total", true) == false + error("MPC: Site.include_exported_elec_emissions_in_total = false is not supported in MPC runs.") + end + if get(_site_input, "include_exported_renewable_electricity_in_total", true) == false + error("MPC: Site.include_exported_renewable_electricity_in_total = false is not supported in MPC runs.") + end + + # TODO: Add warnings for REopt inputs and scenarios that are not modeled in MPC (e.g., coincident peak charges, demand lookback, etc.) + @warn "Using MPC to determine dispatch. MPC does not model: tiered electricity rates; rates will be flattened to the first tier." + + # TODO: Test with outage inputs before enabling this warning. + # # Warning for outage inputs (MPC does not model outages) + # _utility_input = get(d, "ElectricUtility", Dict()) + # if any(k -> haskey(_utility_input, k), ("outage_start_time_step", "outage_start_time_steps", "outage_durations")) + # @warn "MPC: Outage inputs detected (outage_start_time_step, outage_start_time_steps, outage_durations). " * + # "MPC does not model outages; these inputs will be ignored." + # end + + ## Set up MPC inputs ## settings = get!(d, "Settings", Dict()) + solver_name = get(settings, "solver_name", "HiGHS") + if solver_name == "Xpress" && !(xpress_installed=="True") + solver_name = "HiGHS" + @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. + Next time specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'." + end settings["solver_name"] = solver_name # TODO: MPC timeout and optimality tolerance From e25fadd875fd60eef108055911f468630b149d8d Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 11:16:54 -0600 Subject: [PATCH 38/66] rm solver_name from get_mpc_results inputs --- julia_src/http.jl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index 04a4bfa5e..617ef89b7 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -66,14 +66,6 @@ function reopt(req::HTTP.Request) delete!(d, "api_key") end - settings = d["Settings"] - solver_name = get(settings, "solver_name", "HiGHS") - if solver_name == "Xpress" && !(xpress_installed=="True") - solver_name = "HiGHS" - @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. Next time - Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" - end - # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", if needed, first run REopt to get optimal sizing of PV and battery. # Then run the MPC rolling-horizon loop to get a SOC profile (skip MPC dispatch if optimal battery size is 0). @@ -82,7 +74,7 @@ function reopt(req::HTTP.Request) if get(electric_storage, "dispatch_strategy", nothing) == "daily_foresight_optimized" try @info "Running MPC to obtain daily foresight optimized battery dispatch profile." - mpc_results = get_mpc_results(d; solver_name=solver_name) + mpc_results = get_mpc_results(d) # TODO: Cache sizing run results and avoid a second call to REopt? Are those the same results? if get(mpc_results, "skip_mpc", false) == true @info "Cannot execute daily_foresight_optimized battery dispatch because optimal battery size is 0. Setting dispatch strategy to 'optimized'." From 7d4cfaa7dd8d42664964d7c8c0b590d01f4a1ad9 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 16:37:25 -0600 Subject: [PATCH 39/66] condense input processing undo solver_name change --- julia_src/http.jl | 22 ++++- julia_src/mpc.jl | 239 +++++++++++++--------------------------------- 2 files changed, 87 insertions(+), 174 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index 617ef89b7..7d6c2bd40 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -66,6 +66,14 @@ function reopt(req::HTTP.Request) delete!(d, "api_key") end + settings = d["Settings"] + solver_name = get(settings, "solver_name", "HiGHS") + if solver_name == "Xpress" && !(xpress_installed=="True") + solver_name = "HiGHS" + @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. Next time + Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" + end + # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", if needed, first run REopt to get optimal sizing of PV and battery. # Then run the MPC rolling-horizon loop to get a SOC profile (skip MPC dispatch if optimal battery size is 0). @@ -74,7 +82,7 @@ function reopt(req::HTTP.Request) if get(electric_storage, "dispatch_strategy", nothing) == "daily_foresight_optimized" try @info "Running MPC to obtain daily foresight optimized battery dispatch profile." - mpc_results = get_mpc_results(d) + mpc_results = get_mpc_results(d; solver_name=solver_name) # TODO: Cache sizing run results and avoid a second call to REopt? Are those the same results? if get(mpc_results, "skip_mpc", false) == true @info "Cannot execute daily_foresight_optimized battery dispatch because optimal battery size is 0. Setting dispatch strategy to 'optimized'." @@ -846,7 +854,7 @@ Arguments: req::HTTP.Request: REopt inputs dictionary Returns JSON dictionary containing: - - MPC: Metadata (time_steps_per_hour, horizon) + - MPC: Metadata (time_steps_per_hour, horizon_time_steps) - PV: Size and dispatch series (to load, storage, grid, curtailed) - ElectricStorage: Sizes and state-of-charge series - ElectricUtility: Grid dispatch series and emissions @@ -868,7 +876,15 @@ function mpc(req::HTTP.Request) delete!(d, "api_key") end - results = get_mpc_results(d) + settings = d["Settings"] + solver_name = get(settings, "solver_name", "HiGHS") + if solver_name == "Xpress" && !(xpress_installed=="True") + solver_name = "HiGHS" + @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. Next time + Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" + end + + results = get_mpc_results(d; solver_name=solver_name) catch e @error "MPC failed" exception=(e, catch_backtrace()) error_response["error"] = sprint(showerror, e) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index d1b0a12e4..86b4952a2 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -43,61 +43,12 @@ function slice_data(arr::AbstractVector, idx::Int, end_idx::Int) end end -""" - check_series_length(name, series, time_steps_per_hour, length_of_data) - -Validate that a user-entered time series is of length 8760 * time_steps_per_hour (cannot be a leap year). -""" -function check_series_length(name::String, series::AbstractVector, length_of_data::Int) - n = length(series) - if n == length_of_data - return series - end - error("MPC: $name length $n != 8760 * time_steps_per_hour ($length_of_data).") -end - -""" - get_tariff_inputs(electric_tariff, year, time_steps_per_hour) - -Call REopt.ElectricTariff to get energy_rates, monthly_demand_rates, tou_demand_rates, tou_demand_ratchet_time_steps -MPC does not currently model tiers (tiers are flattened), coincident peak charges, or demand lookback. -""" -function get_tariff_inputs(electric_tariff::Dict, year::Union{Int,Nothing}, time_steps_per_hour::Int) - - # TODO: Are all of these relevant? Any missing inputs? - # TODO: Implement lookback? (demand_lookback_months, demand_lookback_percent, demand_lookback_range) Ignoring coincident peak charges for now - # TODO: Need to think through NEM or passing back export values (wholesale_rate, export_rate_beyond_net_metering_limit) - tariff_kwargs = (:urdb_label, :urdb_response, :urdb_utility_name, - :urdb_rate_name, :urdb_metadata, - :wholesale_rate, :export_rate_beyond_net_metering_limit, - :monthly_energy_rates, :monthly_demand_rates, - :blended_annual_energy_rate, :blended_annual_demand_rate, - :add_monthly_rates_to_urdb_rate, - :tou_energy_rates_per_kwh, :add_tou_energy_rates_to_urdb_rate, - :demand_lookback_months, :demand_lookback_percent, :demand_lookback_range) - kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in electric_tariff if Symbol(k) in tariff_kwargs) - - # MPC does not currently model tiered rates - kwargs[:remove_tiers] = true - tariff = reoptjl.ElectricTariff(; year = year, time_steps_per_hour = time_steps_per_hour, kwargs...) - - energy_rates = Float64.(tariff.energy_rates[:, 1]) - monthly_demand_rates = isempty(tariff.monthly_demand_rates) ? - zeros(Float64, 12) : Float64.(tariff.monthly_demand_rates[:, 1]) - tou_demand_rates = isempty(tariff.tou_demand_rates) ? Float64[] : Float64.(tariff.tou_demand_rates[:, 1]) - tou_demand_ratchet_time_steps = [Int.(v) for v in tariff.tou_demand_ratchet_time_steps] - - # TODO: Track lookback variables - demand_lookback_months, demand_lookback_percent, demand_lookback_range? - return (energy_rates = energy_rates, - monthly_demand_rates = monthly_demand_rates, - tou_demand_rates = tou_demand_rates, - tou_demand_ratchet_time_steps = tou_demand_ratchet_time_steps) -end """ generate_pv_production_factors(d, time_steps_per_hour) -Generate a PV production factor series using PVWatts by calling REopt.get_production_factor +Generate a PV production factor series using PVWatts by calling REopt.get_production_factor. +This is called by get_mpc_results only when the user does not provide a custom production_factor_series. """ function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) site = get(d, "Site", Dict()) @@ -122,45 +73,13 @@ function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) return Vector{Float64}(pv_production_factor_series) end -""" - generate_loads_kw(d, time_steps_per_hour) - -Generate an electric load profile using DOE Commercial Reference buildings by calling REopt.ElectricLoad -""" -function generate_loads_kw(d::Dict, time_steps_per_hour::Int) - site = get(d, "Site", Dict()) - if !haskey(site, "latitude") - error("MPC: Site.latitude is required to generate ElectricLoad.loads_kw using a DOE CRB profile.") - end - if !haskey(site, "longitude") - error("MPC: Site.longitude is required to generate ElectricLoad.loads_kw using a DOE CRB profile.") - end - lat = Float64(site["latitude"]) - lon = Float64(site["longitude"]) - - @info "MPC: ElectricLoad.loads_kw not provided; generating via REopt.jl ElectricLoad (lat=$(lat), lon=$(lon))." - - electric_load = get(d, "ElectricLoad", Dict()) - - # TODO: Double check this list for relevance/missing inputs - load_kwargs = (:normalize_and_scale_load_profile_input, - :path_to_csv, :doe_reference_name, - :blended_doe_reference_names, :blended_doe_reference_percents, - :year, :city, :annual_kwh, :monthly_totals_kwh, - :monthly_peaks_kw, :loads_kw_is_net) - kwargs = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in electric_load if Symbol(k) in load_kwargs) - load = reoptjl.ElectricLoad(; latitude = lat, longitude = lon, - time_steps_per_hour = time_steps_per_hour, kwargs...) - return Vector{Float64}(load.loads_kw) -end - """ get_technology_sizes!(d) Determine PV and ElectricStorage sizes for the MPC loop. If min_kw != max_kw and/or min_kwh != max_kwh, call REopt to size technologies. User input battery sizes must also be greater than zero. """ -function get_technology_sizes!(d::Dict) +function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") pv = get!(d, "PV", Dict()) batt = get!(d, "ElectricStorage", Dict()) @@ -176,6 +95,7 @@ function get_technology_sizes!(d::Dict) "to run the daily_foresight_optimized dispatch option.") end + # Check if both PV and BESS sizes fixed pv_fixed = is_fixed(pv, "min_kw", "max_kw") batt_fixed = is_fixed(batt, "min_kw", "max_kw") && is_fixed(batt, "min_kwh", "max_kwh") @@ -184,7 +104,6 @@ function get_technology_sizes!(d::Dict) pv_kw = Float64(pv["min_kw"]) batt_kw = Float64(batt["min_kw"]) batt_kwh = Float64(batt["min_kwh"]) - @info "MPC: fixed technology sizes entered, PV = $(pv_kw) kW, battery = $(batt_kw) kW / $(batt_kwh) kWh." return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false) end @@ -197,25 +116,15 @@ function get_technology_sizes!(d::Dict) delete!(sizing_post["ElectricStorage"], "fixed_soc_series_fraction") end - # TODO: Avoid redefining defaults here? + # TODO: Should we "remove tiers" here for sizing or allow for optimizing with tiers? + settings = get(sizing_post, "Settings", Dict()) - sizing_solver_name = get(settings, "solver_name", "HiGHS") - sizing_timeout = Float64(get(settings, "timeout_seconds", 420)) - sizing_opt_tol = Float64(get(settings, "optimality_tolerance", 0.001)) - - solver_attributes = SolverAttributes(sizing_timeout, sizing_opt_tol) - m = get_solver_model(get_solver_model_type(sizing_solver_name), solver_attributes) - - # Delete Settings inputs specific to the API - api_only_settings_keys = ("timeout_seconds", "optimality_tolerance", "run_bau") - if haskey(sizing_post, "Settings") - for k in api_only_settings_keys - delete!(sizing_post["Settings"], k) - end - if isempty(sizing_post["Settings"]) - delete!(sizing_post, "Settings") - end - end + delete!(settings, "run_bau") # Remove run_bau from sizing run + timeout_seconds = pop!(settings, "timeout_seconds", 420) + optimality_tolerance = pop!(settings, "optimality_tolerance", 0.001) + solver_attributes = SolverAttributes(timeout_seconds, optimality_tolerance) + + m = get_solver_model(get_solver_model_type(solver_name), solver_attributes) model_inputs = reoptjl.REoptInputs(sizing_post) sizing_results = reoptjl.run_reopt(m, model_inputs) @@ -249,15 +158,24 @@ function get_technology_sizes!(d::Dict) return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false) end -function get_mpc_results(d::Dict)::Dict +function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict """ Run a full-year rolling-horizon MPC dispatch for PV + ElectricStorage by calling `REopt.run_mpc` once per timestep with a 24-hour look-ahead. Inputs: - d::Dict, REopt inputs dictionary + d::Dict, REopt inputs dictionary + + Returns JSON dictionary containing: + - MPC: Metadata (time_steps_per_hour, horizon_time_steps) + - PV: Size and dispatch series (to load, storage, grid, curtailed) + - ElectricStorage: Sizes and state-of-charge series + - ElectricUtility: Grid dispatch series and emissions + - ElectricLoad: Load profile used + - ElectricTariff: Energy and demand costs, peak demands by month/ratchet + - status: "optimal" + - reopt_version: Version of REopt.jl used - Returns a Dict with PV and ElectricStorage sizes, dispatch time series, and cost metrics """ ## Validation on allowable inputs for MPC ## @@ -297,12 +215,6 @@ function get_mpc_results(d::Dict)::Dict ## Set up MPC inputs ## settings = get!(d, "Settings", Dict()) - solver_name = get(settings, "solver_name", "HiGHS") - if solver_name == "Xpress" && !(xpress_installed=="True") - solver_name = "HiGHS" - @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. - Next time specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'." - end settings["solver_name"] = solver_name # TODO: MPC timeout and optimality tolerance @@ -314,38 +226,10 @@ function get_mpc_results(d::Dict)::Dict horizon = 24 * time_steps_per_hour per_iter_timeout_s = 30.0 - # TODO: Should MPC handle multiple PVs? - pv = get!(d, "PV", Dict()) - electric_storage = get!(d, "ElectricStorage", Dict()) - electric_load = get!(d, "ElectricLoad", Dict()) - electric_tariff = get(d, "ElectricTariff", Dict()) - electric_utility = get(d, "ElectricUtility", Dict()) - - # Read timeseries PV production factors or generate using PVWatts, save generated values to reduce PVWatts calls - if haskey(pv, "production_factor_series") && !isempty(pv["production_factor_series"]) - pv_prod_factor = Float64.(pv["production_factor_series"]) - else - # TODO: These production factors don't consider degradation, problem? - # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? - pv_prod_factor = generate_pv_production_factors(d, time_steps_per_hour) - pv["production_factor_series"] = pv_prod_factor - end - - # Read timeseries electric load inputs or generate using CRBs - if haskey(electric_load, "loads_kw") && !isempty(electric_load["loads_kw"]) - loads_kw = Float64.(electric_load["loads_kw"]) - else - loads_kw = generate_loads_kw(d, time_steps_per_hour) - - # Previously caching to save an extra CRB call but loads_kw conflicts with other load inputs - # electric_load["loads_kw"] = loads_kw - end - - # Check data series lengths - # TODO: Do API inputs validation when calling the MPC endpoint directly? - # Should we even have an MPC endpoint? - pv_prod_factor = check_series_length("PV.production_factor_series", pv_prod_factor, length_of_data) - loads_kw = check_series_length("ElectricLoad.loads_kw", loads_kw, length_of_data) + # Call REoptInputs once upfront to process and validate inputs + # This handles: load profile generation, tariff processing (including tier removal), emissions defaults, storage efficiency defaults + i = reoptjl.REoptInputs(d) + s = i.s # Access the processed Scenario struct # MPC requires fixed PV and battery sizes. If not provided, call REopt first in a sizing run. technology_sizes = get_technology_sizes!(d) @@ -355,41 +239,54 @@ function get_mpc_results(d::Dict)::Dict return Dict("skip_mpc" => true) end - pv_kw = technology_sizes.pv_kw - batt_kw = technology_sizes.batt_kw - batt_kwh = technology_sizes.batt_kwh - - # TODO: Should these defaults be read from somewhere so that they don't have to be updated in various places? - soc_0 = Float64(get(electric_storage, "soc_init_fraction", 0.5)) - soc_min = Float64(get(electric_storage, "soc_min_fraction", 0.2)) - rect_eff = Float64(get(electric_storage, "rectifier_efficiency_fraction", 0.96)) - inv_eff = Float64(get(electric_storage, "inverter_efficiency_fraction", 0.96)) - int_eff = Float64(get(electric_storage, "internal_efficiency_fraction", 0.975)) + # Note: REoptInputs does not provide PV production factors if user doesn't specify custom values + if !isempty(s.pvs[1].production_factor_series) + pv_prod_factor = Float64.(s.pvs[1].production_factor_series) + else + # TODO: These production factors don't consider degradation, problem? + # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? + pv_prod_factor = generate_pv_production_factors(d, time_steps_per_hour) + end + + loads_kw = Float64.(s.electric_load.loads_kw) + + # Extract tariff inputs relevant to MPC (use first tier only if tiered rates) + # TODO: Are all of these relevant? Any missing inputs? + # TODO: Implement lookback? (demand_lookback_months, demand_lookback_percent, demand_lookback_range) Ignoring coincident peak charges for now + # TODO: Need to think through NEM or passing back export values (wholesale_rate, export_rate_beyond_net_metering_limit) + # TODO: Track lookback variables - demand_lookback_months, demand_lookback_percent, demand_lookback_range? + energy_rates = Float64.(s.electric_tariff.energy_rates[:, 1]) + monthly_demand_rates = isempty(s.electric_tariff.monthly_demand_rates) ? + zeros(Float64, 12) : Float64.(s.electric_tariff.monthly_demand_rates[:, 1]) + tou_demand_rates = isempty(s.electric_tariff.tou_demand_rates) ? Float64[] : Float64.(s.electric_tariff.tou_demand_rates[:, 1]) + tou_demand_ratchet_time_steps = [Int.(v) for v in s.electric_tariff.tou_demand_ratchet_time_steps] + + # Extract storage efficiency and SOC defaults from processed inputs + rect_eff = Float64(s.storage.attr["ElectricStorage"].rectifier_efficiency_fraction) + inv_eff = Float64(s.storage.attr["ElectricStorage"].inverter_efficiency_fraction) + int_eff = Float64(s.storage.attr["ElectricStorage"].internal_efficiency_fraction) charge_eff = rect_eff * sqrt(int_eff) discharge_eff = inv_eff * sqrt(int_eff) + soc_0 = Float64(s.storage.attr["ElectricStorage"].soc_init_fraction) + soc_min = Float64(s.storage.attr["ElectricStorage"].soc_min_fraction) + + # Extract emissions defaults (or use user input if provided) + co2_grid_emissions_series = Float64.(s.electric_utility.emissions_factor_series_lb_CO2_per_kwh) - # Process utility rate - year = haskey(electric_load, "year") ? Int(electric_load["year"]) : nothing - tariff = get_tariff_inputs(electric_tariff, year, time_steps_per_hour) - energy_rates = tariff.energy_rates - tou_demand_rates = tariff.tou_demand_rates - tou_demand_ratchet_time_steps = tariff.tou_demand_ratchet_time_steps - monthly_demand_rates = tariff.monthly_demand_rates - # TODO: MPC loop cherry picked one specific emissions type - remove or keep and add others? - # TODO: If keep, add function to pull defaults? Currently only allowing user upload - emissions = haskey(electric_utility, "emissions_factor_series_lb_CO2_per_kwh") ? - Float64.(electric_utility["emissions_factor_series_lb_CO2_per_kwh"]) : zeros(Float64, length_of_data) - emissions = check_series_length("ElectricUtility.emissions_factor_series_lb_CO2_per_kwh", emissions, length_of_data) + + pv_kw = technology_sizes.pv_kw + batt_kw = technology_sizes.batt_kw + batt_kwh = technology_sizes.batt_kwh month_starts = get_month_transition_timesteps(time_steps_per_hour) # ts_to_month = 8760 array specifying which month each timestep falls in (1-12) ts_to_month = Vector{Int}(undef, length_of_data) for m in 1:12 - s = month_starts[m] + s_idx = month_starts[m] e = m < 12 ? month_starts[m+1] - 1 : length_of_data - ts_to_month[s:e] .= m + ts_to_month[s_idx:e] .= m end # ts_to_ratchet = 8760 array specifying which ratchet each timestep falls in @@ -472,7 +369,7 @@ function get_mpc_results(d::Dict)::Dict current_horizon_pv = slice_data(pv_prod_factor, idx, end_ts) current_horizon_load = slice_data(loads_kw, idx, end_ts) current_horizon_energy_rates = slice_data(energy_rates, idx, end_ts) - current_horizon_emissions = slice_data(emissions, idx, end_ts) + current_horizon_emissions = slice_data(co2_grid_emissions_series, idx, end_ts) # List of length n_tou_ratchets, specifies which ts of the current horizon are in each TOU ratchet # by placing values 1 to horizon into the corresponding element of the array based on ratchet number @@ -525,7 +422,7 @@ function get_mpc_results(d::Dict)::Dict push!(dispatch_series["ElectricUtility"]["electric_to_load_series_kw"], util_to_load) push!(dispatch_series["ElectricUtility"]["electric_to_storage_series_kw"], util_to_batt) push!(dispatch_series["ElectricUtility"]["emissions_series_lb_CO2"], - emissions[idx] * grid_power / time_steps_per_hour) + co2_grid_emissions_series[idx] * grid_power / time_steps_per_hour) push!(dispatch_series["ElectricLoad"]["load_series_kw"], loads_kw[idx]) # Running energy costs @@ -558,7 +455,7 @@ function get_mpc_results(d::Dict)::Dict return Dict( "MPC" => Dict( "time_steps_per_hour" => time_steps_per_hour, - "horizon" => horizon, + "horizon_time_steps" => horizon, ), "PV" => merge(Dict("size_kw" => pv_kw), dispatch_series["PV"]), "ElectricStorage" => merge(Dict("size_kw" => batt_kw, "size_kwh" => batt_kwh), dispatch_series["ElectricStorage"]), From 9fb665df0918273a27b894378ca32ff608aff3f3 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 16:39:39 -0600 Subject: [PATCH 40/66] rmv lat long validation --- julia_src/mpc.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 86b4952a2..f9ec0a8e8 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -52,12 +52,6 @@ This is called by get_mpc_results only when the user does not provide a custom p """ function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) site = get(d, "Site", Dict()) - if !haskey(site, "latitude") - error("MPC: Site.latitude is required to generate PV.production_factor_series using PVWatts.") - end - if !haskey(site, "longitude") - error("MPC: Site.longitude is required to generate PV.production_factor_series using PVWatts.") - end lat = Float64(site["latitude"]) lon = Float64(site["longitude"]) From 7518e3606a79d971c8424b709a22fda334d5ee01 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 17:04:54 -0600 Subject: [PATCH 41/66] get pv prod from REopt run too --- julia_src/mpc.jl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index f9ec0a8e8..c3d55f49f 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -135,6 +135,7 @@ function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") pv_kw = Float64(get(get(sizing_results, "PV", Dict()), "size_kw", 0.0)) batt_kw = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kw", 0.0)) batt_kwh = Float64(get(get(sizing_results, "ElectricStorage", Dict()), "size_kwh", 0.0)) + pv_production_factor_series = get(get(sizing_results, "PV", Dict()), "production_factor_series", nothing) # Skip the MPC loop if no battery is sized if batt_kw <= 0.0 || batt_kwh <= 0.0 @@ -149,7 +150,7 @@ function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") batt["max_kwh"] = batt_kwh @info "MPC: REopt sizing solved with PV = $(pv_kw) kW and battery = $(batt_kw) kW / $(batt_kwh) kWh." - return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false) + return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false, pv_production_factor_series = pv_production_factor_series) end function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict @@ -220,10 +221,16 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict horizon = 24 * time_steps_per_hour per_iter_timeout_s = 30.0 - # Call REoptInputs once upfront to process and validate inputs - # This handles: load profile generation, tariff processing (including tier removal), emissions defaults, storage efficiency defaults - i = reoptjl.REoptInputs(d) - s = i.s # Access the processed Scenario struct + # Process and validate inputs using REoptInputs + try + model_inputs = reoptjl.REoptInputs(d) + @info "Successfully processed REopt inputs." + catch e + @error "Something went wrong during REopt inputs processing!" exception=(e, catch_backtrace()) + error_response["error"] = sprint(showerror, e) + end + + s = model_inputs.s # Access the processed Scenario struct # MPC requires fixed PV and battery sizes. If not provided, call REopt first in a sizing run. technology_sizes = get_technology_sizes!(d) From 9c255efb083447d314bba3353d339717af714891 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 17:26:13 -0600 Subject: [PATCH 42/66] set pv prod for final run --- julia_src/http.jl | 2 +- julia_src/mpc.jl | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index 7d6c2bd40..1d7094ff8 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -77,7 +77,7 @@ function reopt(req::HTTP.Request) # ---- API-only battery heuristic dispatch strategy: "daily_foresight_optimized" ---- # When ElectricStorage.dispatch_strategy == "daily_foresight_optimized", if needed, first run REopt to get optimal sizing of PV and battery. # Then run the MPC rolling-horizon loop to get a SOC profile (skip MPC dispatch if optimal battery size is 0). - # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC before running the main REopt optimization. + # Then set ElectricStorage.fixed_soc_series_fraction = MPC SOC and fix PV and BESS sizing before running the main REopt optimization. electric_storage = get(d, "ElectricStorage", Dict()) if get(electric_storage, "dispatch_strategy", nothing) == "daily_foresight_optimized" try diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index c3d55f49f..d5c9c1dfe 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -139,18 +139,22 @@ function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") # Skip the MPC loop if no battery is sized if batt_kw <= 0.0 || batt_kwh <= 0.0 - return (pv_kw = pv_kw, batt_kw = 0.0, batt_kwh = 0.0, skip_mpc = true) + return (pv_kw = pv_kw, batt_kw = 0.0, batt_kwh = 0.0, skip_mpc = true, pv_production_factor_series = pv_production_factor_series) end + println(pv_kw, batt_kw, batt_kwh) + + # Fix inputs for final REopt run in http.jl pv["min_kw"] = pv_kw pv["max_kw"] = pv_kw + pv["production_factor_series"] = pv_production_factor_series batt["min_kw"] = batt_kw batt["max_kw"] = batt_kw batt["min_kwh"] = batt_kwh batt["max_kwh"] = batt_kwh @info "MPC: REopt sizing solved with PV = $(pv_kw) kW and battery = $(batt_kw) kW / $(batt_kwh) kWh." - return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false, pv_production_factor_series = pv_production_factor_series) + return (; pv_kw, batt_kw, batt_kwh, skip_mpc = false, pv_production_factor_series) end function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict @@ -243,6 +247,8 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict # Note: REoptInputs does not provide PV production factors if user doesn't specify custom values if !isempty(s.pvs[1].production_factor_series) pv_prod_factor = Float64.(s.pvs[1].production_factor_series) + elseif technology_sizes.pv_production_factor_series !== nothing + pv_prod_factor = Float64.(technology_sizes.pv_production_factor_series) else # TODO: These production factors don't consider degradation, problem? # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? @@ -274,8 +280,6 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict # Extract emissions defaults (or use user input if provided) co2_grid_emissions_series = Float64.(s.electric_utility.emissions_factor_series_lb_CO2_per_kwh) - - pv_kw = technology_sizes.pv_kw batt_kw = technology_sizes.batt_kw batt_kwh = technology_sizes.batt_kwh From 01c0140b9f4d525e8e266464bc1981b07f919322 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 17:43:21 -0600 Subject: [PATCH 43/66] update pv prod in final REopt run --- julia_src/http.jl | 4 ++-- julia_src/mpc.jl | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index 1d7094ff8..c4a73dbd6 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -82,7 +82,7 @@ function reopt(req::HTTP.Request) if get(electric_storage, "dispatch_strategy", nothing) == "daily_foresight_optimized" try @info "Running MPC to obtain daily foresight optimized battery dispatch profile." - mpc_results = get_mpc_results(d; solver_name=solver_name) + mpc_results = get_mpc_results!(d; solver_name=solver_name) # TODO: Cache sizing run results and avoid a second call to REopt? Are those the same results? if get(mpc_results, "skip_mpc", false) == true @info "Cannot execute daily_foresight_optimized battery dispatch because optimal battery size is 0. Setting dispatch strategy to 'optimized'." @@ -884,7 +884,7 @@ function mpc(req::HTTP.Request) Specify Settings.solver_name = 'HiGHS' or 'Cbc' or 'SCIP'" end - results = get_mpc_results(d; solver_name=solver_name) + results = get_mpc_results!(d; solver_name=solver_name) catch e @error "MPC failed" exception=(e, catch_backtrace()) error_response["error"] = sprint(showerror, e) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index d5c9c1dfe..bb3229a7a 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -48,7 +48,7 @@ end generate_pv_production_factors(d, time_steps_per_hour) Generate a PV production factor series using PVWatts by calling REopt.get_production_factor. -This is called by get_mpc_results only when the user does not provide a custom production_factor_series. +This is called by get_mpc_results only when the user does not provide a custom production_factor_series and REopt is not called for sizing. """ function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) site = get(d, "Site", Dict()) @@ -142,12 +142,9 @@ function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") return (pv_kw = pv_kw, batt_kw = 0.0, batt_kwh = 0.0, skip_mpc = true, pv_production_factor_series = pv_production_factor_series) end - println(pv_kw, batt_kw, batt_kwh) - # Fix inputs for final REopt run in http.jl pv["min_kw"] = pv_kw pv["max_kw"] = pv_kw - pv["production_factor_series"] = pv_production_factor_series batt["min_kw"] = batt_kw batt["max_kw"] = batt_kw batt["min_kwh"] = batt_kwh @@ -157,7 +154,7 @@ function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") return (; pv_kw, batt_kw, batt_kwh, skip_mpc = false, pv_production_factor_series) end -function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict +function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict """ Run a full-year rolling-horizon MPC dispatch for PV + ElectricStorage by calling `REopt.run_mpc` once per timestep with a 24-hour look-ahead. @@ -244,6 +241,11 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict return Dict("skip_mpc" => true) end + # Sizes from user or initial REopt run + pv_kw = technology_sizes.pv_kw + batt_kw = technology_sizes.batt_kw + batt_kwh = technology_sizes.batt_kwh + # Note: REoptInputs does not provide PV production factors if user doesn't specify custom values if !isempty(s.pvs[1].production_factor_series) pv_prod_factor = Float64.(s.pvs[1].production_factor_series) @@ -254,6 +256,8 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? pv_prod_factor = generate_pv_production_factors(d, time_steps_per_hour) end + # Avoid another PVWatts call in the final REopt run. + d["PV"]["production_factor_series"] = pv_prod_factor loads_kw = Float64.(s.electric_load.loads_kw) @@ -267,6 +271,10 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict zeros(Float64, 12) : Float64.(s.electric_tariff.monthly_demand_rates[:, 1]) tou_demand_rates = isempty(s.electric_tariff.tou_demand_rates) ? Float64[] : Float64.(s.electric_tariff.tou_demand_rates[:, 1]) tou_demand_ratchet_time_steps = [Int.(v) for v in s.electric_tariff.tou_demand_ratchet_time_steps] + # TODO: Monthly demand has never been tested + n_tou_ratchets = length(tou_demand_rates) # Number of TOU ratchets + tou_previous_peak_demands = zeros(Float64, n_tou_ratchets) # Tracks past TOU peak demand per ratchet + monthly_previous_peak_demands = zeros(Float64, 12) # Tracks past monthly peak demand # Extract storage efficiency and SOC defaults from processed inputs rect_eff = Float64(s.storage.attr["ElectricStorage"].rectifier_efficiency_fraction) @@ -280,10 +288,6 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict # Extract emissions defaults (or use user input if provided) co2_grid_emissions_series = Float64.(s.electric_utility.emissions_factor_series_lb_CO2_per_kwh) - pv_kw = technology_sizes.pv_kw - batt_kw = technology_sizes.batt_kw - batt_kwh = technology_sizes.batt_kwh - month_starts = get_month_transition_timesteps(time_steps_per_hour) # ts_to_month = 8760 array specifying which month each timestep falls in (1-12) @@ -302,11 +306,6 @@ function get_mpc_results(d::Dict; solver_name::String="HiGHS")::Dict end end - # TODO: Monthly demand has never been tested - n_tou_ratchets = length(tou_demand_rates) # Number of TOU ratchets - tou_previous_peak_demands = zeros(Float64, n_tou_ratchets) # Tracks past TOU peak demand per ratchet - monthly_previous_peak_demands = zeros(Float64, 12) # Tracks past monthly peak demand - # Saved dispatch series (first timestep of each MPC loop) dispatch_series = Dict( "PV" => Dict( From 569b894271a0fadcc3cc8eace7eb749d77c55df0 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 17:47:53 -0600 Subject: [PATCH 44/66] Create 0118_merge_20260629_2347.py --- reoptjl/migrations/0118_merge_20260629_2347.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 reoptjl/migrations/0118_merge_20260629_2347.py diff --git a/reoptjl/migrations/0118_merge_20260629_2347.py b/reoptjl/migrations/0118_merge_20260629_2347.py new file mode 100644 index 000000000..e55228f28 --- /dev/null +++ b/reoptjl/migrations/0118_merge_20260629_2347.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.26 on 2026-06-29 23:47 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('reoptjl', '0117_merge_20260530_1640'), + ('reoptjl', '0117_merge_20260601_2056'), + ] + + operations = [ + ] From 13bbbed630fbd3be1443da6be8232c767cf89ff2 Mon Sep 17 00:00:00 2001 From: adfarth Date: Mon, 29 Jun 2026 17:52:15 -0600 Subject: [PATCH 45/66] add back in TODO --- julia_src/mpc.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index bb3229a7a..c1e1f9cc2 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -222,6 +222,8 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict horizon = 24 * time_steps_per_hour per_iter_timeout_s = 30.0 + # TODO: Should MPC handle multiple PVs? + # Process and validate inputs using REoptInputs try model_inputs = reoptjl.REoptInputs(d) From 2fc5065788ccc6d5226b249b8032f9348ea9416f Mon Sep 17 00:00:00 2001 From: adfarth Date: Wed, 1 Jul 2026 08:17:12 -0600 Subject: [PATCH 46/66] pv prod and solver --- julia_src/mpc.jl | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index c1e1f9cc2..fad40726c 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -68,7 +68,7 @@ function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) end """ - get_technology_sizes!(d) + get_technology_sizes!(d, solver_name="HiGHS") Determine PV and ElectricStorage sizes for the MPC loop. If min_kw != max_kw and/or min_kwh != max_kwh, call REopt to size technologies. User input battery sizes must also be greater than zero. @@ -236,7 +236,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict s = model_inputs.s # Access the processed Scenario struct # MPC requires fixed PV and battery sizes. If not provided, call REopt first in a sizing run. - technology_sizes = get_technology_sizes!(d) + technology_sizes = get_technology_sizes!(d, solver_name=solver_name) # Skip MPC if no battery is optimally sized if technology_sizes.skip_mpc @@ -249,17 +249,22 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict batt_kwh = technology_sizes.batt_kwh # Note: REoptInputs does not provide PV production factors if user doesn't specify custom values - if !isempty(s.pvs[1].production_factor_series) - pv_prod_factor = Float64.(s.pvs[1].production_factor_series) - elseif technology_sizes.pv_production_factor_series !== nothing - pv_prod_factor = Float64.(technology_sizes.pv_production_factor_series) + if !isempty(s.pvs) # Get prod factors if PV considered. + if !isempty(s.pvs[1].production_factor_series) + pv_prod_factor = Float64.(s.pvs[1].production_factor_series) + elseif technology_sizes.pv_production_factor_series !== nothing + pv_prod_factor = Float64.(technology_sizes.pv_production_factor_series) + else + # TODO: These production factors don't consider degradation, problem? + # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? + # AF: none of these production factors consider degradation and I don't think it's a problem? + pv_prod_factor = generate_pv_production_factors(d, time_steps_per_hour) + end + # Avoid another PVWatts call in the final REopt run. + d["PV"]["production_factor_series"] = pv_prod_factor else - # TODO: These production factors don't consider degradation, problem? - # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? - pv_prod_factor = generate_pv_production_factors(d, time_steps_per_hour) + pv_prod_factor = zeros(Float64, length_of_data) end - # Avoid another PVWatts call in the final REopt run. - d["PV"]["production_factor_series"] = pv_prod_factor loads_kw = Float64.(s.electric_load.loads_kw) From 919d30cb7c5026f5c4e71556277e7d122e8a5985 Mon Sep 17 00:00:00 2001 From: adfarth Date: Wed, 1 Jul 2026 08:23:27 -0600 Subject: [PATCH 47/66] utf-8 --- julia_src/http.jl | 2 +- julia_src/mpc.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index c4a73dbd6..8c23f8ef6 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -1,4 +1,4 @@ -using HTTP, JSON, JuMP +using HTTP, JSON, JuMP using HiGHS, Cbc, SCIP using GhpGhx import REopt as reoptjl # For REopt.jl, needed because we still have local REopt.jl module for V1/V2 diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index fad40726c..0ca203d6e 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -1,4 +1,4 @@ -# REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NatLabRockies/REopt_API/blob/master/LICENSE. +# REopt®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NatLabRockies/REopt_API/blob/master/LICENSE. # ============================================================================ # MPC (Model Predictive Control) endpoint # ---------------------------------------------------------------------------- From 0a557b234c71b1a93bfa8ec37a96951c55bcd41f Mon Sep 17 00:00:00 2001 From: adfarth Date: Wed, 1 Jul 2026 08:35:42 -0600 Subject: [PATCH 48/66] consistent response --- julia_src/http.jl | 2 +- julia_src/mpc.jl | 79 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 55 insertions(+), 26 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index 8c23f8ef6..e857ba30f 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -848,7 +848,7 @@ HTTP endpoint for rolling-horizon Model Predictive Control (MPC) dispatch optimi This endpoint performs a full-year rolling-horizon MPC dispatch for PV + ElectricStorage systems, optimizing daily dispatch using a 24-hour look-ahead window. Runs the MPC dispatch loop via -`get_mpc_results` and returns dispatch results and cost metrics as JSON. +`get_mpc_results!` and returns dispatch results and cost metrics as JSON. Arguments: req::HTTP.Request: REopt inputs dictionary diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 0ca203d6e..5bf0e58c2 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -48,7 +48,7 @@ end generate_pv_production_factors(d, time_steps_per_hour) Generate a PV production factor series using PVWatts by calling REopt.get_production_factor. -This is called by get_mpc_results only when the user does not provide a custom production_factor_series and REopt is not called for sizing. +This is called by get_mpc_results! only when the user does not provide a custom production_factor_series and REopt is not called for sizing. """ function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) site = get(d, "Site", Dict()) @@ -67,6 +67,23 @@ function generate_pv_production_factors(d::Dict, time_steps_per_hour::Int) return Vector{Float64}(pv_production_factor_series) end +""" + build_mpc_response(status; skip_mpc=false, messages=Dict(), result_dict=Dict()) + +Build a consistent MPC response envelope with status, version info, and messages. +Ensures all response paths (success, error, skip) have uniform structure. +""" +function build_mpc_response(status::String; skip_mpc=false, messages=Dict(), result_dict=Dict()) + response = Dict( + "status" => status, + "reopt_version" => string(pkgversion(reoptjl)), + "Messages" => messages, + "skip_mpc" => skip_mpc, + ) + # Merge result data if provided (for success case) + return merge(response, result_dict) +end + """ get_technology_sizes!(d, solver_name="HiGHS") @@ -160,17 +177,21 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict calling `REopt.run_mpc` once per timestep with a 24-hour look-ahead. Inputs: - d::Dict, REopt inputs dictionary + d::Dict, REopt inputs dictionary (will be modified in-place to store computed fixed sizes and PV production factors.) Returns JSON dictionary containing: + - status: "optimal", "error", or "skipped" + - reopt_version: Version of REopt.jl used + - Messages: Dict with optional errors, warnings, or info + - skip_mpc: Boolean indicating if MPC was skipped + + For "optimal" status, also includes: - MPC: Metadata (time_steps_per_hour, horizon_time_steps) - PV: Size and dispatch series (to load, storage, grid, curtailed) - ElectricStorage: Sizes and state-of-charge series - ElectricUtility: Grid dispatch series and emissions - ElectricLoad: Load profile used - ElectricTariff: Energy and demand costs, peak demands by month/ratchet - - status: "optimal" - - reopt_version: Version of REopt.jl used """ @@ -230,7 +251,10 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict @info "Successfully processed REopt inputs." catch e @error "Something went wrong during REopt inputs processing!" exception=(e, catch_backtrace()) - error_response["error"] = sprint(showerror, e) + return build_mpc_response( + "error", + messages = Dict("errors" => [sprint(showerror, e)]) + ) end s = model_inputs.s # Access the processed Scenario struct @@ -240,7 +264,11 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # Skip MPC if no battery is optimally sized if technology_sizes.skip_mpc - return Dict("skip_mpc" => true) + return build_mpc_response( + "skipped", + skip_mpc = true, + messages = Dict("info" => ["No battery was optimally sized in REopt pre-step; MPC dispatch not needed."]) + ) end # Sizes from user or initial REopt run @@ -463,24 +491,25 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict tou_demand_cost_total = n_tou_ratchets > 0 ? sum(tou_previous_peak_demands .* tou_demand_rates) : 0.0 - return Dict( - "MPC" => Dict( - "time_steps_per_hour" => time_steps_per_hour, - "horizon_time_steps" => horizon, - ), - "PV" => merge(Dict("size_kw" => pv_kw), dispatch_series["PV"]), - "ElectricStorage" => merge(Dict("size_kw" => batt_kw, "size_kwh" => batt_kwh), dispatch_series["ElectricStorage"]), - "ElectricUtility" => dispatch_series["ElectricUtility"], - "ElectricLoad" => dispatch_series["ElectricLoad"], - "ElectricTariff" => Dict( - "total_energy_cost" => total_energy_cost, - "energy_cost_series_per_timestep" => energy_cost_series, - "total_tou_demand_cost" => tou_demand_cost_total, - "total_monthly_demand_cost" => monthly_demand_cost_total, - "tou_peaks_by_ratchet_kw" => tou_previous_peak_demands, - "monthly_peaks_kw" => monthly_previous_peak_demands, - ), - "status" => "optimal", - "reopt_version" => string(pkgversion(reoptjl)), + return build_mpc_response( + "optimal", + result_dict = Dict( + "MPC" => Dict( + "time_steps_per_hour" => time_steps_per_hour, + "horizon_time_steps" => horizon, + ), + "PV" => merge(Dict("size_kw" => pv_kw), dispatch_series["PV"]), + "ElectricStorage" => merge(Dict("size_kw" => batt_kw, "size_kwh" => batt_kwh), dispatch_series["ElectricStorage"]), + "ElectricUtility" => dispatch_series["ElectricUtility"], + "ElectricLoad" => dispatch_series["ElectricLoad"], + "ElectricTariff" => Dict( + "total_energy_cost" => total_energy_cost, + "energy_cost_series_per_timestep" => energy_cost_series, + "total_tou_demand_cost" => tou_demand_cost_total, + "total_monthly_demand_cost" => monthly_demand_cost_total, + "tou_peaks_by_ratchet_kw" => tou_previous_peak_demands, + "monthly_peaks_kw" => monthly_previous_peak_demands, + ), + ) ) end From 513c1141bb9b227360a7695fe0e7d7b8d705c450 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:32:17 -0600 Subject: [PATCH 49/66] Fix for environments where user running migrations lacks CREATE perms If the user doesn't have CREATE privileges at the database level, then even if the schema already exists, the `CREATE SCHEMA IF NOT EXISTS` will fail. --- reopt_api/settings.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/reopt_api/settings.py b/reopt_api/settings.py index 2efa87d06..e0da84cc1 100644 --- a/reopt_api/settings.py +++ b/reopt_api/settings.py @@ -182,7 +182,23 @@ def create_postgres_schema(sender, **kwargs): using = kwargs.get('using', 'default') connection = connections[using] with connection.cursor() as cursor: - cursor.execute(f"CREATE SCHEMA IF NOT EXISTS reopt_api;") + # Wrap in extra conditions so it can execute even if user doesn't have + # "CREATE" privileges and schema already exists. + cursor.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = 'reopt_api') THEN + CREATE SCHEMA IF NOT EXISTS reopt_api; + END IF; + IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'reopt_api') THEN + ALTER SCHEMA reopt_api OWNER TO reopt_api; + END IF; + END; + $$; + """ + ) + pre_migrate.connect(create_postgres_schema) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.settings") From 0e2efc04b00c687a254ab8ae64006abb58745983 Mon Sep 17 00:00:00 2001 From: adfarth Date: Thu, 9 Jul 2026 13:02:33 -0600 Subject: [PATCH 50/66] fix inputs processing --- julia_src/mpc.jl | 57 ++++++++++--------- ...uts_fixed_soc_series_fraction_tolerance.py | 19 +++++++ 2 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 reoptjl/migrations/0119_alter_electricstorageinputs_fixed_soc_series_fraction_tolerance.py diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 5bf0e58c2..3e7845717 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -85,12 +85,15 @@ function build_mpc_response(status::String; skip_mpc=false, messages=Dict(), res end """ - get_technology_sizes!(d, solver_name="HiGHS") + get_technology_sizes!(d::Dict, model_inputs::reoptjl.REoptInputs, solver_settings::Dict) Determine PV and ElectricStorage sizes for the MPC loop. If min_kw != max_kw and/or min_kwh != max_kwh, call REopt to size technologies. User input battery sizes must also be greater than zero. + d is mutated in-place to update PV and ElectricStorage sizes. + model_inputs has already been processed to remove inputs not used in REopt.jl """ -function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") +function get_technology_sizes!(d::Dict, model_inputs::reoptjl.REoptInputs, solver_settings::Dict) + # pv and batt are updated in-place from the dictionary `d` and used in the final REopt run pv = get!(d, "PV", Dict()) batt = get!(d, "ElectricStorage", Dict()) @@ -119,25 +122,11 @@ function get_technology_sizes!(d::Dict; solver_name::String="HiGHS") end @info "MPC: PV and/or ElectricStorage sizes are not specified — running REopt sizing first." - sizing_post = deepcopy(d) - - # Delete inputs specific to the heuristic battery dispatch run - if haskey(sizing_post, "ElectricStorage") - delete!(sizing_post["ElectricStorage"], "dispatch_strategy") - delete!(sizing_post["ElectricStorage"], "fixed_soc_series_fraction") - end - # TODO: Should we "remove tiers" here for sizing or allow for optimizing with tiers? - settings = get(sizing_post, "Settings", Dict()) - delete!(settings, "run_bau") # Remove run_bau from sizing run - timeout_seconds = pop!(settings, "timeout_seconds", 420) - optimality_tolerance = pop!(settings, "optimality_tolerance", 0.001) - solver_attributes = SolverAttributes(timeout_seconds, optimality_tolerance) - - m = get_solver_model(get_solver_model_type(solver_name), solver_attributes) + m = get_solver_model(get_solver_model_type(solver_settings["solver_name"]), solver_settings["solver_attributes"]) - model_inputs = reoptjl.REoptInputs(sizing_post) + # model_inputs = reoptjl.REoptInputs(sizing_post) sizing_results = reoptjl.run_reopt(m, model_inputs) if get(sizing_results, "status", "") != "optimal" @@ -232,10 +221,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict ## Set up MPC inputs ## settings = get!(d, "Settings", Dict()) - settings["solver_name"] = solver_name - - # TODO: MPC timeout and optimality tolerance - optimality_tolerance = Float64(get(settings, "optimality_tolerance", 0.001)) + settings["solver_name"] = solver_name # TODO: remove? # TODO: MPC horizons and timeout are currently hard coded time_steps_per_hour = Int(get(settings, "time_steps_per_hour", 1)) @@ -244,10 +230,26 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict per_iter_timeout_s = 30.0 # TODO: Should MPC handle multiple PVs? + # TODO: MPC timeout and optimality tolerance + # Update sizing_post to remove solver settings and dispatch inputs, to be able to validate inputs using REoptInputs + sizing_post = deepcopy(d) + settings = get(sizing_post, "Settings", Dict()) + solver_settings=Dict() + delete!(settings, "run_bau") # Remove run_bau from sizing run + solver_settings["timeout_seconds"] = pop!(settings, "timeout_seconds", 420) + solver_settings["optimality_tolerance"] = pop!(settings, "optimality_tolerance", 0.001) + solver_settings["solver_attributes"] = SolverAttributes(solver_settings["timeout_seconds"], solver_settings["optimality_tolerance"]) + solver_settings["solver_name"] = solver_name + # Delete inputs specific to the heuristic battery dispatch run + if haskey(sizing_post, "ElectricStorage") + delete!(sizing_post["ElectricStorage"], "dispatch_strategy") + delete!(sizing_post["ElectricStorage"], "fixed_soc_series_fraction") + end # Process and validate inputs using REoptInputs + model_inputs = nothing try - model_inputs = reoptjl.REoptInputs(d) + model_inputs = reoptjl.REoptInputs(sizing_post) @info "Successfully processed REopt inputs." catch e @error "Something went wrong during REopt inputs processing!" exception=(e, catch_backtrace()) @@ -256,11 +258,10 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict messages = Dict("errors" => [sprint(showerror, e)]) ) end - - s = model_inputs.s # Access the processed Scenario struct - + # MPC requires fixed PV and battery sizes. If not provided, call REopt first in a sizing run. - technology_sizes = get_technology_sizes!(d, solver_name=solver_name) + technology_sizes = get_technology_sizes!(d, model_inputs, solver_settings) + s = model_inputs.s # Access the processed Scenario struct # Skip MPC if no battery is optimally sized if technology_sizes.skip_mpc @@ -434,7 +435,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac) model = get_solver_model(get_solver_model_type(solver_name), - SolverAttributes(per_iter_timeout_s, optimality_tolerance)) + SolverAttributes(per_iter_timeout_s, solver_settings["optimality_tolerance"])) result = reoptjl.run_mpc(model, post) # Assume perfect forecast; save first timestep of results as the executed state diff --git a/reoptjl/migrations/0119_alter_electricstorageinputs_fixed_soc_series_fraction_tolerance.py b/reoptjl/migrations/0119_alter_electricstorageinputs_fixed_soc_series_fraction_tolerance.py new file mode 100644 index 000000000..2378b63f3 --- /dev/null +++ b/reoptjl/migrations/0119_alter_electricstorageinputs_fixed_soc_series_fraction_tolerance.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.26 on 2026-07-09 18:09 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reoptjl', '0118_merge_20260629_2347'), + ] + + operations = [ + migrations.AlterField( + model_name='electricstorageinputs', + name='fixed_soc_series_fraction_tolerance', + field=models.FloatField(blank=True, help_text='Absolute tolerance on fixed_soc_series_fraction to avoid infeasible solutions when fixed_soc_series_fraction is provided.', null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(1)]), + ), + ] From 41a68ab2cd3815fe9162f42dc8c0480e82c97035 Mon Sep 17 00:00:00 2001 From: adfarth Date: Thu, 9 Jul 2026 14:33:13 -0600 Subject: [PATCH 51/66] add inputs to models.py --- CHANGELOG.md | 5 ++++ ...torageinputs_dispatch_strategy_and_more.py | 24 +++++++++++++++++++ reoptjl/models.py | 18 +++++++++++++- reoptjl/test/posts/all_inputs_test.json | 1 + reoptjl/validators.py | 6 +++++ 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 reoptjl/migrations/0120_electricstorageinputs_dispatch_strategy_and_more.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a2e0d1944..8a9c2292c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,11 @@ Classify the change according to the following categories: ##### Removed ### Patches +## heuristic-dispatch-option +### Minor udpates +#### Added +- Add **ElectricStorage** inputs field **dispatch_options** with heuristic options + ## fixed-soc ### Minor udpates #### Added diff --git a/reoptjl/migrations/0120_electricstorageinputs_dispatch_strategy_and_more.py b/reoptjl/migrations/0120_electricstorageinputs_dispatch_strategy_and_more.py new file mode 100644 index 000000000..486adb62d --- /dev/null +++ b/reoptjl/migrations/0120_electricstorageinputs_dispatch_strategy_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.26 on 2026-07-09 20:32 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reoptjl', '0119_alter_electricstorageinputs_fixed_soc_series_fraction_tolerance'), + ] + + operations = [ + migrations.AddField( + model_name='electricstorageinputs', + name='dispatch_strategy', + field=models.TextField(blank=True, choices=[('optimized', 'Optimized'), ('peak_shaving_look_ahead', 'Peak Shaving Look Ahead'), ('peak_shaving_look_behind', 'Peak Shaving Look Behind'), ('self_consumption', 'Self Consumption'), ('backup', 'Backup'), ('custom_soc', 'Custom Soc'), ('daily_foresight_optimized', 'Daily Foresight Optimized')], default='optimized', help_text='Electric storage dispatch strategy can be one of: optimized, peak_shaving_look_ahead, peak_shaving_look_behind, self_consumption, backup, custom_soc, daily_foresight_optimized', null=True), + ), + migrations.AlterField( + model_name='electricstorageinputs', + name='soc_min_fraction', + field=models.FloatField(blank=True, help_text='Minimum allowable battery state of charge as fraction of energy capacity.', validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(1.0)]), + ), + ] diff --git a/reoptjl/models.py b/reoptjl/models.py index cee7bfaa1..3efe51998 100644 --- a/reoptjl/models.py +++ b/reoptjl/models.py @@ -3668,6 +3668,16 @@ class ElectricStorageInputs(BaseModel, models.Model): primary_key=True ) + ELECTRICSTORAGE_DISPATCH_STRATEGY = models.TextChoices('ELECTRICSTORAGE_DISPATCH_STRATEGY', ( + "optimized", + "peak_shaving_look_ahead", + "peak_shaving_look_behind", + "self_consumption", + "backup", + "custom_soc", + "daily_foresight_optimized" + )) + min_kw = models.FloatField( default=0, validators=[ @@ -3731,8 +3741,14 @@ class ElectricStorageInputs(BaseModel, models.Model): blank=True, help_text="Battery rectifier efficiency" ) + dispatch_strategy = models.TextField( + default=ELECTRICSTORAGE_DISPATCH_STRATEGY.optimized, + choices=ELECTRICSTORAGE_DISPATCH_STRATEGY.choices, + null=True, + blank=True, + help_text="Electric storage dispatch strategy can be one of: optimized, peak_shaving_look_ahead, peak_shaving_look_behind, self_consumption, backup, custom_soc, daily_foresight_optimized" + ) soc_min_fraction = models.FloatField( - default=0.2, validators=[ MinValueValidator(0), MaxValueValidator(1.0) diff --git a/reoptjl/test/posts/all_inputs_test.json b/reoptjl/test/posts/all_inputs_test.json index d53def1ad..3cc26956b 100644 --- a/reoptjl/test/posts/all_inputs_test.json +++ b/reoptjl/test/posts/all_inputs_test.json @@ -167,6 +167,7 @@ "internal_efficiency_fraction": 0.975, "inverter_efficiency_fraction": 0.96, "rectifier_efficiency_fraction": 0.96, + "dispatch_strategy": "optimized", "soc_min_fraction": 0.2, "soc_min_applies_during_outages": true, "soc_init_fraction": 0.5, diff --git a/reoptjl/validators.py b/reoptjl/validators.py index 99cdc456a..76dbd852a 100644 --- a/reoptjl/validators.py +++ b/reoptjl/validators.py @@ -363,6 +363,12 @@ def update_pv_defaults_offgrid(self, pvmodel): else: self.models["ElectricStorage"].soc_init_fraction = 1.0 + if self.models["ElectricStorage"].__getattribute__("soc_min_fraction") == None: + if self.models["ElectricStorage"].dispatch_strategy=="backup": + self.models["ElectricStorage"].soc_min_fraction = 0.8 + else: + self.models["ElectricStorage"].soc_min_fraction = 0.2 + if self.models["ElectricStorage"].__getattribute__("can_grid_charge") == None: if self.models["Settings"].off_grid_flag==False: self.models["ElectricStorage"].can_grid_charge = True From e2eb18a1d1f2276d561f1222f0b6caff869358e2 Mon Sep 17 00:00:00 2001 From: adfarth Date: Wed, 15 Jul 2026 13:47:51 -0600 Subject: [PATCH 52/66] add bess export io --- ...ts_can_export_beyond_nem_limit_and_more.py | 34 +++++++++++++++++++ reoptjl/models.py | 22 ++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 reoptjl/migrations/0121_electricstorageinputs_can_export_beyond_nem_limit_and_more.py diff --git a/reoptjl/migrations/0121_electricstorageinputs_can_export_beyond_nem_limit_and_more.py b/reoptjl/migrations/0121_electricstorageinputs_can_export_beyond_nem_limit_and_more.py new file mode 100644 index 000000000..d1c39e257 --- /dev/null +++ b/reoptjl/migrations/0121_electricstorageinputs_can_export_beyond_nem_limit_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.26 on 2026-07-15 19:47 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reoptjl', '0120_electricstorageinputs_dispatch_strategy_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='electricstorageinputs', + name='can_export_beyond_nem_limit', + field=models.BooleanField(blank=True, default=False, help_text='True/False for if technology can export energy beyond the annual site load (and be compensated for that energy at the export_rate_beyond_net_metering_limit).'), + ), + migrations.AddField( + model_name='electricstorageinputs', + name='can_net_meter', + field=models.BooleanField(blank=True, default=False, help_text='True/False for if technology has option to participate in net metering agreement with utility. Note that a technology can only participate in either net metering or wholesale rates (not both).'), + ), + migrations.AddField( + model_name='electricstorageinputs', + name='can_wholesale', + field=models.BooleanField(blank=True, default=False, help_text='True/False for if technology has option to export energy that is compensated at the wholesale_rate. Note that a technology can only participate in either net metering or wholesale rates (not both).'), + ), + migrations.AddField( + model_name='electricstorageoutputs', + name='storage_to_grid_series_kw', + field=django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(blank=True, null=True), blank=True, default=list, size=None), + ), + ] diff --git a/reoptjl/models.py b/reoptjl/models.py index 3efe51998..b3d0e14f0 100644 --- a/reoptjl/models.py +++ b/reoptjl/models.py @@ -3793,6 +3793,24 @@ class ElectricStorageInputs(BaseModel, models.Model): blank=True, help_text="Flag to set whether the battery can be charged from the grid, or just onsite generation." ) + can_net_meter = models.BooleanField( + default=False, + blank=True, + help_text=("True/False for if technology has option to participate in net metering agreement with utility. " + "Note that a technology can only participate in either net metering or wholesale rates (not both).") + ) + can_wholesale = models.BooleanField( + default=False, + blank=True, + help_text=("True/False for if technology has option to export energy that is compensated at the wholesale_rate. " + "Note that a technology can only participate in either net metering or wholesale rates (not both).") + ) + can_export_beyond_nem_limit = models.BooleanField( + default=False, + blank=True, + help_text=("True/False for if technology can export energy beyond the annual site load (and be compensated for " + "that energy at the export_rate_beyond_net_metering_limit).") + ) installed_cost_per_kw = models.FloatField( default=968.0, validators=[ @@ -3978,6 +3996,10 @@ class ElectricStorageOutputs(BaseModel, models.Model): models.FloatField(null=True, blank=True), blank=True, default=list ) + storage_to_grid_series_kw = ArrayField( + models.FloatField(null=True, blank=True), + blank=True, default=list + ) initial_capital_cost = models.FloatField(null=True, blank=True) maintenance_cost = models.FloatField(null=True, blank=True) state_of_health_series_fraction = ArrayField( From 3f6d57ea255c656fa8d5260fe3d1e4b9d287e230 Mon Sep 17 00:00:00 2001 From: adfarth Date: Wed, 15 Jul 2026 13:59:12 -0600 Subject: [PATCH 53/66] Update custom_table_config.py --- reoptjl/custom_table_config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/reoptjl/custom_table_config.py b/reoptjl/custom_table_config.py index 4cd3ec479..4d52f13cc 100644 --- a/reoptjl/custom_table_config.py +++ b/reoptjl/custom_table_config.py @@ -804,6 +804,12 @@ "bau_value" : lambda df: safe_get(df, "outputs.ElectricStorage.storage_to_load_series_kw_bau"), "scenario_value": lambda df: safe_get(df, "outputs.ElectricStorage.storage_to_load_series_kw") }, + { + "label" : "Battery Exported to Grid (kWh/yr)", + "key" : "battery_exported_to_grid", + "bau_value" : lambda df: safe_get(df, "outputs.ElectricStorage.storage_to_grid_series_kw_bau"), + "scenario_value": lambda df: safe_get(df, "outputs.ElectricStorage.storage_to_grid_series_kw") + }, { "label" : "Generator Serving Load (kWh/yr)", "key" : "generator_serving_load", From f1254560484234ad69a82e84642cad78940c3ed5 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:27:32 -0600 Subject: [PATCH 54/66] Fix database restore not restoring to same database used by API. Align everything on the `reopt` database name as the default. This matches the other `DB_NAME` environment variables being passed into the Celery and Django containers. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index aefe8ee6f..b2e6cae21 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,7 +101,7 @@ services: environment: - LOAD_VAULT_ENV_SECRETS=true - DEV_DB_HOST=db - - DEV_DB_NAME=reopt_api + - DEV_DB_NAME=reopt - DEV_DB_USERNAME=postgres - DEV_DB_PASSWORD=reopt From 756df197e2df32191987deb0be36995c58347cc0 Mon Sep 17 00:00:00 2001 From: adfarth Date: Thu, 16 Jul 2026 13:19:34 -0600 Subject: [PATCH 55/66] fix settings and pv processing --- julia_src/http.jl | 5 ++--- julia_src/mpc.jl | 9 +++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index e857ba30f..3326447df 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -875,9 +875,8 @@ function mpc(req::HTTP.Request) ENV["NREL_DEVELOPER_API_KEY"] = test_nrel_developer_api_key delete!(d, "api_key") end - - settings = d["Settings"] - solver_name = get(settings, "solver_name", "HiGHS") + + solver_name = get(get(d, "Settings", Dict()), "solver_name", "HiGHS") if solver_name == "Xpress" && !(xpress_installed=="True") solver_name = "HiGHS" @warn "Changing solver_name from Xpress to $solver_name because Xpress is not installed. Next time diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 3e7845717..cb8d4204f 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -118,7 +118,7 @@ function get_technology_sizes!(d::Dict, model_inputs::reoptjl.REoptInputs, solve pv_kw = Float64(pv["min_kw"]) batt_kw = Float64(batt["min_kw"]) batt_kwh = Float64(batt["min_kwh"]) - return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false) + return (pv_kw = pv_kw, batt_kw = batt_kw, batt_kwh = batt_kwh, skip_mpc = false, pv_production_factor_series = nothing) end @info "MPC: PV and/or ElectricStorage sizes are not specified — running REopt sizing first." @@ -250,7 +250,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict model_inputs = nothing try model_inputs = reoptjl.REoptInputs(sizing_post) - @info "Successfully processed REopt inputs." + @info "Successfully processed REopt inputs." catch e @error "Something went wrong during REopt inputs processing!" exception=(e, catch_backtrace()) return build_mpc_response( @@ -279,7 +279,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # Note: REoptInputs does not provide PV production factors if user doesn't specify custom values if !isempty(s.pvs) # Get prod factors if PV considered. - if !isempty(s.pvs[1].production_factor_series) + if !isnothing(s.pvs[1].production_factor_series) pv_prod_factor = Float64.(s.pvs[1].production_factor_series) elseif technology_sizes.pv_production_factor_series !== nothing pv_prod_factor = Float64.(technology_sizes.pv_production_factor_series) @@ -432,7 +432,8 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict post = build_mpc_post(current_horizon_pv, current_horizon_load, current_horizon_energy_rates, current_horizon_emissions, current_horizon_tou_ts, current_horizon_monthly_ts, - tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac) + tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac + ) model = get_solver_model(get_solver_model_type(solver_name), SolverAttributes(per_iter_timeout_s, solver_settings["optimality_tolerance"])) From e470e9ce00c4efb12d624d45ad986081920e8987 Mon Sep 17 00:00:00 2001 From: adfarth Date: Thu, 16 Jul 2026 14:41:58 -0600 Subject: [PATCH 56/66] update todos --- julia_src/http.jl | 1 - julia_src/mpc.jl | 23 ++++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index 3326447df..c89172a49 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -101,7 +101,6 @@ function reopt(req::HTTP.Request) end end - #TODO: What timeout and optimality tolerance should MPC use? timeout_seconds = pop!(settings, "timeout_seconds") optimality_tolerance = pop!(settings, "optimality_tolerance") solver_attributes = SolverAttributes(timeout_seconds, optimality_tolerance) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index cb8d4204f..675e42fdb 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -192,6 +192,10 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict error("When using MPC (daily_foresight_optimized dispatch), only PV and ElectricStorage are supported technologies. " * "Unsupported inputs found: $(join(unsupported_keys, ", ")).") end + # Error if multiple PVs + if haskey(d, "PV") && length(d["PV"]) > 1 + error("MPC: Multiple PV systems are not supported.") + end # Error if unsupported CO2/renewable-fraction constraints are set _site_input = get(d, "Site", Dict()) @@ -211,6 +215,8 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # TODO: Add warnings for REopt inputs and scenarios that are not modeled in MPC (e.g., coincident peak charges, demand lookback, etc.) @warn "Using MPC to determine dispatch. MPC does not model: tiered electricity rates; rates will be flattened to the first tier." + # TODO: Error if rate tariff contains lookbacks. + # TODO: Test with outage inputs before enabling this warning. # # Warning for outage inputs (MPC does not model outages) # _utility_input = get(d, "ElectricUtility", Dict()) @@ -220,24 +226,22 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # end ## Set up MPC inputs ## - settings = get!(d, "Settings", Dict()) - settings["solver_name"] = solver_name # TODO: remove? # TODO: MPC horizons and timeout are currently hard coded + settings = get!(d, "Settings", Dict()) time_steps_per_hour = Int(get(settings, "time_steps_per_hour", 1)) length_of_data = 8760 * time_steps_per_hour horizon = 24 * time_steps_per_hour per_iter_timeout_s = 30.0 - # TODO: Should MPC handle multiple PVs? - # TODO: MPC timeout and optimality tolerance + # TODO: Handle multiple PVs # Update sizing_post to remove solver settings and dispatch inputs, to be able to validate inputs using REoptInputs sizing_post = deepcopy(d) settings = get(sizing_post, "Settings", Dict()) solver_settings=Dict() delete!(settings, "run_bau") # Remove run_bau from sizing run - solver_settings["timeout_seconds"] = pop!(settings, "timeout_seconds", 420) - solver_settings["optimality_tolerance"] = pop!(settings, "optimality_tolerance", 0.001) + solver_settings["timeout_seconds"] = pop!(settings, "timeout_seconds", 600) # only gets used in sizing run. + solver_settings["optimality_tolerance"] = pop!(settings, "optimality_tolerance", 0.001) # Update to a higher value if solve time becomes an issue solver_settings["solver_attributes"] = SolverAttributes(solver_settings["timeout_seconds"], solver_settings["optimality_tolerance"]) solver_settings["solver_name"] = solver_name # Delete inputs specific to the heuristic battery dispatch run @@ -278,15 +282,13 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict batt_kwh = technology_sizes.batt_kwh # Note: REoptInputs does not provide PV production factors if user doesn't specify custom values + # PV production is NOT levelized in MPC but IS levelized in REopt. This will result in a slight mis-match. if !isempty(s.pvs) # Get prod factors if PV considered. if !isnothing(s.pvs[1].production_factor_series) pv_prod_factor = Float64.(s.pvs[1].production_factor_series) elseif technology_sizes.pv_production_factor_series !== nothing pv_prod_factor = Float64.(technology_sizes.pv_production_factor_series) else - # TODO: These production factors don't consider degradation, problem? - # Does MPCPV need a degradation input to calculate the levelization factor used in the optimization? - # AF: none of these production factors consider degradation and I don't think it's a problem? pv_prod_factor = generate_pv_production_factors(d, time_steps_per_hour) end # Avoid another PVWatts call in the final REopt run. @@ -299,9 +301,8 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # Extract tariff inputs relevant to MPC (use first tier only if tiered rates) # TODO: Are all of these relevant? Any missing inputs? - # TODO: Implement lookback? (demand_lookback_months, demand_lookback_percent, demand_lookback_range) Ignoring coincident peak charges for now + # TODO: Implement lookback (demand_lookback_months, demand_lookback_percent, demand_lookback_range) Ignoring coincident peak charges for now # TODO: Need to think through NEM or passing back export values (wholesale_rate, export_rate_beyond_net_metering_limit) - # TODO: Track lookback variables - demand_lookback_months, demand_lookback_percent, demand_lookback_range? energy_rates = Float64.(s.electric_tariff.energy_rates[:, 1]) monthly_demand_rates = isempty(s.electric_tariff.monthly_demand_rates) ? zeros(Float64, 12) : Float64.(s.electric_tariff.monthly_demand_rates[:, 1]) From 41514227f47250f02ed0cf3cfe6bedb2bfa900d4 Mon Sep 17 00:00:00 2001 From: adfarth Date: Fri, 17 Jul 2026 13:27:52 -0700 Subject: [PATCH 57/66] account for nem and whl --- julia_src/mpc.jl | 110 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 86 insertions(+), 24 deletions(-) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 675e42fdb..a85616e04 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -180,7 +180,9 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict - ElectricStorage: Sizes and state-of-charge series - ElectricUtility: Grid dispatch series and emissions - ElectricLoad: Load profile used - - ElectricTariff: Energy and demand costs, peak demands by month/ratchet + - ElectricTariff: Separate cost components (total_energy_cost, total_export_benefit, + total_tou_demand_cost, total_monthly_demand_cost), a combined total_electricity_bill, + per-timestep energy/export series, and peak demands by month/ratchet """ @@ -193,6 +195,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict "Unsupported inputs found: $(join(unsupported_keys, ", ")).") end # Error if multiple PVs + # TODO: Handle multiple PVs if haskey(d, "PV") && length(d["PV"]) > 1 error("MPC: Multiple PV systems are not supported.") end @@ -234,7 +237,6 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict horizon = 24 * time_steps_per_hour per_iter_timeout_s = 30.0 - # TODO: Handle multiple PVs # Update sizing_post to remove solver settings and dispatch inputs, to be able to validate inputs using REoptInputs sizing_post = deepcopy(d) settings = get(sizing_post, "Settings", Dict()) @@ -325,6 +327,27 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # Extract emissions defaults (or use user input if provided) co2_grid_emissions_series = Float64.(s.electric_utility.emissions_factor_series_lb_CO2_per_kwh) + # --- Export / net metering setup (mirror the sizing-run scenario) --- + # NEM is enabled in MPC when ElectricUtility.net_metering_limit_kw > 0. + nm_limit_kw = Float64(s.electric_utility.net_metering_limit_kw) + + # WHL (net billing) rate: export_rates[:WHL] in the processed scenario is a negative "cost"; + # MPCElectricTariff expects a positive wholesale_rate (it negates internally). + whl_rate_full = (:WHL in s.electric_tariff.export_bins) ? + -1.0 .* Float64.(s.electric_tariff.export_rates[:WHL]) : nothing + + # PV export capability comes from the processed PV struct. + pv_can_net_meter = !isempty(s.pvs) ? Bool(s.pvs[1].can_net_meter) : false + pv_can_wholesale = !isempty(s.pvs) ? Bool(s.pvs[1].can_wholesale) : false + + # Battery export capability comes from the processed ElectricStorage struct + _batt_attr = s.storage.attr["ElectricStorage"] + batt_can_net_meter = Bool(_batt_attr.can_net_meter) + batt_can_wholesale = Bool(_batt_attr.can_wholesale) + + # NEM export is credited (approximately) at the retail energy rate; used for cost reporting below. + nem_active = nm_limit_kw > 0 && (pv_can_net_meter || batt_can_net_meter) + month_starts = get_month_transition_timesteps(time_steps_per_hour) # ts_to_month = 8760 array specifying which month each timestep falls in (1-12) @@ -353,6 +376,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict ), "ElectricStorage" => Dict( "storage_to_load_series_kw" => Float64[], + "storage_to_grid_series_kw" => Float64[], "soc_series_fraction" => Float64[], ), "ElectricUtility" => Dict( @@ -364,18 +388,37 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict "load_series_kw" => Float64[], ), ) - energy_cost_series = Float64[] + energy_charge_series = Float64[] # grid purchase (energy) charges per timestep + export_benefit_series = Float64[] # NEM/WHL export credits per timestep (positive = revenue) total_energy_cost = 0.0 + total_export_benefit = 0.0 soc_init_frac = soc_0 # Build MPC post function build_mpc_post(current_horizon_pv, current_horizon_load, current_horizon_energy_rates, current_horizon_emissions, current_horizon_tou_ts, current_horizon_monthly_ts, - tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac) + tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac, + current_horizon_whl_rate) + tariff = Dict( + "energy_rates" => current_horizon_energy_rates, + "tou_demand_rates" => tou_demand_rates, + "tou_demand_ratchet_time_steps" => current_horizon_tou_ts, + "tou_previous_peak_demands" => tou_previous_peak_demands, + "monthly_demand_rates" => monthly_demand_rates, + "time_steps_monthly" => current_horizon_monthly_ts, + "monthly_previous_peak_demands" => monthly_previous_peak_demands, + ) + # WHL (net billing) export: MPCElectricTariff reads a positive `wholesale_rate` and + # builds the :WHL export bin when it is provided. + if current_horizon_whl_rate !== nothing + tariff["wholesale_rate"] = current_horizon_whl_rate + end return Dict( "PV" => Dict( "size_kw" => pv_kw, "production_factor_series" => current_horizon_pv, + "can_net_meter" => pv_can_net_meter, + "can_wholesale" => pv_can_wholesale, ), "ElectricStorage" => Dict( "size_kw" => batt_kw, @@ -384,20 +427,16 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict "discharge_efficiency" => discharge_eff, "soc_init_fraction" => soc_init_frac, "soc_min_fraction" => soc_min, + "can_net_meter" => batt_can_net_meter, + "can_wholesale" => batt_can_wholesale, ), "ElectricLoad" => Dict( "loads_kw" => current_horizon_load, ), - "ElectricTariff" => Dict( - "energy_rates" => current_horizon_energy_rates, - "tou_demand_rates" => tou_demand_rates, - "tou_demand_ratchet_time_steps" => current_horizon_tou_ts, - "tou_previous_peak_demands" => tou_previous_peak_demands, - "monthly_demand_rates" => monthly_demand_rates, - "time_steps_monthly" => current_horizon_monthly_ts, - "monthly_previous_peak_demands" => monthly_previous_peak_demands, - ), + "ElectricTariff" => tariff, + # net_metering_limit_kw drives the NEM export bin in MPCElectricTariff (NEM on if > 0) "ElectricUtility" => Dict( + "net_metering_limit_kw" => nm_limit_kw, "emissions_factor_series_lb_CO2_per_kwh" => current_horizon_emissions, ), ) @@ -411,6 +450,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict current_horizon_load = slice_data(loads_kw, idx, end_ts) current_horizon_energy_rates = slice_data(energy_rates, idx, end_ts) current_horizon_emissions = slice_data(co2_grid_emissions_series, idx, end_ts) + current_horizon_whl_rate = whl_rate_full === nothing ? nothing : slice_data(whl_rate_full, idx, end_ts) # List of length n_tou_ratchets, specifies which ts of the current horizon are in each TOU ratchet # by placing values 1 to horizon into the corresponding element of the array based on ratchet number @@ -433,7 +473,8 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict post = build_mpc_post(current_horizon_pv, current_horizon_load, current_horizon_energy_rates, current_horizon_emissions, current_horizon_tou_ts, current_horizon_monthly_ts, - tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac + tou_previous_peak_demands, monthly_previous_peak_demands, soc_init_frac, + current_horizon_whl_rate ) model = get_solver_model(get_solver_model_type(solver_name), @@ -450,6 +491,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict pv_to_grid = haskey(pv_res, "electric_to_grid_series_kw") ? pv_res["electric_to_grid_series_kw"][1] : 0.0 pv_curtailed = haskey(pv_res, "electric_curtailed_series_kw") ? pv_res["electric_curtailed_series_kw"][1] : 0.0 batt_to_load = batt_res["storage_to_load_series_kw"][1] + batt_to_grid = haskey(batt_res, "storage_to_grid_series_kw") ? batt_res["storage_to_grid_series_kw"][1] : 0.0 batt_soc = batt_res["soc_series_fraction"][1] util_to_load = util_res["electric_to_load_series_kw"][1] util_to_batt = util_res["electric_to_storage_series_kw"][1] @@ -460,6 +502,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict push!(dispatch_series["PV"]["electric_to_grid_series_kw"], pv_to_grid) push!(dispatch_series["PV"]["electric_curtailed_series_kw"], pv_curtailed) push!(dispatch_series["ElectricStorage"]["storage_to_load_series_kw"], batt_to_load) + push!(dispatch_series["ElectricStorage"]["storage_to_grid_series_kw"], batt_to_grid) push!(dispatch_series["ElectricStorage"]["soc_series_fraction"], batt_soc) push!(dispatch_series["ElectricUtility"]["electric_to_load_series_kw"], util_to_load) push!(dispatch_series["ElectricUtility"]["electric_to_storage_series_kw"], util_to_batt) @@ -467,10 +510,22 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict co2_grid_emissions_series[idx] * grid_power / time_steps_per_hour) push!(dispatch_series["ElectricLoad"]["load_series_kw"], loads_kw[idx]) - # Running energy costs - step_energy_cost = grid_power * energy_rates[idx] / time_steps_per_hour - push!(energy_cost_series, step_energy_cost) - total_energy_cost += step_energy_cost + # Running electricity costs (reported as separate components; see results below) + # Energy (grid purchase) charge for this timestep + step_energy_charge = grid_power * energy_rates[idx] / time_steps_per_hour + + # Export credit for this timestep. NEM credits at the retail energy rate; otherwise WHL credits + # at the wholesale rate. NOTE: when both NEM and WHL are available the model chooses per horizon + # and the executed-timestep bin split is not returned, so this can misprice such timesteps. + step_export_kw = pv_to_grid + batt_to_grid + export_rate_idx = nem_active ? energy_rates[idx] : + (whl_rate_full !== nothing ? whl_rate_full[idx] : 0.0) + step_export_benefit = step_export_kw * export_rate_idx / time_steps_per_hour + + push!(energy_charge_series, step_energy_charge) + push!(export_benefit_series, step_export_benefit) + total_energy_cost += step_energy_charge + total_export_benefit += step_export_benefit soc_init_frac = batt_soc @@ -506,12 +561,19 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict "ElectricUtility" => dispatch_series["ElectricUtility"], "ElectricLoad" => dispatch_series["ElectricLoad"], "ElectricTariff" => Dict( - "total_energy_cost" => total_energy_cost, - "energy_cost_series_per_timestep" => energy_cost_series, - "total_tou_demand_cost" => tou_demand_cost_total, - "total_monthly_demand_cost" => monthly_demand_cost_total, - "tou_peaks_by_ratchet_kw" => tou_previous_peak_demands, - "monthly_peaks_kw" => monthly_previous_peak_demands, + # --- Cost components (before tax); combine for the total bill below --- + "total_energy_cost" => total_energy_cost, # grid purchases (energy) only + "total_export_benefit" => total_export_benefit, # NEM/WHL credits (positive = revenue) + "total_tou_demand_cost" => tou_demand_cost_total, + "total_monthly_demand_cost" => monthly_demand_cost_total, + # --- Total electricity bill = energy charge - export benefit + demand charges --- + "total_electricity_bill" => total_energy_cost - total_export_benefit + + tou_demand_cost_total + monthly_demand_cost_total, + # --- Per-timestep series (energy/exports only; demand is a peak-based charge) --- + "energy_charge_series_per_timestep" => energy_charge_series, + "export_benefit_series_per_timestep" => export_benefit_series, + "tou_peaks_by_ratchet_kw" => tou_previous_peak_demands, + "monthly_peaks_kw" => monthly_previous_peak_demands, ), ) ) From 181a316ddeced82db04204c59f605c6177423bdd Mon Sep 17 00:00:00 2001 From: adfarth Date: Fri, 17 Jul 2026 14:42:38 -0700 Subject: [PATCH 58/66] updt branch --- julia_src/Manifest.toml | 4 ++-- julia_src/mpc.jl | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/julia_src/Manifest.toml b/julia_src/Manifest.toml index 47ac83bda..1cb08ee62 100644 --- a/julia_src/Manifest.toml +++ b/julia_src/Manifest.toml @@ -948,11 +948,11 @@ version = "1.11.0" [[deps.REopt]] deps = ["ArchGDAL", "CSV", "CoolProp", "DataFrames", "Dates", "DelimitedFiles", "HTTP", "JLD", "JSON", "JuMP", "LinDistFlow", "LinearAlgebra", "Logging", "MathOptInterface", "Requires", "Roots", "Statistics", "TestEnv"] -git-tree-sha1 = "1e54ef8f58ddcf9d66f392cb76fdd78123d7d2ad" +git-tree-sha1 = "539750260abec1f7b965670e498ee2618548e148" repo-rev = "dispatch-options" repo-url = "https://github.com/NatLabRockies/REopt.jl.git" uuid = "d36ad4e8-d74a-4f7a-ace1-eaea049febf6" -version = "0.59.2" +version = "0.59.3" [[deps.Random]] deps = ["SHA"] diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index a85616e04..dcbea8fa8 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -304,7 +304,6 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # Extract tariff inputs relevant to MPC (use first tier only if tiered rates) # TODO: Are all of these relevant? Any missing inputs? # TODO: Implement lookback (demand_lookback_months, demand_lookback_percent, demand_lookback_range) Ignoring coincident peak charges for now - # TODO: Need to think through NEM or passing back export values (wholesale_rate, export_rate_beyond_net_metering_limit) energy_rates = Float64.(s.electric_tariff.energy_rates[:, 1]) monthly_demand_rates = isempty(s.electric_tariff.monthly_demand_rates) ? zeros(Float64, 12) : Float64.(s.electric_tariff.monthly_demand_rates[:, 1]) @@ -388,7 +387,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict "load_series_kw" => Float64[], ), ) - energy_charge_series = Float64[] # grid purchase (energy) charges per timestep + energy_cost_series = Float64[] # grid purchase (energy) charges per timestep export_benefit_series = Float64[] # NEM/WHL export credits per timestep (positive = revenue) total_energy_cost = 0.0 total_export_benefit = 0.0 @@ -522,7 +521,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict (whl_rate_full !== nothing ? whl_rate_full[idx] : 0.0) step_export_benefit = step_export_kw * export_rate_idx / time_steps_per_hour - push!(energy_charge_series, step_energy_charge) + push!(energy_cost_series, step_energy_charge) push!(export_benefit_series, step_export_benefit) total_energy_cost += step_energy_charge total_export_benefit += step_export_benefit @@ -570,7 +569,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict "total_electricity_bill" => total_energy_cost - total_export_benefit + tou_demand_cost_total + monthly_demand_cost_total, # --- Per-timestep series (energy/exports only; demand is a peak-based charge) --- - "energy_charge_series_per_timestep" => energy_charge_series, + "energy_cost_series_per_timestep" => energy_cost_series, "export_benefit_series_per_timestep" => export_benefit_series, "tou_peaks_by_ratchet_kw" => tou_previous_peak_demands, "monthly_peaks_kw" => monthly_previous_peak_demands, From ac1b14954b01e2409a3708c763c4669972497c53 Mon Sep 17 00:00:00 2001 From: adfarth Date: Fri, 17 Jul 2026 15:47:58 -0700 Subject: [PATCH 59/66] update REopt --- julia_src/Manifest.toml | 2 +- julia_src/mpc.jl | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/julia_src/Manifest.toml b/julia_src/Manifest.toml index 1cb08ee62..ceb0887ae 100644 --- a/julia_src/Manifest.toml +++ b/julia_src/Manifest.toml @@ -948,7 +948,7 @@ version = "1.11.0" [[deps.REopt]] deps = ["ArchGDAL", "CSV", "CoolProp", "DataFrames", "Dates", "DelimitedFiles", "HTTP", "JLD", "JSON", "JuMP", "LinDistFlow", "LinearAlgebra", "Logging", "MathOptInterface", "Requires", "Roots", "Statistics", "TestEnv"] -git-tree-sha1 = "539750260abec1f7b965670e498ee2618548e148" +git-tree-sha1 = "ec4dad03ea876407e69a384cecc62c29be858405" repo-rev = "dispatch-options" repo-url = "https://github.com/NatLabRockies/REopt.jl.git" uuid = "d36ad4e8-d74a-4f7a-ace1-eaea049febf6" diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index dcbea8fa8..22f524da5 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -194,10 +194,18 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict error("When using MPC (daily_foresight_optimized dispatch), only PV and ElectricStorage are supported technologies. " * "Unsupported inputs found: $(join(unsupported_keys, ", ")).") end - # Error if multiple PVs + # PV can be provided as a Dict (single system) or an array of Dicts. + # MPC only supports a single PV, so error on multiple PVs and normalize a + # one-element array down to a Dict so downstream code can treat d["PV"] as a Dict. # TODO: Handle multiple PVs - if haskey(d, "PV") && length(d["PV"]) > 1 - error("MPC: Multiple PV systems are not supported.") + if haskey(d, "PV") && isa(d["PV"], AbstractArray) + if length(d["PV"]) > 1 + error("MPC: Multiple PV systems are not supported.") + elseif length(d["PV"]) == 1 + d["PV"] = d["PV"][1] + else + delete!(d, "PV") # empty PV array -> treat as no PV + end end # Error if unsupported CO2/renewable-fraction constraints are set @@ -218,7 +226,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict # TODO: Add warnings for REopt inputs and scenarios that are not modeled in MPC (e.g., coincident peak charges, demand lookback, etc.) @warn "Using MPC to determine dispatch. MPC does not model: tiered electricity rates; rates will be flattened to the first tier." - # TODO: Error if rate tariff contains lookbacks. + # TODO: Error if rate tariff contains lookbacks or coincident peak charges. # TODO: Test with outage inputs before enabling this warning. # # Warning for outage inputs (MPC does not model outages) @@ -314,6 +322,9 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict tou_previous_peak_demands = zeros(Float64, n_tou_ratchets) # Tracks past TOU peak demand per ratchet monthly_previous_peak_demands = zeros(Float64, 12) # Tracks past monthly peak demand + println("monthly_demand_rates: ", monthly_demand_rates) + + # Extract storage efficiency and SOC defaults from processed inputs rect_eff = Float64(s.storage.attr["ElectricStorage"].rectifier_efficiency_fraction) inv_eff = Float64(s.storage.attr["ElectricStorage"].inverter_efficiency_fraction) From 41468b13ecad12ae46a8bc68faadd44539c13090 Mon Sep 17 00:00:00 2001 From: adfarth Date: Fri, 17 Jul 2026 16:56:30 -0700 Subject: [PATCH 60/66] update for demand rates --- julia_src/mpc.jl | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/julia_src/mpc.jl b/julia_src/mpc.jl index 22f524da5..e9f8b5c1d 100644 --- a/julia_src/mpc.jl +++ b/julia_src/mpc.jl @@ -181,7 +181,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict - ElectricUtility: Grid dispatch series and emissions - ElectricLoad: Load profile used - ElectricTariff: Separate cost components (total_energy_cost, total_export_benefit, - total_tou_demand_cost, total_monthly_demand_cost), a combined total_electricity_bill, + total_tou_demand_cost, total_non_tou_monthly_demand_cost), a combined total_electricity_bill, per-timestep energy/export series, and peak demands by month/ratchet """ @@ -264,7 +264,19 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict model_inputs = nothing try model_inputs = reoptjl.REoptInputs(sizing_post) - @info "Successfully processed REopt inputs." + + # REoptInputs returns an error Dict (rather than throwing) when input validation fails. + # Surface those messages instead of falling through to get_technology_sizes!, which expects + # a REoptInputs and would otherwise raise a confusing MethodError. + if isa(model_inputs, Dict) + @error "REopt input validation failed during MPC pre-solve." messages=get(model_inputs, "Messages", Dict()) + return build_mpc_response( + "error", + messages = get(model_inputs, "Messages", Dict("errors" => ["REopt input validation failed."])) + ) + else + @info "Successfully processed REopt inputs." + end catch e @error "Something went wrong during REopt inputs processing!" exception=(e, catch_backtrace()) return build_mpc_response( @@ -272,7 +284,9 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict messages = Dict("errors" => [sprint(showerror, e)]) ) end - + + + # MPC requires fixed PV and battery sizes. If not provided, call REopt first in a sizing run. technology_sizes = get_technology_sizes!(d, model_inputs, solver_settings) s = model_inputs.s # Access the processed Scenario struct @@ -412,7 +426,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict tariff = Dict( "energy_rates" => current_horizon_energy_rates, "tou_demand_rates" => tou_demand_rates, - "tou_demand_ratchet_time_steps" => current_horizon_tou_ts, + "tou_demand_time_steps" => current_horizon_tou_ts, "tou_previous_peak_demands" => tou_previous_peak_demands, "monthly_demand_rates" => monthly_demand_rates, "time_steps_monthly" => current_horizon_monthly_ts, @@ -575,7 +589,7 @@ function get_mpc_results!(d::Dict; solver_name::String="HiGHS")::Dict "total_energy_cost" => total_energy_cost, # grid purchases (energy) only "total_export_benefit" => total_export_benefit, # NEM/WHL credits (positive = revenue) "total_tou_demand_cost" => tou_demand_cost_total, - "total_monthly_demand_cost" => monthly_demand_cost_total, + "total_non_tou_monthly_demand_cost" => monthly_demand_cost_total, # --- Total electricity bill = energy charge - export benefit + demand charges --- "total_electricity_bill" => total_energy_cost - total_export_benefit + tou_demand_cost_total + monthly_demand_cost_total, From 43f4cce03010f6523e22cccf0d17f8f9adef9cf1 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:54:15 -0600 Subject: [PATCH 61/66] Preparing for production deploy. --- config/vault_secrets.json.tmpl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/vault_secrets.json.tmpl b/config/vault_secrets.json.tmpl index 483cdcba8..6572c5e27 100644 --- a/config/vault_secrets.json.tmpl +++ b/config/vault_secrets.json.tmpl @@ -35,8 +35,8 @@ {{ with (datasource "vault" (printf "reopt-api/%s/web" $deploy_env)).data }} {{ $secrets = coll.Merge (coll.Dict "SECRET_ASHRAE_TMY_NLR_API_KEY" .ashrae_tmy_nlr_api_key - "SECRET_DB_HOST" .db_host_v18 - "SECRET_DB_NAME" .db_name_v18 + "SECRET_DB_HOST" .db_host + "SECRET_DB_NAME" .db_name "SECRET_DEVELOPER_NLR_GOV_API_KEY" .developer_nlr_gov_api_key "SECRET_DJANGO_SECRET_KEY" .django_secret_key "SECRET_PVWATTS_NLR_API_KEY" .pvwatts_nlr_api_key @@ -47,16 +47,16 @@ {{ with (datasource "vault" (printf "reopt-api/%s/db-migrate" $deploy_env)).data }} {{ $secrets = coll.Merge (coll.Dict - "SECRET_DB_PASSWORD" .db_password_v18 - "SECRET_DB_USERNAME" .db_username_v18 + "SECRET_DB_PASSWORD" .db_password + "SECRET_DB_USERNAME" .db_username ) $secrets }} {{ end }} {{ if (eq (env.Getenv "DB_MIGRATE") "true") }} {{ with (datasource "vault" (printf "reopt-db/%s/db-superuser" $deploy_env)).data }} {{ $secrets = coll.Merge (coll.Dict - "SECRET_DB_SUPERUSER_PASSWORD" .db_password_v18 - "SECRET_DB_SUPERUSER_USERNAME" .db_username_v18 + "SECRET_DB_SUPERUSER_PASSWORD" .db_password + "SECRET_DB_SUPERUSER_USERNAME" .db_username ) $secrets }} {{ end }} {{ end }} From fed227f9473781a7885f63476db1565b13a00c13 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:02:00 -0600 Subject: [PATCH 62/66] Perform dry-run deploys to see how new deployments will work in main. --- .github/workflows/deploy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fe42733d0..8020eaf10 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -99,6 +99,9 @@ jobs: metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} vault-registry-credentials-path: secret/data/deploy/common/aws-ecr vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + # FIXME: Temporarily disable deploys and perform a dry run so we can see + # how all this would get rolled out once we merge with `main`. + dry-run: true secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} @@ -114,6 +117,9 @@ jobs: metadata: ${{ needs.deploy-metadata.outputs.production-metadata }} vault-registry-credentials-path: secret/data/deploy/common/aws-ecr vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt + # FIXME: Temporarily disable deploys and perform a dry run so we can see + # how all this would get rolled out once we merge with `main`. + dry-run: true secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} From b24bca681d806dc32de65b76838e149a70241b8f Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:41:14 -0600 Subject: [PATCH 63/66] Reenable staging deploys after dry-run tests. Part of the larger, manual rollout of the deployment changes in https://github.com/NatLabRockies/REopt_API/pull/714 --- .github/workflows/deploy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8020eaf10..9be14ee7a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -99,9 +99,6 @@ jobs: metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} vault-registry-credentials-path: secret/data/deploy/common/aws-ecr vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt - # FIXME: Temporarily disable deploys and perform a dry run so we can see - # how all this would get rolled out once we merge with `main`. - dry-run: true secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} From 43f1aff93c69dd2704d273ef96e141edc26a98f3 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:13:14 -0600 Subject: [PATCH 64/66] Reenable production deploys after dry-run tests. Part of the larger, manual rollout of the deployment changes in https://github.com/NatLabRockies/REopt_API/pull/714 --- .github/workflows/deploy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9be14ee7a..fe42733d0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -114,9 +114,6 @@ jobs: metadata: ${{ needs.deploy-metadata.outputs.production-metadata }} vault-registry-credentials-path: secret/data/deploy/common/aws-ecr vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt - # FIXME: Temporarily disable deploys and perform a dry run so we can see - # how all this would get rolled out once we merge with `main`. - dry-run: true secrets: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} From 8847441c1db2b633c73a6f5fbd471355ca0a8884 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:38:09 -0600 Subject: [PATCH 65/66] Remove nrel.gov ingress. Remove the duplicative nrel.gov ingress now that everything with developer.nlr.gov is cutover to use the nlr.gov ingress. --- config/deploy/WebDeployment.pkl | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl index afbdc38a7..a3ec1cf94 100644 --- a/config/deploy/WebDeployment.pkl +++ b/config/deploy/WebDeployment.pkl @@ -366,34 +366,6 @@ resources: Listing = new { new TadaWebIngress { tadaName = "django" tadaServicePort = 8000 - - // Configure the ingress to listen on the old `*.nrel.gov` domains (in - // addition to the new `nlr.gov` domains). We can remove this once the - // nrel.gov domain is fully gone. - local nrelHost = deployMetadata.`url-host`.replaceLast(".nlr.gov", ".nrel.gov") - spec { - rules { - new { - host = nrelHost!! - http { - paths { - new { - path = "/" - pathType = "Prefix" - backend { - service { - name = djangoService.metadata.name - port { - number = 8000 - } - } - } - } - } - } - } - } - } } new TadaWebPodDisruptionBudget {} From 31ca82237e21bfe4b023530714c55628206d364d Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:07:13 -0600 Subject: [PATCH 66/66] Fix ingress to set explicit host for production ingress This was a copy-paste issue from staging, but the production variable name differs. --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fe42733d0..74cab7037 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -73,7 +73,7 @@ jobs: app-name: reopt-api rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} - branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_url_host }} + url-host: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_url_host }} undeploy-staging: name: Undeploy Staging