diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 000000000..1c7281722 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,17 @@ +name: smoke + +on: + pull_request: + +jobs: + smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install runtime dependencies + run: python -m pip install -r requirements.txt + - name: Run clean-base verification + run: python scripts/verify_clean_base.py diff --git a/API/Classes/Base/Config.py b/API/Classes/Base/Config.py index 49a889043..3d14b7d4d 100644 --- a/API/Classes/Base/Config.py +++ b/API/Classes/Base/Config.py @@ -36,6 +36,23 @@ def validate_path(base_dir, user_input): SYSTEM = platform.system() + +def _default_runtime_dir(): + override = os.environ.get("MUIOGO_RUNTIME_DIR", "").strip() + if override: + return Path(override).expanduser() + + if SYSTEM == "Windows": + base_dir = Path(os.environ.get("LOCALAPPDATA", str(Path.home()))) + elif SYSTEM == "Darwin": + base_dir = Path.home() / "Library" / "Logs" + else: + base_dir = Path( + os.environ.get("XDG_STATE_HOME", str(Path.home() / ".local" / "state")) + ) + + return base_dir / "MUIOGO" + # S3_BUCKET = os.environ.get("S3_BUCKET") # S3_KEY = os.environ.get("S3_KEY") # S3_SECRET = os.environ.get("S3_SECRET") @@ -62,6 +79,10 @@ def validate_path(base_dir, user_input): CLASS_FOLDER = WEBAPP_PATH / "Classes" SOLVERs_FOLDER = WEBAPP_PATH / "SOLVERs" EXTRACT_FOLDER = BASE_DIR +RUNTIME_DIR = _default_runtime_dir() +LOG_DIR = RUNTIME_DIR / "logs" +APP_LOG_FILE = LOG_DIR / "app.log" +SOLVER_MODEL_FILE = SOLVERs_FOLDER / "model.v.5.4.txt" # Ensure DataStorage exists DATA_STORAGE.mkdir(parents=True, exist_ok=True) diff --git a/API/Classes/Case/DataFileClass.py b/API/Classes/Case/DataFileClass.py index bd5cc38b1..470ab7963 100644 --- a/API/Classes/Case/DataFileClass.py +++ b/API/Classes/Case/DataFileClass.py @@ -1,4 +1,5 @@ from pathlib import Path +import logging import pandas as pd import traceback import json, shutil, os, time, subprocess @@ -9,6 +10,11 @@ from Classes.Case.OsemosysClass import Osemosys from Classes.Base.FileClass import File from Classes.Base.CustomThreadClass import CustomThread + + +logger = logging.getLogger(__name__) + + class DataFile(Osemosys): # def __init__(self, case): # Osemosys.__init__(self, case) @@ -790,19 +796,16 @@ def generateDatafile( self, caserunname ): def createCaseRun(self, caserunname, data): try: - caseRunPath = Path(Config.DATA_STORAGE,self.case,'res', caserunname) - csvPath = Path(Config.DATA_STORAGE,self.case,'res', caserunname, 'csv') - resDataPath = Path(Config.DATA_STORAGE,self.case,'view', 'resData.json') - - if not os.path.exists(caseRunPath): - os.makedirs(caseRunPath) - os.makedirs(csvPath) - if not os.path.exists(resDataPath): - File.writeFile( data, resDataPath) - else: - resData = File.readFile(resDataPath) - resData['osy-cases'].append(data) - File.writeFile( resData, resDataPath) + caseRunPath = self.resultsPath / caserunname + csvPath = caseRunPath / "csv" + + if not caseRunPath.exists(): + caseRunPath.mkdir(parents=True, exist_ok=False) + csvPath.mkdir(parents=True, exist_ok=True) + case_runs = self._case_run_entries() + case_runs.append(data) + self.resDataPath.parent.mkdir(parents=True, exist_ok=True) + File.writeFile(self.resData, self.resDataPath) response = { "message": "You have created a case run!", "status_code": "success" @@ -822,8 +825,7 @@ def createCaseRun(self, caserunname, data): def deleteScenarioCaseRuns(self, scenarioId): try: - resData = File.readFile(self.resDataPath) - cases = resData['osy-cases'] + cases = self._case_run_entries() for cs in cases: for sc in cs['Scenarios']: @@ -831,7 +833,7 @@ def deleteScenarioCaseRuns(self, scenarioId): cs['Scenarios'].remove(sc) - File.writeFile(resData, self.resDataPath) + File.writeFile(self.resData, self.resDataPath) response = { "message": "You have deleted scenario from caseruns!", "status_code": "success" @@ -847,41 +849,36 @@ def deleteScenarioCaseRuns(self, scenarioId): def updateCaseRun(self, caserunname, oldcaserunname, data): try: - caseRunPath = Path(Config.DATA_STORAGE,self.case,'res', oldcaserunname) - newcaseRunPath = Path(Config.DATA_STORAGE,self.case,'res', caserunname) - csvPath = Path(Config.DATA_STORAGE,self.case,'res', caserunname, 'csv') - resDataPath = Path(Config.DATA_STORAGE,self.case,'view', 'resData.json') + caseRunPath = self.resultsPath / oldcaserunname + newcaseRunPath = self.resultsPath / caserunname + csvPath = newcaseRunPath / "csv" - if not os.path.exists(newcaseRunPath): + if not newcaseRunPath.exists(): os.rename(caseRunPath, newcaseRunPath) - if not os.path.exists(csvPath): - os.makedirs(csvPath) + if not csvPath.exists(): + csvPath.mkdir(parents=True, exist_ok=True) - resData = File.readFile(resDataPath) - - resdata = resData['osy-cases'] + resdata = self._case_run_entries() for i, case in enumerate(resdata): if case['Case'] == oldcaserunname: - resData['osy-cases'][i] = data + self.resData['osy-cases'][i] = data - File.writeFile( resData, resDataPath) + File.writeFile(self.resData, self.resDataPath) response = { "message": "You have updated a case run!", "status_code": "success" } - elif os.path.exists(newcaseRunPath) and caserunname==oldcaserunname: - if not os.path.exists(csvPath): - os.makedirs(csvPath) - - resData = File.readFile(resDataPath) + elif newcaseRunPath.exists() and caserunname == oldcaserunname: + if not csvPath.exists(): + csvPath.mkdir(parents=True, exist_ok=True) - resdata = resData['osy-cases'] + resdata = self._case_run_entries() for i, case in enumerate(resdata): if case['Case'] == oldcaserunname: - resData['osy-cases'][i] = data + self.resData['osy-cases'][i] = data - File.writeFile( resData, resDataPath) + File.writeFile(self.resData, self.resDataPath) response = { "message": "You have updated a case run!", "status_code": "success" @@ -899,6 +896,24 @@ def updateCaseRun(self, caserunname, oldcaserunname, data): except OSError: raise OSError + def _case_run_entries(self): + if not isinstance(self.resData, dict): + self.resData = {"osy-cases": []} + + case_runs = self.resData.get("osy-cases") + if not isinstance(case_runs, list): + case_runs = [] + self.resData["osy-cases"] = case_runs + + return case_runs + + def _default_view_data(self): + view_definitions = {} + for group, lists in self.VARIABLES.items(): + for item in lists: + view_definitions[item["id"]] = [] + return {"osy-views": view_definitions} + def deleteCaseResultsJSON(self, caserunname): try: csvPath = Path(self.resultsPath, caserunname, "csv") @@ -923,18 +938,24 @@ def deleteCaseResultsJSON(self, caserunname): def deleteCaseRun(self, caserunname, resultsOnly): try: - #caseRunPath = Path(Config.DATA_STORAGE,self.case,'res', caserunname) - #resDataPath = Path(Config.DATA_STORAGE,self.case,'view', 'resData.json') - + caseRunPath = self.resultsPath / caserunname + if caseRunPath.exists(): + if not resultsOnly: + shutil.rmtree(caseRunPath) + else: + for item in caseRunPath.iterdir(): + if item.is_file() or item.is_symlink(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) if not resultsOnly: - resData = File.readFile(self.resDataPath) - - for obj in resData['osy-cases']: - if obj['Case'] == caserunname: - resData['osy-cases'].remove(obj) - - File.writeFile( resData, self.resDataPath) + self.resData["osy-cases"] = [ + case + for case in self._case_run_entries() + if case.get("Case") != caserunname + ] + File.writeFile(self.resData, self.resDataPath) #delete from view folder for group, array in self.VARIABLES.items(): @@ -960,8 +981,9 @@ def deleteCaseRun(self, caserunname, resultsOnly): except OSError: raise OSError - def cleanUp(self): + def _legacy_cleanUp_unused(self): try: + return self.cleanUp() #delete from view folder # moramo izbrisati res i view folder ostaviti samo resData.json i viewDefinitions.json @@ -970,11 +992,10 @@ def cleanUp(self): # self.viewFolderPath = Path(Config.DATA_STORAGE,case,'view') # folder_path = "C:/putanja/do/foldera" - for caserunname in os.listdir( self.resultsPath): - caserunname_path = os.path.join(self.resultsPath, caserunname) - # Skip files such as .DS_Store that can appear on macOS. - if not os.path.isdir(caserunname_path): - continue + if self.resultsPath.exists() and self.resultsPath.is_dir(): + for case_run_path in self.resultsPath.iterdir(): + if not case_run_path.is_dir(): + continue for carerunData in os.listdir( caserunname_path): file_path = os.path.join(caserunname_path, carerunData) try: @@ -1022,6 +1043,54 @@ def cleanUp(self): raise IndexError except OSError: raise OSError + + def cleanUp(self): + try: + if self.resultsPath.exists() and self.resultsPath.is_dir(): + for case_run_path in self.resultsPath.iterdir(): + if not case_run_path.is_dir(): + continue + for result_item in case_run_path.iterdir(): + try: + if result_item.is_file() or result_item.is_symlink(): + result_item.unlink() + elif result_item.is_dir(): + shutil.rmtree(result_item) + except OSError as exc: + logger.warning("Failed to clean result item %s: %s", result_item, exc) + + self.viewFolderPath.mkdir(parents=True, exist_ok=True) + for item in self.viewFolderPath.iterdir(): + if item.name in {"resData.json", "viewDefinitions.json"}: + continue + try: + if item.is_file() or item.is_symlink(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + except OSError as exc: + logger.warning("Failed to clean view item %s: %s", item, exc) + + viewDefPath = self.viewFolderPath / "viewDefinitions.json" + if not viewDefPath.exists(): + File.writeFile(self._default_view_data(), viewDefPath) + + self.resultsPath.mkdir(parents=True, exist_ok=True) + for case_run in self._case_run_entries(): + case_name = case_run.get("Case") + if case_name: + (self.resultsPath / case_name).mkdir(parents=True, exist_ok=True) + + response = { + "message": "You have recycled results!", + "status_code": "success" + } + + return response + except(IOError, IndexError): + raise IndexError + except OSError: + raise OSError def saveView(self, data, param): try: @@ -1069,20 +1138,12 @@ def updateViews(self, data, param): def readDataFile( self, caserunname ): try: - - #f = open(self.dataFile, mode="r") - dataFilePath = Path(Config.DATA_STORAGE, self.case, 'res',caserunname,'data.txt') - if os.path.exists(dataFilePath): - f = open(dataFilePath, mode="r", encoding='utf-8-sig') - data = f.read() - f.close - else: - data = None + dataFilePath = self.resultsPath / caserunname / "data.txt" + if not dataFilePath.exists(): + return None - # f = open(self.dataFile, 'r') - # file_contents = f.read() - # f.close() - return data + with dataFilePath.open(mode="r", encoding="utf-8-sig") as handle: + return handle.read() except(IOError, IndexError): raise IndexError except OSError: diff --git a/API/Classes/Case/OsemosysClass.py b/API/Classes/Case/OsemosysClass.py index 5ef3a5e53..93533df9a 100644 --- a/API/Classes/Case/OsemosysClass.py +++ b/API/Classes/Case/OsemosysClass.py @@ -8,52 +8,51 @@ class Osemosys(): def __init__(self, case): self.case = case - self.PARAMETERS = File.readParamFile(Path(Config.DATA_STORAGE, 'Parameters.json')) - self.VARIABLES = File.readParamFile(Path(Config.DATA_STORAGE, 'Variables.json')) - self.genData = File.readFile(Path(Config.DATA_STORAGE,case,'genData.json')) - self.resData = File.readFile( Path(Config.DATA_STORAGE, case,'view', 'resData.json')) - - #Case.__init__(self, case) - self.casePath = Path(Config.DATA_STORAGE,case) - self.zipPath = Path(Config.DATA_STORAGE,case+'.zip') - - #self.genData = Path(Config.DATA_STORAGE,case,'genData.json') - - self.rPath = Path(Config.DATA_STORAGE,case,'R.json') - self.ryPath = Path(Config.DATA_STORAGE,case,'RY.json') - self.rtPath = Path(Config.DATA_STORAGE,case,'RT.json') - self.rePath = Path(Config.DATA_STORAGE,case,'RE.json') - self.rsPath = Path(Config.DATA_STORAGE,case,'RS.json') - self.rycnPath = Path(Config.DATA_STORAGE,case,'RYCn.json') - self.rytPath = Path(Config.DATA_STORAGE,case,'RYT.json') - self.rysPath = Path(Config.DATA_STORAGE,case,'RYS.json') - self.rytcnPath = Path(Config.DATA_STORAGE,case,'RYTCn.json') - self.rytmPath = Path(Config.DATA_STORAGE,case,'RYTM.json') - self.rytcPath = Path(Config.DATA_STORAGE,case,'RYTC.json') - self.rytcmPath = Path(Config.DATA_STORAGE,case,'RYTCM.json') - self.rytsmPath = Path(Config.DATA_STORAGE,case,'RYTSM.json') - self.rtsmPath = Path(Config.DATA_STORAGE,case,'RTSM.json') - self.rytsPath = Path(Config.DATA_STORAGE,case,'RYTs.json') - self.rydtbPath = Path(Config.DATA_STORAGE,case,'RYDtb.json') - self.rysedtPath = Path(Config.DATA_STORAGE,case,'RYSeDt.json') - self.rycPath = Path(Config.DATA_STORAGE,case,'RYC.json') - self.ryePath = Path(Config.DATA_STORAGE,case,'RYE.json') - self.ryttsPath = Path(Config.DATA_STORAGE,case,'RYTTs.json') - self.ryctsPath = Path(Config.DATA_STORAGE,case,'RYCTs.json') - self.rytePath = Path(Config.DATA_STORAGE,case,'RYTE.json') - self.rytemPath = Path(Config.DATA_STORAGE,case,'RYTEM.json') - - - self.osemosysFile = Path(Config.SOLVERs_FOLDER,'model.v.5.4.txt') - self.osemosysFileOriginal = Path(Config.SOLVERs_FOLDER,'osemosys.txt') + self.storagePath = Path(Config.DATA_STORAGE) + self.casePath = self.storagePath / case + self.resultsPath = self.casePath / "res" + self.viewFolderPath = self.casePath / "view" + self.resDataPath = self.viewFolderPath / "resData.json" + + self.PARAMETERS = File.readParamFile(self.storagePath / "Parameters.json") + self.VARIABLES = File.readParamFile(self.storagePath / "Variables.json") + self.genData = File.readFile(self.casePath / "genData.json") + if self.resDataPath.exists(): + self.resData = File.readFile(self.resDataPath) + else: + self.resData = {"osy-cases": []} + + self.zipPath = self.storagePath / f"{case}.zip" + + self.rPath = self.casePath / "R.json" + self.ryPath = self.casePath / "RY.json" + self.rtPath = self.casePath / "RT.json" + self.rePath = self.casePath / "RE.json" + self.rsPath = self.casePath / "RS.json" + self.rycnPath = self.casePath / "RYCn.json" + self.rytPath = self.casePath / "RYT.json" + self.rysPath = self.casePath / "RYS.json" + self.rytcnPath = self.casePath / "RYTCn.json" + self.rytmPath = self.casePath / "RYTM.json" + self.rytcPath = self.casePath / "RYTC.json" + self.rytcmPath = self.casePath / "RYTCM.json" + self.rytsmPath = self.casePath / "RYTSM.json" + self.rtsmPath = self.casePath / "RTSM.json" + self.rytsPath = self.casePath / "RYTs.json" + self.rydtbPath = self.casePath / "RYDtb.json" + self.rysedtPath = self.casePath / "RYSeDt.json" + self.rycPath = self.casePath / "RYC.json" + self.ryePath = self.casePath / "RYE.json" + self.ryttsPath = self.casePath / "RYTTs.json" + self.ryctsPath = self.casePath / "RYCTs.json" + self.rytePath = self.casePath / "RYTE.json" + self.rytemPath = self.casePath / "RYTEM.json" + + self.osemosysFile = Path(Config.SOLVER_MODEL_FILE) + self.osemosysFileOriginal = Path(Config.SOLVERs_FOLDER, "osemosys.txt") self._glpkFolder = None self._cbcFolder = None - self.resultsPath = Path(Config.DATA_STORAGE,case,'res') - self.viewFolderPath = Path(Config.DATA_STORAGE,case,'view') - - self.resDataPath = Path(Config.DATA_STORAGE,case,'view', 'resData.json') - # self.resPath = Path(Config.DATA_STORAGE,case,'res', 'csv') #self.dataFile = Path(Config.DATA_STORAGE,case, 'res','data.txt') diff --git a/API/Routes/DataFile/DataFileRoute.py b/API/Routes/DataFile/DataFileRoute.py index 2b7e98eff..f1cf18f8e 100644 --- a/API/Routes/DataFile/DataFileRoute.py +++ b/API/Routes/DataFile/DataFileRoute.py @@ -1,10 +1,32 @@ -from flask import Blueprint, jsonify, request, send_file, session +from flask import Blueprint, Response, jsonify, request, send_file, session from pathlib import Path -import shutil, datetime, time, os +import logging +import time from Classes.Case.DataFileClass import DataFile from Classes.Base import Config datafile_api = Blueprint('DataFileRoute', __name__) +logger = logging.getLogger(__name__) + + +def _resolve_allowed_file(allowed_root: Path, candidate: Path) -> Path: + safe_root = allowed_root.resolve(strict=False) + safe_path = candidate.resolve(strict=False) + try: + safe_path.relative_to(safe_root) + except ValueError as exc: + raise PermissionError(f"Refusing to read outside {safe_root}") from exc + + if not safe_path.is_file(): + raise FileNotFoundError(safe_path) + + return safe_path + + +def _read_text_response(allowed_root: Path, candidate: Path) -> Response: + safe_path = _resolve_allowed_file(allowed_root, candidate) + text = safe_path.read_text(encoding="utf-8", errors="replace") + return Response(text, content_type="text/plain; charset=utf-8") @datafile_api.route("/generateDataFile", methods=['POST']) def generateDataFile(): @@ -64,17 +86,6 @@ def deleteCaseRun(): if not casename: return jsonify({'message': 'No model selected.', 'status_code': 'error'}), 400 - casePath = Path(Config.DATA_STORAGE, casename, 'res', caserunname) - if not resultsOnly: - shutil.rmtree(casePath) - else: - for item in os.listdir(casePath): - item_path = os.path.join(casePath, item) - if os.path.isfile(item_path) or os.path.islink(item_path): - os.remove(item_path) - elif os.path.isdir(item_path): - shutil.rmtree(item_path) - caserun = DataFile(casename) response = caserun.deleteCaseRun(caserunname, resultsOnly) return jsonify(response), 200 @@ -141,6 +152,26 @@ def readDataFile(): return jsonify(response), 200 except(IOError): return jsonify('No existing cases!'), 404 + + +@datafile_api.route("/readModelFile", methods=['GET']) +def readModelFile(): + try: + return _read_text_response(Config.SOLVERs_FOLDER, Config.SOLVER_MODEL_FILE) + except FileNotFoundError: + return jsonify({'message': 'Model file not found.', 'status_code': 'error'}), 404 + except PermissionError: + return jsonify({'message': 'Access denied.', 'status_code': 'error'}), 403 + + +@datafile_api.route("/readLogFile", methods=['GET']) +def readLogFile(): + try: + return _read_text_response(Config.LOG_DIR, Config.APP_LOG_FILE) + except FileNotFoundError: + return jsonify({'message': 'Log file not found.', 'status_code': 'error'}), 404 + except PermissionError: + return jsonify({'message': 'Access denied.', 'status_code': 'error'}), 403 @datafile_api.route("/validateInputs", methods=['POST']) def validateInputs(): @@ -218,8 +249,20 @@ def run(): casename = request.json['casename'] caserunname = request.json['caserunname'] solver = request.json['solver'] + logger.info( + "Starting optimization for model=%s caserun=%s solver=%s", + casename, + caserunname, + solver, + ) txtFile = DataFile(casename) - response = txtFile.run(solver, caserunname) + response = txtFile.run(solver, caserunname) + logger.info( + "Finished optimization for model=%s caserun=%s status=%s", + casename, + caserunname, + response.get("status_code", "unknown"), + ) return jsonify(response), 200 # except Exception as ex: # print(ex) @@ -236,11 +279,18 @@ def batchRun(): cases = request.json['cases'] if modelname != None: + logger.info("Starting batch run for model=%s case_count=%s", modelname, len(cases)) txtFile = DataFile(modelname) for caserun in cases: + logger.info("Generating data file for model=%s caserun=%s", modelname, caserun) txtFile.generateDatafile(caserun) response = txtFile.batchRun( 'CBC', cases) + logger.info( + "Finished batch run for model=%s status=%s", + modelname, + response.get("status", "unknown"), + ) end = time.time() response['time'] = end-start return jsonify(response), 200 @@ -254,8 +304,10 @@ def cleanUp(): if modelname != None: model = DataFile(modelname) - response = model.cleanUp() + logger.info("Starting clean up for model=%s", modelname) + response = model.cleanUp() + logger.info("Finished clean up for model=%s", modelname) return jsonify(response), 200 except(IOError): - return jsonify('Error!'), 404 \ No newline at end of file + return jsonify('Error!'), 404 diff --git a/API/app.py b/API/app.py index 458aad034..bf76dc8ab 100644 --- a/API/app.py +++ b/API/app.py @@ -3,6 +3,8 @@ import os import secrets import sys +import warnings +from logging.handlers import TimedRotatingFileHandler # Fail fast: unsupported Python hits cryptic pandas/numpy import errors without this. SUPPORTED_PYTHON_MIN = (3, 10) @@ -60,13 +62,57 @@ # template_dir = 'WebAPP' # static_dir = '../WebAPP' + +def _configure_logging(): + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") + + has_console_handler = any( + isinstance(handler, logging.StreamHandler) + and not isinstance(handler, logging.FileHandler) + for handler in root_logger.handlers + ) + if not has_console_handler: + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + root_logger.addHandler(console_handler) + + try: + Config.LOG_DIR.mkdir(parents=True, exist_ok=True) + log_path = Config.APP_LOG_FILE.resolve(strict=False) + has_file_handler = any( + isinstance(handler, logging.FileHandler) + and Path(getattr(handler, "baseFilename", "")).resolve(strict=False) == log_path + for handler in root_logger.handlers + ) + if not has_file_handler: + file_handler = TimedRotatingFileHandler( + log_path, + when="midnight", + interval=1, + backupCount=7, + encoding="utf-8", + ) + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + except OSError as exc: + root_logger.warning("File logging disabled for %s: %s", Config.APP_LOG_FILE, exc) + + logging.captureWarnings(True) + warnings.simplefilter("default") + return logging.getLogger(__name__) + + +logger = _configure_logging() + app = Flask(__name__, static_url_path='', static_folder=static_dir, template_folder=template_dir) app.permanent_session_lifetime = timedelta(days=5) secret_key = os.environ.get("MUIOGO_SECRET_KEY", "").strip() if not secret_key: secret_key = secrets.token_hex(32) - logging.warning( + logger.warning( "MUIOGO_SECRET_KEY is not configured. Using a temporary in-memory key. " "Run setup to create a persistent secret in .env." ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 498e472c4..de8edc0cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,6 +73,12 @@ The `pr-intake` workflow is advisory. If required structure is missing, it may a 5. Update docs for any setup, architecture, or workflow change 6. Open a PR into `EAPD-DRB/MUIOGO:main` using the repository PR template +## Clean-base verification + +Run `python scripts/verify_clean_base.py` before upstream sync merges and before requesting review on guardrail work. It checks unresolved git operation state, conflict markers, Python compilation, and the stdlib smoke harness. + +See `docs/dev/upstream_sync_playbook.md` for the v5.5 sync order, overlap inventory, and rejected upstream patterns. + ## Required branching rule Every implementation contribution must use: diff --git a/WebAPP/SOLVERs/model.v.5.4.txt b/WebAPP/SOLVERs/model.v.5.4.txt index 749ace889..54aafd2a8 100644 --- a/WebAPP/SOLVERs/model.v.5.4.txt +++ b/WebAPP/SOLVERs/model.v.5.4.txt @@ -23,19 +23,14 @@ set MODExTECHNOLOGYperSTORAGEto{STORAGE} within MODE_OF_OPERATION cross TECHNOLO set MODExTECHNOLOGYperSTORAGEfrom{STORAGE} within MODE_OF_OPERATION cross TECHNOLOGY; set MODExTECHNOLOGYperEMISSION{e in EMISSION} within MODE_OF_OPERATION cross TECHNOLOGY; set MODExTECHNOLOGYperEMISSIONChange{e in EMISSION} within MODE_OF_OPERATION cross TECHNOLOGY; - set INPUTxNEWxCAPACITYperFUEL{COMMODITY} within TECHNOLOGY; set INPUTxTOTALxCAPACITYperFUEL{COMMODITY} within TECHNOLOGY; set INPUTxFUEL; - # Build pair sets from your per-fuel mappings set INPUT_TF_PAIRS_NEW := setof {f in INPUTxFUEL, t in INPUTxNEWxCAPACITYperFUEL[f]} (t,f); - set INPUT_TF_PAIRS_TOTAL := setof {f in INPUTxFUEL, t in INPUTxTOTALxCAPACITYperFUEL[f]} (t,f); - - # ##################### # Parameters # @@ -87,7 +82,6 @@ param UDCMultiplierNewCapacity{r in REGION, t in TECHNOLOGY, u in UDC, y in YEAR param UDCMultiplierActivity{r in REGION, t in TECHNOLOGY, u in UDC, y in YEAR}; param UDCConstant{r in REGION, u in UDC, y in YEAR}; param UDCTag{r in REGION, u in UDC}; - param CapitalRecoveryFactor{r in REGION, t in TECHNOLOGY}; param PvAnnuity{r in REGION, t in TECHNOLOGY}; ########################Storage added VK @@ -96,10 +90,8 @@ param CapitalCostStorage{r in REGION, s in STORAGE, y in YEAR}; param ResidualStorageCapacity{r in REGION, s in STORAGE, y in YEAR}; param TechnologyToStorage{r in REGION, t in TECHNOLOGY, s in STORAGE, m in MODE_OF_OPERATION}; param TechnologyFromStorage{r in REGION, t in TECHNOLOGY, s in STORAGE, m in MODE_OF_OPERATION}; - param StorageLevelStart{r in REGION, s in STORAGE}; param MinStorageCharge{r in REGION, s in STORAGE, y in YEAR}; - param Conversionls{l in TIMESLICE, ls in SEASON}; param Conversionld{l in TIMESLICE, ld in DAYTYPE}; param Conversionlh{l in TIMESLICE, lh in DAILYTIMEBRACKET}; @@ -111,8 +103,6 @@ set TIMESLICEofSEASON{ls in SEASON} within TIMESLICE := {l in TIMESLICE : Conver set TIMESLICEofDAYTYPE{ld in DAYTYPE} within TIMESLICE := {l in TIMESLICE : Conversionld[l,ld] = 1}; set TIMESLICEofDAILYTIMEBRACKET{lh in DAILYTIMEBRACKET} within TIMESLICE := {l in TIMESLICE : Conversionlh[l,lh] = 1}; set TIMESLICEofSDB{ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET} within TIMESLICE := TIMESLICEofSEASON[ls] inter TIMESLICEofDAYTYPE[ld] inter TIMESLICEofDAILYTIMEBRACKET[lh]; - - # ########################## # Model Variables # @@ -128,14 +118,10 @@ var Demand{r in REGION, l in TIMESLICE, f in COMMODITY, y in YEAR} >= 0; var DiscountedSalvageValue{r in REGION, t in TECHNOLOGY, y in YEAR} >= 0; #var InputToNewCapacity{r in REGION, t in TECHNOLOGY, f in COMMODITY, y in YEAR} >= 0; #var InputToTotalCapacity{r in REGION, t in TECHNOLOGY, f in COMMODITY, y in YEAR} >= 0; - - - # Sparse variables: only for technologies that actually use fuel f #var InputToNewCapacity{r in REGION, f in COMMODITY, t in INPUTxNEWxCAPACITYperFUEL[f], y in YEAR} >= 0; #var InputToTotalCapacity{r in REGION, f in COMMODITY, t in INPUTxTOTALxCAPACITYperFUEL[f], y in YEAR} >= 0; - # Variables over sparse domains (choose a consistent index order) var InputToNewCapacity {r in REGION, t in TECHNOLOGY,f in COMMODITY, y in YEAR: @@ -145,7 +131,6 @@ var InputToTotalCapacity {r in REGION, t in TECHNOLOGY, f in COMMODITY,y in YEAR: (t,f) in INPUT_TF_PAIRS_TOTAL} >= 0; - var NewCapacity{r in REGION, t in TECHNOLOGY, y in YEAR} >= 0; var NumberOfNewTechnologyUnits{r in REGION, t in TECHNOLOGY, y in YEAR} >= 0,integer; var ProductionByTechnology{r in REGION, l in TIMESLICE, t in TECHNOLOGY, f in COMMODITY, y in YEAR} >= 0; @@ -220,13 +205,7 @@ s.t. TAC1_TotalModelHorizonTechnologyActivity{r in REGION, t in TECHNOLOGY}: sum # InputToCapacityRatios # s.t. EBb4_EnergyBalanceEachYear4_ICR{r in REGION, f in COMMODITY, y in YEAR}: sum{(m,t) in MODExTECHNOLOGYperFUELout[f], l in TIMESLICE} RateOfActivity[r,l,t,m,y]*OutputActivityRatio[r,t,f,m,y]*YearSplit[l,y] >= sum{(m,t) in MODExTECHNOLOGYperFUELin[f], l in TIMESLICE} RateOfActivity[r,l,t,m,y]*InputActivityRatio[r,t,f,m,y]*YearSplit[l,y] + sum{l in TIMESLICE, rr in REGION} Trade[r,rr,l,f,y]*TradeRoute[r,rr,f,y] + AccumulatedAnnualDemand[r,f,y] + sum{t in TECHNOLOGY} InputToNewCapacity [r, t, f, y] + sum{t in TECHNOLOGY} InputToTotalCapacity [r, t, f, y]; - - - - - - -# Energy balance – include only pairs that exist (others implicitly 0) +# Energy balance – include only pairs that exist (others implicitly 0) s.t. EBb4_EnergyBalanceEachYear4_ICR {r in REGION, f in COMMODITY, y in YEAR}: sum { (m,t) in MODExTECHNOLOGYperFUELout[f], l in TIMESLICE } @@ -236,17 +215,10 @@ s.t. EBb4_EnergyBalanceEachYear4_ICR RateOfActivity[r,l,t,m,y] * InputActivityRatio[r,t,f,m,y] * YearSplit[l,y] + sum { l in TIMESLICE, rr in REGION } Trade[r,rr,l,f,y] * TradeRoute[r,rr,f,y] + AccumulatedAnnualDemand[r,f,y] - + sum { t in TECHNOLOGY : (t,f) in INPUT_TF_PAIRS_NEW } InputToNewCapacity[r,t,f,y] + sum { t in TECHNOLOGY : (t,f) in INPUT_TF_PAIRS_TOTAL } InputToTotalCapacity[r,t,f,y]; - - - - # s.t. INC1_InputToNewCapacity{r in REGION, t in TECHNOLOGY, f in COMMODITY, y in YEAR: InputToNewCapacityRatio [r, t, f, y] <> 0}: InputToNewCapacityRatio [r, t, f, y] * NewCapacity [r, t, y] = InputToNewCapacity [r, t, f, y]; - - s.t. INC1_InputToNewCapacity {r in REGION, y in YEAR, #f in INPUTxFUEL, @@ -256,11 +228,8 @@ s.t. INC1_InputToNewCapacity InputToNewCapacityRatio[r,t,f,y] * NewCapacity[r,t,y] = InputToNewCapacity[r,t,f,y]; - - #s.t. ITC1_InputToTotalCapacity{r in REGION, t in TECHNOLOGY, f in COMMODITY, y in YEAR: InputToTotalCapacityRatio [r, t, f, y] <> 0}: InputToTotalCapacityRatio [r, t, f, y] * TotalCapacityAnnual [r, t, y] = InputToTotalCapacity [r, t, f, y]; - s.t. ITC1_InputToTotalCapacity {r in REGION, y in YEAR, #f in INPUTxFUEL, @@ -270,8 +239,6 @@ s.t. ITC1_InputToTotalCapacity InputToTotalCapacityRatio[r,t,f,y] * TotalCapacityAnnual[r,t,y] = InputToTotalCapacity[r,t,f,y]; - - # Long_Code_Equations s.t. AAC1_TotalAnnualTechnologyActivity{r in REGION, t in TECHNOLOGY, y in YEAR}: sum{l in TIMESLICE, m in MODEperTECHNOLOGY[t]} RateOfActivity[r,l,t,m,y]*YearSplit[l,y] = TotalTechnologyAnnualActivity[r,t,y]; s.t. CAa3_TotalActivityOfEachTechnology{r in REGION, t in TECHNOLOGY, l in TIMESLICE, y in YEAR}: sum{m in MODEperTECHNOLOGY[t]} RateOfActivity[r,l,t,m,y] = RateOfTotalActivity[r,t,l,y]; @@ -318,11 +285,8 @@ s.t. E10_InterYearActivityEmissionChange{r in REGION, e in EMISSION, (m, t) in M # TN Changed 2024 01 s.t. E11_InterYearActivityEmissionChange{r in REGION, e in EMISSION, (m, t) in MODExTECHNOLOGYperEMISSIONChange[e], y in YEAR, yy in YEAR: y == min{yyy in YEAR} min(yyy)}: 0 = EmissionByActivityChange[r, t, e, m, y]; - - ######### Storage Equations ############# # - s.t. S14_RateOfNetStorageActivity{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: (sum{t in TECHNOLOGY, m in MODEperTECHNOLOGY[t], l in TIMESLICE:TechnologyToStorage[r,t,s,m]>0} RateOfActivity[r,l,t,m,y] * TechnologyToStorage[r,t,s,m] * Conversionls[l,ls] * Conversionld[l,ld] * Conversionlh[l,lh]) - (sum{t in TECHNOLOGY, m in MODEperTECHNOLOGY[t], l in TIMESLICE:TechnologyFromStorage[r,t,s,m]>0} RateOfActivity[r,l,t,m,y] * TechnologyFromStorage[r,t,s,m] * Conversionls[l,ls] * Conversionld[l,ld] * Conversionlh[l,lh]) = RateOfNetStorageActivity[r,s,ls,ld,lh,y]; s.t. S3_NetChargeWithinYear{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: sum{l in TIMESLICE:Conversionls[l,ls]>0&&Conversionld[l,ld]>0&&Conversionlh[l,lh]>0} (sum{t in TECHNOLOGY, m in MODEperTECHNOLOGY[t]:TechnologyToStorage[r,t,s,m]>0} (RateOfActivity[r,l,t,m,y] * TechnologyToStorage[r,t,s,m] * Conversionls[l,ls] * Conversionld[l,ld] * Conversionlh[l,lh]) - (sum{t in TECHNOLOGY, m in MODEperTECHNOLOGY[t]:TechnologyFromStorage[r,t,s,m]>0} RateOfActivity[r,l,t,m,y] * TechnologyFromStorage[r,t,s,m] * Conversionls[l,ls] * Conversionld[l,ld] * Conversionlh[l,lh])) * YearSplit[l,y] * Conversionls[l,ls] * Conversionld[l,ld] * Conversionlh[l,lh] = NetChargeWithinYear[r,s,ls,ld,lh,y]; s.t. S4_NetChargeWithinDay{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: ((sum{t in TECHNOLOGY, m in MODEperTECHNOLOGY[t], l in TIMESLICE:TechnologyToStorage[r,t,s,m]>0} RateOfActivity[r,l,t,m,y] * TechnologyToStorage[r,t,s,m] * Conversionls[l,ls] * Conversionld[l,ld] * Conversionlh[l,lh]) - (sum{t in TECHNOLOGY, m in MODEperTECHNOLOGY[t], l in TIMESLICE:TechnologyFromStorage[r,t,s,m]>0} RateOfActivity[r,l,t,m,y] * TechnologyFromStorage[r,t,s,m] * Conversionls[l,ls] * Conversionld[l,ld] * Conversionlh[l,lh])) * DaySplit[lh,y] = NetChargeWithinDay[r,s,ls,ld,lh,y]; @@ -354,7 +318,6 @@ s.t. S39_StorageIntrayear{r in REGION, s in STORAGEINTRAYEAR, y in YEAR}: sum{ls #v.k. SC4_LowerLimit_BeginningOfDailyTimeBracketOfFirstInstanceOfDayTypeInLastWeekConstraint = SC4_LLBDFILW #v.k. SC4_UpperLimit_BeginningOfDailyTimeBracketOfFirstInstanceOfDayTypeInLastWeekConstraint = SC4_ULBDFILW - s.t. SC1_LLBDFIFW{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: 0 <= (StorageLevelDayTypeStart[r,s,ls,ld,y]+sum{lhlh in DAILYTIMEBRACKET:lh-lhlh>0} (((sum{(m,t) in MODExTECHNOLOGYperSTORAGEto[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyToStorage[r,t,s,m]) - (sum{(m,t) in MODExTECHNOLOGYperSTORAGEfrom[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyFromStorage[r,t,s,m])) * DaySplit[lhlh,y]))-MinStorageCharge[r,s,y]*(sum{yy in YEAR: y-yy < OperationalLifeStorage[r,s] && y-yy>=0} NewStorageCapacity[r,s,yy]+ResidualStorageCapacity[r,s,y]); s.t. SC1_ULBDFIFW{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: @@ -365,19 +328,14 @@ s.t. SC3_LLEDLILW{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in s.t. SC3_ULEDLILW{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: (StorageLevelDayTypeFinish[r,s,ls,ld,y] - sum{lhlh in DAILYTIMEBRACKET:lh-lhlh<0} (((sum{(m,t) in MODExTECHNOLOGYperSTORAGEto[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyToStorage[r,t,s,m]) - (sum{(m,t) in MODExTECHNOLOGYperSTORAGEfrom[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyFromStorage[r,t,s,m])) * DaySplit[lhlh,y]))-(sum{yy in YEAR: y-yy < OperationalLifeStorage[r,s] && y-yy>=0} NewStorageCapacity[r,s,yy]+ResidualStorageCapacity[r,s,y]) <= 0; s.t. SC4_LLBDFILW{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: 0 <= if ld > min{ldld in DAYTYPE} min(ldld) then (StorageLevelDayTypeFinish[r,s,ls,ld-1,y]+sum{lhlh in DAILYTIMEBRACKET:lh-lhlh>0} (((sum{(m,t) in MODExTECHNOLOGYperSTORAGEto[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyToStorage[r,t,s,m]) - (sum{(m,t) in MODExTECHNOLOGYperSTORAGEfrom[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyFromStorage[r,t,s,m])) * DaySplit[lhlh,y]))-MinStorageCharge[r,s,y]*(sum{yy in YEAR: y-yy < OperationalLifeStorage[r,s] && y-yy>=0} NewStorageCapacity[r,s,yy]+ResidualStorageCapacity[r,s,y]); s.t. SC4_ULBDFILW{r in REGION, s in STORAGE, ls in SEASON, ld in DAYTYPE, lh in DAILYTIMEBRACKET, y in YEAR}: if ld > min{ldld in DAYTYPE} min(ldld) then (StorageLevelDayTypeFinish[r,s,ls,ld-1,y]+sum{lhlh in DAILYTIMEBRACKET:lh-lhlh>0} (((sum{(m,t) in MODExTECHNOLOGYperSTORAGEto[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyToStorage[r,t,s,m]) - (sum{(m,t) in MODExTECHNOLOGYperSTORAGEfrom[s], l in TIMESLICEofSDB[ls,ld,lhlh]} RateOfActivity[r,l,t,m,y] * TechnologyFromStorage[r,t,s,m])) * DaySplit[lhlh,y]))-(sum{yy in YEAR: y-yy < OperationalLifeStorage[r,s] && y-yy>=0} NewStorageCapacity[r,s,yy]+ResidualStorageCapacity[r,s,y]) <= 0; - # ######### Storage Investments ############# # s.t. SI6_SalvageValueStorageAtEndOfPeriod1{r in REGION, s in STORAGE, y in YEAR: (y+OperationalLifeStorage[r,s]-1) <= (max{yy in YEAR} max(yy))}: 0 = SalvageValueStorage[r,s,y]; - - - s.t. SI7_SalvageValueStorageAtEndOfPeriod2{r in REGION, s in STORAGE, y in YEAR: (y+OperationalLifeStorage[r,s]-1) > (max{yy in YEAR} max(yy))}: CapitalCostStorage[r,s,y] * NewStorageCapacity[r,s,y]*(1-(max{yy in YEAR} max(yy) - y+1)/OperationalLifeStorage[r,s]) = SalvageValueStorage[r,s,y]; # 20240625 vk ta DepreciationMethod default 2 #s.t. SI7_SalvageValueStorageAtEndOfPeriod2{r in REGION, s in STORAGE, y in YEAR: (DepreciationMethod[r]=1 && (y+OperationalLifeStorage[r,s]-1) > (max{yy in YEAR} max(yy)) && DiscountRate[r]=0) || (DepreciationMethod[r]=2 && (y+OperationalLifeStorage[r,s]-1) > (max{yy in YEAR} max(yy)))}: CapitalCostStorage[r,s,y] * NewStorageCapacity[r,s,y]*(1-(max{yy in YEAR} max(yy) - y+1)/OperationalLifeStorage[r,s]) = SalvageValueStorage[r,s,y]; #s.t. SI8_SalvageValueStorageAtEndOfPeriod3{r in REGION, s in STORAGE, y in YEAR: DepreciationMethod[r]=1 && (y+OperationalLifeStorage[r,s]-1) > (max{yy in YEAR} max(yy)) && DiscountRate[r]>0}: CapitalCostStorage[r,s,y] * NewStorageCapacity[r,s,y]*(1-(((1+DiscountRate[r])^(max{yy in YEAR} max(yy) - y+1)-1)/((1+DiscountRate[r])^OperationalLifeStorage[r,s]-1))) = SalvageValueStorage[r,s,y]; - #s.t. SI1_StorageUpperLimit{r in REGION, s in STORAGE, y in YEAR}: sum{yy in YEAR: y-yy < OperationalLifeStorage[r,s] && y-yy>=0} NewStorageCapacity[r,s,yy]+ResidualStorageCapacity[r,s,y] = StorageUpperLimit[r,s,y]; s.t. SI2_StorageLowerLimit{r in REGION, s in STORAGE, y in YEAR}: MinStorageCharge[r,s,y]*(sum{yy in YEAR: y-yy < OperationalLifeStorage[r,s] && y-yy>=0} NewStorageCapacity[r,s,yy]+ResidualStorageCapacity[r,s,y]) = StorageLowerLimit[r,s,y]; s.t. SI3_TotalNewStorage{r in REGION, s in STORAGE, y in YEAR}: sum{yy in YEAR: y-yy < OperationalLifeStorage[r,s] && y-yy>=0} NewStorageCapacity[r,s,yy]=AccumulatedNewStorageCapacity[r,s,y]; @@ -386,10 +344,6 @@ s.t. SI4_UndiscountedCapitalInvestmentStorage{r in REGION, s in STORAGE, y in YE s.t. SI9_SalvageValueStorageDiscountedToStartYear{r in REGION, s in STORAGE, y in YEAR}: SalvageValueStorage[r,s,y]/((1+DiscountRate[r])^(max{yy in YEAR} max(yy)-min{yy in YEAR} min(yy)+1)) = DiscountedSalvageValueStorage[r,s,y]; #s.t. SI10_TotalDiscountedCostByStorage{r in REGION, s in STORAGE, y in YEAR}: (CapitalCostStorage[r,s,y] * NewStorageCapacity[r,s,y]/((1+DiscountRate[r])^(y-min{yy in YEAR} min(yy)))-CapitalCostStorage[r,s,y] * NewStorageCapacity[r,s,y]/((1+DiscountRate[r])^(y-min{yy in YEAR} min(yy)))) = TotalDiscountedStorageCost[r,s,y]; # - - - - # User-defined constraints s.t. UDC1_UserDefinedConstraintInequality{r in REGION, u in UDC, y in YEAR: UDCTag[r,u] = 0}: sum{t in TECHNOLOGY}UDCMultiplierTotalCapacity[r,t,u,y]*TotalCapacityAnnual[r,t,y] + diff --git a/docs/dev/upstream_sync_playbook.md b/docs/dev/upstream_sync_playbook.md new file mode 100644 index 000000000..db4ecb87d --- /dev/null +++ b/docs/dev/upstream_sync_playbook.md @@ -0,0 +1,65 @@ +# Upstream Sync Playbook + +This repository tracks upstream MUIO releases. For sync work such as the v5.5 stack in [#388](https://github.com/EAPD-DRB/MUIOGO/issues/388), follow upstream by default and only diverge when MUIOGO needs portability, security, runtime reliability, or downstream integration fixes. + +## Sync Order + +Land the v5.5 stack in this order: + +1. [#389](https://github.com/EAPD-DRB/MUIOGO/issues/389): guardrails and smoke harness. This PR is the gate. +2. Backend and runtime safe changes from the remaining [#388](https://github.com/EAPD-DRB/MUIOGO/issues/388) child work. +3. Diagnostics UI follow-up work. +4. Result metadata bundle follow-up work, including `Variables.json` and rendering alignment. + +Use [#390](https://github.com/EAPD-DRB/MUIOGO/issues/390) as the overlap inventory before resolving conflicts or choosing a deliberate downstream divergence. + +## Protected Overlap Surface + +Review these files carefully in every upstream sync PR because they are the highest-overlap backend/runtime touch points: + +- `API/Classes/Base/Config.py` +- `API/Classes/Base/FileClass.py` +- `API/Classes/Case/DataFileClass.py` +- `API/Classes/Case/OsemosysClass.py` +- `API/Routes/DataFile/DataFileRoute.py` +- `API/app.py` + +If a PR touches any of them, state whether the change follows upstream as-is or intentionally diverges and why. + +## Rejected Upstream Patterns + +Do not land these patterns without an explicit maintainer decision: + +- writing logs under `WebAPP/`, including `WebAPP/app.log` +- deleting logs on startup +- current-working-directory-relative paths for important I/O +- `shell=True` subprocess usage +- compact JSON as the default write format +- `FileClassCompressed.py` +- similar compression or logging rewrites that reduce portability or make runtime behavior harder to reason about + +## Clean-Base Verification + +Before starting an upstream merge, after resolving conflicts, and before requesting review, run: + +```bash +python scripts/verify_clean_base.py +``` + +This command checks: + +- unresolved git operation state from `.git` control files such as `MERGE_HEAD` and rebase markers +- conflict markers in common tracked text files +- Python source compilation without assuming the repo root is writable +- stdlib `unittest` smoke coverage for app import and fixed routes + +Fix every reported failure before continuing with the sync. + +## Validation Scope For PR 1 + +For [#389](https://github.com/EAPD-DRB/MUIOGO/issues/389), keep validation intentionally small: + +- run `python scripts/verify_clean_base.py` +- confirm the smoke harness passes on a supported Python interpreter + +The CBC demo and broader backend/runtime validation belong to later PRs in the [#388](https://github.com/EAPD-DRB/MUIOGO/issues/388) stack, not this guardrail PR. diff --git a/scripts/verify_clean_base.py b/scripts/verify_clean_base.py new file mode 100644 index 000000000..0c3636db1 --- /dev/null +++ b/scripts/verify_clean_base.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import os +from pathlib import Path +import py_compile +import subprocess +import sys +import tempfile +from typing import Iterable + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TEXT_EXTENSIONS = { + ".css", + ".html", + ".js", + ".json", + ".md", + ".py", + ".txt", + ".yaml", + ".yml", +} +SKIP_DIR_NAMES = { + ".git", + ".mypy_cache", + ".pytest_cache", + ".venv", + "__pycache__", + "node_modules", + "venv", +} + + +def resolve_git_dir(root: Path) -> Path | None: + git_path = root / ".git" + if git_path.is_dir(): + return git_path + if not git_path.is_file(): + return None + + try: + first_line = git_path.read_text(encoding="utf-8").splitlines()[0].strip() + except (IndexError, OSError): + return None + + prefix = "gitdir:" + if not first_line.lower().startswith(prefix): + return None + + git_dir = first_line[len(prefix) :].strip() + candidate = Path(git_dir) + if not candidate.is_absolute(): + candidate = (root / candidate).resolve() + return candidate + + +def iter_repo_files(root: Path, suffixes: set[str]) -> Iterable[Path]: + git_list = subprocess.run( + [ + "git", + "-C", + str(root), + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + ], + capture_output=True, + check=False, + ) + if git_list.returncode == 0: + for raw_path in git_list.stdout.split(b"\x00"): + if not raw_path: + continue + path = root / raw_path.decode("utf-8", "surrogateescape") + if path.suffix.lower() not in suffixes: + continue + if any(part in SKIP_DIR_NAMES for part in path.parts): + continue + yield path + return + + for path in root.rglob("*"): + if path.is_dir(): + continue + if path.suffix.lower() not in suffixes: + continue + if any(part in SKIP_DIR_NAMES for part in path.parts): + continue + yield path + + +def check_unresolved_git_state(root: Path) -> list[str]: + git_dir = resolve_git_dir(root) + if git_dir is None: + return ["Git metadata not found; cannot verify merge or rebase state."] + + issues: list[str] = [] + markers = ( + "MERGE_HEAD", + "CHERRY_PICK_HEAD", + "REVERT_HEAD", + "REBASE_HEAD", + ) + for marker in markers: + if (git_dir / marker).exists(): + issues.append(f"Git operation in progress: {git_dir / marker}") + + for rebase_dir in ("rebase-apply", "rebase-merge"): + if (git_dir / rebase_dir).exists(): + issues.append(f"Git rebase in progress: {git_dir / rebase_dir}") + + return issues + + +def check_conflict_markers(root: Path) -> list[str]: + hits: list[str] = [] + + for path in iter_repo_files(root, TEXT_EXTENSIONS): + try: + with path.open("r", encoding="utf-8", errors="ignore") as handle: + for line_number, line in enumerate(handle, start=1): + stripped = line.lstrip() + if stripped.startswith("<<<<<<<"): + hits.append(f"{path.relative_to(root)}:{line_number}") + break + except OSError as exc: + hits.append(f"{path.relative_to(root)}: unreadable ({exc})") + + return hits + + +def run_py_compile(root: Path) -> list[str]: + failures: list[str] = [] + python_files = list(iter_repo_files(root, {".py"})) + + for path in python_files: + file_descriptor, temp_path = tempfile.mkstemp( + prefix="verify-clean-base-", + suffix=".pyc", + ) + os.close(file_descriptor) + try: + py_compile.compile( + str(path), + cfile=temp_path, + doraise=True, + ) + except (OSError, py_compile.PyCompileError) as exc: + failures.append(f"{path.relative_to(root)}: {exc}") + finally: + try: + os.remove(temp_path) + except OSError: + pass + + return failures + + +def run_smoke_tests(root: Path) -> int: + command = [ + sys.executable, + "-m", + "unittest", + "discover", + "-s", + "tests_smoke", + "-p", + "test_*.py", + ] + process = subprocess.run(command, cwd=root) + return process.returncode + + +def main() -> int: + print(f"[verify] repo root: {REPO_ROOT}") + + git_state = check_unresolved_git_state(REPO_ROOT) + if git_state: + print("[FAIL] unresolved git state detected:") + for issue in git_state: + print(f" - {issue}") + return 1 + print("[OK] git state: clean") + + conflicts = check_conflict_markers(REPO_ROOT) + if conflicts: + print("[FAIL] conflict markers found:") + for conflict in conflicts: + print(f" - {conflict}") + return 1 + print("[OK] no conflict markers found") + + compile_failures = run_py_compile(REPO_ROOT) + if compile_failures: + print("[FAIL] python compile step failed:") + for failure in compile_failures: + print(f" - {failure}") + return 1 + print("[OK] python compile step passed") + + smoke_status = run_smoke_tests(REPO_ROOT) + if smoke_status != 0: + print(f"[FAIL] smoke tests failed (rc={smoke_status})") + return smoke_status + print("[OK] smoke tests passed") + + print("[SUCCESS] clean-base verification passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests_smoke/test_smoke_app.py b/tests_smoke/test_smoke_app.py new file mode 100644 index 000000000..76c412ffa --- /dev/null +++ b/tests_smoke/test_smoke_app.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +import unittest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +API_DIR = REPO_ROOT / "API" + +if str(API_DIR) not in sys.path: + sys.path.insert(0, str(API_DIR)) + + +def load_app_module(): + return importlib.import_module("app") + + +class SmokeAppTest(unittest.TestCase): + def test_import_app(self) -> None: + app_module = load_app_module() + self.assertTrue(hasattr(app_module, "app")) + + def test_fixed_routes(self) -> None: + app_module = load_app_module() + client = app_module.app.test_client() + + response = client.get("/getSession") + self.assertEqual(response.status_code, 200) + self.assertTrue(response.is_json) + self.assertIn("session", response.get_json()) + + response = client.get("/") + self.assertEqual(response.status_code, 200) + + def test_set_session_clear(self) -> None: + app_module = load_app_module() + client = app_module.app.test_client() + + response = client.post("/setSession", json={"case": None}) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.is_json) + self.assertEqual(response.get_json(), {"osycase": None}) + + def test_set_session_missing_case(self) -> None: + app_module = load_app_module() + client = app_module.app.test_client() + + response = client.post("/setSession", json={}) + self.assertEqual(response.status_code, 404) + self.assertTrue(response.is_json) + + +if __name__ == "__main__": + unittest.main()