Scripts for building SMLP in virtual environment

venv/python3.11/Dockerfile

@@ -0,0 +1,40 @@
+# Use ubuntu 24.04 as an image
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Layer 1: base tools and Python PPA
+RUN apt-get update && apt-get install -y \
+    software-properties-common wget git gzip vim xvfb \
+    && add-apt-repository ppa:deadsnakes/ppa -y
+
+# Layer 2: Python 3.11
+RUN apt-get install -y \
+    libgomp1 tcsh \
+    python3.11-dev python3.11-tk python3.11-venv python3.11-distutils python3.11
+
+# Set working directory
+WORKDIR /app
+
+# Build SMLP and run DORA test
+RUN touch /.dockerenv
+
+#tkdiff patch for https://bugs.launchpad.net/bugs/2139062
+COPY tkagg_patch.sh .
+RUN chmod +x tkagg_patch.sh
+
+#Install smlp and run DORA test
+COPY run_git_clone .
+ARG CACHE_BUST_SMLP
+ARG GIT_BRANCH=master
+RUN echo "Building image for branch: $GIT_BRANCH"
+RUN  ./run_git_clone $GIT_BRANCH && \
+     cd smlp/venv/python3.11 && \
+     /app/tkagg_patch.sh && \
+    ./run_venv_build
+
+## Change working directory to venv root
+WORKDIR /app/smlp/venv/python3.11
+
+## Default command
+CMD ["/bin/bash"]

venv/python3.11/README.md

@@ -0,0 +1,52 @@
+# SMLP installation instructions in virtual environment
+
+## 1.  Dependencies installation instructions for Ubuntu 24.04
+
+```bash
+export DEBIAN_FRONTEND=noninteractive
+sudo apt update
+sudo apt-get install software-properties-common wget git gzip vim xvfb -y
+sudo add-apt-repository ppa:deadsnakes/ppa -y
+sudo apt-get install libgomp1 tcsh python3.11-dev python3.11-tk python3.11-venv python3.11-distutils python3.11 -y
+```
+
+## 2. Clean SMLP installation and running Dead-Or-Alive Test
+
+```bash
+./run_venv_build
+```
+
+## 3. Build Validation in virtual enviroment
+
+```bash
+source venv/bin/activate
+./run_smlp_dora
+```
+
+## 4. Build validation in Docker
+
+```bash
+./run_docker_build
+./run_venv_container
+source venv/bin/activate
+./run_smlp_dora
+```
+## 5. Regression in virtual enviroment
+
+```bash
+source venv/bin/activate
+\cp -p ../../docker/python3.11/run_mathsat_build ../../..
+../../../run_mathsat_build
+rm -rf /tmp/mathsat*
+cd smlp_regression
+./run_smlp_regression
+```
+
+## 6. Regression in Docker
+
+```bash
+./run_venv_container
+cd smlp_regression
+./run_smlp_regression
+```
+

venv/python3.11/requirements.txt

@@ -0,0 +1,13 @@
+doepy
+jenkspy 
+keras_tuner 
+mrmr-selection 
+pandas 
+pycaret 
+pyDOE 
+pysubgroup 
+scikit-learn 
+scipy==1.11.4 
+seaborn 
+tensorflow==2.15.1 
+z3-solver==4.8.12

venv/python3.11/run_docker_build

@@ -0,0 +1,2 @@
+#!/usr/bin/tcsh -f
+env DOCKER_BUILDKIT=1 docker build --no-cache --build-arg GIT_BRANCH=`git branch --show-current` --progress=plain -t smlp-dev .

venv/python3.11/run_docker_build_incremental

@@ -0,0 +1,11 @@
+#!/usr/bin/tcsh -f
+set build_args=""
+if($#argv > 0) then
+    if("-rebuild_smlp" == "$argv[1]" ) then
+        set build_args="--build-arg CACHE_BUST_SMLP=`date +%s`"
+    else
+        echo "\nUsage: `realpath $0 | xargs basename` [-rebuild_smlp]\n"
+        exit 0
+    endif
+endif
+env DOCKER_BUILDKIT=1 docker build --progress=plain $build_args --build-arg GIT_BRANCH=`git branch --show-current` -t smlp-dev .

venv/python3.11/run_git_clone

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+git clone https://github.com/SMLP-Systems/smlp
+GIT_BRANCH=$1
+cd smlp
+if [ $(git branch -r --list origin/$GIT_BRANCH) ]; then
+    git switch $GIT_BRANCH
+fi

venv/python3.11/run_smlp_dora

@@ -0,0 +1,63 @@
+#!/usr/bin/tcsh -f
+set script_path=`realpath $0 | xargs dirname`
+set script_name=`realpath $0 | xargs basename`
+set test=smlp_toy_num_resp_mult
+set log=${test}.log
+set pref=Test83
+\rm -f logs.log  >& /dev/null
+\rm -f $script_path:h:h/regr_smlp/code/logs.log  >& /dev/null
+\rm -f $script_path:h:h/regr_smlp/code/$log >& /dev/null
+\rm -f $script_path:h:h/regr_smlp/code/${pref}_* >& /dev/null
+if($#argv > 0 ) then
+    if( '-clean' == "$argv[1]" ) then
+        exit(0)
+    endif
+endif
+if( 1 == `$script_path:h:h/src/run_smlp.py -h |& grep -c ^usage`) then
+    echo "\nSMLP installation succeeded\n"
+else
+    echo "\nSMLP installation failed\n"
+    $script_path:h:h/src/run_smlp.py -h
+    cat $script_path:h:h/src/run_smlp.py
+    exit(1)
+endif
+cd $script_path:h:h/regr_smlp/code
+xvfb-run $script_path:h:h/src/run_smlp.py \
+    -data ../data/${test}.csv \
+    -out_dir ./ \
+    -pref $pref \
+    -mode optimize -pareto t \
+    -resp y1,y2 \
+    -feat x,p1,p2 \
+    -model dt_sklearn \
+    -dt_sklearn_max_depth 15 \
+    -spec ../specs/${test}_free_inps.spec \
+    -data_scaler min_max \
+    -beta "y1>7 and y2>6" \
+    -objv_names obj1,objv2,objv3 \
+    -objv_exprs "(y1+y2)/2;y1/2-y2;y2" \
+    -epsilon 0.05 \
+    -delta_rel 0.01 \
+    -save_model_config f \
+    -mrmr_pred 0 \
+    -plots f \
+    -seed 10 \
+    -log_time f \
+    >& $log
+set new_results=${pref}_smlp_toy_num_resp_mult_optimization_results.json 
+set expected_results=${script_path}/smlp_dora_expected_results.json
+if(-e $new_results) then
+    set diff_cmd="diff `realpath $new_results` $expected_results"
+    echo $diff_cmd
+    $diff_cmd
+    if($status) then
+        echo "\nSMLP passed, but results do not match expected. DORA test failed\n"
+        exit 1
+    else
+        echo "\nDORA test PASSED\n"
+        exit 0
+    endif
+else
+    echo "\nERROR: Can't find file $new_results\nDORA test failed\n" 
+    exit 1
+endif

venv/python3.11/run_venv_build

@@ -0,0 +1,12 @@
+#!/usr/bin/bash
+script_path=$(realpath $0 | xargs dirname)
+venv_dir=venv
+\rm -rf $venv_dir logs.log > /dev/null
+if [[ $# -gt 0 ]]; then
+   if [ "$1" == "-clean" ]; then
+       exit 0
+   fi
+fi
+python3.11 -m venv $venv_dir && source venv/bin/activate
+pip install $script_path/../../dist/smlp-0.1.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+$script_path/run_smlp_dora

venv/python3.11/run_venv_build.benchmark.txt

@@ -0,0 +1,4 @@
+Build from pip cache:
+83.937u 7.795s 2:46.64 55.0%	0+0k 328416+7613296io 0pf+0w
+Build from scratch using Docker:
+2.727u 2.522s 18:21.44 0.4%	0+0k 19040+24io 148pf+0w

venv/python3.11/run_venv_container

@@ -0,0 +1,4 @@
+#!/usr/bin/tcsh -f
+mkdir -p $HOME/smlp_shared
+if( ! $?TZ ) setenv TZ `readlink /etc/localtime | sed 's@/usr/share/zoneinfo/@@'`
+docker run -e TZ=$TZ -v $HOME/smlp_shared:/shared -it mdmitry1/smlp-dev:latest $*

venv/python3.11/smlp_dora_expected_results.json

@@ -0,0 +1,63 @@
+{
+	"obj1": {
+		"value_in_config": 9.0,
+		"threshold_scaled": 1.0,
+		"threshold": 9.0,
+		"max_in_data": 9.0,
+		"min_in_data": 5.0
+	},
+	"objv2": {
+		"value_in_config": -4.5,
+		"threshold_scaled": 0.3125,
+		"threshold": -4.625,
+		"max_in_data": -0.5,
+		"min_in_data": -6.5
+	},
+	"objv3": {
+		"value_in_config": 9.0,
+		"threshold_scaled": 1.0,
+		"threshold": 9.0,
+		"max_in_data": 9.0,
+		"min_in_data": 5.0
+	},
+	"p1": {
+		"value_in_config": 7.0
+	},
+	"x": {
+		"value_in_config": 11.020833348234495
+	},
+	"y1": {
+		"value_in_config": 9.0
+	},
+	"p2": {
+		"value_in_config": 5.222833400388558
+	},
+	"y2": {
+		"value_in_config": 9.0
+	},
+	"obj1_scaled_objv3_scaled": {
+		"value_in_config": 1.0
+	},
+	"threshold_lo_scaled": {
+		"value_in_config": 1.0
+	},
+	"threshold_lo": {
+		"value_in_config": 1.0
+	},
+	"max_in_data": {
+		"value_in_config": 1.0
+	},
+	"min_in_data": {
+		"value_in_config": 0.0
+	},
+	"threshold_up_scaled": {
+		"value_in_config": 1.03984375
+	},
+	"threshold_up": {
+		"value_in_config": 1.03984375
+	},
+	"smlp_execution": "completed",
+	"interface_consistent": "true",
+	"model_consistent": "true",
+	"synthesis_feasible": "true"
+}

venv/python3.11/smlp_regression/create_diff_report

@@ -0,0 +1,9 @@
+#!/usr/bin/tcsh -f
+set script_path=`realpath $0 | xargs dirname`
+set root_path=`git rev-parse --show-toplevel`/regr_smlp
+foreach f (`grep Failed $root_path/code/all_log.txt | grep -v : | grep -v "master file does not exist" | awk '{print $1}'`)
+    set new_results="$root_path/code/${f}" 
+    echo =================== Diff report for: $new_results:t ==================================
+    diff -w $new_results $new_results:h:h/master
+    echo =================== End of $new_results:t diff report ================================
+end

venv/python3.11/smlp_regression/run_smlp_regression

@@ -0,0 +1,40 @@
+#!/usr/bin/tcsh -f
+set script_path=`realpath $0 | xargs dirname`
+set script_name=`realpath $0 | xargs basename`
+set log=$PWD/${script_name}.log
+set diff_report=$log:h/${script_name}_diff_report.log
+git rev-parse --show-toplevel >& /dev/null
+if($status) then
+    echo "\nERROR: Current directory is outside git directory tree\n"
+    exit(1)
+endif
+\rm -f $log >& /dev/null
+cd `git rev-parse --show-toplevel`
+\rm -f regr_smlp/models/test{63,101}_model_y*_smlp_* >& /dev/null
+\rm -rf regr_smlp/code/*{test,Test}* >& /dev/null
+\rm -rf regr_smlp/code/{__pycache__,all_log.txt,logs.log,_doe.csv} >& /dev/null
+\rm -f logs.log >& /dev/null
+if($#argv > 0) then
+    if("-clean" == "$argv[1]") then
+        exit(0)
+    endif
+endif
+if(-f "/.dockerenv" ) then
+    echo "Regression is running in Docker container"
+    if(! -f $script_path:h/external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat ) then
+        echo "Installing mathsat"
+        \cp -p $script_path:h:h:h/docker/python3.11/run_mathsat_build ..
+         ../run_mathsat_build
+        \rm -rf /tmp/mathsat* >& /dev/null
+    endif
+endif
+cd `git rev-parse --show-toplevel`/regr_smlp/code
+echo "Log file: $log"
+set prompt='%U%m%u:%B%~%b%#'
+source $script_path:h/venv/bin/activate.csh
+echo "Virtual environment has been activated"
+echo Python version: `echo 'import sys; print(sys.version)' | python`
+set regression_script=" ./smlp_regr.py" 
+if(-f "/.dockerenv" ) set regression_script="xvfb-run $regression_script"
+echo n | env CUDA_VISIBLE_DEVICES=-1 $regression_script -w 8 -def n -t all -tol 7 -g |& tee $log
+${script_path}/create_diff_report >& $diff_report