From 3c76ecdf8516ae52881d54c53586068badfd018b Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 14:12:47 +0100 Subject: [PATCH 01/18] Add CD to compile --- .github/workflows/app-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index d2bddd2..3b62e62 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -50,6 +50,10 @@ jobs: ./release/generate_linux_binary.sh chmod +x ./dist/varc ./dist/varc # check it doesn't return non 0 exit status, i.e. crash + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc centos: runs-on: ubuntu-latest From 91c9a3e3f7c0b2b0585efc7271b83902f5dfddd7 Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 14:20:41 +0100 Subject: [PATCH 02/18] Add build for macos and windows --- .github/workflows/app-ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 3b62e62..0a6b80b 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -118,6 +118,11 @@ jobs: - name: Run varc Python run: | python3 varc.py + ./release/generate_windows_binary.bat + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc.exe macos: runs-on: macos-12 @@ -133,3 +138,15 @@ jobs: - name: Run varc Python run: | python3 varc.py + - name: Build & Run Binary + run: | + pip3 install pyinstaller + chmod +x ./release/generate_osx_binary.sh + ./release/generate_osx_binary.sh + chmod +x ./dist/varc + ./dist/varc # check it doesn't return non 0 exit status, i.e. crash + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc + From c8497ca284385d55f78e97c55429cb4df36f95bc Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 17:34:16 +0100 Subject: [PATCH 03/18] Change buildpath --- .github/workflows/app-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 0a6b80b..95b1197 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -118,7 +118,8 @@ jobs: - name: Run varc Python run: | python3 varc.py - ./release/generate_windows_binary.bat + cd release + generate_windows_binary.bat - uses: actions/upload-artifact@v3 with: name: varc @@ -142,7 +143,8 @@ jobs: run: | pip3 install pyinstaller chmod +x ./release/generate_osx_binary.sh - ./release/generate_osx_binary.sh + cd release + ./generate_osx_binary.sh chmod +x ./dist/varc ./dist/varc # check it doesn't return non 0 exit status, i.e. crash - uses: actions/upload-artifact@v3 From 796f64e38da15f07eb3009060a4fb0431fe72068 Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 18:02:24 +0100 Subject: [PATCH 04/18] Looks like m1/2 not in github hosted runners so have to run in an ec2 for now --- .github/workflows/app-ci.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 95b1197..2d83f7f 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -119,7 +119,7 @@ jobs: run: | python3 varc.py cd release - generate_windows_binary.bat + ./generate_windows_binary.bat - uses: actions/upload-artifact@v3 with: name: varc @@ -142,13 +142,3 @@ jobs: - name: Build & Run Binary run: | pip3 install pyinstaller - chmod +x ./release/generate_osx_binary.sh - cd release - ./generate_osx_binary.sh - chmod +x ./dist/varc - ./dist/varc # check it doesn't return non 0 exit status, i.e. crash - - uses: actions/upload-artifact@v3 - with: - name: varc - path: ./dist/varc - From 8c6d84e4a9ef95fb0e4e375cf5096a8d43ba66ae Mon Sep 17 00:00:00 2001 From: chriscado Date: Sat, 1 Oct 2022 13:07:37 +0100 Subject: [PATCH 05/18] Add timeout --- .github/workflows/app-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 2d83f7f..b486bd5 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -106,6 +106,7 @@ jobs: windows: runs-on: windows-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v3 - name: Set up Python From 36171aa7ab1d1cfa352d7e1ffc33eadac9ea3949 Mon Sep 17 00:00:00 2001 From: chriscado Date: Sat, 1 Oct 2022 13:31:51 +0100 Subject: [PATCH 06/18] Try fixing fat binary --- release/generate_osx_binary.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/release/generate_osx_binary.sh b/release/generate_osx_binary.sh index 4be76fd..0fa24ae 100644 --- a/release/generate_osx_binary.sh +++ b/release/generate_osx_binary.sh @@ -1,4 +1,13 @@ # Tested with python 3.10 and PyInstaller 5.4.1 -pip3 install -r requirements.txt -pyinstaller --onefile --clean --target-arch universal2 varc.py +PATHS="../venv/lib/python3.10/site-packages" + +python3 -m venv venv +source ./venv/bin/activate +cd cado-host-python + + +python3 -m pip install --upgrade pip +python3 -m pip install pyinstaller==4.10 +python3 -m pip install -r requirements.txt +python3 -m PyInstaller --onefile --clean --paths $PATHS --target-arch universal2 varc.py From d3937e47b4d75b77f939965f68060b68038d96b8 Mon Sep 17 00:00:00 2001 From: chriscado Date: Sat, 1 Oct 2022 13:57:01 +0100 Subject: [PATCH 07/18] Try older pyinstaller --- .github/workflows/app-ci.yml | 10 +++++++++- release/generate_osx_binary.sh | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index b486bd5..2f8ba9a 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -120,6 +120,7 @@ jobs: run: | python3 varc.py cd release + pip3 install pyinstaller=5.0 ./generate_windows_binary.bat - uses: actions/upload-artifact@v3 with: @@ -142,4 +143,11 @@ jobs: python3 varc.py - name: Build & Run Binary run: | - pip3 install pyinstaller + chmod +x ./release/generate_linux_binary.sh + ./release/generate_linux_binary.sh + chmod +x ./dist/varc + ./dist/varc # check it doesn't return non 0 exit status, i.e. crash + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc \ No newline at end of file diff --git a/release/generate_osx_binary.sh b/release/generate_osx_binary.sh index 0fa24ae..e99aea2 100644 --- a/release/generate_osx_binary.sh +++ b/release/generate_osx_binary.sh @@ -8,6 +8,8 @@ cd cado-host-python python3 -m pip install --upgrade pip -python3 -m pip install pyinstaller==4.10 +python3 -m pip install pyinstaller==5.4.1 python3 -m pip install -r requirements.txt -python3 -m PyInstaller --onefile --clean --paths $PATHS --target-arch universal2 varc.py +# Universal target currently fails +# python3 -m PyInstaller --onefile --clean --paths $PATHS --target-arch universal2 varc.py +python3 -m PyInstaller --onefile --clean --paths $PATHS varc.py From d8bc3314fe6d6e2347b037eda1cceaaf01cabc7a Mon Sep 17 00:00:00 2001 From: chriscado Date: Sun, 2 Oct 2022 20:59:18 +0100 Subject: [PATCH 08/18] Skip error encountered on t2.micro --- varc_core/systems/linux.py | 45 ++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/varc_core/systems/linux.py b/varc_core/systems/linux.py index 04002e3..4b72da1 100644 --- a/varc_core/systems/linux.py +++ b/varc_core/systems/linux.py @@ -95,27 +95,30 @@ def dump_processes(self) -> None: """Dumps all processes to temp files, adds temp file to output archive then removes the temp file""" archive_out = self.output_path with zipfile.ZipFile(archive_out, "a", compression=zipfile.ZIP_DEFLATED) as zip_file: - for proc in tqdm(self.process_info, desc="Process dump progess", unit=" procs"): - pid = proc["Process ID"] - p_name = proc["Name"] - maps = self.parse_mem_map(pid, p_name) - if not maps: - continue - with NamedTemporaryFile(mode="w+b", buffering=0, delete=True) as tmpfile: - try: - for map in maps: - page_start = map[0] - page_len = map[1] - map[0] - mem_page_content = self.read_bytes(pid, page_start, page_len) - if mem_page_content: - tmpfile.write(mem_page_content) - zip_file.write(tmpfile.name, f"process_dumps{sep}{p_name}_{pid}.mem") - except PermissionError: - logging.warning(f"Permission denied opening process memory for {p_name} (pid {pid}). Cannot dump this process.") + try: + for proc in tqdm(self.process_info, desc="Process dump progess", unit=" procs"): + pid = proc["Process ID"] + p_name = proc["Name"] + maps = self.parse_mem_map(pid, p_name) + if not maps: continue - except OSError as oserror: - logging.warning(f"Error opening process memory page for {p_name} (pid {pid}). Error was {oserror}. Dump may be incomplete.") - pass - + with NamedTemporaryFile(mode="w+b", buffering=0, delete=True) as tmpfile: + try: + for map in maps: + page_start = map[0] + page_len = map[1] - map[0] + mem_page_content = self.read_bytes(pid, page_start, page_len) + if mem_page_content: + tmpfile.write(mem_page_content) + zip_file.write(tmpfile.name, f"process_dumps{sep}{p_name}_{pid}.mem") + except PermissionError: + logging.warning(f"Permission denied opening process memory for {p_name} (pid {pid}). Cannot dump this process.") + continue + except OSError as oserror: + logging.warning(f"Error opening process memory page for {p_name} (pid {pid}). Error was {oserror}. Dump may be incomplete.") + pass + except MemoryError: + logging.warning("Exceeded available memory, skipping further memory collection") + logging.info(f"Dumping processing has completed. Output file is located: {archive_out}") \ No newline at end of file From ae3c72e5af181645262a098d3ddd4c131e345a57 Mon Sep 17 00:00:00 2001 From: chriscado Date: Sun, 2 Oct 2022 21:00:37 +0100 Subject: [PATCH 09/18] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e5ae53d..94fb7b1 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text() -VERSION = '1.0.5' +VERSION = '1.0.6' setup( name='varc', From 7e1153280e6d21542df79f1041fb615ff0d28ff7 Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 14:12:47 +0100 Subject: [PATCH 10/18] Add CD to compile --- .github/workflows/app-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index d2bddd2..3b62e62 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -50,6 +50,10 @@ jobs: ./release/generate_linux_binary.sh chmod +x ./dist/varc ./dist/varc # check it doesn't return non 0 exit status, i.e. crash + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc centos: runs-on: ubuntu-latest From 31240c9e4b808aaf8ecef36fce072a45e6462e3c Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 14:20:41 +0100 Subject: [PATCH 11/18] Add build for macos and windows --- .github/workflows/app-ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 3b62e62..0a6b80b 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -118,6 +118,11 @@ jobs: - name: Run varc Python run: | python3 varc.py + ./release/generate_windows_binary.bat + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc.exe macos: runs-on: macos-12 @@ -133,3 +138,15 @@ jobs: - name: Run varc Python run: | python3 varc.py + - name: Build & Run Binary + run: | + pip3 install pyinstaller + chmod +x ./release/generate_osx_binary.sh + ./release/generate_osx_binary.sh + chmod +x ./dist/varc + ./dist/varc # check it doesn't return non 0 exit status, i.e. crash + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc + From ac188b7a3c3b7d09ac9c68cbe5df424c8d8bc90b Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 17:34:16 +0100 Subject: [PATCH 12/18] Change buildpath --- .github/workflows/app-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 0a6b80b..95b1197 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -118,7 +118,8 @@ jobs: - name: Run varc Python run: | python3 varc.py - ./release/generate_windows_binary.bat + cd release + generate_windows_binary.bat - uses: actions/upload-artifact@v3 with: name: varc @@ -142,7 +143,8 @@ jobs: run: | pip3 install pyinstaller chmod +x ./release/generate_osx_binary.sh - ./release/generate_osx_binary.sh + cd release + ./generate_osx_binary.sh chmod +x ./dist/varc ./dist/varc # check it doesn't return non 0 exit status, i.e. crash - uses: actions/upload-artifact@v3 From d04f957e48c4713e718781d7254f63cea6968d6d Mon Sep 17 00:00:00 2001 From: chriscado Date: Fri, 30 Sep 2022 18:02:24 +0100 Subject: [PATCH 13/18] Looks like m1/2 not in github hosted runners so have to run in an ec2 for now --- .github/workflows/app-ci.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 95b1197..2d83f7f 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -119,7 +119,7 @@ jobs: run: | python3 varc.py cd release - generate_windows_binary.bat + ./generate_windows_binary.bat - uses: actions/upload-artifact@v3 with: name: varc @@ -142,13 +142,3 @@ jobs: - name: Build & Run Binary run: | pip3 install pyinstaller - chmod +x ./release/generate_osx_binary.sh - cd release - ./generate_osx_binary.sh - chmod +x ./dist/varc - ./dist/varc # check it doesn't return non 0 exit status, i.e. crash - - uses: actions/upload-artifact@v3 - with: - name: varc - path: ./dist/varc - From a04ccdeeaac30ee9b3ca247895e2fc865b7387dc Mon Sep 17 00:00:00 2001 From: chriscado Date: Sat, 1 Oct 2022 13:07:37 +0100 Subject: [PATCH 14/18] Add timeout --- .github/workflows/app-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 2d83f7f..b486bd5 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -106,6 +106,7 @@ jobs: windows: runs-on: windows-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v3 - name: Set up Python From 819b2cbd1642c7450377f75cbe06c41401c7da32 Mon Sep 17 00:00:00 2001 From: chriscado Date: Sat, 1 Oct 2022 13:31:51 +0100 Subject: [PATCH 15/18] Try fixing fat binary --- release/generate_osx_binary.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/release/generate_osx_binary.sh b/release/generate_osx_binary.sh index 4be76fd..0fa24ae 100644 --- a/release/generate_osx_binary.sh +++ b/release/generate_osx_binary.sh @@ -1,4 +1,13 @@ # Tested with python 3.10 and PyInstaller 5.4.1 -pip3 install -r requirements.txt -pyinstaller --onefile --clean --target-arch universal2 varc.py +PATHS="../venv/lib/python3.10/site-packages" + +python3 -m venv venv +source ./venv/bin/activate +cd cado-host-python + + +python3 -m pip install --upgrade pip +python3 -m pip install pyinstaller==4.10 +python3 -m pip install -r requirements.txt +python3 -m PyInstaller --onefile --clean --paths $PATHS --target-arch universal2 varc.py From bbf3a075bb900f7ea85795e46846e3d22fa44c52 Mon Sep 17 00:00:00 2001 From: chriscado Date: Sat, 1 Oct 2022 13:57:01 +0100 Subject: [PATCH 16/18] Try older pyinstaller --- .github/workflows/app-ci.yml | 10 +++++++++- release/generate_osx_binary.sh | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index b486bd5..2f8ba9a 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -120,6 +120,7 @@ jobs: run: | python3 varc.py cd release + pip3 install pyinstaller=5.0 ./generate_windows_binary.bat - uses: actions/upload-artifact@v3 with: @@ -142,4 +143,11 @@ jobs: python3 varc.py - name: Build & Run Binary run: | - pip3 install pyinstaller + chmod +x ./release/generate_linux_binary.sh + ./release/generate_linux_binary.sh + chmod +x ./dist/varc + ./dist/varc # check it doesn't return non 0 exit status, i.e. crash + - uses: actions/upload-artifact@v3 + with: + name: varc + path: ./dist/varc \ No newline at end of file diff --git a/release/generate_osx_binary.sh b/release/generate_osx_binary.sh index 0fa24ae..e99aea2 100644 --- a/release/generate_osx_binary.sh +++ b/release/generate_osx_binary.sh @@ -8,6 +8,8 @@ cd cado-host-python python3 -m pip install --upgrade pip -python3 -m pip install pyinstaller==4.10 +python3 -m pip install pyinstaller==5.4.1 python3 -m pip install -r requirements.txt -python3 -m PyInstaller --onefile --clean --paths $PATHS --target-arch universal2 varc.py +# Universal target currently fails +# python3 -m PyInstaller --onefile --clean --paths $PATHS --target-arch universal2 varc.py +python3 -m PyInstaller --onefile --clean --paths $PATHS varc.py From 9a210dc8ff05579448c66e63e67035684ecae61c Mon Sep 17 00:00:00 2001 From: chriscado Date: Tue, 4 Oct 2022 15:43:48 +0100 Subject: [PATCH 17/18] Skip memory for windows as its slow --- .github/workflows/app-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 2f8ba9a..5c9e895 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -118,7 +118,7 @@ jobs: pip3 install -r ./requirements.txt - name: Run varc Python run: | - python3 varc.py + python3 varc.py --skip-memory cd release pip3 install pyinstaller=5.0 ./generate_windows_binary.bat From 12e30a978d7f13aae5395bc29077f5e5196b94de Mon Sep 17 00:00:00 2001 From: chriscado Date: Tue, 4 Oct 2022 16:20:56 +0100 Subject: [PATCH 18/18] Typo --- .github/workflows/app-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index 5c9e895..98bbee4 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -120,7 +120,7 @@ jobs: run: | python3 varc.py --skip-memory cd release - pip3 install pyinstaller=5.0 + pip3 install pyinstaller==5.0 ./generate_windows_binary.bat - uses: actions/upload-artifact@v3 with: