Upgrade and idx2apos 32bit

.gitea/workflows/CI-nix.yaml

@@ -0,0 +1,58 @@
+name: Gitea Actions Demo
+run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
+on: [push]
+
+jobs:
+  Explore-Gitea-Actions:
+    runs-on: nixos-latest
+    steps:
+      - run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event."
+      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
+      - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
+      - name: install node
+        run: nix-env -iA nixpkgs.nodejs_20
+      - name: Check out repository code
+        #uses: actions/checkout@v4
+        uses: https://gitea.com/ScMi1/checkout@v1.4
+        with:
+          submodules: false
+      - run: |
+          git submodule update --init
+      - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
+      - run: echo "🖥️ The workflow is now ready to test your code on the runner."
+      - name: List files in the repository
+        run: |
+          printenv | sort
+          ls ${{ gitea.workspace }}
+      - name: compute dependencies cache hash
+        id: cache_hash
+        shell: bash
+        run: |
+          HASH=$(sha256sum <(git submodule | cut -d ' ' -f 2) | cut -d ' ' -f 1)
+          echo "hash=$HASH" >> $GITHUB_OUTPUT
+          echo "output: hash=$HASH"
+      - name: cache dependencies
+        id: caching
+        uses: actions/cache@v4
+        with:
+          key: ${{ steps.cache_hash.outputs.hash }}
+          path: |
+            ext/Linux_*
+            ext/Darwin_*
+            ext/xWindows_*
+      - name: compile dependencies
+        run: |
+          nix-shell --run "make -C ext"
+      - name: build project
+        shell: nix-shell --run "{0}"
+        run: |
+          mkdir -vp build
+          cd build
+          cmake --fresh -G Ninja -DCMAKE_BUILD_TYPE=Debug ..
+          ninja
+      - name: test project
+        shell: nix-shell --run "{0}"
+        run: |
+          cd build
+          ./tests/cpp/utelykseer-cpp --report_level=detailed
+      - run: echo "🍏 This job's status is ${{ job.status }}."

.github/workflows/CI.yml

@@ -29,48 +29,43 @@ jobs:
         sudo apt update
         sudo apt-get -y install cmake ninja-build clang-19 clang-tools-19 cppcheck m4 busybox libdigest-sha3-perl libboost-all-dev libboost-contract-dev gettext pkg-config automake autoconf libtool texinfo
 
-#    - name: Set up cache on external code
-#      uses: actions/cache@v2
-#      with:
-#        path: ext
-#        key: ${{ runner.os }}-ext-${{ hashFiles('**/CMakeLists.txt') }}
-#        restore-keys: |
-#          ${{ runner.os }}-ext-
-
     - name: Update submodules
       shell: bash
       working-directory: ${{github.workspace}}/
       run: |
         mkdir -vp build
         git submodule update --init
 
+    - name: compute dependencies cache hash
+      id: cache_hash
+      shell: bash
+      run: |
+        HASH=$(sha256sum <(git submodule | cut -d ' ' -f 2) | cut -d ' ' -f 1)
+        echo "hash=$HASH" >> $GITHUB_OUTPUT
+        echo "output: hash=$HASH"
+    - name: cache dependencies
+      id: caching
+      uses: actions/cache@v4
+      with:
+        key: ${{ steps.cache_hash.outputs.hash }}
+        path: |
+          ext/Linux_*
+          ext/Darwin_*
+          ext/xWindows_*
+
     - name: Build dependencies
       shell: bash
       working-directory: ${{github.workspace}}/ext
       run: make
 
-    - name: Set up cache on build dir
-      uses: actions/cache@v2
-      with:
-        path: build
-        key: ${{ runner.os }}-build-${{ hashFiles('**/CMakeLists.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-build-
-
     - name: Configure CMake
-      # Use a bash shell so we can use the same syntax for environment variable
-      # access regardless of the host operating system
       shell: bash
       working-directory: ${{github.workspace}}/build
-      # Note the current convention is to use the -S and -B options here to specify source 
-      # and build directories, but this is only available with CMake 3.13 and higher.  
-      # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
       run: cmake --fresh -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
 
     - name: Build
       working-directory: ${{github.workspace}}/build
       shell: bash
-      # Execute the build.  You can specify a specific target with "--target <NAME>"
       run: cmake --build . --config $BUILD_TYPE
 
     - name: Test
@@ -79,7 +74,7 @@ jobs:
       # Execute tests defined by the CMake configuration.  
       # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
       #run: ctest -C $BUILD_TYPE
-      run: ctest
+      run: ./tests/cpp/utelykseer-cpp --report_level=detailed
 
     - name: Packaging
       working-directory: ${{github.workspace}}/build

CMakeLists.txt

@@ -24,8 +24,8 @@ project(elykseer-cpp LANGUAGES CXX)
 
 # The version number.
 set (${PROJECT_NAME}_VERSION_MAJOR 4)
-set (${PROJECT_NAME}_VERSION_MINOR 0)
-set (${PROJECT_NAME}_VERSION_PATCH 1)
+set (${PROJECT_NAME}_VERSION_MINOR 1)
+set (${PROJECT_NAME}_VERSION_PATCH 0)
 set (${PROJECT_NAME}_VERSION_STRING ${elykseer-cpp_VERSION_MAJOR}.${elykseer-cpp_VERSION_MINOR}.${elykseer-cpp_VERSION_PATCH})
 
 # set to '-DDEBUG' to turn on debugging output

ext/Makefile

@@ -54,6 +54,8 @@ makedirs: $(CCACHE_DIR) $(HOST_TARGETDIR) $(WIN64_TARGETDIR)
 
 $(CCACHE_DIR) $(HOST_TARGETDIR) $(WIN64_TARGETDIR):
 	mkdir -vp $@
+	mkdir -vp $@/lib
+	ln -sf $@/lib $@/lib64
 
 check_cross_compilers: check_cc check_cxx check_ar check_ranlib
 check_ar:
@@ -116,11 +118,11 @@ $(fset_elykseer_crypto):
 	if [ -z "$(CROSS_COMPILATION)" ]; then \
 	$(MAKE) -C ext; \
 	mkdir -vp build; cd build; \
-	cmake --fresh -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} --install-prefix "${TARGETDIR}" -G Ninja .. ; \
+	cmake --fresh -DUSE_OPENSSL=ON -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} --install-prefix "${TARGETDIR}" -G Ninja .. ; \
 	else \
 	$(MAKE) -C ext xWin64; \
 	cd build; \
-	cmake --fresh -DCMAKE_TOOLCHAIN_FILE=${XCOMPILE_TOOLCHAIN} -DCMAKE_BUILD_TYPE=Release --install-prefix "${TARGETDIR}" -G "${CMAKE_GENERATOR}" . ; \
+	cmake --fresh -DUSE_OPENSSL=ON -DCMAKE_TOOLCHAIN_FILE=${XCOMPILE_TOOLCHAIN} -DCMAKE_BUILD_TYPE=Release --install-prefix "${TARGETDIR}" -G "${CMAKE_GENERATOR}" . ; \
 	fi; \
 	ninja; \
 	ninja install;

shell.nix

@@ -8,17 +8,21 @@ stdenv.mkDerivation rec {
     # Customizable development requirements
     nativeBuildInputs = [
         cmake
+        ninja
+        ccache
         git
         gnused
-        pandoc
-        html2text
+        gettext
+        texinfo
         swig
-        #gcc
-        clang
+        gcc
+        llvmPackages_19.clang-tools
+        llvmPackages_19.openmp
+        lldb_19
+        clang_19
         opam m4
-        #global
         cppcheck
-        #busybox
+        perl
         perl538Packages.DigestSHA3
         pkg-config
         autoconf automake libtool
@@ -29,7 +33,8 @@ stdenv.mkDerivation rec {
     buildInputs = [
         openssl
         zlib
-        boost
+        zstd
+        boost186
     ];
 
     shellHook = ''
@@ -39,7 +44,6 @@ stdenv.mkDerivation rec {
       else
         export SED=sed
       fi
-      export SED=sed
       export CC=clang
       export CXX=clang++
     '';

src/cpp/cli/lxrbackup.cpp

@@ -95,7 +95,7 @@ void output_help() {
   std::cout << "--dry-run      just validate arguments" << std::endl;
   std::cout << "-x --pX        sets output path for encrypted chunks" << std::endl;
   std::cout << "-o --pD        sets output path for meta data" << std::endl;
-  std::cout << "-n --nchunks   sets number of chunks (16-256) per assembly" << std::endl;
+  std::cout << "-n --nchunks   sets number of chunks (16-128) per assembly" << std::endl;
   // std::cout << "-c --compression  sets compression (0 or 1)" << std::endl;
   // std::cout << "-u --deduplication sets duplication mode (0, 1 or 2)" << std::endl;
   std::cout << "--refdbfi      adds reference dbfi for deduplication (*)" << std::endl;
@@ -183,10 +183,10 @@ void set_backup_dir_rec(std::string const fp) {
 
 void set_n_chunks(std::optional<int> &nchunks, std::string const p) {
   int n = atoi(p.c_str());
-  if (n >= 16 && n <= 256) {
+  if (n >= 16 && n <= 128) {
     nchunks = n;
   } else {
-    std::clog << "number of chunks, out of range (16-256): " << p << std::endl;
+    std::clog << "number of chunks, out of range (16-128): " << p << std::endl;
     output_error();
   }
 }

src/cpp/cli/lxrrestore.cpp

@@ -18,6 +18,7 @@
 */
 
 #include <chrono>
+#include <fstream>
 #include <iostream>
 #include <filesystem>
 #include <fstream>
@@ -188,10 +189,10 @@ void set_myid(std::optional<std::string> &myid, std::string const p) {
 
 void set_n_chunks(std::optional<int> &nchunks, std::string const p) {
   int n = atoi(p.c_str());
-  if (n >= 16 && n <= 256) {
+  if (n >= 16 && n <= 128) {
     nchunks = n;
   } else {
-    std::clog << "number of chunks, out of range (16-256): " << p << std::endl;
+    std::clog << "number of chunks, out of range (16-128): " << p << std::endl;
     output_error();
   }
 }
@@ -322,7 +323,7 @@ int main (int argc, char * const argv[]) {
 #ifdef DEBUG
         std::clog << "  open file " << fp.c_str() << std::endl;
 #endif
-        FILE *tgt = fopen(fp.c_str(), "wx");
+        std::ofstream tgt(fp, std::ios::binary);
         if (tgt) {
 #ifdef DEBUG
           std::clog << "  file ready." << std::endl;
@@ -333,7 +334,7 @@ int main (int argc, char * const argv[]) {
 #ifdef DEBUG
                 std::clog << "     fseek to " << rqr._readrequest._fpos << std::endl;
 #endif
-                fseek(tgt, rqr._readrequest._fpos, SEEK_SET);
+                tgt.seekp(rqr._readrequest._fpos);
 #ifdef DEBUG
                 std::clog << "     writing " << rqr._rresult->len() << " bytes" << std::endl;
 #endif
@@ -354,7 +355,7 @@ int main (int argc, char * const argv[]) {
             }
           }
           process_read_blocks(qac->iterate_read_queue());
-          fclose(tgt);
+          tgt.close();
         }
       }
     }

src/cpp/coqassembly.cpp

@@ -20,6 +20,7 @@ module;
 
 #include <algorithm>
 #include <cassert>
+#include <cstdint>
 #include <chrono>
 #include <filesystem>
 #include <fstream>
@@ -118,12 +119,17 @@ CoqAssemblyPlainWritable::CoqAssemblyPlainWritable(const CoqConfiguration &c)
   // std::clog << "CoqAssemblyPlainWritable::CoqAssemblyPlainWritable(const CoqConfiguration &c)" << std::endl;
   const Nchunks& nchunks = c.nchunks();
   _buffer.reset(new CoqBufferPlain(assemblysize(nchunks)));
-  Key128 _rand128{false};
-  const char *r = (const char*)_rand128.bytes();
-  for (int i = 0; i < 16; i++) {
-    _buffer->at(idx2apos(i,nchunks), r[i]);
+  uint32_t _rand32;
+  uint32_t idx{0};
+  while (idx < nchunks.u()) {
+    _rand32 = Random::rng().random();
+    const unsigned char *r = (const unsigned char[4])_rand32;
+    for (int i = 0; i < 4; i++) {
+      _buffer->at(idx, r[i]);
+    }
+    idx += 4;
   }
-  _assemblyinformation._apos = 16;
+  _assemblyinformation._apos = nchunks.u();
   _assemblyinformation._nchunks = nchunks;
   mk_assembly_id();
 }
@@ -357,13 +363,12 @@ CoqAssembly::BlockInformation CoqAssemblyPlainWritable::backup(const CoqBufferPl
 CoqAssembly::BlockInformation CoqAssemblyPlainWritable::backup(const CoqBufferPlain &b, const uint32_t offset, const uint32_t dlen)
 {
     uint32_t apos_n = _assemblyinformation._apos;
-    const Nchunks nchunks = _config.nchunks();
     if (dlen + offset > b.len()) { return {}; }
     auto const chksum = b.calc_checksum(offset, dlen);
     if (chksum) {
         // TODO parallelise !
         for (int i = 0; i < dlen; i++) {
-            uint32_t apos = idx2apos(i + apos_n, nchunks);
+            uint32_t apos = idx2apos(i + apos_n);
             _buffer->at(apos, b.at(i + offset));
         }
         _assemblyinformation._apos += dlen;
@@ -398,7 +403,7 @@ uint32_t CoqAssemblyEncrypted::extract()
         if (const auto cfpathopt = chunk_path(cid, aid); cfpathopt) {
             const std::filesystem::path cfpath = cfpathopt.value();
             if (! std::filesystem::exists(cfpath)) {
-                if (FILE * fstr = fopen(cfpath.string().c_str(), "wb"); fstr) {
+                if (std::ofstream fstr(cfpath, std::ios::binary); fstr.good()) {
 #ifdef DEBUG
                     std::clog << "    extract chunk " << cid << " to path " << cfpath << std::endl;
 #endif
@@ -407,7 +412,7 @@ uint32_t CoqAssemblyEncrypted::extract()
                     } else {
                         std::clog << "only wrote " << n << " bytes to chunk with path " << cfpath.string() << std::endl;
                     }
-                    fclose(fstr);
+                    fstr.close();
                 } else {
                     std::clog << "cannot write to chunk with path " << cfpath.string() << std::endl;
                 }
@@ -459,7 +464,7 @@ std::shared_ptr<CoqAssemblyEncrypted> CoqAssemblyEncrypted::recall(const CoqConf
         if (const auto cfpathopt = newassembly->chunk_path(cid, aid); cfpathopt) {
             const std::filesystem::path cfpath = cfpathopt.value();
             if (std::filesystem::exists(cfpath)) {
-                if (FILE * fstr = fopen(cfpath.string().c_str(), "rb"); fstr) {
+                if (std::ifstream fstr(cfpath, std::ios::binary); fstr.good()) {
 #ifdef DEBUG
                     std::clog << "    recall chunk " << cid << " from path " << cfpath << std::endl;
 #endif
@@ -468,7 +473,7 @@ std::shared_ptr<CoqAssemblyEncrypted> CoqAssemblyEncrypted::recall(const CoqConf
                     } else {
                         std::clog << "only read " << n << " bytes from chunk with path " << cfpath.string() << std::endl;
                     }
-                    fclose(fstr);
+                    fstr.close();
                 } else {
                     std::clog << "cannot read from chunk with path " << cfpath.string() << std::endl;
                 }
@@ -499,9 +504,8 @@ Program Definition restore (b : AssemblyPlainFull.B) (bi : blockinformation) : o
 std::shared_ptr<CoqBufferPlain> CoqAssemblyPlainFull::restore(const CoqAssembly::BlockInformation & bi) const
 {
     std::shared_ptr<CoqBufferPlain> b{new CoqBufferPlain(bi.blocksize)};
-    const Nchunks& nchunks = _config.nchunks();
     for (uint32_t idx = 0; idx < bi.blocksize; idx++) {
-        uint32_t apos = idx2apos(idx + bi.blockapos, nchunks);
+        uint32_t apos = idx2apos(idx + bi.blockapos);
         b->at(idx, _buffer->at(apos)); 
     }
     return b;