middleware multi-node support

libqflex/libqflex-legacy-api.h

@@ -237,24 +237,31 @@ typedef size_t (*QEMU_GET_NUM_CORES_t)(void);
 typedef logical_address_t (*QEMU_GET_PC_t)(size_t core_index);
 typedef bool (*QEMU_GET_IRQ_t)(size_t core_index);
 typedef uint64_t (*QEMU_CPU_EXEC_t)(size_t core_index, bool count);
-typedef void (*QEMU_TICK_t)(void);
+typedef void (*QEMU_TICK_t)(bool paused);
 typedef void (*QEMU_GET_MEM_t)(uint8_t* buffer, physical_address_t pa, size_t nb_bytes);
 typedef void (*QEMU_STOP_t)(char const* const msg);
 typedef char* (*QEMU_DISASS_t)(size_t core_index, uint64_t addr, size_t size);
 typedef bool (*QEMU_CPU_BUSY_t)(size_t core_index);
+typedef bool (*QEMU_CAN_STOP_t)();
 // ─────────────────────────────────────────────────────────────────────────────
 
 typedef void (*FLEXUS_START_t)(uint64_t);
 typedef void (*FLEXUS_STOP_t)(void);
 typedef void (*FLEXUS_QMP_t)(qmp_flexus_cmd_t, const char*);
 typedef void (*FLEXUS_TRACE_MEM_t)(uint64_t, memory_transaction_t*);
+typedef void (*FLEXUS_PAUSE_t)(void);
+typedef void (*FLEXUS_RESUME_t)(void);
+typedef bool (*FLEXUS_IS_PAUSED_t)(void);
 
 typedef struct FLEXUS_API_t
 {
     FLEXUS_START_t start;
     FLEXUS_STOP_t stop;
     FLEXUS_QMP_t qmp;
     FLEXUS_TRACE_MEM_t trace_mem;
+    FLEXUS_PAUSE_t pause;
+    FLEXUS_RESUME_t resume;
+    FLEXUS_IS_PAUSED_t is_paused;
 } FLEXUS_API_t;
 
 typedef struct QEMU_API_t
@@ -269,6 +276,7 @@ typedef struct QEMU_API_t
     QEMU_GET_MEM_t get_mem;
     QEMU_STOP_t stop;
     QEMU_TICK_t tick;
+    QEMU_CAN_STOP_t can_stop;
     QEMU_DISASS_t disassembly;
     QEMU_CPU_BUSY_t is_busy;
 } QEMU_API_t;
@@ -284,6 +292,8 @@ void
 FLEXUS_qmp(qmp_flexus_cmd_t, const char*);
 void
 FLEXUS_trace_mem(uint64_t, memory_transaction_t*);
+void FLEXUS_pause(void);
+void FLEXUS_resume(void);
 
 void
 QEMU_get_api(QEMU_API_t* api);

libqflex/libqflex-module.c

@@ -133,6 +133,7 @@ libqflex_flexus_init(void)
         .stop               = libqflex_stop,
         .get_mem            = libqflex_read_main_memory,
         .tick               = libqflex_tick,
+        .can_stop           = libqflex_can_stop,
         .disassembly        = libqflex_disas,
         .is_busy            = libqflex_is_core_busy,
     };

libqflex/libqflex.c

@@ -9,6 +9,7 @@
 #include "qapi/qapi-commands-control.h"
 #include "sysemu/runstate.h"
 #include "include/disas/disas.h"
+#include "qemu/main-loop.h"
 
 #include "libqflex.h"
 #include "libqflex-module.h"
@@ -17,6 +18,7 @@
 
 #include "cpu.h"
 #include "internals.h"
+#include "net/pdes-engine.h"
 
 #include "target/arm/cpregs.h" // Need to be last
 // ─────────────────────────────────────────────────────────────────────────────
@@ -311,22 +313,55 @@ libqflex_has_interrupt(size_t cpu_index)
                 cpu_wrapper->state->interrupt_request);
 }
 
+bool libqflex_can_stop(void)
+{
+    PDESEngine* engine = get_singleton_engine();
+    if(engine == NULL){
+        return true;
+    }
+    return can_stop(engine);
+}
 
 void
-libqflex_tick(void)
+libqflex_tick(bool paused)
 {
     g_assert(qemu_libqflex_state.is_configured);
     g_assert(qemu_libqflex_state.is_running);
     g_assert(qemu_libqflex_state.mode == MODE_TIMING);
+    if (icount_enabled()) {
+        if (!paused) {
+            seqlock_write_lock(&timers_state.vm_clock_seqlock,
+                               &timers_state.vm_clock_lock);
+            // TODO remove accumulated icount between cycles
+            int64_t icount = icount_drain_executed();
+            qatomic_set_i64(&timers_state.qemu_icount,
+                            timers_state.qemu_icount + 1);
+            seqlock_write_unlock(&timers_state.vm_clock_seqlock,
+                                 &timers_state.vm_clock_lock);
+        }
+    } else {
+        assert(false && "Tick should not be called when icount is disabled");
+    }
 
-    // proceed one clock cycle
-    seqlock_write_lock(&timers_state.vm_clock_seqlock, &timers_state.vm_clock_lock);
-    qatomic_set_i64(&timers_state.qemu_icount, timers_state.qemu_icount + 1);
-    seqlock_write_unlock(&timers_state.vm_clock_seqlock, &timers_state.vm_clock_lock);
-    // fire qemu timers to generate guest timer interrupts
-    icount_account_warp_timer();
-    icount_handle_deadline();
+
+    qemu_mutex_unlock_iothread();
+    replay_mutex_lock();
+    qemu_mutex_lock_iothread();
+    if (!paused) {
+        if (icount_enabled()) {
+            icount_account_warp_timer();
+            icount_handle_deadline();
+        }
+    }
+    replay_mutex_unlock();
+
+    if (icount_enabled() && all_cpu_threads_idle()) {
+        qemu_notify_event();
+    }
 
+    // ALWAYS call this on vCPU thread — this is how the vCPU
+    // acknowledges vm_stop and blocks until vm_start
+    rr_wait_io_event();
 }
 
 uint64_t
@@ -348,6 +383,19 @@ libqflex_stop(char const * const msg)
 {
     Error* err = NULL;
     qemu_log("> [Libqflex] Stopping: %s\n", msg);
+    printf("> [Libqflex] Stopping: %s\n", msg);
+    // get engine
+    PDESEngine* engine = get_singleton_engine();
+    if (engine != NULL) {
+        if(!engine->notified_neighbors_for_exit){
+            // TODO check if extra message is ok
+            notify_neighbours_of_end(engine);
+            destroy_strategy();
+            return;
+        }
+        destroy_strategy();
+    }
+
 
     qmp_stop(&err);
 

libqflex/libqflex.h

@@ -175,7 +175,11 @@ uint64_t
 libqflex_advance(size_t, bool);
 
 void
-libqflex_tick(void);
+libqflex_tick(bool paused);
+
+bool
+libqflex_can_stop(void);
+
 
 uint64_t
 libqflex_step(struct CPUState*);