diff --git a/src/kernel/hal/arch/riscv/pagefault.h b/src/kernel/hal/arch/riscv/pagefault.h new file mode 100644 index 00000000..bddf4aae --- /dev/null +++ b/src/kernel/hal/arch/riscv/pagefault.h @@ -0,0 +1,39 @@ +#ifndef HAL_RISCV_PAGEFAULT +#define HAL_RISCV_PAGEFAULT + +#include +#include +#include + +#include "hal/arch/riscv/pmap.h" +#include "hal/arch/riscv/pte.h" +#include "hal/include/address_space.h" +#include "libcore/error.h" + +/// Mock process address space +extern address_space_t as; +/** + * @brief Mock pagefault resolution in virtual address space map. + */ +error_t vm_handle_pagefault(uintptr_t va, paddr_t pt, hal_address_region_flag_t fault_prot, [[maybe_unused]] bool usermode) { + for(size_t i = 0; i < as.regions_count; ++i) { + address_space_region_t as_region = as.regions[i]; + if(va >= (uintptr_t)as_region.addr && va < ((uintptr_t)as_region.addr + as_region.size)) { + if((as_region.flags & fault_prot) != 0) { + pte_t* ptep = pmap_pte_ptr_lookup(va, pt, PMAP_PAGE); + pte_t pte_flags = as_flags_to_pte((as_region.flags)); + + paddr_t pa; + error_t err = pmap_allocate_page(&pa, PMAP_PAGE); + if(err != ERR_NONE) + return err; + *ptep = pa_to_pte(pa) | pte_flags; + return ERR_NONE; + } + //Violated permission + return ERR_NOT_VALID; + } + } + return ERR_OUT_OF_BOUNDS; +} +#endif /*!HAL_RISCV_PAGEFAULT */ diff --git a/src/kernel/hal/arch/riscv/pmap.c b/src/kernel/hal/arch/riscv/pmap.c new file mode 100644 index 00000000..4ee97758 --- /dev/null +++ b/src/kernel/hal/arch/riscv/pmap.c @@ -0,0 +1,195 @@ +#include "pmap.h" + +#include +#include +#include +#include + +#include "hal/include/address_space.h" +#include "pte.h" + +error_t pmap_allocate_page(paddr_t* pa, pmap_page_t page_type) { + physical_memory_region_t phys_region; + error_t err = phys_mem_alloc_frame(g_pmap_frame_order[page_type], &phys_region); + if (err == ERR_NONE) { + memset(physical_to_effective(phys_region.addr), 0, g_pmap_frame_order[page_type]); + *pa = (paddr_t)phys_region.addr; + } + return err; +} + +error_t pmap_unallocate_page(paddr_t pa, pmap_page_t page_type) { + physical_memory_region_t phys_region = {.addr = (__phys void*)pa, .size = g_pmap_frame_order[page_type]}; + error_t err = phys_mem_free_frame(phys_region); + if (err == ERR_NOT_VALID) { + /* TODO: Deal with unallocating invalid frame */ + return ERR_NOT_VALID; + } + return ERR_NONE; +} + +static error_t pmap_free_ptes_l0(pagetable_t* pt_l0) { + for (int i = 0; i < PTE_COUNT; i++) { + pte_t pte = pt_l0[i]; + if (!pte_is_valid(pte)) + continue; + error_t err = pmap_unallocate_page(pte, PMAP_PAGE); + if (err != ERR_NONE) { + return err; + } + pte = pte_mark_invalid(pte); + pt_l0[i] = pte; + } + return ERR_NONE; +}; + +static error_t pmap_free_ptes_l1(pagetable_t* pt_l1) { + error_t err = ERR_NONE; + for (int i = 0; i < PTE_COUNT; i++) { + pte_t pte = pt_l1[i]; + if (!pte_is_valid(pte)) + continue; + if (pte_is_leaf(pte)) { + err = pmap_unallocate_page(pte_to_pa(pte), PMAP_MEGAPAGE); + if (err == ERR_NOT_VALID) { + return ERR_NOT_VALID; + } + pt_l1[i] = pte_mark_invalid(pte); + continue; + } + pagetable_t* pt = (pagetable_t*)pa_to_da(pte_to_pa(pte)); + // go one level down + err = pmap_free_ptes_l0(pt); + if (err == ERR_NOT_VALID) { + return ERR_NOT_VALID; + } + // actual pte + err = pmap_unallocate_page(pte_to_pa(pte), PMAP_PAGE); + if (err == ERR_NOT_VALID) { + return ERR_NOT_VALID; + } + pte = pte_mark_invalid(pte); + pt_l1[i] = pte; + } + return ERR_NONE; +}; + +static error_t pmap_free_ptes_l2(pagetable_t* pt_l2) { + error_t err = ERR_NONE; + for (int i = 0; i < PTE_COUNT; i++) { + pte_t pte = pt_l2[i]; + if (!pte_is_valid(pte)) + continue; + if (pte_is_leaf(pte)) { + err = pmap_unallocate_page(pte_to_pa(pte), PMAP_HUGEPAGE); + if (err == ERR_NOT_VALID) { + return ERR_NOT_VALID; + } + pt_l2[i] = pte_mark_invalid(pte); + continue; + } + pagetable_t* pt = (pagetable_t*)pa_to_da(pte_to_pa(pte)); + // go level down + err = pmap_free_ptes_l1(pt); + if (err != ERR_NONE) { + return err; + } + // actual pte + err = pmap_unallocate_page(pte_to_pa(pte), PMAP_PAGE); + if (err == ERR_NOT_VALID) { + return ERR_NOT_VALID; + } + pte = pte_mark_invalid(pte); + pt_l2[i] = pte; + } + return ERR_NONE; +}; + +error_t pmap_free_pagetable(paddr_t pt) { + pagetable_t* root_pt = pa_to_da(pt); + if (root_pt == nullptr) + return ERR_BAD_ARG; + error_t err = pmap_free_ptes_l2(root_pt); + if(err != ERR_NONE) + return err; + err = pmap_unallocate_page(pt, PMAP_PAGE); + return err; +} + +pte_t* pmap_pte_ptr_lookup_valid(pagetable_t pt, vaddr_t va) { + for (int level = PT_LEVELS - 1; level >= 0; --level) { + pte_t* ptep = pte_ptr(pt, va_vpn(level, va)); + if (pte_is_valid(*ptep)) { + if (pte_is_leaf(*ptep)) { + return ptep; + } + pt = pte_to_pa(*ptep); + } else { + return nullptr; + } + } + return nullptr; +} + +pte_t* pmap_pte_ptr_lookup(pagetable_t pt, vaddr_t va, pmap_page_t page_type) { + int desired_page_level = (int)page_type; + for (int level = PT_LEVELS - 1; level > desired_page_level; --level) { + pte_t* ptep = pte_ptr(pt, va_vpn(level, va)); + if (pte_is_valid(*ptep)) { + if (pte_is_leaf(*ptep)) { + if (level == desired_page_level) { + return ptep; + } + return nullptr; + } + pt = pte_to_pa(*ptep); + } else { + if(level == desired_page_level) { + return ptep; + } + paddr_t pte_new_pa; + error_t err = pmap_allocate_page(&pte_new_pa, PMAP_PAGE); + if(err != ERR_NONE) { + /* TODO: Deal with allocation failure */ + return nullptr; + } + *ptep = pte_mark_valid(pa_to_pte(pte_new_pa)); + pt = pte_new_pa; + } + } + return pte_ptr(pt, va_vpn(desired_page_level, va)); +} +/** + * + */ +bool pmap_pagefault_fixup(pagetable_t pt, vaddr_t va, hal_address_region_flag_t prot) { + pte_t* ptep = pmap_pte_ptr_lookup_valid(pt, va); + if (ptep == nullptr) { + return false; + } + pte_t pte = *ptep; + bool modified = false; + switch (prot) { + case HAL_ASR_FLAGS_READ: + if ((pte & (PTE_R | PTE_A)) == PTE_R) { + pte |= PTE_A; + modified = true; + } + break; + case HAL_ASR_FLAGS_WRITE: + if (((pte & PTE_W) != 0) && ((pte & PTE_DA) != PTE_DA)) { + pte |= PTE_A | PTE_D; + modified = true; + } + break; + case HAL_ASR_FLAGS_EXECUTE: + if ((pte & (PTE_X | PTE_A)) == PTE_X) { + pte |= PTE_A; + modified = true; + } + } + if (modified) { + *ptep = pte; + } + return modified; +} diff --git a/src/kernel/hal/arch/riscv/pmap.h b/src/kernel/hal/arch/riscv/pmap.h new file mode 100644 index 00000000..2ee71bd5 --- /dev/null +++ b/src/kernel/hal/arch/riscv/pmap.h @@ -0,0 +1,117 @@ +#ifndef HAL_RISCV_PMAP_H +#define HAL_RISCV_PMAP_H + +#include +#include +#include +#include +#include + +#include "pte.h" + +/** + * @brief Physical page types supported in SV39 mode. + * Underlying value specifies which level Pagetable contains it's + * Pagetable Entry. + * */ +typedef enum : u8 { + PMAP_PAGE = 0, + PMAP_MEGAPAGE = 1, + PMAP_HUGEPAGE = 2, +} pmap_page_t; + +/** + * @brief Correspondence between RISC V physical page and physical frame order + */ +static const frame_order_t g_pmap_frame_order[] = { + [PMAP_PAGE] = FRAME_ORDER_4KiB, [PMAP_HUGEPAGE] = FRAME_ORDER_2MiB, [PMAP_MEGAPAGE] = FRAME_ORDER_1GiB}; + +#define SATP_MODE_SV39 (8UL << 60) +#define SATP_MODE SATP_MODE_SV39 +#define SATP_ASID_S 44 +#define SATP_ASID_MASK 0x3FUL +#define PTE_COUNT 512 /* Number of PTEs in a pagetable */ +#define PT_LEVELS 3 /* Number of pagetable indirection levels */ + +static inline reg_t make_satp(paddr_t pa, u64 asid) { + return SATP_MODE | (asid << SATP_ASID_S) | (pa >> PPN_OFFSET); +} + +static inline paddr_t satp_to_pa(reg_t satp) { + return (satp & PPN_MASK) << PPN_OFFSET; +} + +static inline u64 satp_to_asid(reg_t satp) { + return (satp >> SATP_ASID_S) & SATP_ASID_MASK; +} + +/** + * @brief Translates between physical address and it's mapping in kernel + * address space (direct address). + */ +static inline void* pa_to_da(paddr_t pa) { + return physical_to_effective((__phys void*)pa); +} + +/** + * @brief Extract pointer to a PTE from the pagetable. + * + * @param pt Pagetable physical address + * @param n Physical Page Number + * @returns Pointer to the n-th PTE in the pagetable. + */ +static inline pte_t* pte_ptr(pagetable_t pt, u64 n) { + pagetable_t* ptp = (pagetable_t*)pa_to_da(pt); + ptp = ptp + n; + return (pte_t*)ptp; +} + +/** + * @brief Fixes pagefault caused by memory accesses to a page + * without appropriate Accessed and/or Dirty bits set. + */ +bool pmap_pagefault_fixup(pagetable_t, vaddr_t, hal_address_region_flag_t); + +/** + * @brief Find valid PTE corresponding to the virtual address. + */ +pte_t* pmap_pte_ptr_lookup_valid(pagetable_t, vaddr_t); + +/** + * @brief Find PTE corresponding to the virtual address and page type. + * Allocates intermediate Pagetable pages if necessary. + * + * @return Pointer to the PTE or nullptr if either allocation proved + * impossible or there already exists a valid PTE correspoding to virtual + * address, but it's type is different than expected. + * + * @note Does not allocate Leaf PTE and associated page frame, + * to allow mapping of already existing resource. + */ +pte_t* pmap_pte_ptr_lookup(pagetable_t, vaddr_t, pmap_page_t); +/** + * @brief Unallocate all pages associated with the pagetable. + * + */ +error_t pmap_free_pagetable(paddr_t); + +/** + * @brief Allocate physical page + * @param[out] pa, physical address of the allocated page. + * @param[in] page_type, page_type . + * @return ERR_NOT_VALID, if allocation of physical memory fails + * ERR_NONE otherwise. + * @todo Physical page bookkeeping. + */ +error_t pmap_allocate_page(paddr_t* pa, pmap_page_t page_type); + +/** + * @brief Unallocate physical page + * @param[in] pa, physical address of page to be unallocated. + * @param[in] page_type, page_type specifies size of the allocation. + * @return ERR_NOT_VALID, if unallocation of physical memory fails + * ERR_NONE otherwise. + * @todo Physical page bookkeeping. + */ +error_t pmap_unallocate_page(paddr_t pa, pmap_page_t page_type); +#endif /* !HAL_RISCV_PMAP_H */ diff --git a/src/kernel/hal/arch/riscv/pte.h b/src/kernel/hal/arch/riscv/pte.h new file mode 100644 index 00000000..29526329 --- /dev/null +++ b/src/kernel/hal/arch/riscv/pte.h @@ -0,0 +1,155 @@ +/** + * @file pte.h + * @brief Internal RISC-V Pagetable macros and functions + * + * This header is internal to HAL and should not be included directly by user code. + */ + +#ifndef HAL_ARCH_RISCV_PAGE_TABLE_H +#define HAL_ARCH_RISCV_PAGE_TABLE_H + +#include +#include +#include "hal/include/address_space.h" + +typedef u64 vaddr_t; +typedef u64 paddr_t; +typedef u64 pte_t; +typedef pte_t pagetable_t; + +/* MODE bits; 4 MSB in 64bit register satp */ +#define SATP_SV39 (8UL << 60) +#define PPN_MASK 0xF'FF'FF'FF'FF'FFull /* 44 low bits */ +#define PPN_OFFSET 12 + +#define VA_VPN_WIDTH 9 +#define VA_VPN_OFFSET 12 +#define VA_VPN_MASK 0x1'FFull +#define VA_VPN0_S 12 +#define VA_VPN1_S 21 +#define VA_VPN2_S 30 + +#define PA_PPN_OFFSET 12 +#define PA_PPN0_S 12 +#define PA_PPN1_S 21 +#define PA_PPN2_S 30 + +#define PTE_PPN_OFFSET 10 +#define PTE_PPN0_S 10 +#define PTE_PPN1_S 19 +#define PTE_PPN2_S 28 + +/* */ +#define PTE_RSW (3ull << 8) /* RSW */ +#define PTE_D (1ull << 7) /* Dirty */ +#define PTE_A (1ull << 6) /* Accessed */ +#define PTE_G (1ull << 5) /* Global */ +#define PTE_U (1ull << 4) /* User */ +#define PTE_X (1ull << 3) /* Execute */ +#define PTE_W (1ull << 2) /* Write */ +#define PTE_R (1ull << 1) /* Read */ +#define PTE_V (1ull << 0) /* Valid */ +#define PTE_RWX (PTE_R | PTE_W | PTE_X) +#define PTE_RW (PTE_R | PTE_W) +#define PTE_RX (PTE_R | PTE_X) +#define PTE_WX (PTE_W | PTE_X) +#define PTE_DA (PTE_D | PTE_A) + +static inline bool pte_is_valid(pte_t pte) { + return ((pte & PTE_V) != 0); +} + +static inline pte_t pte_mark_valid(pte_t pte) { + return (pte | PTE_V); +} + +static inline pte_t pte_mark_invalid(pte_t pte) { + return (pte & ~PTE_V); +} + +static inline paddr_t pte_to_pa(pte_t pte) { + return (((pte >> PTE_PPN_OFFSET) & PPN_MASK) << PA_PPN_OFFSET); +} + +static inline pte_t pa_to_pte(paddr_t pa) { + return (((pa >> PA_PPN_OFFSET) & PPN_MASK) << PTE_PPN_OFFSET); +} + +static inline bool pte_is_leaf(pte_t pte) { + return ((pte & PTE_R) || ((pte & PTE_WX) == PTE_X)); +} + +static inline bool pte_is_branch(pte_t pte) { + return ((pte & PTE_RWX) == 0); +} + +static inline pte_t pte_make_branch(paddr_t pa) { + return pa_to_pte(pa) | PTE_V; +} + +static inline u64 va_vpn2(vaddr_t va) { + return ((va >> VA_VPN2_S) & VA_VPN_MASK); +} + +static inline u64 va_vpn1(vaddr_t va) { + return ((va >> VA_VPN1_S) & VA_VPN_MASK); +} + +static inline u64 va_vpn0(vaddr_t va) { + return ((va >> VA_VPN0_S) & VA_VPN_MASK); +} + +static inline u64 va_vpn(uint8_t level, vaddr_t va) { + switch (level) { + case 0: return va_vpn0(va); + case 1: return va_vpn1(va); + case 2: + default: return va_vpn2(va); + } +} + +static inline hal_address_region_flag_t pte_to_asr_flags(pte_t pte) { + hal_address_region_flag_t asr_flags = {0}; + if(pte & PTE_R) + asr_flags |= HAL_ASR_FLAGS_READ; + if(pte & PTE_W) + asr_flags |= HAL_ASR_FLAGS_WRITE; + if(pte & PTE_X) + asr_flags |= HAL_ASR_FLAGS_EXECUTE; + return asr_flags; +} + +static inline hal_address_space_flag_t pte_to_as_flags(pte_t pte) { + hal_address_space_flag_t as_flags = {0}; + if(pte & PTE_G) + as_flags |= HAL_AS_GLOBAL; + if(pte & PTE_U) + as_flags |= HAL_AS_USER; + return as_flags; +} + +static inline u64 as_flags_to_pte(address_space_region_flags_t as_flags) { + u64 flags = 0; + if (as_flags & HAL_AS_GLOBAL) flags |= PTE_G; + if (as_flags & HAL_AS_USER) flags |= PTE_U; + + if (as_flags & HAL_ASR_FLAGS_READ) flags |= PTE_R; + if (as_flags & HAL_ASR_FLAGS_WRITE) flags |= PTE_W; + if (as_flags & HAL_ASR_FLAGS_EXECUTE) flags |= PTE_X; + + return flags; +} + +static inline u64 hal_as_flags_to_pte(hal_address_space_flag_t as_flags, hal_address_region_flag_t asr_flags) { + u64 flags = 0; + if (as_flags & HAL_AS_GLOBAL) flags |= PTE_G; + if (as_flags & HAL_AS_USER) flags |= PTE_U; + + if (asr_flags & HAL_ASR_FLAGS_READ) flags |= PTE_R; + if (asr_flags & HAL_ASR_FLAGS_WRITE) flags |= PTE_W; + if (asr_flags & HAL_ASR_FLAGS_EXECUTE) flags |= PTE_X; + + return flags; +} + +#endif // !HAL_ARCH_RISCV_PAGE_TABLE_H diff --git a/src/kernel/hal/arch/riscv/tlb.h b/src/kernel/hal/arch/riscv/tlb.h new file mode 100644 index 00000000..7240ab4e --- /dev/null +++ b/src/kernel/hal/arch/riscv/tlb.h @@ -0,0 +1,10 @@ +#ifndef HAL_RISCV_TLB_H +#define HAL_RISCV_TLB_H + +/** + * @brief Invalidates all address-translation cache entries, for all address spaces + */ +void hal_riscv_tlb_invalidate(void) { + __asm __volatile("sfence.vma" ::: "memory"); +} +#endif /* !HAL_RISCV_TLB_H */ diff --git a/src/kernel/hal/arch/riscv/trap.c b/src/kernel/hal/arch/riscv/trap.c index c1037339..5757c9a9 100644 --- a/src/kernel/hal/arch/riscv/trap.c +++ b/src/kernel/hal/arch/riscv/trap.c @@ -1,13 +1,18 @@ #include "hal/include/trap.h" +#include #include #include #include #include #include +#include #include "csr.h" #include "csr_vals.h" +#include "hal/include/address_space.h" +#include "pmap.h" +#include "pagefault.h" #include "trap.h" extern void hal_riscv_trap_entry(); @@ -117,6 +122,37 @@ static void hal_riscv_trap_interrupt_handler(hal_riscv_trap_interrupt_t code) { } } +static void hal_riscv_page_fault_handler(hal_riscv_trap_exception_t code, riscv_trap_frame_t *ctx) { + bool usermode = ((ctx->sstatus & CSR_SSTATUS_SPP) == 0); + uintptr_t vaddr = (uintptr_t)ctx->stval; + /* For the timebeing, address of the pagetable is taken directly + * from the satp register. */ + uintptr_t pagetable_addr = (uintptr_t)(satp_to_pa(CSR_READ(satp))); + hal_address_region_flag_t fault_prot; + switch(code) { + case HAL_RISCV_TRAP_EXC_STORE_PAGE_FAULT: + fault_prot = HAL_ASR_FLAGS_WRITE; + break; + case HAL_RISCV_TRAP_EXC_LOAD_PAGE_FAULT: + fault_prot = HAL_ASR_FLAGS_READ; + break; + case HAL_RISCV_TRAP_EXC_INSTR_PAGE_FAULT: + fault_prot = HAL_ASR_FLAGS_EXECUTE; + break; + default: + return; + } + KLOG_TRACE("Pagefault at %p, ", (void*)vaddr); + /* If page is present, fix Accessed, Dirty bits */ + if(pmap_pagefault_fixup(pagetable_addr, vaddr, fault_prot)){ + return; + } + if(vm_handle_pagefault(vaddr, pagetable_addr, fault_prot, usermode) == ERR_NONE){ + return; + } + //TODO: Oops if in kernel, send SIGSEGV equivalent if in usermode. +} + static void hal_riscv_trap_exception_handler(hal_riscv_trap_exception_t code, riscv_trap_frame_t* ctx) { switch (code) { case HAL_RISCV_TRAP_EXC_ENV_CALL_U: @@ -140,6 +176,11 @@ static void hal_riscv_trap_exception_handler(hal_riscv_trap_exception_t code, ri ctx->a1 = result.value; } break; + case HAL_RISCV_TRAP_EXC_STORE_ADDRESS_FAULT: + case HAL_RISCV_TRAP_EXC_LOAD_ACCESS_FAULT: + case HAL_RISCV_TRAP_EXC_INSTR_PAGE_FAULT: + hal_riscv_page_fault_handler(code, ctx); + break; default: hal_riscv_trap_unhandled_exception(code, ctx->stval); break; } }