Added initramfs image format, builder and kernel parser

include/stdbigos/bitutils.h

@@ -8,10 +8,14 @@
 /// @addtogroup bitutils
 /// @{
 
+u16 read_be16(const void* addr);
+
 u32 read_be32(const void* addr);
 
 u64 read_be64(const void* addr);
 
+u16 read_le16(const void* addr);
+
 u32 read_le32(const void* addr);
 
 u64 read_le64(const void* addr);

include/stdbigos/buffer.h

@@ -31,17 +31,26 @@ static inline bool buffer_is_empty(buffer_t buf) {
 }
 
 [[nodiscard]]
-bool buffer_read_u32_be(buffer_t buf, size_t offset, u32* out);
+bool buffer_read_u8(buffer_t buf, size_t offset, u8* out);
 
 [[nodiscard]]
-bool buffer_read_u64_be(buffer_t buf, size_t offset, u64* out);
+bool buffer_read_u16_le(buffer_t buf, size_t offset, u16* out);
 
 [[nodiscard]]
 bool buffer_read_u32_le(buffer_t buf, size_t offset, u32* out);
 
 [[nodiscard]]
 bool buffer_read_u64_le(buffer_t buf, size_t offset, u64* out);
 
+[[nodiscard]]
+bool buffer_read_u16_be(buffer_t buf, size_t offset, u16* out);
+
+[[nodiscard]]
+bool buffer_read_u32_be(buffer_t buf, size_t offset, u32* out);
+
+[[nodiscard]]
+bool buffer_read_u64_be(buffer_t buf, size_t offset, u64* out);
+
 [[nodiscard]]
 bool buffer_read_cstring_len(buffer_t buf, size_t offset, const char** out_str, u64* len);
 

include/stdbigos/error.h

@@ -17,6 +17,8 @@ typedef enum [[nodiscard]] {
 	ERR_NOT_ENOUGH_MEMORY,
 	ERR_REPEATED_INITIALIZATION,
 	ERR_ALLOCATION_FAILED,
+	ERR_DT_NODE_NOT_FOUND,
+	ERR_DT_PROP_NOT_FOUND,
 } error_t;
 
 #endif // !BIGOS_INCLUDE_STDBIGOS_ERROR

scripts/create_initramfs.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# usage: ./create_initramfs.sh file1 file2 file3 ... -o outfile.img
+# input file names must be shorter then 255 chars
+# input files must be smaller then 4GiB
+# input files combined size must be smaller then 4GiB
+
+GIT_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)
+
+if [ -z "$GIT_ROOT" ]; then
+    echo "Not inside a Git repository."
+	exit 1
+fi
+
+python3 "$GIT_ROOT/scripts/internal/create_initramfs.py" "$@"

scripts/internal/create_initramfs.py

@@ -0,0 +1,113 @@
+import argparse
+import struct
+from pathlib import Path
+
+
+def parse():
+    parser = argparse.ArgumentParser( description="Create initramfs with provided files")
+    parser.add_argument("files", nargs="+", help="Files to add")
+    parser.add_argument("-o", "--output", required=True, help="Output initramfs file")
+    args = parser.parse_args()
+
+    files = [Path(f) for f in args.files]
+
+    missing = [f for f in files if not f.is_file()]
+    if missing:
+        print("These files are missing or not regular files:")
+        for f in missing:
+            print(" -", f)
+        return
+
+    too_long = [f for f in files if len(f.name) > 255]
+    if too_long:
+        print("These files have too long names (max 255 chars):")
+        for f in too_long:
+            print(" -", f)
+        return
+
+    oversized = [f for f in files if f.stat().st_size > 2**32 - 1]
+    if oversized:
+        print("These files have are too big (max 4GiB):")
+        for f in oversized:
+            print(" -", f)
+        return
+
+    totalSize = 0
+    for f in files:
+        totalSize += f.stat().st_size
+    if totalSize > 2**32 - 1:
+        print(f"The total size of all files must be at most 4GiB ({ 2**32 - 1:,}) and is {totalSize:,}")
+
+    entries = [
+        (path, open(path, "rb"))
+        for path in files
+    ]
+    outfile = open(args.output, "wb")
+
+    return entries, outfile
+
+
+def align2(x):
+    return (x + 1) & ~1
+
+
+def align16(x):
+    return (x + 15) & ~15
+
+
+def calcHeaderSize(entries):
+    MAGIC_SIZE = 4
+    VERSION_SIZE = 8
+    FILE_COUNT_SIZE = 2
+    out = MAGIC_SIZE + VERSION_SIZE + FILE_COUNT_SIZE
+    for f, _ in entries:
+        OFFSET_SIZE = 4
+        SIZE_SIZE = 4
+        NAME_LENGTH_SIZE = 1
+        out += align2(OFFSET_SIZE + SIZE_SIZE + NAME_LENGTH_SIZE + len(f.name))
+    return out
+
+
+def main():
+    parsed = parse()
+    if parsed is None:
+        return
+
+    entries, outfile = parsed
+
+    headerSize = calcHeaderSize(entries)
+    offset = align16(headerSize)
+
+    MAGIC = 0xB16B00B5
+    VERSION = 0x00000000  # TODO:
+
+    outfile.write(struct.pack(">IQH", MAGIC, VERSION, len(entries)))
+
+    for path, file in entries:
+        file_offset = offset
+        file_size = path.stat().st_size
+        file_name = path.name.encode()
+        file_name_size = len(file_name)
+
+        fmt = f">IIB{file_name_size}s"
+
+        outfile.write(struct.pack(fmt, file_offset, file_size, file_name_size, file_name))
+
+        if (file_name_size + 1) % 2 == 1:
+            outfile.write(b"\x00")
+
+        offset += align16(file_size)
+
+    for i in range((16 - headerSize % 16) % 16):
+        outfile.write(b"\x00")
+
+    for path, file in entries:
+        outfile.write(file.read())
+        for i in range((16 - path.stat().st_size % 16) % 16):
+            outfile.write(b"\x00")
+
+        file.close()
+
+
+if __name__ == "__main__":
+    main()

src/kernel/hal/include/initramfs.h

@@ -0,0 +1,9 @@
+#ifndef HAL_INITRAMFS
+#define HAL_INITRAMFS
+
+#include "stdbigos/buffer.h"
+#include "stdbigos/error.h"
+
+error_t hal_initramfs_read(const char* filename, buffer_t* buffOUT);
+
+#endif // !HAL_INITRAMFS

src/kernel/hal/initramfs.c

@@ -0,0 +1,114 @@
+#include "hal/include/initramfs.h"
+
+#include "dt/dt.h"
+#include "hal_internal.h"
+#include "stdbigos/buffer.h"
+#include "stdbigos/math.h"
+#include "stdbigos/string.h"
+
+/*
+ * initramfs layout (big endian)
+ *
+ * u32 magic
+ * u64 version
+ * u16 - file_count
+ *
+ * aligned to 16bit boundry relative to the start of initramfs
+ *	u32 - offset (from the start of initramfs)
+ *	u32 - size
+ *	u8 - namesize
+ *	char[] - name
+ *	...
+ *
+ * data blob
+ * */
+
+static error_t validate(buffer_t initramfs_buff, u64* cursorOUT) {
+	const u32 INITRAMFS_MAGIC = 0xB16B00B5;
+	u32 magic = 0;
+	u64 version = 0;
+	*cursorOUT = 0;
+	if (!buffer_read_u32_be(initramfs_buff, *cursorOUT, &magic))
+		return ERR_INVALID_MEMORY_REGION;
+	*cursorOUT += sizeof(u32);
+	if (magic != INITRAMFS_MAGIC)
+		return ERR_INVALID_MEMORY_REGION;
+	if (!buffer_read_u64_be(initramfs_buff, *cursorOUT, &version))
+		return ERR_INVALID_MEMORY_REGION;
+	*cursorOUT += sizeof(u64);
+	// TODO: Handle version
+	return ERR_NONE;
+}
+
+static error_t find_initramfs_buffer(buffer_t* buffOUT) {
+	void* dtbptr = nullptr;
+	if (ihal_get_dtb(&dtbptr) != ERR_NONE)
+		return ERR_NOT_INITIALIZED;
+	fdt_t fdt;
+	(void)dt_init(dtbptr, &fdt); // This error is checkd on hal init
+	dt_node_t initramfs_node;
+	error_t err = dt_get_node_by_path(&fdt, "initramfs", &initramfs_node);
+	if (err)
+		return ERR_DT_NODE_NOT_FOUND;
+	dt_prop_t initramfs_reg;
+	err = dt_get_prop_by_name(&fdt, initramfs_node, "reg", &initramfs_reg);
+	if (err)
+		return ERR_DT_PROP_NOT_FOUND;
+	return dt_get_prop_buffer(&fdt, initramfs_reg, buffOUT);
+}
+
+typedef struct {
+	u32 offset;
+	u32 size;
+} file_header_t;
+
+// TODO: Change filename to string_view
+static error_t find_file_by_name(const char* filename, buffer_t initramfs_buff, file_header_t* headerOUT, u64 cursor) {
+	u16 filecount = 0;
+	if (!buffer_read_u16_be(initramfs_buff, cursor, &filecount))
+		return ERR_INVALID_MEMORY_REGION;
+	cursor += sizeof(u16);
+	for (u16 i = 0; i < filecount; ++i) {
+		if (!buffer_read_u32_be(initramfs_buff, cursor, &headerOUT->offset))
+			return ERR_INVALID_MEMORY_REGION;
+		cursor += sizeof(u32);
+		if (!buffer_read_u32_be(initramfs_buff, cursor, &headerOUT->size))
+			return ERR_INVALID_MEMORY_REGION;
+		cursor += sizeof(u32);
+		u64 file_name_size = 0; // This is of type u8 but buffer_t interface wants u64
+		if (!buffer_read_u8(initramfs_buff, cursor, (u8*)&file_name_size))
+			return ERR_INVALID_MEMORY_REGION;
+		cursor += sizeof(u8);
+
+		buffer_t target_filename_buff = make_buffer(filename, strlen(filename));
+		buffer_t filename_buff = make_buffer(initramfs_buff.data + cursor, file_name_size);
+		cursor += file_name_size;
+		cursor = ALIGN_UP(cursor, sizeof(u16));
+
+		if (buffer_memcmp(target_filename_buff, filename_buff) == 0) {
+			if (headerOUT->offset + headerOUT->size > initramfs_buff.size)
+				return ERR_OUT_OF_BOUNDS;
+			return ERR_NONE;
+		}
+	}
+	headerOUT->size = 0;
+	headerOUT->offset = 0;
+	return ERR_NOT_FOUND;
+}
+
+error_t hal_initramfs_read(const char* filename, buffer_t* buffOUT) {
+	buffer_t initramfs_buff;
+	error_t err = find_initramfs_buffer(&initramfs_buff);
+	if (err)
+		return err;
+	u64 cursor = 0;
+	err = validate(initramfs_buff, &cursor);
+	if (err)
+		return err;
+	file_header_t file_header;
+	err = find_file_by_name(filename, initramfs_buff, &file_header, cursor);
+	if (err)
+		return err;
+	*buffOUT = make_buffer(initramfs_buff.data + file_header.offset, file_header.size);
+	return ERR_NONE;
+}

src/lib/stdbigos/bitutils.c

@@ -1,16 +1,26 @@
 #include <stdbigos/bitutils.h>
 #include <stdbigos/types.h>
 
+u16 read_be16(const void* addr) {
+	const u8* bytes = (const u8*)addr;
+	return ((u16)bytes[0] << 8) | ((u16)bytes[1] << 0);
+}
+
 u32 read_be32(const void* addr) {
 	const u8* bytes = (const u8*)addr;
 	return ((u32)bytes[0] << 24) | ((u32)bytes[1] << 16) | ((u32)bytes[2] << 8) | ((u32)bytes[3] << 0);
-};
+}
 
 u64 read_be64(const void* addr) {
 	const u8* bytes = (const u8*)addr;
 	return ((u64)bytes[0] << 56) | ((u64)bytes[1] << 48) | ((u64)bytes[2] << 40) | ((u64)bytes[3] << 32) |
 	       ((u64)bytes[4] << 24) | ((u64)bytes[5] << 16) | ((u64)bytes[6] << 8) | ((u64)bytes[7] << 0);
-};
+}
+
+u16 read_le16(const void* addr) {
+	const u8* bytes = (const u8*)addr;
+	return ((u16)bytes[1] << 8) | ((u16)bytes[0] << 0);
+}
 
 u32 read_le32(const void* addr) {
 	const u8* bytes = (const u8*)addr;

src/lib/stdbigos/buffer.c

@@ -5,32 +5,55 @@
 #include <stdbigos/types.h>
 #include <stddef.h>
 
-bool buffer_read_u32_be(buffer_t buf, size_t offset, u32* out) {
+bool buffer_read_u8(buffer_t buf, size_t offset, u8* out) {
 	bool ok = buffer_is_valid(buf) && offset + sizeof(*out) <= buf.size;
 	if (ok)
-		*out = read_be32((const u8*)buf.data + offset);
+		*out = *((const u8*)buf.data + offset);
 	return ok;
 }
 
-bool buffer_read_u32_le(buffer_t buf, size_t offset, u32* out) {
+bool buffer_read_u16_le(buffer_t buf, size_t offset, u16* out) {
 	bool ok = buffer_is_valid(buf) && offset + sizeof(*out) <= buf.size;
 	if (ok)
-		*out = read_le32((const u8*)buf.data + offset);
+		*out = read_le16((const u8*)buf.data + offset);
 	return ok;
 }
-bool buffer_read_u64_be(buffer_t buf, size_t offset, u64* out) {
+
+bool buffer_read_u32_le(buffer_t buf, size_t offset, u32* out) {
 	bool ok = buffer_is_valid(buf) && offset + sizeof(*out) <= buf.size;
 	if (ok)
-		*out = read_be64((const u8*)buf.data + offset);
+		*out = read_le32((const u8*)buf.data + offset);
 	return ok;
 }
+
 bool buffer_read_u64_le(buffer_t buf, size_t offset, u64* out) {
 	bool ok = buffer_is_valid(buf) && offset + sizeof(*out) <= buf.size;
 	if (ok)
 		*out = read_le64((const u8*)buf.data + offset);
 	return ok;
 }
 
+bool buffer_read_u16_be(buffer_t buf, size_t offset, u16* out) {
+	bool ok = buffer_is_valid(buf) && offset + sizeof(*out) <= buf.size;
+	if (ok)
+		*out = read_be16((const u8*)buf.data + offset);
+	return ok;
+}
+
+bool buffer_read_u32_be(buffer_t buf, size_t offset, u32* out) {
+	bool ok = buffer_is_valid(buf) && offset + sizeof(*out) <= buf.size;
+	if (ok)
+		*out = read_be32((const u8*)buf.data + offset);
+	return ok;
+}
+
+bool buffer_read_u64_be(buffer_t buf, size_t offset, u64* out) {
+	bool ok = buffer_is_valid(buf) && offset + sizeof(*out) <= buf.size;
+	if (ok)
+		*out = read_be64((const u8*)buf.data + offset);
+	return ok;
+}
+
 bool buffer_read_cstring_len(buffer_t buf, size_t offset, const char** out_str, u64* len) {
 	if (!buffer_is_valid(buf) || !out_str || offset >= buf.size)
 		return false;