convolution improvements [wip]

pycontroller/app/(tabs)/video.tsx

@@ -6,6 +6,7 @@ import ModeOne from '../../components/magnetovision/modeOne';
 import ModeTwo from '../../components/magnetovision/modeTwo';
 import ModeThree from '../../components/magnetovision/modeThree';
 import ModeFour from '../../components/magnetovision/modeFour';
+import ModeFive from '../../components/magnetovision/modeFive';
 import { Dimensions, View, Animated, PanResponder, TouchableOpacity } from 'react-native';
 import { useNavigation } from '@react-navigation/native';
 
@@ -53,6 +54,8 @@ export default function VideoScreen() {
         return <ModeThree key="mode-three" />;
       case 4:
         return <ModeFour key="mode-four" />;
+      case 5:
+        return <ModeFive key="mode-five" />;
       default:
         return <ModeZero key="mode-zero" />;
     }

pycontroller/components/magnetovision/modeFive.tsx

@@ -0,0 +1,22 @@
+
+import { View, StyleSheet } from 'react-native';
+import { requireNativeViewManager } from "expo-modules-core";
+
+const ConvolutionView = requireNativeViewManager('ConvolutionView');
+
+export default function ModeFive() {
+  return (
+    <View style={styles.container}>
+      <ConvolutionView style={{ flex: 1 }} />
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    backgroundColor: '#000',
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+});

pycontroller/sensorlib/ios/SensorlibModule.swift

@@ -2,6 +2,13 @@
 import CoreHaptics
 import CoreHaptics
 import ExpoModulesCore
+import Foundation
+import MetalKit
+import CoreMotion
+import simd
+import UIKit
+
+
 // Error type for haptics
 enum HapticError: Error {
   case missingDuration
@@ -103,5 +110,111 @@ public class SensorlibModule: Module {
 
       Events("onLoad")
     }
+
+    // Register ConvolutionView as a native view
+    View(ConvolutionView.self) {
+      // Example: add props/events as needed later
+    }
   }
 }
+
+
+// Metal-based convolution view for full-screen RGB grid
+public class ConvolutionView: MTKView {
+    // Grid dimensions (full screen)
+    var gridWidth: Int = 0
+    var gridHeight: Int = 0
+    var gridChannels: Int = 3 // RGB
+    var gridState: [Float] = [] // [R,G,B,R,G,B,...]
+    var kernelSize: Int = 3 // 3x3x3
+    var kernel: [Float] = [] // [kx,ky,kz,...]
+    var deviceMotion: CMMotionManager = CMMotionManager()
+    var commandQueue: MTLCommandQueue!
+    var pipelineState: MTLComputePipelineState!
+    var texture: MTLTexture!
+    var initialized: Bool = false
+
+    required init(coder: NSCoder) {
+        super.init(coder: coder)
+        self.framebufferOnly = false
+        self.device = MTLCreateSystemDefaultDevice()
+        self.commandQueue = self.device?.makeCommandQueue()
+        self.isPaused = false
+        self.enableSetNeedsDisplay = false
+        self.framebufferOnly = false
+        self.setupMetal()
+        self.startSensors()
+    }
+
+    override init(frame: CGRect, device: MTLDevice?) {
+        super.init(frame: frame, device: device)
+        self.framebufferOnly = false
+        self.device = device ?? MTLCreateSystemDefaultDevice()
+        self.commandQueue = self.device?.makeCommandQueue()
+        self.isPaused = false
+        self.enableSetNeedsDisplay = false
+        self.framebufferOnly = false
+        self.setupMetal()
+        self.startSensors()
+    }
+
+    func setupMetal() {
+        guard let device = self.device else { return }
+        // Set grid size to view size
+        gridWidth = Int(self.bounds.width)
+        gridHeight = Int(self.bounds.height)
+        gridState = (0..<(gridWidth * gridHeight * gridChannels)).map { _ in Float.random(in: 0...1) }
+        kernel = (0..<(kernelSize * kernelSize * gridChannels)).map { _ in Float.random(in: -1...1) }
+        // Create texture for rendering
+        let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: gridWidth, height: gridHeight, mipmapped: false)
+        desc.usage = [.shaderWrite, .shaderRead, .renderTarget]
+        texture = device.makeTexture(descriptor: desc)
+        // Load compute shader
+        let library = device.makeDefaultLibrary()
+        let function = library?.makeFunction(name: "convolveKernel")
+        pipelineState = try? device.makeComputePipelineState(function: function!)
+        initialized = true
+    }
+
+    func startSensors() {
+        // Start magnetometer updates
+        if deviceMotion.isMagnetometerAvailable {
+            deviceMotion.magnetometerUpdateInterval = 0.03
+            deviceMotion.startMagnetometerUpdates(to: OperationQueue.current ?? OperationQueue.main) { [weak self] (data, error) in
+                guard let self = self, let mag = data?.magneticField else { return }
+                // Use magnetometer data to update kernel
+                self.updateKernel(with: mag)
+            }
+        }
+    }
+
+    func updateKernel(with mag: CMMagneticField) {
+        // Example: update kernel values with magnetometer
+        for i in 0..<kernel.count {
+            kernel[i] = Float(mag.x + mag.y + mag.z) * Float.random(in: -1...1)
+        }
+    }
+
+  public override func draw(_ rect: CGRect) {
+        guard initialized, let device = self.device, let commandQueue = self.commandQueue, let pipelineState = self.pipelineState else { return }
+        guard let drawable = self.currentDrawable else { return }
+        let commandBuffer = commandQueue.makeCommandBuffer()
+        let encoder = commandBuffer?.makeComputeCommandEncoder()
+        encoder?.setComputePipelineState(pipelineState)
+        // Pass grid state and kernel as buffers
+        let gridBuffer = device.makeBuffer(bytes: gridState, length: gridState.count * MemoryLayout<Float>.size, options: [])
+        let kernelBuffer = device.makeBuffer(bytes: kernel, length: kernel.count * MemoryLayout<Float>.size, options: [])
+        encoder?.setBuffer(gridBuffer, offset: 0, index: 0)
+        encoder?.setBuffer(kernelBuffer, offset: 0, index: 1)
+        encoder?.setTexture(texture, index: 0)
+        // Dispatch threads
+        let w = pipelineState.threadExecutionWidth
+        let h = pipelineState.maxTotalThreadsPerThreadgroup / w
+        let threadsPerGroup = MTLSize(width: w, height: h, depth: 1)
+        let threadsPerGrid = MTLSize(width: gridWidth, height: gridHeight, depth: 1)
+        encoder?.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerGroup)
+        encoder?.endEncoding()
+        commandBuffer?.present(drawable)
+        commandBuffer?.commit()
+    }
+}