diff --git a/pycontroller/app/(tabs)/video.tsx b/pycontroller/app/(tabs)/video.tsx index 8c0c861..ac3173f 100644 --- a/pycontroller/app/(tabs)/video.tsx +++ b/pycontroller/app/(tabs)/video.tsx @@ -6,6 +6,7 @@ import ModeOne from '../../components/magnetovision/modeOne'; import ModeTwo from '../../components/magnetovision/modeTwo'; import ModeThree from '../../components/magnetovision/modeThree'; import ModeFour from '../../components/magnetovision/modeFour'; +import ModeFive from '../../components/magnetovision/modeFive'; import { Dimensions, View, Animated, PanResponder, TouchableOpacity } from 'react-native'; import { useNavigation } from '@react-navigation/native'; @@ -53,6 +54,8 @@ export default function VideoScreen() { return ; case 4: return ; + case 5: + return ; default: return ; } diff --git a/pycontroller/components/magnetovision/modeFive.tsx b/pycontroller/components/magnetovision/modeFive.tsx new file mode 100644 index 0000000..267f4b0 --- /dev/null +++ b/pycontroller/components/magnetovision/modeFive.tsx @@ -0,0 +1,22 @@ + +import { View, StyleSheet } from 'react-native'; +import { requireNativeViewManager } from "expo-modules-core"; + +const ConvolutionView = requireNativeViewManager('ConvolutionView'); + +export default function ModeFive() { + return ( + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: '#000', + justifyContent: 'center', + alignItems: 'center', + }, +}); \ No newline at end of file diff --git a/pycontroller/sensorlib/ios/ConvolutionView.swift b/pycontroller/sensorlib/ios/ConvolutionView.swift new file mode 100644 index 0000000..e69de29 diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index 0a50a64..7a3b642 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -2,6 +2,13 @@ import CoreHaptics import CoreHaptics import ExpoModulesCore +import Foundation +import MetalKit +import CoreMotion +import simd +import UIKit + + // Error type for haptics enum HapticError: Error { case missingDuration @@ -103,5 +110,111 @@ public class SensorlibModule: Module { Events("onLoad") } + + // Register ConvolutionView as a native view + View(ConvolutionView.self) { + // Example: add props/events as needed later + } } } + + +// Metal-based convolution view for full-screen RGB grid +public class ConvolutionView: MTKView { + // Grid dimensions (full screen) + var gridWidth: Int = 0 + var gridHeight: Int = 0 + var gridChannels: Int = 3 // RGB + var gridState: [Float] = [] // [R,G,B,R,G,B,...] + var kernelSize: Int = 3 // 3x3x3 + var kernel: [Float] = [] // [kx,ky,kz,...] + var deviceMotion: CMMotionManager = CMMotionManager() + var commandQueue: MTLCommandQueue! + var pipelineState: MTLComputePipelineState! + var texture: MTLTexture! + var initialized: Bool = false + + required init(coder: NSCoder) { + super.init(coder: coder) + self.framebufferOnly = false + self.device = MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + override init(frame: CGRect, device: MTLDevice?) { + super.init(frame: frame, device: device) + self.framebufferOnly = false + self.device = device ?? MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + func setupMetal() { + guard let device = self.device else { return } + // Set grid size to view size + gridWidth = Int(self.bounds.width) + gridHeight = Int(self.bounds.height) + gridState = (0..<(gridWidth * gridHeight * gridChannels)).map { _ in Float.random(in: 0...1) } + kernel = (0..<(kernelSize * kernelSize * gridChannels)).map { _ in Float.random(in: -1...1) } + // Create texture for rendering + let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: gridWidth, height: gridHeight, mipmapped: false) + desc.usage = [.shaderWrite, .shaderRead, .renderTarget] + texture = device.makeTexture(descriptor: desc) + // Load compute shader + let library = device.makeDefaultLibrary() + let function = library?.makeFunction(name: "convolveKernel") + pipelineState = try? device.makeComputePipelineState(function: function!) + initialized = true + } + + func startSensors() { + // Start magnetometer updates + if deviceMotion.isMagnetometerAvailable { + deviceMotion.magnetometerUpdateInterval = 0.03 + deviceMotion.startMagnetometerUpdates(to: OperationQueue.current ?? OperationQueue.main) { [weak self] (data, error) in + guard let self = self, let mag = data?.magneticField else { return } + // Use magnetometer data to update kernel + self.updateKernel(with: mag) + } + } + } + + func updateKernel(with mag: CMMagneticField) { + // Example: update kernel values with magnetometer + for i in 0...size, options: []) + let kernelBuffer = device.makeBuffer(bytes: kernel, length: kernel.count * MemoryLayout.size, options: []) + encoder?.setBuffer(gridBuffer, offset: 0, index: 0) + encoder?.setBuffer(kernelBuffer, offset: 0, index: 1) + encoder?.setTexture(texture, index: 0) + // Dispatch threads + let w = pipelineState.threadExecutionWidth + let h = pipelineState.maxTotalThreadsPerThreadgroup / w + let threadsPerGroup = MTLSize(width: w, height: h, depth: 1) + let threadsPerGrid = MTLSize(width: gridWidth, height: gridHeight, depth: 1) + encoder?.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerGroup) + encoder?.endEncoding() + commandBuffer?.present(drawable) + commandBuffer?.commit() + } +} \ No newline at end of file