From 79fd2653c6d2d974f289e9a7f657f8fdf5dd9595 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 07:01:48 +0900 Subject: [PATCH 01/10] begin convolutional view --- .../sensorlib/ios/ConvolutionView.swift | 119 ++++++++++++++++++ .../sensorlib/ios/SensorlibModule.swift | 5 + 2 files changed, 124 insertions(+) create mode 100644 pycontroller/sensorlib/ios/ConvolutionView.swift diff --git a/pycontroller/sensorlib/ios/ConvolutionView.swift b/pycontroller/sensorlib/ios/ConvolutionView.swift new file mode 100644 index 00000000..8d39fe0a --- /dev/null +++ b/pycontroller/sensorlib/ios/ConvolutionView.swift @@ -0,0 +1,119 @@ +import Foundation +import MetalKit +import CoreMotion +import simd +import UIKit + +// Metal-based convolution view for full-screen RGB grid +class ConvolutionView: MTKView { + // Grid dimensions (full screen) + var gridWidth: Int = 0 + var gridHeight: Int = 0 + var gridChannels: Int = 3 // RGB + var gridState: [Float] = [] // [R,G,B,R,G,B,...] + var kernelSize: Int = 3 // 3x3x3 + var kernel: [Float] = [] // [kx,ky,kz,...] + var deviceMotion: CMMotionManager = CMMotionManager() + var commandQueue: MTLCommandQueue! + var pipelineState: MTLComputePipelineState! + var texture: MTLTexture! + var initialized: Bool = false + + required init(coder: NSCoder) { + super.init(coder: coder) + self.framebufferOnly = false + self.device = MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + override init(frame: CGRect, device: MTLDevice?) { + super.init(frame: frame, device: device) + self.framebufferOnly = false + self.device = device ?? MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + func setupMetal() { + guard let device = self.device else { return } + // Set grid size to view size + gridWidth = Int(self.bounds.width) + gridHeight = Int(self.bounds.height) + gridState = (0..<(gridWidth * gridHeight * gridChannels)).map { _ in Float.random(in: 0...1) } + kernel = (0..<(kernelSize * kernelSize * gridChannels)).map { _ in Float.random(in: -1...1) } + // Create texture for rendering + let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: gridWidth, height: gridHeight, mipmapped: false) + desc.usage = [.shaderWrite, .shaderRead, .renderTarget] + texture = device.makeTexture(descriptor: desc) + // Load compute shader + let library = device.makeDefaultLibrary() + let function = library?.makeFunction(name: "convolveKernel") + pipelineState = try? device.makeComputePipelineState(function: function!) + initialized = true + } + + func startSensors() { + // Start magnetometer updates + if deviceMotion.isMagnetometerAvailable { + deviceMotion.magnetometerUpdateInterval = 0.03 + deviceMotion.startMagnetometerUpdates(to: OperationQueue.current ?? OperationQueue.main) { [weak self] (data, error) in + guard let self = self, let mag = data?.magneticField else { return } + // Use magnetometer data to update kernel + self.updateKernel(with: mag) + } + } + } + + func updateKernel(with mag: CMMagneticField) { + // Example: update kernel values with magnetometer + for i in 0...size, options: []) + let kernelBuffer = device.makeBuffer(bytes: kernel, length: kernel.count * MemoryLayout.size, options: []) + encoder?.setBuffer(gridBuffer, offset: 0, index: 0) + encoder?.setBuffer(kernelBuffer, offset: 0, index: 1) + encoder?.setTexture(texture, index: 0) + // Dispatch threads + let w = pipelineState.threadExecutionWidth + let h = pipelineState.maxTotalThreadsPerThreadgroup / w + let threadsPerGroup = MTLSize(width: w, height: h, depth: 1) + let threadsPerGrid = MTLSize(width: gridWidth, height: gridHeight, depth: 1) + encoder?.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerGroup) + encoder?.endEncoding() + commandBuffer?.present(drawable) + commandBuffer?.commit() + } +} + +// Metal shader (to be placed in a .metal file in the bundle) +/* +kernel void convolveKernel( + device float *grid [[ buffer(0) ]], + device float *kernel [[ buffer(1) ]], + texture2d outTexture [[ texture(0) ]], + uint2 gid [[ thread_position_in_grid ]] +) { + // Example: 3x3x3 convolution for RGB + // ... implement convolution logic here ... + // Write result to outTexture +} +*/ diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index 0a50a64d..598dd113 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -103,5 +103,10 @@ public class SensorlibModule: Module { Events("onLoad") } + + // Register ConvolutionView as a native view + View(ConvolutionView.self) { + // Example: add props/events as needed later + } } } From 9e1d6aef6eb0a479d9b20c6999a2b749b744de75 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 07:09:17 +0900 Subject: [PATCH 02/10] removing mod4 replacing with swift convolutional eng --- .../components/magnetovision/modeFour.tsx | 252 ++---------------- 1 file changed, 16 insertions(+), 236 deletions(-) diff --git a/pycontroller/components/magnetovision/modeFour.tsx b/pycontroller/components/magnetovision/modeFour.tsx index 08828372..7ced8814 100644 --- a/pycontroller/components/magnetovision/modeFour.tsx +++ b/pycontroller/components/magnetovision/modeFour.tsx @@ -1,242 +1,22 @@ -import React, { useRef, useState, useEffect } from 'react'; -import { TouchableWithoutFeedback } from 'react-native'; -import { Dimensions, View } from 'react-native'; -import { Text } from 'react-native'; -import { useFocusEffect } from '@react-navigation/native'; -import { Magnetometer } from 'expo-sensors'; -const PIXEL_WIDTH = 256; -const BUFFER_SIZE = 64; +import { requireNativeComponent } from 'react-native'; +import { View, StyleSheet } from 'react-native'; -export default function ModeFour() { - const { width: screenWidth, height: screenHeight } = Dimensions.get('window'); - const pixelHeight = Math.round((screenHeight / screenWidth) * PIXEL_WIDTH); - const pixelSize = screenWidth / PIXEL_WIDTH; - const canvasHeight = pixelHeight * pixelSize; - - const bufferRef = useRef<{x: number, y: number, z: number}[]>([]); - const magnetometerRef = useRef<{x: number, y: number, z: number} | null>(null); - const [buffer, setBuffer] = useState<{x: number, y: number, z: number}[]>([]); - const [magnetometer, setMagnetometer] = useState<{x: number, y: number, z: number} | null>(null); - - - const [isFocused, setIsFocused] = useState(true); - useFocusEffect( - React.useCallback(() => { - setIsFocused(true); - const sub = Magnetometer.addListener(data => { - bufferRef.current.push(data); - if (bufferRef.current.length > BUFFER_SIZE) bufferRef.current.shift(); - magnetometerRef.current = data; - }); - Magnetometer.setUpdateInterval(24); - const interval = setInterval(() => { - setBuffer([...bufferRef.current]); - setMagnetometer(magnetometerRef.current); - }, 33); - return () => { - setIsFocused(false); - sub && sub.remove(); - clearInterval(interval); - }; - }, []) - ); - - const [minMax, setMinMax] = useState({ - minX: 0, maxX: 1, - minY: 0, maxY: 1, - minZ: 0, maxZ: 1, - }); - - useEffect(() => { - if (buffer.length === 0) return; - let minX = buffer[0].x, maxX = buffer[0].x; - let minY = buffer[0].y, maxY = buffer[0].y; - let minZ = buffer[0].z, maxZ = buffer[0].z; - for (const v of buffer) { - if (v.x < minX) minX = v.x; - if (v.x > maxX) maxX = v.x; - if (v.y < minY) minY = v.y; - if (v.y > maxY) maxY = v.y; - if (v.z < minZ) minZ = v.z; - if (v.z > maxZ) maxZ = v.z; - } - setMinMax({ minX, maxX, minY, maxY, minZ, maxZ }); - }, [buffer]); - - // Helper to normalize magnetometer values to [-1, +1] - const norm = (val: number, min: number, max: number) => { - if (max === min) return 0; - // Map val from [min, max] to [-1, +1] - return ((val - min) / (max - min)) * 2 - 1; - }; - - // Kernel is last 9 magnetometer samples (flattened) - const KERNEL_SIZE = 3; - // No need for random kernel, will use convBuffer - - // Buffer for last 9 magnetometer readings - const [convBuffer, setConvBuffer] = useState<{x: number, y: number, z: number}[]>([]); +const NativeConvolutionView = requireNativeComponent('ConvolutionView'); - // On each magnetometer update, add to convBuffer, keep last 9 - useEffect(() => { - if (magnetometer) { - setConvBuffer(prev => { - const next = [...prev, magnetometer]; - if (next.length > 9) next.shift(); - return next; - }); - } - }, [magnetometer]); - - - // 16x16 grid setup - // 24x24 grid setup - const GRID_SIZE = 24; - const squareSize = screenWidth / GRID_SIZE; - - // Helper to create a new random grid - const createRandomGrid = () => - Array.from({ length: GRID_SIZE }, () => - Array.from({ length: GRID_SIZE }, () => Math.random()) - ); - - // Initial randomized grid - const [imageGrid, setImageGrid] = useState(createRandomGrid); - - // Double-tap handler to reset grid - const lastTapRef = useRef(0); - const handleGridTap = () => { - const now = Date.now(); - if (now - lastTapRef.current < 300) { - // Double tap detected - setImageGrid(createRandomGrid()); - } - lastTapRef.current = now; - }; - - // On each update, apply convolution using convBuffer as kernel - useEffect(() => { - if (convBuffer.length < 9) return; - // Normalize kernel values to [-1, 1] using instantaneous group of 9 - const xs = convBuffer.map(sample => sample.x); - const minK = Math.min(...xs); - const maxK = Math.max(...xs); - const flatKernel = xs.map(x => { - if (maxK === minK) return 0; - return ((x - minK) / (maxK - minK)) * 2 - 1; - }); - setImageGrid(prevGrid => { - // For each cell, apply 3x3 conv with kernel - const newGrid = prevGrid.map((row, r) => - row.map((val, c) => { - let acc = 0; - let k = 0; - for (let dr = -1; dr <= 1; dr++) { - for (let dc = -1; dc <= 1; dc++) { - const rr = r + dr; - const cc = c + dc; - if (rr >= 0 && rr < GRID_SIZE && cc >= 0 && cc < GRID_SIZE) { - acc += prevGrid[rr][cc] * flatKernel[k]; - } - k++; - } - } - // Clamp and normalize - return Math.max(0, Math.min(1, acc)); - }) - ); - // If all values are zero, reinitialize - const allZero = newGrid.every(row => row.every(v => v === 0)); - if (allZero) { - return createRandomGrid(); - } - return newGrid; - }); - }, [convBuffer, minMax]); - - if (!isFocused) { - return ; - } - - // Map [0, 1] to [0, 255] for display - const mapColor = (v: number) => Math.round(v * 255); - - // Render grid +export default function ModeFour() { return ( - - {/* Main grid with double-tap gesture */} - - - {imageGrid.map((row, r) => ( - - {row.map((val, c) => { - // Use grayscale for now, could extend to RGB - const color = `rgb(${mapColor(val)},${mapColor(val)},${mapColor(val)})`; - return ( - - ); - })} - - ))} - - - {/* Kernel grid below main grid */} - - - {/* Show kernel as 3x3 grid of last 9 magnetometer x values */} - {convBuffer.length === 9 && ( - - {Array.from({ length: 3 }).map((_, row) => ( - - {Array.from({ length: 3 }).map((_, col) => { - const idx = row * 3 + col; - const sample = convBuffer[idx]; - return ( - - {/* Show normalized value between -1 and 1 */} - {convBuffer.length === 9 && ( - - {(() => { - const xs = convBuffer.map(s => s.x); - const minK = Math.min(...xs); - const maxK = Math.max(...xs); - if (maxK === minK) return '0.00'; - const normVal = ((sample.x - minK) / (maxK - minK)) * 2 - 1; - return normVal.toFixed(2); - })()} - - )} - - ); - })} - - ))} - - )} - - + + ); -} \ No newline at end of file +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: '#000', + justifyContent: 'center', + alignItems: 'center', + }, +}); \ No newline at end of file From a48a1f7c06985a116b675cb2e639b4f4a6cc724a Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 07:11:05 +0900 Subject: [PATCH 03/10] import --- pycontroller/sensorlib/ios/SensorlibModule.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index 598dd113..7e596697 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -2,6 +2,7 @@ import CoreHaptics import CoreHaptics import ExpoModulesCore +import ConvolutionView // Error type for haptics enum HapticError: Error { case missingDuration From dab3900ef98f40e78d55ae89398a79f2d973be10 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 07:37:07 +0900 Subject: [PATCH 04/10] merge to one file because swift sucks at imports --- .../sensorlib/ios/ConvolutionView.swift | 119 ------------------ .../sensorlib/ios/SensorlibModule.swift | 108 +++++++++++++++- 2 files changed, 107 insertions(+), 120 deletions(-) diff --git a/pycontroller/sensorlib/ios/ConvolutionView.swift b/pycontroller/sensorlib/ios/ConvolutionView.swift index 8d39fe0a..e69de29b 100644 --- a/pycontroller/sensorlib/ios/ConvolutionView.swift +++ b/pycontroller/sensorlib/ios/ConvolutionView.swift @@ -1,119 +0,0 @@ -import Foundation -import MetalKit -import CoreMotion -import simd -import UIKit - -// Metal-based convolution view for full-screen RGB grid -class ConvolutionView: MTKView { - // Grid dimensions (full screen) - var gridWidth: Int = 0 - var gridHeight: Int = 0 - var gridChannels: Int = 3 // RGB - var gridState: [Float] = [] // [R,G,B,R,G,B,...] - var kernelSize: Int = 3 // 3x3x3 - var kernel: [Float] = [] // [kx,ky,kz,...] - var deviceMotion: CMMotionManager = CMMotionManager() - var commandQueue: MTLCommandQueue! - var pipelineState: MTLComputePipelineState! - var texture: MTLTexture! - var initialized: Bool = false - - required init(coder: NSCoder) { - super.init(coder: coder) - self.framebufferOnly = false - self.device = MTLCreateSystemDefaultDevice() - self.commandQueue = self.device?.makeCommandQueue() - self.isPaused = false - self.enableSetNeedsDisplay = false - self.framebufferOnly = false - self.setupMetal() - self.startSensors() - } - - override init(frame: CGRect, device: MTLDevice?) { - super.init(frame: frame, device: device) - self.framebufferOnly = false - self.device = device ?? MTLCreateSystemDefaultDevice() - self.commandQueue = self.device?.makeCommandQueue() - self.isPaused = false - self.enableSetNeedsDisplay = false - self.framebufferOnly = false - self.setupMetal() - self.startSensors() - } - - func setupMetal() { - guard let device = self.device else { return } - // Set grid size to view size - gridWidth = Int(self.bounds.width) - gridHeight = Int(self.bounds.height) - gridState = (0..<(gridWidth * gridHeight * gridChannels)).map { _ in Float.random(in: 0...1) } - kernel = (0..<(kernelSize * kernelSize * gridChannels)).map { _ in Float.random(in: -1...1) } - // Create texture for rendering - let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: gridWidth, height: gridHeight, mipmapped: false) - desc.usage = [.shaderWrite, .shaderRead, .renderTarget] - texture = device.makeTexture(descriptor: desc) - // Load compute shader - let library = device.makeDefaultLibrary() - let function = library?.makeFunction(name: "convolveKernel") - pipelineState = try? device.makeComputePipelineState(function: function!) - initialized = true - } - - func startSensors() { - // Start magnetometer updates - if deviceMotion.isMagnetometerAvailable { - deviceMotion.magnetometerUpdateInterval = 0.03 - deviceMotion.startMagnetometerUpdates(to: OperationQueue.current ?? OperationQueue.main) { [weak self] (data, error) in - guard let self = self, let mag = data?.magneticField else { return } - // Use magnetometer data to update kernel - self.updateKernel(with: mag) - } - } - } - - func updateKernel(with mag: CMMagneticField) { - // Example: update kernel values with magnetometer - for i in 0...size, options: []) - let kernelBuffer = device.makeBuffer(bytes: kernel, length: kernel.count * MemoryLayout.size, options: []) - encoder?.setBuffer(gridBuffer, offset: 0, index: 0) - encoder?.setBuffer(kernelBuffer, offset: 0, index: 1) - encoder?.setTexture(texture, index: 0) - // Dispatch threads - let w = pipelineState.threadExecutionWidth - let h = pipelineState.maxTotalThreadsPerThreadgroup / w - let threadsPerGroup = MTLSize(width: w, height: h, depth: 1) - let threadsPerGrid = MTLSize(width: gridWidth, height: gridHeight, depth: 1) - encoder?.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerGroup) - encoder?.endEncoding() - commandBuffer?.present(drawable) - commandBuffer?.commit() - } -} - -// Metal shader (to be placed in a .metal file in the bundle) -/* -kernel void convolveKernel( - device float *grid [[ buffer(0) ]], - device float *kernel [[ buffer(1) ]], - texture2d outTexture [[ texture(0) ]], - uint2 gid [[ thread_position_in_grid ]] -) { - // Example: 3x3x3 convolution for RGB - // ... implement convolution logic here ... - // Write result to outTexture -} -*/ diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index 7e596697..a2d5905c 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -2,7 +2,113 @@ import CoreHaptics import CoreHaptics import ExpoModulesCore -import ConvolutionView +import Foundation +import MetalKit +import CoreMotion +import simd +import UIKit + + +// Metal-based convolution view for full-screen RGB grid +class ConvolutionView: MTKView { + // Grid dimensions (full screen) + var gridWidth: Int = 0 + var gridHeight: Int = 0 + var gridChannels: Int = 3 // RGB + var gridState: [Float] = [] // [R,G,B,R,G,B,...] + var kernelSize: Int = 3 // 3x3x3 + var kernel: [Float] = [] // [kx,ky,kz,...] + var deviceMotion: CMMotionManager = CMMotionManager() + var commandQueue: MTLCommandQueue! + var pipelineState: MTLComputePipelineState! + var texture: MTLTexture! + var initialized: Bool = false + + required init(coder: NSCoder) { + super.init(coder: coder) + self.framebufferOnly = false + self.device = MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + override init(frame: CGRect, device: MTLDevice?) { + super.init(frame: frame, device: device) + self.framebufferOnly = false + self.device = device ?? MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + func setupMetal() { + guard let device = self.device else { return } + // Set grid size to view size + gridWidth = Int(self.bounds.width) + gridHeight = Int(self.bounds.height) + gridState = (0..<(gridWidth * gridHeight * gridChannels)).map { _ in Float.random(in: 0...1) } + kernel = (0..<(kernelSize * kernelSize * gridChannels)).map { _ in Float.random(in: -1...1) } + // Create texture for rendering + let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: gridWidth, height: gridHeight, mipmapped: false) + desc.usage = [.shaderWrite, .shaderRead, .renderTarget] + texture = device.makeTexture(descriptor: desc) + // Load compute shader + let library = device.makeDefaultLibrary() + let function = library?.makeFunction(name: "convolveKernel") + pipelineState = try? device.makeComputePipelineState(function: function!) + initialized = true + } + + func startSensors() { + // Start magnetometer updates + if deviceMotion.isMagnetometerAvailable { + deviceMotion.magnetometerUpdateInterval = 0.03 + deviceMotion.startMagnetometerUpdates(to: OperationQueue.current ?? OperationQueue.main) { [weak self] (data, error) in + guard let self = self, let mag = data?.magneticField else { return } + // Use magnetometer data to update kernel + self.updateKernel(with: mag) + } + } + } + + func updateKernel(with mag: CMMagneticField) { + // Example: update kernel values with magnetometer + for i in 0...size, options: []) + let kernelBuffer = device.makeBuffer(bytes: kernel, length: kernel.count * MemoryLayout.size, options: []) + encoder?.setBuffer(gridBuffer, offset: 0, index: 0) + encoder?.setBuffer(kernelBuffer, offset: 0, index: 1) + encoder?.setTexture(texture, index: 0) + // Dispatch threads + let w = pipelineState.threadExecutionWidth + let h = pipelineState.maxTotalThreadsPerThreadgroup / w + let threadsPerGroup = MTLSize(width: w, height: h, depth: 1) + let threadsPerGrid = MTLSize(width: gridWidth, height: gridHeight, depth: 1) + encoder?.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerGroup) + encoder?.endEncoding() + commandBuffer?.present(drawable) + commandBuffer?.commit() + } +} + // Error type for haptics enum HapticError: Error { case missingDuration From 133d070d800cd2090e09f1a6fb63ae4420fc7234 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 07:55:33 +0900 Subject: [PATCH 05/10] closer --- .../components/magnetovision/modeFour.tsx | 7 +- .../sensorlib/ios/SensorlibModule.swift | 201 +++++++++--------- 2 files changed, 104 insertions(+), 104 deletions(-) diff --git a/pycontroller/components/magnetovision/modeFour.tsx b/pycontroller/components/magnetovision/modeFour.tsx index 7ced8814..0d657079 100644 --- a/pycontroller/components/magnetovision/modeFour.tsx +++ b/pycontroller/components/magnetovision/modeFour.tsx @@ -1,13 +1,12 @@ - -import { requireNativeComponent } from 'react-native'; +import { requireNativeModule } from 'expo-modules-core'; import { View, StyleSheet } from 'react-native'; -const NativeConvolutionView = requireNativeComponent('ConvolutionView'); +const sensorLib = requireNativeModule('Sensorlib'); export default function ModeFour() { return ( - + ); } diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index a2d5905c..d1d47d60 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -9,106 +9,6 @@ import simd import UIKit -// Metal-based convolution view for full-screen RGB grid -class ConvolutionView: MTKView { - // Grid dimensions (full screen) - var gridWidth: Int = 0 - var gridHeight: Int = 0 - var gridChannels: Int = 3 // RGB - var gridState: [Float] = [] // [R,G,B,R,G,B,...] - var kernelSize: Int = 3 // 3x3x3 - var kernel: [Float] = [] // [kx,ky,kz,...] - var deviceMotion: CMMotionManager = CMMotionManager() - var commandQueue: MTLCommandQueue! - var pipelineState: MTLComputePipelineState! - var texture: MTLTexture! - var initialized: Bool = false - - required init(coder: NSCoder) { - super.init(coder: coder) - self.framebufferOnly = false - self.device = MTLCreateSystemDefaultDevice() - self.commandQueue = self.device?.makeCommandQueue() - self.isPaused = false - self.enableSetNeedsDisplay = false - self.framebufferOnly = false - self.setupMetal() - self.startSensors() - } - - override init(frame: CGRect, device: MTLDevice?) { - super.init(frame: frame, device: device) - self.framebufferOnly = false - self.device = device ?? MTLCreateSystemDefaultDevice() - self.commandQueue = self.device?.makeCommandQueue() - self.isPaused = false - self.enableSetNeedsDisplay = false - self.framebufferOnly = false - self.setupMetal() - self.startSensors() - } - - func setupMetal() { - guard let device = self.device else { return } - // Set grid size to view size - gridWidth = Int(self.bounds.width) - gridHeight = Int(self.bounds.height) - gridState = (0..<(gridWidth * gridHeight * gridChannels)).map { _ in Float.random(in: 0...1) } - kernel = (0..<(kernelSize * kernelSize * gridChannels)).map { _ in Float.random(in: -1...1) } - // Create texture for rendering - let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: gridWidth, height: gridHeight, mipmapped: false) - desc.usage = [.shaderWrite, .shaderRead, .renderTarget] - texture = device.makeTexture(descriptor: desc) - // Load compute shader - let library = device.makeDefaultLibrary() - let function = library?.makeFunction(name: "convolveKernel") - pipelineState = try? device.makeComputePipelineState(function: function!) - initialized = true - } - - func startSensors() { - // Start magnetometer updates - if deviceMotion.isMagnetometerAvailable { - deviceMotion.magnetometerUpdateInterval = 0.03 - deviceMotion.startMagnetometerUpdates(to: OperationQueue.current ?? OperationQueue.main) { [weak self] (data, error) in - guard let self = self, let mag = data?.magneticField else { return } - // Use magnetometer data to update kernel - self.updateKernel(with: mag) - } - } - } - - func updateKernel(with mag: CMMagneticField) { - // Example: update kernel values with magnetometer - for i in 0...size, options: []) - let kernelBuffer = device.makeBuffer(bytes: kernel, length: kernel.count * MemoryLayout.size, options: []) - encoder?.setBuffer(gridBuffer, offset: 0, index: 0) - encoder?.setBuffer(kernelBuffer, offset: 0, index: 1) - encoder?.setTexture(texture, index: 0) - // Dispatch threads - let w = pipelineState.threadExecutionWidth - let h = pipelineState.maxTotalThreadsPerThreadgroup / w - let threadsPerGroup = MTLSize(width: w, height: h, depth: 1) - let threadsPerGrid = MTLSize(width: gridWidth, height: gridHeight, depth: 1) - encoder?.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerGroup) - encoder?.endEncoding() - commandBuffer?.present(drawable) - commandBuffer?.commit() - } -} - // Error type for haptics enum HapticError: Error { case missingDuration @@ -217,3 +117,104 @@ public class SensorlibModule: Module { } } } + + +// Metal-based convolution view for full-screen RGB grid +public class ConvolutionView: MTKView { + // Grid dimensions (full screen) + var gridWidth: Int = 0 + var gridHeight: Int = 0 + var gridChannels: Int = 3 // RGB + var gridState: [Float] = [] // [R,G,B,R,G,B,...] + var kernelSize: Int = 3 // 3x3x3 + var kernel: [Float] = [] // [kx,ky,kz,...] + var deviceMotion: CMMotionManager = CMMotionManager() + var commandQueue: MTLCommandQueue! + var pipelineState: MTLComputePipelineState! + var texture: MTLTexture! + var initialized: Bool = false + + required init(coder: NSCoder) { + super.init(coder: coder) + self.framebufferOnly = false + self.device = MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + override init(frame: CGRect, device: MTLDevice?) { + super.init(frame: frame, device: device) + self.framebufferOnly = false + self.device = device ?? MTLCreateSystemDefaultDevice() + self.commandQueue = self.device?.makeCommandQueue() + self.isPaused = false + self.enableSetNeedsDisplay = false + self.framebufferOnly = false + self.setupMetal() + self.startSensors() + } + + func setupMetal() { + guard let device = self.device else { return } + // Set grid size to view size + gridWidth = Int(self.bounds.width) + gridHeight = Int(self.bounds.height) + gridState = (0..<(gridWidth * gridHeight * gridChannels)).map { _ in Float.random(in: 0...1) } + kernel = (0..<(kernelSize * kernelSize * gridChannels)).map { _ in Float.random(in: -1...1) } + // Create texture for rendering + let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: gridWidth, height: gridHeight, mipmapped: false) + desc.usage = [.shaderWrite, .shaderRead, .renderTarget] + texture = device.makeTexture(descriptor: desc) + // Load compute shader + let library = device.makeDefaultLibrary() + let function = library?.makeFunction(name: "convolveKernel") + pipelineState = try? device.makeComputePipelineState(function: function!) + initialized = true + } + + func startSensors() { + // Start magnetometer updates + if deviceMotion.isMagnetometerAvailable { + deviceMotion.magnetometerUpdateInterval = 0.03 + deviceMotion.startMagnetometerUpdates(to: OperationQueue.current ?? OperationQueue.main) { [weak self] (data, error) in + guard let self = self, let mag = data?.magneticField else { return } + // Use magnetometer data to update kernel + self.updateKernel(with: mag) + } + } + } + + func updateKernel(with mag: CMMagneticField) { + // Example: update kernel values with magnetometer + for i in 0...size, options: []) + let kernelBuffer = device.makeBuffer(bytes: kernel, length: kernel.count * MemoryLayout.size, options: []) + encoder?.setBuffer(gridBuffer, offset: 0, index: 0) + encoder?.setBuffer(kernelBuffer, offset: 0, index: 1) + encoder?.setTexture(texture, index: 0) + // Dispatch threads + let w = pipelineState.threadExecutionWidth + let h = pipelineState.maxTotalThreadsPerThreadgroup / w + let threadsPerGroup = MTLSize(width: w, height: h, depth: 1) + let threadsPerGrid = MTLSize(width: gridWidth, height: gridHeight, depth: 1) + encoder?.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerGroup) + encoder?.endEncoding() + commandBuffer?.present(drawable) + commandBuffer?.commit() + } +} \ No newline at end of file From a8ca9069505362827575e888172a0ca65d861986 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 07:59:29 +0900 Subject: [PATCH 06/10] bring back four as backup and 5 is main area now --- pycontroller/app/(tabs)/video.tsx | 3 + .../components/magnetovision/modeFive.tsx | 21 ++ .../components/magnetovision/modeFour.tsx | 251 ++++++++++++++++-- .../sensorlib/ios/SensorlibModule.swift | 5 +- 4 files changed, 263 insertions(+), 17 deletions(-) create mode 100644 pycontroller/components/magnetovision/modeFive.tsx diff --git a/pycontroller/app/(tabs)/video.tsx b/pycontroller/app/(tabs)/video.tsx index 8c0c8619..ac3173f6 100644 --- a/pycontroller/app/(tabs)/video.tsx +++ b/pycontroller/app/(tabs)/video.tsx @@ -6,6 +6,7 @@ import ModeOne from '../../components/magnetovision/modeOne'; import ModeTwo from '../../components/magnetovision/modeTwo'; import ModeThree from '../../components/magnetovision/modeThree'; import ModeFour from '../../components/magnetovision/modeFour'; +import ModeFive from '../../components/magnetovision/modeFive'; import { Dimensions, View, Animated, PanResponder, TouchableOpacity } from 'react-native'; import { useNavigation } from '@react-navigation/native'; @@ -53,6 +54,8 @@ export default function VideoScreen() { return ; case 4: return ; + case 5: + return ; default: return ; } diff --git a/pycontroller/components/magnetovision/modeFive.tsx b/pycontroller/components/magnetovision/modeFive.tsx new file mode 100644 index 00000000..cbd42d91 --- /dev/null +++ b/pycontroller/components/magnetovision/modeFive.tsx @@ -0,0 +1,21 @@ + +import { View, StyleSheet, requireNativeComponent } from 'react-native'; + +const ConvolutionView = requireNativeComponent('ConvolutionView'); + +export default function ModeFive() { + return ( + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: '#000', + justifyContent: 'center', + alignItems: 'center', + }, +}); \ No newline at end of file diff --git a/pycontroller/components/magnetovision/modeFour.tsx b/pycontroller/components/magnetovision/modeFour.tsx index 0d657079..08828372 100644 --- a/pycontroller/components/magnetovision/modeFour.tsx +++ b/pycontroller/components/magnetovision/modeFour.tsx @@ -1,21 +1,242 @@ -import { requireNativeModule } from 'expo-modules-core'; -import { View, StyleSheet } from 'react-native'; +import React, { useRef, useState, useEffect } from 'react'; +import { TouchableWithoutFeedback } from 'react-native'; +import { Dimensions, View } from 'react-native'; +import { Text } from 'react-native'; +import { useFocusEffect } from '@react-navigation/native'; +import { Magnetometer } from 'expo-sensors'; -const sensorLib = requireNativeModule('Sensorlib'); +const PIXEL_WIDTH = 256; +const BUFFER_SIZE = 64; export default function ModeFour() { + const { width: screenWidth, height: screenHeight } = Dimensions.get('window'); + const pixelHeight = Math.round((screenHeight / screenWidth) * PIXEL_WIDTH); + const pixelSize = screenWidth / PIXEL_WIDTH; + const canvasHeight = pixelHeight * pixelSize; + + const bufferRef = useRef<{x: number, y: number, z: number}[]>([]); + const magnetometerRef = useRef<{x: number, y: number, z: number} | null>(null); + const [buffer, setBuffer] = useState<{x: number, y: number, z: number}[]>([]); + const [magnetometer, setMagnetometer] = useState<{x: number, y: number, z: number} | null>(null); + + + const [isFocused, setIsFocused] = useState(true); + useFocusEffect( + React.useCallback(() => { + setIsFocused(true); + const sub = Magnetometer.addListener(data => { + bufferRef.current.push(data); + if (bufferRef.current.length > BUFFER_SIZE) bufferRef.current.shift(); + magnetometerRef.current = data; + }); + Magnetometer.setUpdateInterval(24); + const interval = setInterval(() => { + setBuffer([...bufferRef.current]); + setMagnetometer(magnetometerRef.current); + }, 33); + return () => { + setIsFocused(false); + sub && sub.remove(); + clearInterval(interval); + }; + }, []) + ); + + const [minMax, setMinMax] = useState({ + minX: 0, maxX: 1, + minY: 0, maxY: 1, + minZ: 0, maxZ: 1, + }); + + useEffect(() => { + if (buffer.length === 0) return; + let minX = buffer[0].x, maxX = buffer[0].x; + let minY = buffer[0].y, maxY = buffer[0].y; + let minZ = buffer[0].z, maxZ = buffer[0].z; + for (const v of buffer) { + if (v.x < minX) minX = v.x; + if (v.x > maxX) maxX = v.x; + if (v.y < minY) minY = v.y; + if (v.y > maxY) maxY = v.y; + if (v.z < minZ) minZ = v.z; + if (v.z > maxZ) maxZ = v.z; + } + setMinMax({ minX, maxX, minY, maxY, minZ, maxZ }); + }, [buffer]); + + // Helper to normalize magnetometer values to [-1, +1] + const norm = (val: number, min: number, max: number) => { + if (max === min) return 0; + // Map val from [min, max] to [-1, +1] + return ((val - min) / (max - min)) * 2 - 1; + }; + + // Kernel is last 9 magnetometer samples (flattened) + const KERNEL_SIZE = 3; + // No need for random kernel, will use convBuffer + + // Buffer for last 9 magnetometer readings + const [convBuffer, setConvBuffer] = useState<{x: number, y: number, z: number}[]>([]); + + // On each magnetometer update, add to convBuffer, keep last 9 + useEffect(() => { + if (magnetometer) { + setConvBuffer(prev => { + const next = [...prev, magnetometer]; + if (next.length > 9) next.shift(); + return next; + }); + } + }, [magnetometer]); + + + // 16x16 grid setup + // 24x24 grid setup + const GRID_SIZE = 24; + const squareSize = screenWidth / GRID_SIZE; + + // Helper to create a new random grid + const createRandomGrid = () => + Array.from({ length: GRID_SIZE }, () => + Array.from({ length: GRID_SIZE }, () => Math.random()) + ); + + // Initial randomized grid + const [imageGrid, setImageGrid] = useState(createRandomGrid); + + // Double-tap handler to reset grid + const lastTapRef = useRef(0); + const handleGridTap = () => { + const now = Date.now(); + if (now - lastTapRef.current < 300) { + // Double tap detected + setImageGrid(createRandomGrid()); + } + lastTapRef.current = now; + }; + + // On each update, apply convolution using convBuffer as kernel + useEffect(() => { + if (convBuffer.length < 9) return; + // Normalize kernel values to [-1, 1] using instantaneous group of 9 + const xs = convBuffer.map(sample => sample.x); + const minK = Math.min(...xs); + const maxK = Math.max(...xs); + const flatKernel = xs.map(x => { + if (maxK === minK) return 0; + return ((x - minK) / (maxK - minK)) * 2 - 1; + }); + setImageGrid(prevGrid => { + // For each cell, apply 3x3 conv with kernel + const newGrid = prevGrid.map((row, r) => + row.map((val, c) => { + let acc = 0; + let k = 0; + for (let dr = -1; dr <= 1; dr++) { + for (let dc = -1; dc <= 1; dc++) { + const rr = r + dr; + const cc = c + dc; + if (rr >= 0 && rr < GRID_SIZE && cc >= 0 && cc < GRID_SIZE) { + acc += prevGrid[rr][cc] * flatKernel[k]; + } + k++; + } + } + // Clamp and normalize + return Math.max(0, Math.min(1, acc)); + }) + ); + // If all values are zero, reinitialize + const allZero = newGrid.every(row => row.every(v => v === 0)); + if (allZero) { + return createRandomGrid(); + } + return newGrid; + }); + }, [convBuffer, minMax]); + + if (!isFocused) { + return ; + } + + // Map [0, 1] to [0, 255] for display + const mapColor = (v: number) => Math.round(v * 255); + + // Render grid return ( - - + + {/* Main grid with double-tap gesture */} + + + {imageGrid.map((row, r) => ( + + {row.map((val, c) => { + // Use grayscale for now, could extend to RGB + const color = `rgb(${mapColor(val)},${mapColor(val)},${mapColor(val)})`; + return ( + + ); + })} + + ))} + + + {/* Kernel grid below main grid */} + + + {/* Show kernel as 3x3 grid of last 9 magnetometer x values */} + {convBuffer.length === 9 && ( + + {Array.from({ length: 3 }).map((_, row) => ( + + {Array.from({ length: 3 }).map((_, col) => { + const idx = row * 3 + col; + const sample = convBuffer[idx]; + return ( + + {/* Show normalized value between -1 and 1 */} + {convBuffer.length === 9 && ( + + {(() => { + const xs = convBuffer.map(s => s.x); + const minK = Math.min(...xs); + const maxK = Math.max(...xs); + if (maxK === minK) return '0.00'; + const normVal = ((sample.x - minK) / (maxK - minK)) * 2 - 1; + return normVal.toFixed(2); + })()} + + )} + + ); + })} + + ))} + + )} + + ); -} - -const styles = StyleSheet.create({ - container: { - flex: 1, - backgroundColor: '#000', - justifyContent: 'center', - alignItems: 'center', - }, -}); \ No newline at end of file +} \ No newline at end of file diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index d1d47d60..6500d12d 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -111,8 +111,9 @@ public class SensorlibModule: Module { Events("onLoad") } - // Register ConvolutionView as a native view + // Register ConvolutionView as a native view with explicit name View(ConvolutionView.self) { + .name("ConvolutionView") // Example: add props/events as needed later } } @@ -195,7 +196,7 @@ public class ConvolutionView: MTKView { } } - override func draw(_ rect: CGRect) { + public override func draw(_ rect: CGRect) { guard initialized, let device = self.device, let commandQueue = self.commandQueue, let pipelineState = self.pipelineState else { return } guard let drawable = self.currentDrawable else { return } let commandBuffer = commandQueue.makeCommandBuffer() From 6ac1dbca447021259eaef50193f4b6cef13f7304 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 08:00:16 +0900 Subject: [PATCH 07/10] syntax? --- pycontroller/sensorlib/ios/SensorlibModule.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index 6500d12d..bd164372 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -112,8 +112,7 @@ public class SensorlibModule: Module { } // Register ConvolutionView as a native view with explicit name - View(ConvolutionView.self) { - .name("ConvolutionView") + View(ConvolutionView.self, "ConvolutionView") { // Example: add props/events as needed later } } From a5138a707100d542fce2fd7d0b89d48d2bc22c7a Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 08:06:32 +0900 Subject: [PATCH 08/10] hmmm. . still failing --- pycontroller/sensorlib/ios/SensorlibModule.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pycontroller/sensorlib/ios/SensorlibModule.swift b/pycontroller/sensorlib/ios/SensorlibModule.swift index bd164372..7a3b6428 100644 --- a/pycontroller/sensorlib/ios/SensorlibModule.swift +++ b/pycontroller/sensorlib/ios/SensorlibModule.swift @@ -111,8 +111,8 @@ public class SensorlibModule: Module { Events("onLoad") } - // Register ConvolutionView as a native view with explicit name - View(ConvolutionView.self, "ConvolutionView") { + // Register ConvolutionView as a native view + View(ConvolutionView.self) { // Example: add props/events as needed later } } From c2e4d5e87faa1397bc5db4dea4e227c038b78cf1 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 08:10:25 +0900 Subject: [PATCH 09/10] at least this doesn't crash --- pycontroller/components/magnetovision/modeFive.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pycontroller/components/magnetovision/modeFive.tsx b/pycontroller/components/magnetovision/modeFive.tsx index cbd42d91..16ac8242 100644 --- a/pycontroller/components/magnetovision/modeFive.tsx +++ b/pycontroller/components/magnetovision/modeFive.tsx @@ -1,7 +1,8 @@ -import { View, StyleSheet, requireNativeComponent } from 'react-native'; +import { View, StyleSheet } from 'react-native'; +import { requireNativeViewManager } from "expo-modules-core"; -const ConvolutionView = requireNativeComponent('ConvolutionView'); +const ConvolutionView = requireNativeViewManager('ConvolutionView'); export default function ModeFive() { return ( From 96dc9037ce129bf6f279dc0c296a6c0df83be0c9 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Fri, 21 Nov 2025 08:12:36 +0900 Subject: [PATCH 10/10] ok it's not crashing?? --- pycontroller/components/magnetovision/modeFive.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycontroller/components/magnetovision/modeFive.tsx b/pycontroller/components/magnetovision/modeFive.tsx index 16ac8242..267f4b00 100644 --- a/pycontroller/components/magnetovision/modeFive.tsx +++ b/pycontroller/components/magnetovision/modeFive.tsx @@ -7,7 +7,7 @@ const ConvolutionView = requireNativeViewManager('ConvolutionView'); export default function ModeFive() { return ( - + ); }