Skip to content

Add newer implementation of sidewalk attributes #397

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
himanshunaidu wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Add newer implementation of sidewalk attributes #397

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6d9a192
Add legacy attributes for future comparison with older implementations
himanshunaidu Jan 24, 2026
1131ffc
Start adding logic for plane fitting
himanshunaidu Jan 25, 2026
97aea75
Move the delta flip logic in LocalizationProcessor outside the delta …
himanshunaidu Jan 25, 2026
5494cf8
Add kernel for computing plane 3d world points
himanshunaidu Jan 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions IOSAccessAssessment.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@
A35E051E2EDFB09A003C26CF /* OSMWay.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35E051D2EDFB099003C26CF /* OSMWay.swift */; };
A364B5332F25576000325E5C /* DepthFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = A364B5322F25575D00325E5C /* DepthFilter.swift */; };
A364B5352F25589B00325E5C /* DepthFiltering.metal in Sources */ = {isa = PBXBuildFile; fileRef = A364B5342F25589600325E5C /* DepthFiltering.metal */; };
A364B5D92F259AD700325E5C /* PlaneFit.swift in Sources */ = {isa = PBXBuildFile; fileRef = A364B5D82F259AD600325E5C /* PlaneFit.swift */; };
A364B5DD2F259AFE00325E5C /* PlaneFitting.metal in Sources */ = {isa = PBXBuildFile; fileRef = A364B5DC2F259AF900325E5C /* PlaneFitting.metal */; };
A36C6E022E134CE600A86004 /* bisenetv2_35_640_640.mlpackage in Sources */ = {isa = PBXBuildFile; fileRef = A36C6E012E134CE600A86004 /* bisenetv2_35_640_640.mlpackage */; };
A374FAB72EE0173600055268 /* OSMResponseElement.swift in Sources */ = {isa = PBXBuildFile; fileRef = A374FAB62EE0173200055268 /* OSMResponseElement.swift */; };
A37E3E3C2EED60F300B07B77 /* PngEncoder.mm in Sources */ = {isa = PBXBuildFile; fileRef = A37E3E3B2EED60F300B07B77 /* PngEncoder.mm */; };
Expand Down Expand Up @@ -243,6 +245,8 @@
A35E051D2EDFB099003C26CF /* OSMWay.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OSMWay.swift; sourceTree = "<group>"; };
A364B5322F25575D00325E5C /* DepthFilter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DepthFilter.swift; sourceTree = "<group>"; };
A364B5342F25589600325E5C /* DepthFiltering.metal */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.metal; path = DepthFiltering.metal; sourceTree = "<group>"; };
A364B5D82F259AD600325E5C /* PlaneFit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlaneFit.swift; sourceTree = "<group>"; };
A364B5DC2F259AF900325E5C /* PlaneFitting.metal */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.metal; path = PlaneFitting.metal; sourceTree = "<group>"; };
A36C6E012E134CE600A86004 /* bisenetv2_35_640_640.mlpackage */ = {isa = PBXFileReference; lastKnownFileType = folder.mlpackage; path = bisenetv2_35_640_640.mlpackage; sourceTree = "<group>"; };
A374FAB62EE0173200055268 /* OSMResponseElement.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OSMResponseElement.swift; sourceTree = "<group>"; };
A37E3E382EED60F300B07B77 /* lodepng.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = lodepng.h; sourceTree = "<group>"; };
Expand Down Expand Up @@ -602,6 +606,7 @@
A35E050B2EDE359C003C26CF /* AttributeEstimation */ = {
isa = PBXGroup;
children = (
A364B5D72F259A7800325E5C /* Plane */,
A35E050E2EDE35ED003C26CF /* Localization */,
A35E05172EDEA470003C26CF /* AttributeEstimationPipeline.swift */,
);
Expand Down Expand Up @@ -663,6 +668,15 @@
path = Helpers;
sourceTree = "<group>";
};
A364B5D72F259A7800325E5C /* Plane */ = {
isa = PBXGroup;
children = (
A364B5DC2F259AF900325E5C /* PlaneFitting.metal */,
A364B5D82F259AD600325E5C /* PlaneFit.swift */,
);
path = Plane;
sourceTree = "<group>";
};
A37E3E352EED60C100B07B77 /* CHelpers */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -1185,6 +1199,7 @@
A35E05182EDEA476003C26CF /* AttributeEstimationPipeline.swift in Sources */,
CAA947792CDE700A000C6918 /* AuthService.swift in Sources */,
A3C22FD32CF194A600533BF7 /* CGImageUtils.swift in Sources */,
A364B5DD2F259AFE00325E5C /* PlaneFitting.metal in Sources */,
A3DC22E92DCF0F9A0020CE84 /* ImageProcessing.metal in Sources */,
A37E72142ED95C0C00CFE4EF /* MeshHelpers.swift in Sources */,
A30C67E62EE27331006E4321 /* EditableAccessibilityFeature.swift in Sources */,
Expand Down Expand Up @@ -1249,6 +1264,7 @@
A3FFAA832DE5253E002B99BD /* bisenetv2_53_640_640.mlpackage in Sources */,
A3FFAA7A2DE01A0F002B99BD /* ARCameraView.swift in Sources */,
A37E3E9E2EFBAA8700B07B77 /* AccessibilityFeatureSnapshot.swift in Sources */,
A364B5D92F259AD700325E5C /* PlaneFit.swift in Sources */,
A3FFAA782DE01637002B99BD /* ARCameraUtils.swift in Sources */,
A3FE166E2E1C2AF200DAE5BE /* SegmentationEncoder.swift in Sources */,
A30BED3A2ED162F1004A5B51 /* ConnectedComponents.swift in Sources */,
Expand Down Expand Up @@ -1442,6 +1458,7 @@
LD_RUNPATH_SEARCH_PATHS = "$(inherited)";
MARKETING_VERSION = 0.3;
MTL_HEADER_SEARCH_PATHS = "$(SRCROOT)/IOSAccessAssessment/**";
OTHER_CFLAGS = "";
PRODUCT_BUNDLE_IDENTIFIER = edu.uw.pointmapper;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
Expand Down Expand Up @@ -1485,6 +1502,7 @@
LD_RUNPATH_SEARCH_PATHS = "$(inherited)";
MARKETING_VERSION = 0.3;
MTL_HEADER_SEARCH_PATHS = "$(SRCROOT)/IOSAccessAssessment/**";
OTHER_CFLAGS = "";
PRODUCT_BUNDLE_IDENTIFIER = edu.uw.pointmapper;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
Expand Down
43 changes: 43 additions & 0 deletions IOSAccessAssessment/AccessibilityFeature/Attributes/AccessibilityFeatureAttribute.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ enum AccessibilityFeatureAttribute: String, Identifiable, CaseIterable, Codable,
case lidarDepth
case latitudeDelta
case longitudeDelta
/**
- NOTE:
Legacy attributes for comparison with older data
*/
case widthLegacy
case runningSlopeLegacy
case crossSlopeLegacy

enum Value: Sendable, Codable, Equatable {
case length(Measurement<UnitLength>)
Expand Down Expand Up @@ -97,6 +104,24 @@ enum AccessibilityFeatureAttribute: String, Identifiable, CaseIterable, Codable,
valueType: .length(Measurement(value: 0, unit: .meters)),
osmTagKey: APIConstants.TagKeys.longitudeDeltaKey
)
case .widthLegacy:
return Metadata(
id: 10, name: "Width Legacy", unit: UnitLength.meters,
valueType: .length(Measurement(value: 0, unit: .meters)),
osmTagKey: "width_legacy"
)
case .runningSlopeLegacy:
return Metadata(
id: 20, name: "Running Slope Legacy", unit: UnitAngle.degrees,
valueType: .angle(Measurement(value: 0, unit: .degrees)),
osmTagKey: "incline_legacy"
)
case .crossSlopeLegacy:
return Metadata(
id: 30, name: "Cross Slope Legacy", unit: UnitAngle.degrees,
valueType: .angle(Measurement(value: 0, unit: .degrees)),
osmTagKey: "cross_slope_legacy"
)
}
}

Expand Down Expand Up @@ -150,6 +175,12 @@ extension AccessibilityFeatureAttribute {
return true
case (.longitudeDelta, .length):
return true
case (.widthLegacy, .length):
return true
case (.runningSlopeLegacy, .angle):
return true
case (.crossSlopeLegacy, .angle):
return true
default:
return false
}
Expand Down Expand Up @@ -198,6 +229,12 @@ extension AccessibilityFeatureAttribute {
return .length(Measurement(value: value, unit: .meters))
case .longitudeDelta:
return .length(Measurement(value: value, unit: .meters))
case .widthLegacy:
return .length(Measurement(value: value, unit: .meters))
case .runningSlopeLegacy:
return .angle(Measurement(value: value, unit: .degrees))
case .crossSlopeLegacy:
return .angle(Measurement(value: value, unit: .degrees))
}
}

Expand Down Expand Up @@ -229,6 +266,12 @@ extension AccessibilityFeatureAttribute {
return String(format: "%.2f", measurement.converted(to: .meters).value)
case (.longitudeDelta, .length(let measurement)):
return String(format: "%.2f", measurement.converted(to: .meters).value)
case (.widthLegacy, .length(let measurement)):
return String(format: "%.2f", measurement.converted(to: .meters).value)
case (.runningSlopeLegacy, .angle(let measurement)):
return String(format: "%.2f", measurement.converted(to: .degrees).value)
case (.crossSlopeLegacy, .angle(let measurement)):
return String(format: "%.2f", measurement.converted(to: .degrees).value)
default:
return nil
}
Expand Down
5 changes: 4 additions & 1 deletion IOSAccessAssessment/AccessibilityFeature/Config/MapillaryCustom11ClassConfig.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ extension AccessibilityFeatureConfig {
x: 0.0, y: 0.5, width: 1.0, height: 0.4
),
meshClassification: [.floor],
attributes: [.width, .runningSlope, .crossSlope, .surfaceIntegrity],
attributes: [
.width, .runningSlope, .crossSlope, .surfaceIntegrity,
.widthLegacy, .runningSlopeLegacy, .crossSlopeLegacy
],
oswPolicy: OSWPolicy(oswElementClass: .Sidewalk)
),

Expand Down
9 changes: 9 additions & 0 deletions IOSAccessAssessment/AttributeEstimation/AttributeEstimationPipeline.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,15 @@ class AttributeEstimationPipeline: ObservableObject {
case .crossSlope:
let crossSlopeAttributeValue = try self.calculateCrossSlope(accessibilityFeature: accessibilityFeature)
try accessibilityFeature.setAttributeValue(crossSlopeAttributeValue, for: .crossSlope, isCalculated: true)
case .widthLegacy:
let widthAttributeValue = try self.calculateWidth(accessibilityFeature: accessibilityFeature)
try accessibilityFeature.setAttributeValue(widthAttributeValue, for: .widthLegacy, isCalculated: true)
case .runningSlopeLegacy:
let runningSlopeAttributeValue = try self.calculateRunningSlope(accessibilityFeature: accessibilityFeature)
try accessibilityFeature.setAttributeValue(runningSlopeAttributeValue, for: .runningSlopeLegacy, isCalculated: true)
case .crossSlopeLegacy:
let crossSlopeAttributeValue = try self.calculateCrossSlope(accessibilityFeature: accessibilityFeature)
try accessibilityFeature.setAttributeValue(crossSlopeAttributeValue, for: .crossSlopeLegacy, isCalculated: true)
default:
continue
}
Expand Down
23 changes: 16 additions & 7 deletions IOSAccessAssessment/AttributeEstimation/Localization/LocalizationProcessor.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,10 @@ struct LocalizationProcessor {
cameraTransform: cameraTransform,
cameraIntrinsics: cameraIntrinsics
)
let latitudeDelta = -delta.z
let longitudeDelta = delta.x
return self.calculateLocation(
latitudeDelta: delta.z, longitudeDelta: delta.x,
latitudeDelta: latitudeDelta, longitudeDelta: longitudeDelta,
deviceLocation: deviceLocation
)
}
Expand All @@ -81,7 +83,7 @@ struct LocalizationProcessor {
cameraTransform: cameraTransform,
cameraIntrinsics: cameraIntrinsics
)
return SIMD2<Float>(delta.z, delta.x)
return SIMD2<Float>( -delta.z, delta.x )
}

func calculateLocation(
Expand Down Expand Up @@ -149,8 +151,6 @@ struct LocalizationProcessor {
cameraTransform.columns.3.y,
cameraTransform.columns.3.z)
var delta = worldPoint - cameraOriginPoint
/// Fix the z-axis back so that it points north
delta.z = -delta.z

return delta
}
Expand Down Expand Up @@ -194,10 +194,13 @@ extension LocalizationProcessor {
throw LocalizationProcessorError.invalidBounds
}
let trapezoidDeltas = trapezoidBoundsWithDepth.map { pointWithDepth in
return getDeltaFromPoint(
let delta = getDeltaFromPoint(
point: pointWithDepth.point, depth: pointWithDepth.depth, imageSize: imageSize,
cameraTransform: cameraTransform, cameraIntrinsics: cameraIntrinsics
)
var deltaNorth = delta
deltaNorth.z = -deltaNorth.z
return deltaNorth
}
let bottomLeft = trapezoidDeltas[0]
let topLeft = trapezoidDeltas[1]
Expand Down Expand Up @@ -240,10 +243,13 @@ extension LocalizationProcessor {
throw LocalizationProcessorError.invalidBounds
}
let trapezoidDeltas = trapezoidBoundsWithDepth.map { pointWithDepth in
return getDeltaFromPoint(
let delta = getDeltaFromPoint(
point: pointWithDepth.point, depth: pointWithDepth.depth, imageSize: imageSize,
cameraTransform: cameraTransform, cameraIntrinsics: cameraIntrinsics
)
var deltaNorth = delta
deltaNorth.z = -deltaNorth.z
return deltaNorth
}
let bottomLeft = trapezoidDeltas[0]
let topLeft = trapezoidDeltas[1]
Expand Down Expand Up @@ -275,10 +281,13 @@ extension LocalizationProcessor {
throw LocalizationProcessorError.invalidBounds
}
let trapezoidDeltas = trapezoidBoundsWithDepth.map { pointWithDepth in
return getDeltaFromPoint(
let delta = getDeltaFromPoint(
point: pointWithDepth.point, depth: pointWithDepth.depth, imageSize: imageSize,
cameraTransform: cameraTransform, cameraIntrinsics: cameraIntrinsics
)
var deltaNorth = delta
deltaNorth.z = -deltaNorth.z
return deltaNorth
}
let bottomLeft = trapezoidDeltas[0]
let topLeft = trapezoidDeltas[1]
Expand Down
7 changes: 7 additions & 0 deletions IOSAccessAssessment/AttributeEstimation/Plane/PlaneFit.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
//
// PlaneFit.swift
// IOSAccessAssessment
//
// Created by Himanshu on 1/24/26.
//

69 changes: 69 additions & 0 deletions IOSAccessAssessment/AttributeEstimation/Plane/PlaneFitting.metal
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
//
// PlaneFitting.metal
// IOSAccessAssessment
//
// Created by Himanshu on 1/24/26.
//

#include <metal_stdlib>
#include <CoreImage/CoreImage.h>
using namespace metal;
#import "ShaderTypes.h"

inline float3 projectPixelToWorld(
float2 pixelCoord,
float depthValue,
constant float4x4& cameraTransform,
constant float3x3& invIntrinsics
) {
float3 imagePoint = float3(pixelCoord, 1.0);
float3 ray = invIntrinsics * imagePoint;
float3 rayDirection = normalize(ray);

float3 cameraPoint = rayDirection * depthValue;
cameraPoint = float3(cameraPoint.x, -cameraPoint.y, -cameraPoint.z);
float4 cameraPoint4 = float4(cameraPoint, 1.0);

float4 worldPoint4 = cameraTransform * cameraPoint4;
float3 worldPoint = worldPoint4.xyz / worldPoint4.w;

return worldPoint;
}

// Plane Fitting Point Extraction Kernel
// Assumes the depth texture is the same size as the segmentation texture
kernel void computePlanePoints(
texture2d<float, access::read> segmentationTexture [[texture(0)]],
texture2d<float, access::read> depthTexture [[texture(1)]],
constant uint8_t& targetValue [[buffer(0)]],
constant PlanePointsParams& params [[buffer(1)]],
device PlanePoint* points [[buffer(2)]],
device atomic_uint* pointCount [[buffer(3)]],
uint2 gid [[thread_position_in_grid]]
) {
if (gid.x >= segmentationTexture.get_width() || gid.y >= segmentationTexture.get_height())
return;

float4 pixelColor = segmentationTexture.read(gid);
float grayscale = pixelColor.r;

// Normalize grayscale to the range of the LUT
uint index = min(uint(round(grayscale * 255.0)), 255u);
if (index != targetValue) {
return;
}
float depthValue = depthTexture.read(gid).r;
if (depthValue <= params.minDepthThreshold || depthValue >= params.maxDepthThreshold) {
return;
}

float3 worldPoint = projectPixelToWorld(
float2(gid),
depthValue,
params.cameraTransform,
params.invIntrinsics
);

uint idx = atomic_fetch_add_explicit(pointCount, 1u, memory_order_relaxed);
outPoints[idx].p = worldPoint;
}
1 change: 1 addition & 0 deletions IOSAccessAssessment/Image/Depth/DepthMapProcessor.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ struct DepthMapProcessor {
pixelFormatType: kCVPixelFormatType_DepthFloat32,
colorSpace: nil
)
print("DepthMapProcessor initialized with depth image of size: \(depthWidth)x\(depthHeight)")
}

private func getDepthAtPoint(point: CGPoint) throws -> Float {
Expand Down
16 changes: 16 additions & 0 deletions IOSAccessAssessment/ShaderTypes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
typedef uint8_t MTL_UINT8; // 8-bit
typedef uint MTL_UINT; // 32-bit
typedef uint MTL_BOOL; // use 0/1
typedef float3 MTL_FLOAT3;
typedef float4x4 MTL_FLOAT4X4;
typedef float3x3 MTL_FLOAT3X3; // 48 bytes (3 cols, 16B aligned)
typedef uint2 MTL_UINT2;
Expand All @@ -24,6 +25,7 @@
typedef uint8_t MTL_UINT8; // 8-bit
typedef uint32_t MTL_UINT;
typedef uint32_t MTL_BOOL; // 0/1
typedef simd_float3 MTL_FLOAT3;
typedef simd_float4x4 MTL_FLOAT4X4;
typedef simd_float3x3 MTL_FLOAT3X3; // 48 bytes
typedef simd_uint2 MTL_UINT2;
Expand Down Expand Up @@ -59,3 +61,17 @@ typedef struct BoundsParams {
float maxX;
float maxY;
} BoundsParams;

// For PCA Plane Fitting
typedef struct PlanePoint {
MTL_FLOAT3 p;
} PlanePoint;

typedef struct PlanePointsParams {
MTL_UINT2 imageSize;
float minDepthThreshold;
float maxDepthThreshold;
MTL_FLOAT4X4 cameraTransform;
MTL_FLOAT3X3 invIntrinsics;
} PlanePointsParams;

Loading